From a2fb877f12bad753a39435a7db18ac7620d1372d Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 13 Jan 2022 23:11:21 +1100 Subject: [PATCH] implemented job archiving (via a Setting, and just viewing recent or all) --- TODO | 2 +- job.py | 20 +++++++++++++++----- settings.py | 6 +++++- shared.py | 2 +- tables.sql | 10 ++++++---- templates/jobs.html | 9 +++++++++ 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/TODO b/TODO index 1f09b37..2c455dc 100644 --- a/TODO +++ b/TODO @@ -6,7 +6,7 @@ - [DONE] joblog page should show last X logs, , newest X logs, - [DONE/TEST?] need to use sm-txt class more as the space is too constrained - make clickable sort toggles - - need to archive jobs + - [DONE] need to archive jobs * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach? diff --git a/job.py b/job.py index 8c16c60..fda7437 100644 --- a/job.py +++ b/job.py @@ -1,8 +1,9 @@ from wtforms import SubmitField, StringField, FloatField, HiddenField, validators, Form from flask_wtf import FlaskForm from flask import request, render_template +from settings import Settings from main import db, app, ma -from sqlalchemy import Sequence +from sqlalchemy import Sequence, func from sqlalchemy.exc import SQLAlchemyError from status import st, Status from datetime import datetime, timedelta @@ -11,6 +12,8 @@ import pytz import socket from shared import PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, NEWEST_LOG_LIMIT, OLDEST_LOG_LIMIT from flask_login import login_required, current_user +from sqlalchemy.dialects.postgresql import INTERVAL +from sqlalchemy.sql.functions import concat # pylint: disable=no-member @@ -103,13 +106,20 @@ def NewJob(name, num_files="0", wait_for=None, jex=None ): return job ################################################################################ -# /jobs -> show current settings +# /jobs -> show jobs (default to only showing 'non-archived' jobs -- age is in +# settings.job_archive_age ################################################################################ -@app.route("/jobs", methods=["GET"]) +@app.route("/jobs", methods=["GET", "POST"]) @login_required def jobs(): - page_title='Job list' - jobs = Job.query.order_by(Job.id.desc()).all() + settings = Settings.query.first() + print( request.method ) + if request.method == 'POST': + page_title='Job list (all)' + jobs = Job.query.order_by(Job.id.desc()).all() + else: + page_title='Job list (recent)' + jobs = Job.query.filter( Job.last_update >= (func.now() - func.cast(concat(settings.job_archive_age, 'DAYS'), INTERVAL)) ).order_by(Job.id.desc()).all() return render_template("jobs.html", jobs=jobs, page_title=page_title) diff --git a/settings.py b/settings.py index 73f463b..f1dac02 100644 --- a/settings.py +++ b/settings.py @@ -37,9 +37,10 @@ class Settings(db.Model): scheduled_storage_scan = db.Column(db.Integer) scheduled_bin_cleanup = db.Column(db.Integer) bin_cleanup_file_age = db.Column(db.Integer) + job_archive_age = db.Column(db.Integer) def __repr__(self): - return f"" + return f"" ################################################################################ # Helper class that inherits a .dump() method to turn class Settings into json / useful in jinja2 @@ -68,6 +69,7 @@ class SettingsForm(FlaskForm): scheduled_storage_scan = IntegerField('Days between forced scan of storage path', [validators.DataRequired()]) scheduled_bin_cleanup = IntegerField('Days between checking to clean Recycle Bin:', [validators.DataRequired()]) bin_cleanup_file_age = IntegerField('Age of files to clean out of the Recycle Bin', [validators.DataRequired()]) + job_archive_age = IntegerField('Age of jobs to archive', [validators.DataRequired()]) submit = SubmitField('Save' ) ################################################################################ @@ -90,6 +92,7 @@ def settings(): HELP['scheduled_storage_scan']="The # of days between forced scans of the storage path for any file system changes outside of Photo Assistant" HELP['scheduled_bin_cleanup']="The # of days between running a job to delete old files from the Recycle Bin" HELP['bin_cleanup_file_age']="The # of days a file has to exist in the Recycle Bin before it can be really deleted" + HELP['job_archive_age']="The # of days a job has to exist for to be archived" if request.method == 'POST' and form.validate(): try: @@ -110,6 +113,7 @@ def settings(): s.scheduled_storage_scan = request.form['scheduled_storage_scan'] s.scheduled_bin_cleanup = request.form['scheduled_bin_cleanup'] s.bin_cleanup_file_age = request.form['bin_cleanup_file_age'] + s.job_archive_age = request.form['job_archive_age'] db.session.commit() return redirect( url_for( 'settings' ) ) except SQLAlchemyError as e: diff --git a/shared.py b/shared.py index 46cecfe..a8b8197 100644 --- a/shared.py +++ b/shared.py @@ -24,7 +24,7 @@ ICON["Storage"]="db" ICON["Bin"]="trash" SECS_IN_A_DAY = 86400 -NEWEST_LOG_LIMIT = 5 +NEWEST_LOG_LIMIT = 15 OLDEST_LOG_LIMIT = 5 # check where we are running, if laptop, then run web server and db on localhost diff --git a/tables.sql b/tables.sql index 024ab86..1c31791 100644 --- a/tables.sql +++ b/tables.sql @@ -8,7 +8,9 @@ insert into AI_MODEL values ( 2, 'cnn', 'more accurate / much slower' ); create table SETTINGS( ID integer, BASE_PATH varchar, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, DEFAULT_REFIMG_MODEL integer, DEFAULT_SCAN_MODEL integer, DEFAULT_THRESHOLD float, - SCHEDULED_IMPORT_SCAN integer, SCHEDULED_STORAGE_SCAN integer, SCHEDULED_BIN_CLEANUP integer, BIN_CLEANUP_FILE_AGE integer, + SCHEDULED_IMPORT_SCAN integer, SCHEDULED_STORAGE_SCAN integer, + SCHEDULED_BIN_CLEANUP integer, BIN_CLEANUP_FILE_AGE integer, + JOB_ARCHIVE_AGE integer, constraint PK_SETTINGS_ID primary key(ID), constraint FK_DEFAULT_REFIMG_MODEL foreign key (DEFAULT_REFIMG_MODEL) references AI_MODEL(ID), constraint FK_DEFAULT_SCAN_MODEL foreign key (DEFAULT_SCAN_MODEL) references AI_MODEL(ID) ); @@ -126,8 +128,8 @@ insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mum', 'Mandy', ' insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' ); insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' ); -- DEV(ddp): -insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#new_img_dir/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30 ); +insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#new_img_dir/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 3 ); -- DEV(cam): ---insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', 2, 1, '0.55' ); +--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 3 ); -- PROD: ---insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30 ); +--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 4 ); diff --git a/templates/jobs.html b/templates/jobs.html index 630db7b..a8c02d6 100644 --- a/templates/jobs.html +++ b/templates/jobs.html @@ -86,5 +86,14 @@ for(el in completed_rows) $('#job_tbl_body').append(completed_rows[el]) {% endif %} + {% if 'recent' in page_title %} + tr=` + + + + ` + $('#job_tbl_body').append( tr ) + {% endif %} + {% endblock main_content %}