implemented job archiving (via a Setting, and just viewing recent or all)

TODO

@@ -6,7 +6,7 @@
             - [DONE] joblog page should show last X logs, <show all button>, newest X logs, 
                 - [DONE/TEST?] need to use sm-txt class more as the space is too constrained
                 - make clickable sort toggles
-            - need to archive jobs
+            - [DONE] need to archive jobs
 
     * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach?
 

job.py

@@ -1,8 +1,9 @@
 from wtforms import SubmitField, StringField, FloatField, HiddenField, validators, Form
 from flask_wtf import FlaskForm
 from flask import request, render_template
+from settings import Settings
 from main import db, app, ma
-from sqlalchemy import Sequence
+from sqlalchemy import Sequence, func
 from sqlalchemy.exc import SQLAlchemyError
 from status import st, Status
 from datetime import datetime, timedelta
@@ -11,6 +12,8 @@ import pytz
 import socket
 from shared import PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, NEWEST_LOG_LIMIT, OLDEST_LOG_LIMIT
 from flask_login import login_required, current_user
+from sqlalchemy.dialects.postgresql import INTERVAL
+from sqlalchemy.sql.functions import concat
 
 # pylint: disable=no-member
 
@@ -103,13 +106,20 @@ def NewJob(name, num_files="0", wait_for=None, jex=None ):
     return job
 
 ################################################################################
-# /jobs -> show current settings
+# /jobs -> show jobs (default to only showing 'non-archived' jobs -- age is in 
+# settings.job_archive_age
 ################################################################################
-@app.route("/jobs", methods=["GET"])
+@app.route("/jobs", methods=["GET", "POST"])
 @login_required
 def jobs():
-    page_title='Job list'
+    settings = Settings.query.first()
-    jobs = Job.query.order_by(Job.id.desc()).all() 
+    print( request.method )
+    if request.method == 'POST':
+        page_title='Job list (all)'
+        jobs = Job.query.order_by(Job.id.desc()).all() 
+    else:
+        page_title='Job list (recent)'
+        jobs = Job.query.filter( Job.last_update >= (func.now() - func.cast(concat(settings.job_archive_age, 'DAYS'), INTERVAL)) ).order_by(Job.id.desc()).all() 
     return render_template("jobs.html", jobs=jobs, page_title=page_title)
 
 

settings.py

@@ -37,9 +37,10 @@ class Settings(db.Model):
     scheduled_storage_scan = db.Column(db.Integer)
     scheduled_bin_cleanup = db.Column(db.Integer)
     bin_cleanup_file_age = db.Column(db.Integer)
+    job_archive_age = db.Column(db.Integer)
 
     def __repr__(self):
-        return f"<id: {self.id}, import_path: {self.import_path}, storage_path: {self.storage_path}, recycle_bin_path: {self.recycle_bin_path}, default_refimg_model: {self.default_refimg_model}, default_scan_model: {self.default_scan_model}, default_threshold: {self.default_threshold}, scheduled_import_scan:{self.scheduled_import_scan}, scheduled_storage_scan: {self.scheduled_storage_scan}, scheduled_bin_cleanup: {Self.scheduled_bin_cleanup}, bin_cleanup_file_age: {self.bin_cleanup_file_age} >"
+        return f"<id: {self.id}, import_path: {self.import_path}, storage_path: {self.storage_path}, recycle_bin_path: {self.recycle_bin_path}, default_refimg_model: {self.default_refimg_model}, default_scan_model: {self.default_scan_model}, default_threshold: {self.default_threshold}, scheduled_import_scan:{self.scheduled_import_scan}, scheduled_storage_scan: {self.scheduled_storage_scan}, scheduled_bin_cleanup: {self.scheduled_bin_cleanup}, bin_cleanup_file_age: {self.bin_cleanup_file_age}, job_archive_age: {self.job_archive_age}>"
 
 ################################################################################
 # Helper class that inherits a .dump() method to turn class Settings into json / useful in jinja2
@@ -68,6 +69,7 @@ class SettingsForm(FlaskForm):
     scheduled_storage_scan = IntegerField('Days between forced scan of storage path', [validators.DataRequired()])
     scheduled_bin_cleanup = IntegerField('Days between checking to clean Recycle Bin:', [validators.DataRequired()])
     bin_cleanup_file_age = IntegerField('Age of files to clean out of the Recycle Bin', [validators.DataRequired()])
+    job_archive_age = IntegerField('Age of jobs to archive', [validators.DataRequired()])
     submit = SubmitField('Save' )
 
 ################################################################################
@@ -90,6 +92,7 @@ def settings():
     HELP['scheduled_storage_scan']="The # of days between forced scans of the storage path for any file system changes outside of Photo Assistant"
     HELP['scheduled_bin_cleanup']="The # of days between running a job to delete old files from the Recycle Bin"
     HELP['bin_cleanup_file_age']="The # of days a file has to exist in the Recycle Bin before it can be really deleted"
+    HELP['job_archive_age']="The # of days a job has to exist for to be archived"
 
     if request.method == 'POST' and form.validate():
         try:
@@ -110,6 +113,7 @@ def settings():
                 s.scheduled_storage_scan = request.form['scheduled_storage_scan']
                 s.scheduled_bin_cleanup = request.form['scheduled_bin_cleanup']
                 s.bin_cleanup_file_age = request.form['bin_cleanup_file_age']
+                s.job_archive_age = request.form['job_archive_age']
             db.session.commit()
             return redirect( url_for( 'settings' ) )
         except SQLAlchemyError as e:

shared.py

@@ -24,7 +24,7 @@ ICON["Storage"]="db"
 ICON["Bin"]="trash"
 
 SECS_IN_A_DAY = 86400
-NEWEST_LOG_LIMIT = 5
+NEWEST_LOG_LIMIT = 15
 OLDEST_LOG_LIMIT = 5
 
 # check where we are running, if laptop, then run web server and db on localhost

tables.sql

@@ -8,7 +8,9 @@ insert into AI_MODEL values ( 2, 'cnn', 'more accurate / much slower' );
 create table SETTINGS( 
     ID integer, BASE_PATH varchar, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, 
     DEFAULT_REFIMG_MODEL integer, DEFAULT_SCAN_MODEL integer, DEFAULT_THRESHOLD float,
-    SCHEDULED_IMPORT_SCAN integer, SCHEDULED_STORAGE_SCAN integer, SCHEDULED_BIN_CLEANUP integer, BIN_CLEANUP_FILE_AGE integer,
+    SCHEDULED_IMPORT_SCAN integer, SCHEDULED_STORAGE_SCAN integer, 
+    SCHEDULED_BIN_CLEANUP integer, BIN_CLEANUP_FILE_AGE integer,
+    JOB_ARCHIVE_AGE integer,
     constraint PK_SETTINGS_ID primary key(ID),
     constraint FK_DEFAULT_REFIMG_MODEL foreign key (DEFAULT_REFIMG_MODEL) references AI_MODEL(ID),
     constraint FK_DEFAULT_SCAN_MODEL foreign key (DEFAULT_SCAN_MODEL) references AI_MODEL(ID) );
@@ -126,8 +128,8 @@ insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mum', 'Mandy', '
 insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' );
 insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' );
 -- DEV(ddp):
-insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#new_img_dir/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30 );
+insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#new_img_dir/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 3 );
 -- DEV(cam):
---insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', 2, 1, '0.55' );
+--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 3 );
 -- PROD:
---insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30 );
+--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55', 1, 1, 7, 30, 4 );

templates/jobs.html

@@ -86,5 +86,14 @@
         for(el in completed_rows)
             $('#job_tbl_body').append(completed_rows[el])
         {% endif %}
+        {% if 'recent' in page_title %}
+            tr=`
+                <tr>
+                    <td><button type="button" class="btn btn-outline-info my-0 py-1" onClick="document.body.innerHTML+='<form id=_fm method=POST action={{url_for('jobs')}}></form>';document.getElementById('_fm').submit();">Show all jobs</button>
+                    </td>
+                 </tr>`
+            $('#job_tbl_body').append( tr )
+        {% endif %}
+
     </script>
 {% endblock main_content %}