From a29cbb143c415618b9f835bc244eacd6364b379d Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 11 Jan 2023 13:50:05 +1100 Subject: [PATCH] Huge change, removed Status class and all "alert" messages are now shown as BS toast() and are via the DB and handled async in the F/E in jscript via Ajax. Fixed BUG-113 where toasts() were repeating. Removed many of the explicit alert messages (other than errors) and hooked {New|Finish}Job to consistently send messages to the F/E. Other messages (F/E without a job, like save settings) now use this model as well. Finally converted most of the older POST responses to formal json --- BUGs | 3 +- TODO | 11 ++--- ai.py | 10 ++--- dups.py | 2 - files.py | 55 +++++++++---------------- internal/js/files_support.js | 2 +- internal/js/jobs.js | 47 +++++++++------------- internal/js/view_support.js | 1 + job.py | 55 ++++++++++++++++++------- main.py | 8 ---- pa_job_manager.py | 78 +++++++++++++++++------------------- person.py | 74 ++++++++++++++-------------------- settings.py | 7 ++-- templates/base.html | 25 ++---------- user.py | 1 - 15 files changed, 162 insertions(+), 217 deletions(-) diff --git a/BUGs b/BUGs index 08a5a4e..80a7dba 100644 --- a/BUGs +++ b/BUGs @@ -1,4 +1,4 @@ -### Next: 113 +### Next: 114 BUG-100: I managed to get 2 photos matching mich in the NOT_WORKING photo (probably dif refimgs but same p.tag?) = /photos/2012/20120414-damien/IMG_8467.JPG BUG-106: cant add trudy /pat? as refimgs via FaceDBox @@ -6,4 +6,3 @@ BUG-106: cant add trudy /pat? as refimgs via FaceDBox (it came from a face bbox, BUT, I have grown the face seln by 10%?) BUG-109: add mich force override, removed it, then re-added it, then rebuild DB form scratch and metadata has a duplicate - redo disco metadata with md5 not UUID of face dataS -BUG-113: with a stale job, we keep checking each 1 second, and so we keep creating new status messages diff --git a/TODO b/TODO index d78f430..cc6f236 100644 --- a/TODO +++ b/TODO @@ -1,19 +1,14 @@ ### GENERAL - * get all status messages to use toasts AND get func to also increase/descrease the job counter as appropriate) - - [DONE] all (success/creation) status messages use toasts - -- [TODO] ensure all pa_job_mgr jobs are sending messages back to the FE -- SHOULD??? I just hook FinishJob??? - * should be using jsonify to return real json to my API calls, e.g: use make_response( jsonify (... ) ) -- [TODO] all POSTs should follow new status behaviour (unless its a form POST, as that immediately renders the form in flask and will show the Status) all GETs stay as is for now (as they expect a html reply, not data, and then html is base.html+ and it handles the status) - -- ONLY time this will fail is multiple status messages (which can occur I believe if a Wake of job mgr and then a job is created in DB, the success will only be shown) - -- [TODO] simple fix will be to get 'hidden' jobs (wake job_mgr and maybe? metadata) to post failure events to the JobMgr_FE DB queue - -- [TODO] -- change class Status to class UILog - -- [TODO] -- change alert to level + -- [TODO] find persons needs an array returned - test this, also viewlist * delete files should behave like /move_files (stay on same page) as well as the status messages above + * all routes should be consistent naming conventions (with or without _ ) + * change the rotation code to use that jpeg util to reduce/remove compression loss? * ignore face should ignore ALL matching faces (re: Declan) diff --git a/ai.py b/ai.py index 46c0070..416f625 100644 --- a/ai.py +++ b/ai.py @@ -4,7 +4,6 @@ from flask import request, render_template, redirect from main import db, app, ma from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError -from status import st, Status from path import Path, PathType from files import Entry, Dir, File, PathDirLink from person import Refimg, Person, PersonRefimgLink @@ -52,8 +51,7 @@ def run_ai_on(): jex=[] for el in request.form: jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) - job=NewJob( "run_ai_on", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to Look for face(s) in selected file(s)") + job=NewJob( "run_ai_on", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in selected file(s)" ) return redirect("/jobs") @app.route("/run_ai_on_import", methods=["GET"]) @@ -64,8 +62,7 @@ def run_ai_on_import(): ptype=PathType.query.filter(PathType.name=='Import').first() jex.append( JobExtra( name=f"person", value="all" ) ) jex.append( JobExtra( name=f"path_type", value=ptype.id ) ) - job=NewJob( "run_ai_on_path", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to Look for face(s) in import path(s)") + job=NewJob( "run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in import path(s)") return redirect("/jobs") @app.route("/run_ai_on_storage", methods=["GET"]) @@ -75,8 +72,7 @@ def run_ai_on_storage(): ptype=PathType.query.filter(PathType.name=='Storage').first() jex.append( JobExtra( name=f"person", value="all" ) ) jex.append( JobExtra( name=f"path_type", value=ptype.id ) ) - job=NewJob( "run_ai_on_path", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to Look for face(s) in storage path(s)") + job=NewJob( "run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in storage path(s)") return redirect("/jobs") @app.route("/unmatched_faces", methods=["GET"]) diff --git a/dups.py b/dups.py index a3769e6..3106933 100644 --- a/dups.py +++ b/dups.py @@ -4,7 +4,6 @@ from flask import request, render_template, send_from_directory from main import db, app, ma from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError -from status import st, Status import os import glob from PIL import Image @@ -20,7 +19,6 @@ import re ################################################################################ # Local Class imports ################################################################################ -from job import Job, JobExtra, Joblog, NewJob from settings import Settings from shared import SymlinkName, PA from path import PathType diff --git a/files.py b/files.py index dbbf540..385aaa0 100644 --- a/files.py +++ b/files.py @@ -4,7 +4,6 @@ from flask import request, render_template, redirect, send_from_directory, url_f from main import db, app, ma from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError -from status import st, Status import os import glob from PIL import Image @@ -24,7 +23,7 @@ from states import States, PA_UserState ################################################################################ # Local Class imports ################################################################################ -from job import Job, JobExtra, Joblog, NewJob +from job import Job, JobExtra, Joblog, NewJob, SetFELog from path import PathType, Path, MovePathDetails from person import Refimg, Person, PersonRefimgLink from settings import Settings, SettingsIPath, SettingsSPath, SettingsRBPath @@ -463,13 +462,12 @@ def search(search_term): return render_template("files.html", page_title='View Files', search_term=search_term, entry_data=entries, OPT=OPT, move_paths=move_paths ) ################################################################################ -# /files/scannow -> allows us to force a check for new files +# /files/scan_ip -> allows us to force a check for new files ################################################################################ -@app.route("/files/scannow", methods=["GET"]) +@app.route("/files/scan_ip", methods=["GET"]) @login_required -def scannow(): - job=NewJob("scannow" ) - st.SetMessage("scanning for new files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) ) +def scan_ip(): + job=NewJob( name="scan_ip", num_files=0, wait_for=None, jex=None, desc="scan for new files in import path" ) return redirect("/jobs") ################################################################################ @@ -478,8 +476,7 @@ def scannow(): @app.route("/files/forcescan", methods=["GET"]) @login_required def forcescan(): - job=NewJob("forcescan" ) - st.SetMessage("force scan & rebuild data for files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) ) + job=NewJob( name="forcescan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" ) return redirect("/jobs") ################################################################################ @@ -488,8 +485,7 @@ def forcescan(): @app.route("/files/scan_sp", methods=["GET"]) @login_required def scan_sp(): - job=NewJob("scan_sp" ) - st.SetMessage("scanning for new files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) ) + job=NewJob( name="scan_sp", num_files=0, wait_for=None, jex=None, desc="scan for new files in storage path" ) return redirect("/jobs") @@ -504,7 +500,7 @@ def fix_dups(): rows = db.engine.execute( "select e1.id as id1, f1.hash, d1.rel_path as rel_path1, d1.eid as did1, e1.name as fname1, p1.id as path1, p1.type_id as path_type1, e2.id as id2, d2.rel_path as rel_path2, d2.eid as did2, e2.name as fname2, p2.id as path2, p2.type_id as path_type2 from entry e1, file f1, dir d1, entry_dir_link edl1, path_dir_link pdl1, path p1, entry e2, file f2, dir d2, entry_dir_link edl2, path_dir_link pdl2, path p2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and p1.type_id != (select id from path_type where name = 'Bin') and p1.id = pdl1.path_id and pdl1.dir_eid = d1.eid and p2.type_id != (select id from path_type where name = 'Bin') and p2.id = pdl2.path_id and pdl2.dir_eid = d2.eid and f1.hash = f2.hash and e1.id != e2.id and f1.size_mb = f2.size_mb order by path1, rel_path1, fname1"); if rows.returns_rows == False: - st.SetMessage(f"Err, no dups - should now clear the FE 'danger' message?") + SetFELog(f"Err, no dups - should now clear the FE 'danger' message?", "warning") return redirect("/") if 'pagesize' not in request.form: @@ -546,8 +542,7 @@ def rm_dups(): jex.append( JobExtra( name="pagesize", value=10 ) ) - job=NewJob( "rmdups", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to delete duplicate files") + job=NewJob( name="rmdups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" ) return redirect("/jobs") @@ -561,8 +556,7 @@ def restore_files(): for el in request.form: jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) - job=NewJob( "restore_files", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to restore selected file(s)") + job=NewJob( name="restore_files", num_files=0, wait_for=None, jex=jex, desc="to restore selected file(s)" ) return redirect("/jobs") ################################################################################ @@ -575,8 +569,7 @@ def delete_files(): for el in request.form: jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) - job=NewJob( "delete_files", 0, None, jex ) - st.SetMessage( f"Created Job #{job.id} to delete selected file(s)") + job=NewJob( name="delete_files", num_files=0, wait_for=None, jex=jex, desc="to delete selected file(s)" ) return redirect("/jobs") ################################################################################ @@ -589,11 +582,9 @@ def move_files(): jex=[] for el in request.form: jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) - job=NewJob( "move_files", 0, None, jex ) - return make_response( jsonify( - job_id=job.id, - message=f"Created Job #{job.id} to move selected file(s)", - level="success", alert="success", persistent=False, cant_close=False ) ) + job=NewJob( name="move_files", num_files=0, wait_for=None, jex=jex, desc="to move selected file(s)" ) + # data is not used, but send response to trigger CheckForJobs() + return make_response( jsonify( job_id=job.id ) ) @login_required @app.route("/viewlist", methods=["POST"]) @@ -700,7 +691,7 @@ def view(id): msg += "Clearing out all states. This means browser back buttons will not work, please start a new tab and try again" PA_UserState.query.delete() db.session.commit() - st.SetMessage( msg, "warning" ) + SetFELog( msg, "warning" ) return redirect("/") else: NMO_data = FaceOverrideType.query.all() @@ -735,12 +726,8 @@ def transform(): for el in request.form: jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) - job=NewJob( "transform_image", 0, None, jex ) - - resp={} - resp['job_id']=job.id - - return resp + job=NewJob( name="transform_image", num_files=0, wait_for=None, jex=jex, desc="to transform selected file(s)" ) + return make_response( jsonify( job_id=job.id ) ) ################################################################################ # /checktransformjob -> URL that is called repeatedly by front-end waiting for the @@ -753,14 +740,12 @@ def transform(): def checktransformjob(): job_id = request.form['job_id'] job = Job.query.get(job_id) - resp={} - resp['finished']=False + j=jsonify( finished=False ) if job.pa_job_state == 'Completed': id=[jex.value for jex in job.extra if jex.name == "id"][0] e=Entry.query.join(File).filter(Entry.id==id).first() - resp['thumbnail']=e.file_details.thumbnail - resp['finished']=True - return resp + j=jsonify( finished=True, thumbnail=e.file_details.thumbnail ) + return make_response( j ) ################################################################################ # /include -> return contents on /include and does not need a login, so we diff --git a/internal/js/files_support.js b/internal/js/files_support.js index 29d2f74..445875a 100644 --- a/internal/js/files_support.js +++ b/internal/js/files_support.js @@ -64,7 +64,7 @@ function MoveSubmit() // reorder the images via ecnt again, so highlighting can work document.mf_id=0; $('.figure').each( function() { $(this).attr('ecnt', document.mf_id ); document.mf_id++ } ) $('#dbox').modal('hide') - $.ajax({ type: 'POST', data: $('#mv_fm').serialize(), url: '/move_files', success: function(data){ console.log(data); StatusMsg(data); CheckForJobs(); return false; } }) + $.ajax({ type: 'POST', data: $('#mv_fm').serialize(), url: '/move_files', success: function(data){ CheckForJobs(); return false; } }) } // show the DBox for a move file, includes all thumbnails of selected files to move diff --git a/internal/js/jobs.js b/internal/js/jobs.js index f26bab2..c97f6ca 100644 --- a/internal/js/jobs.js +++ b/internal/js/jobs.js @@ -1,30 +1,31 @@ -// global -var next_toast_id=1 - function NewToast(data) { - console.log(data) - // make new div, include data.alert as background colour, and data.message as toast body - d_id='st' + String(next_toast_id) + // toast "id" is based on msg_id, for any persistent/long-lived msg's we will try to reshow them + // dont bother, if it already exists it is visible, just move on + d_id='st' + String(data.id) + if( $('#'+d_id).length !== 0 ) + return + // make new div, include data.level as background colour, and data.message as toast body div=' - - +
+ diff --git a/user.py b/user.py index 807ae9f..933073e 100644 --- a/user.py +++ b/user.py @@ -4,7 +4,6 @@ from flask import request, redirect from flask_login import UserMixin, login_required from main import db, app, ma from sqlalchemy.exc import SQLAlchemyError -from status import st, Status # pylint: disable=no-member