From 4b1bbcb2bf28cd634538d0d524208c67d56a3b3a Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 15 Jan 2023 23:17:59 +1100 Subject: [PATCH] broad (small) changes to make style of all routes and jobs to be consistent, e.g. use underscords between words --- TODO | 8 +++--- ai.py | 8 +++--- files.py | 34 ++++++++++++------------- internal/js/files_support.js | 2 +- internal/js/files_transform.js | 3 ++- internal/js/jobs.js | 4 +-- internal/js/view_transform.js | 3 ++- job.py | 22 ++++++++-------- pa_job_manager.py | 46 +++++++++++++++++----------------- states.py | 16 ++++++------ templates/base.html | 6 ++--- templates/files.html | 4 +-- templates/jobs.html | 4 +-- templates/states.html | 2 +- templates/viewer.html | 2 +- user.py | 10 ++++---- 16 files changed, 88 insertions(+), 86 deletions(-) diff --git a/TODO b/TODO index 26088f2..6d38fc9 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,9 @@ ### GENERAL - * all routes should be consistent naming conventions (with or without _ ) - - * get build process to create a random string for secret for PROD, otherwise use builtin for dev - * think about security - in job_mgr anywhere I can os.replace/remove NEED to protect, etc + - just need to use this I think: + from werkzeug.utils import secure_filename + + secure_filename(xxxx) * change the rotation code to use that jpeg util to reduce/remove compression loss? diff --git a/ai.py b/ai.py index 773a9aa..4e727f8 100644 --- a/ai.py +++ b/ai.py @@ -18,11 +18,11 @@ from face import Face, FaceFileLink, FaceRefimgLink, FaceNoMatchOverride, FaceFo # pylint: disable=no-member ################################################################################ -# /aistats -> placholder for some sort of stats +# /ai_stats -> placholder for some sort of stats ################################################################################ -@app.route("/aistats", methods=["GET"]) +@app.route("/ai_stats", methods=["GET"]) @login_required -def aistats(): +def ai_stats(): stats = db.session.execute( "select p.tag, count(f.id) from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag order by 2 desc" ) cnt_res = db.session.execute( "select count(1) from ( select p.tag from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag ) as foo" ) num_stats=cnt_res.first()[0] @@ -38,7 +38,7 @@ def aistats(): fstats['all_matched_faces'] = db.session.execute( "select count(distinct face_id) as count from face_refimg_link" ).first()[0] fstats['all_unmatched_faces'] = db.session.execute( "select count(f.id) from face f left join face_refimg_link frl on f.id = frl.face_id where frl.refimg_id is null" ).first()[0] - return render_template("aistats.html", page_title='AI Statistics', stats=stats, num_stats=num_stats, fstats=fstats ) + return render_template("ai_stats.html", page_title='AI Statistics', stats=stats, num_stats=num_stats, fstats=fstats ) ################################################################################ diff --git a/files.py b/files.py index 20f4684..7e60955 100644 --- a/files.py +++ b/files.py @@ -320,7 +320,7 @@ def SetOrderStrings( OPT ): ################################################################################ # /GetEntries -> helper function that Gets Entries for required files to show -# for several routes (files_ip, files_sp, files_rbp, search, viewlist) +# for several routes (files_ip, files_sp, files_rbp, search, view_list) ################################################################################ def GetEntries( OPT ): entries=[] @@ -367,9 +367,9 @@ def GetEntries( OPT ): UpdatePref( pref, OPT ) return entries -@app.route("/ChangeFileOpts", methods=["POST"]) +@app.route("/change_file_opts", methods=["POST"]) @login_required -def ChangeFileOpts(): +def change_file_opts(): # reset options based on form post, then redirect back to orig page (with a GET to allow back button to work) OPT=States( request ) return redirect( request.referrer ) @@ -462,12 +462,12 @@ def scan_ip(): return redirect("/jobs") ################################################################################ -# /files/forcescan -> deletes old data in DB, and does a brand new scan +# /files/force_scan -> deletes old data in DB, and does a brand new scan ################################################################################ -@app.route("/files/forcescan", methods=["GET"]) +@app.route("/files/force_scan", methods=["GET"]) @login_required -def forcescan(): - job=NewJob( name="forcescan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" ) +def force_scan(): + job=NewJob( name="force_scan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" ) return redirect("/jobs") ################################################################################ @@ -497,7 +497,7 @@ def fix_dups(): if 'pagesize' not in request.form: # default to 10, see if we have a larger value as someone reset it in the gui, rather than first time invoked pagesize = 10 - jexes = JobExtra.query.join(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all() + jexes = JobExtra.query.join(Job).filter(Job.name=='check_dups').filter(Job.pa_job_state=='New').all() jexes.append( JobExtra( name="pagesize", value=pagesize ) ) else: pagesize=int(request.form['pagesize']) @@ -533,7 +533,7 @@ def rm_dups(): jex.append( JobExtra( name="pagesize", value=10 ) ) - job=NewJob( name="rmdups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" ) + job=NewJob( name="rm_dups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" ) return redirect("/jobs") @@ -578,8 +578,8 @@ def move_files(): return make_response( jsonify( job_id=job.id ) ) @login_required -@app.route("/viewlist", methods=["POST"]) -def viewlist(): +@app.route("/view_list", methods=["POST"]) +def view_list(): OPT=States( request ) # Get next/prev set of data - e.g. if next set, then it will use orig_url # to go forward how_many from offset and then use viewer.html to show that @@ -705,7 +705,7 @@ def view_img_post(id): # route called from front/end - if multiple images are being transformed, each transorm == a separate call # to this route (and therefore a separate transorm job. Each reponse allows the f/e to check the -# specific transorm job is finished (/checktransformjob) which will be called (say) every 1 sec. from f/e +# specific transorm job is finished (/check_transform_job) which will be called (say) every 1 sec. from f/e # with a spinning wheel, then when pa_job_mgr has finished it will return the transformed thumb @app.route("/transform", methods=["POST"]) @login_required @@ -721,14 +721,14 @@ def transform(): return make_response( jsonify( job_id=job.id ) ) ################################################################################ -# /checktransformjob -> URL that is called repeatedly by front-end waiting for the +# /check_transform_job -> URL that is called repeatedly by front-end waiting for the # b/e to finish the transform job. Once done, the new / now # transformed image's thumbnail is returned so the f/e can # update with it ################################################################################ -@app.route("/checktransformjob", methods=["POST"]) +@app.route("/check_transform_job", methods=["POST"]) @login_required -def checktransformjob(): +def check_transform_job(): job_id = request.form['job_id'] job = Job.query.get(job_id) j=jsonify( finished=False ) @@ -779,9 +779,9 @@ def _jinja2_filter_parentpath(path): # json list of existing dir names that could be near it in time. Starting # simple, by using YYYYMM-1, YYYYMM, YYYYMM+1 dirs ############################################################################### -@app.route("/getexistingpaths/
", methods=["POST"]) +@app.route("/get_existing_paths/
", methods=["POST"]) @login_required -def GetExistingPathsAsDiv(dt): +def get_existing_paths(dt): dir_ft=FileType.query.filter(FileType.name=='Directory').first() dirs_arr=[] for delta in range(-7, 8): diff --git a/internal/js/files_support.js b/internal/js/files_support.js index d6c0ae3..4e38b49 100644 --- a/internal/js/files_support.js +++ b/internal/js/files_support.js @@ -49,7 +49,7 @@ function change_rp_sel() function GetExistingDirsAsDiv( dt, divname, ptype ) { $.ajax({ - type: 'POST', data: null, url: '/getexistingpaths/'+dt, + type: 'POST', data: null, url: '/get_existing_paths/'+dt, success: function(data) { $('#'+divname).html(data) dirs = JSON.parse(data) diff --git a/internal/js/files_transform.js b/internal/js/files_transform.js index 273eb5c..4525efc 100644 --- a/internal/js/files_transform.js +++ b/internal/js/files_transform.js @@ -4,9 +4,10 @@ // newly created one that was sent back in the response to the POST function CheckTransformJob(id,job_id) { + CheckForJobs() $.ajax( { - type: 'POST', data: '&job_id='+job_id, url: '/checktransformjob', success: function(data) { + type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', success: function(data) { if( data.finished ) { $('#s'+id).hide() diff --git a/internal/js/jobs.js b/internal/js/jobs.js index e437eaf..ccac0b6 100644 --- a/internal/js/jobs.js +++ b/internal/js/jobs.js @@ -48,7 +48,7 @@ function StatusMsg(st) // clear message only when toast is hidden (either timeout OR user clicks close btn) $('#' + el).on( 'hidden.bs.toast', function() { - $.ajax( { type: 'POST', url: '/clearmsg/'+st.id, success: function(data) {} } ) + $.ajax( { type: 'POST', url: '/clear_msg/'+st.id, success: function(data) {} } ) } ) } @@ -73,7 +73,7 @@ function CheckForJobs() { $.ajax( { - type: 'POST', url: '/checkforjobs', + type: 'POST', url: '/check_for_jobs', success: function(data) { data.sts.forEach( function(el) diff --git a/internal/js/view_transform.js b/internal/js/view_transform.js index c8a7283..35d77d9 100644 --- a/internal/js/view_transform.js +++ b/internal/js/view_transform.js @@ -4,9 +4,10 @@ // newly created one that was sent back in the response to the POST function CheckTransformJob(id,job_id) { + CheckForJobs() $.ajax( { - type: 'POST', data: '&job_id='+job_id, url: '/checktransformjob', success: function(data) { + type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', success: function(data) { if( data.finished ) { // stop throbber, remove grayscale & then force reload with timestamped version of im.src diff --git a/job.py b/job.py index 6d9035a..cfe8991 100644 --- a/job.py +++ b/job.py @@ -236,12 +236,12 @@ def joblog(id): return render_template("joblog.html", job=joblog, logs=logs, duration=duration, display_more=display_more, order=order, estimate=estimate, refresh=refresh) ############################################################################### -# /wakeup -> GET -> forces the job manager to wake up, and check the queue +# /wake_up -> GET -> forces the job manager to wake up, and check the queue # should not be needed, but in DEV can be helpful ################################################################################ -@app.route("/wakeup", methods=["GET"]) +@app.route("/wake_up", methods=["GET"]) @login_required -def wakeup(): +def wake_up(): WakePAJobManager(job_id=None) return redirect("/") @@ -317,12 +317,12 @@ def joblog_search(): ############################################################################### -# / -> POST -> looks for pa_job_manager status to F/E jobs and sends json of -# them back to F/E (called form internal/js/jobs.js:CheckForJobs() +# /check_for_jobs -> POST -> looks for pa_job_manager status to F/E jobs and sends json of +# them back to F/E called form internal/js/jobs.js:check_for_jobs() ################################################################################ -@app.route("/checkforjobs", methods=["POST"]) +@app.route("/check_for_jobs", methods=["POST"]) @login_required -def CheckForJobs(): +def check_for_jobs(): num=GetNumActiveJobs() sts=[] for msg in PA_JobManager_Message.query.all(): @@ -333,12 +333,12 @@ def CheckForJobs(): return make_response( jsonify( num_active_jobs=num, sts=sts ) ) ############################################################################### -# / -> POST -> looks for pa_job_manager status to F/E jobs and sends json of -# them back to F/E (called form internal/js/jobs.js:CheckForJobs() +# /clear_msg -> POST -> clears out a F/E message based on passed in +# called form internal/js/jobs.js:CheckForJobs() ################################################################################ -@app.route("/clearmsg/", methods=["POST"]) +@app.route("/clear_msg/", methods=["POST"]) @login_required -def ClearMessage(id): +def clear_message(id): PA_JobManager_Message.query.filter(PA_JobManager_Message.id==id).delete() db.session.commit() # no real need for this response, as if it succeeded/failed the F/E ignores it diff --git a/pa_job_manager.py b/pa_job_manager.py index 19b960e..598fce1 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -687,22 +687,22 @@ def JobsForPath( parent_job, path, ptype ): jex=[] jex.append( JobExtra( name="path", value=path ) ) jex.append( JobExtra( name="path_type", value=ptype.id ) ) - job1=NewJob( name="importdir", num_files=cfn, wait_for=None, jex=jex, parent_job=parent_job, desc=f"scan for files from {ptype.name} path" ) + job1=NewJob( name="import_dir", num_files=cfn, wait_for=None, jex=jex, parent_job=parent_job, desc=f"scan for files from {ptype.name} path" ) # then get file details (hash/thumbs) jex=[] jex.append( JobExtra( name="path", value=path ) ) - job2=NewJob( name="getfiledetails", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"get details of files from {ptype.name} path" ) + job2=NewJob( name="get_file_details", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"get details of files from {ptype.name} path" ) - # can start straight after importdir - job1, does not need details (job2) + # can start straight after import_dir - job1, does not need details (job2) jex=[] jex.append( JobExtra( name="person", value="all" ) ) jex.append( JobExtra( name="path_type", value=ptype.id ) ) job3=NewJob( name="run_ai_on_path", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"match faces on files from {ptype.name} path" ) - # careful here, wait for getfiledetails (job2), the ai job cannot cause a dup - # but it can fail - in which case the checkdup will be withdrawn - job4=NewJob( name="checkdups", num_files=0, wait_for=job2.id, jex=None, parent_job=parent_job, desc="check for duplicate files" ) + # careful here, wait for get_file_details (job2), the ai job cannot cause a dup + # but it can fail - in which case the check_dup will be withdrawn + job4=NewJob( name="check_dups", num_files=0, wait_for=job2.id, jex=None, parent_job=parent_job, desc="check for duplicate files" ) # okay, now process all the new jobs HandleJobs(False) @@ -835,17 +835,17 @@ def RunJob(job): job.start_time=datetime.now(pytz.utc) if job.name =="scan_ip": JobScanImportDir(job) - elif job.name =="forcescan": + elif job.name =="force_scan": JobForceScan(job) elif job.name =="scan_sp": JobScanStorageDir(job) - elif job.name =="importdir": + elif job.name =="import_dir": JobImportDir(job) - elif job.name =="getfiledetails": + elif job.name =="get_file_details": JobGetFileDetails(job) - elif job.name == "checkdups": + elif job.name == "check_dups": JobCheckForDups(job) - elif job.name == "rmdups": + elif job.name == "rm_dups": JobRemoveDups(job) elif job.name == "delete_files": JobDeleteFiles(job) @@ -1570,7 +1570,7 @@ def WithdrawDependantJobs( job, id, reason ): #################################################################################################################################### # next 3 funcs used to optimise whether to do dependant jobs (i.e. no new files, dont keep doing file details, ai scans -# find last successful importdir job for this path +# find last successful import_dir job for this path #################################################################################################################################### def find_last_time_new_files_found(job): path=[jex.value for jex in job.extra if jex.name == "path"][0] @@ -1581,11 +1581,11 @@ def find_last_time_new_files_found(job): return 0 #################################################################################################################################### -# find time of last getfiledetails job for this path +# find time of last get_file_details job for this path #################################################################################################################################### def find_last_successful_gfd_job(job): path=[jex.value for jex in job.extra if jex.name == "path"][0] - jobs=session.query(Job).join(JobExtra).filter(Job.name=="getfiledetails").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all() + jobs=session.query(Job).join(JobExtra).filter(Job.name=="get_file_details").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all() for j in jobs: return j.last_update.timestamp() return 0 @@ -1731,7 +1731,7 @@ def JobImportDir(job): last_ai_scan=find_last_successful_ai_scan(job) for j in session.query(Job).filter(Job.wait_for==job.id).all(): - if j.name == "getfiledetails" and last_file_details > last_scan: + if j.name == "get_file_details" and last_file_details > last_scan: FinishJob(j, f"Job #{j.id} has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" ) # scan found no new files and last ai scan was after the last file scan if j.name == "run_ai_on_path" and last_ai_scan > last_scan: @@ -2032,10 +2032,10 @@ def GenVideoThumbnail( job, fname): # /removedups, but some other job has since created another dup message... #################################################################################################################################### def ClearOtherDupMessagesAndJobs(): - msgs=session.query(PA_JobManager_FE_Message).join(Job).filter(Job.name=='checkdups') + msgs=session.query(PA_JobManager_FE_Message).join(Job).filter(Job.name=='check_dups') for msg in msgs: session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==msg.id).delete() - cd_jobs=session.query(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all() + cd_jobs=session.query(Job).filter(Job.name=='check_dups').filter(Job.pa_job_state=='New').all() for j in cd_jobs: FinishJob(j, "New CheckForDups job/removal supercedes this job, withdrawing it", "Withdrawn") session.commit() @@ -2065,7 +2065,7 @@ def JobCheckForDups(job): def JobRemoveDups(job): JobProgressState( job, "In Progress" ) AddLogForJob(job, f"INFO: Starting Remove Duplicates job...") - # as checkdups covers all dups, delete all future dups messages, and Withdraw future checkdups jobs + # as check_dups covers all dups, delete all future dups messages, and Withdraw future check_dups jobs ClearOtherDupMessagesAndJobs() dup_cnt=0 @@ -2120,8 +2120,8 @@ def JobRemoveDups(job): dup_cnt += 1 - # Need to put another checkdups job in now to force / validate we have no dups - next_job=NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) + # Need to put another check_dups job in now to force / validate we have no dups + next_job=NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) AddLogForJob(job, f"adding job id={next_job.id} {next_job.name} to confirm there are no more duplicates" ) FinishJob(job, f"Finished removing {dup_cnt} duplicate files" ) return @@ -2152,7 +2152,7 @@ def JobMoveFiles(job): if 'eid-' in jex.name: move_me=session.query(Entry).get(jex.value) MoveEntriesToOtherFolder( job, move_me, dst_storage_path, f"{prefix}{suffix}" ) - NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) + NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) FinishJob(job, f"Finished move selected file(s)") return @@ -2166,7 +2166,7 @@ def JobDeleteFiles(job): if 'eid-' in jex.name: del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() MoveFileToRecycleBin(job,del_me) - NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) + NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) FinishJob(job, f"Finished deleting selected file(s)") return @@ -2180,7 +2180,7 @@ def JobRestoreFiles(job): if 'eid-' in jex.name: restore_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() RestoreFile(job,restore_me) - NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) + NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" ) FinishJob(job, f"Finished restoring selected file(s)") return diff --git a/states.py b/states.py index dd3a0c0..cf0d898 100644 --- a/states.py +++ b/states.py @@ -60,14 +60,14 @@ class States(PA): # this is any next/prev or noo, grouping, etc. change (so use referrer to work out what to do with this) # because this can happen on a view, or files_up, etc. change this FIRST - if 'ChangeFileOpts' in request.path: + if 'change_file_opts' in request.path: ref=request.referrer base=request.base_url - base=base.replace("ChangeFileOpts", "") + base=base.replace("change_file_opts", "") self.url = "/"+ref.replace(base, "" ) - # if viewlist, then we really are a view, and view_eid should be in the form - if 'viewlist' in request.path: + # if view_list, then we really are a view, and view_eid should be in the form + if 'view_list' in request.path: self.path_type = 'View' self.view_eid = request.form['view_eid'] self.url = request.form['orig_url'] @@ -122,7 +122,7 @@ class States(PA): self.view_eid = self.url[6:] self.path_type="View" self.orig_url=self.url - elif 'ChangeFileOpts' not in self.url: + elif 'change_file_opts' not in self.url: print( f"ERROR: DDP messed up, failed to match URL {self.url} for settings this will fail, redirecting to home" ) print( f"referrer={request.referrer}" ) return @@ -205,7 +205,7 @@ class States(PA): if request.method=="POST": if self.path_type != "View" and 'noo' in request.form: # we are changing values based on a POST to the form, if we changed the noo option, we need to reset things - if 'ChangeFileOpts' in request.path and self.noo != request.form['noo']: + if 'change_file_opts' in request.path and self.noo != request.form['noo']: self.noo=request.form['noo'] self.first_eid=0 self.last_eid=0 @@ -222,7 +222,7 @@ class States(PA): # seems html cant do boolean, but uses strings so convert if self.path_type != "View" and 'folders' in request.form: # we are changing values based on a POST to the form, if we are in folder view and we changed the folders option, we need to reset things - if 'ChangeFileOpts' in request.path: + if 'change_file_opts' in request.path: if self.folders and self.folders != request.form['folders']: self.num_entries=0 self.first_eid=0 @@ -283,7 +283,7 @@ class States(PA): pref.first_eid = self.first_eid pref.last_eid = self.last_eid pref.num_entries = self.num_entries - # only passed in (at the moment) in viewlist + # only passed in (at the moment) in view_list pref.current = self.current db.session.add(pref) diff --git a/templates/base.html b/templates/base.html index 1afab3f..96d98a6 100644 --- a/templates/base.html +++ b/templates/base.html @@ -84,7 +84,7 @@