From 0751cc6010b1c488a72e1df69f016091c7722ea8 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 27 Jan 2022 21:44:29 +1100 Subject: [PATCH] clarified bug-82, fixed issue with AI scan optimising still scanning when it should not, finalised back button work for view/viewlist --- BUGs | 4 +-- TODO | 26 ++++---------- files.py | 86 ++++++++++++++++++++++++++--------------------- pa_job_manager.py | 22 ++++++------ states.py | 15 +++------ 5 files changed, 72 insertions(+), 81 deletions(-) diff --git a/BUGs b/BUGs index 2bc4a7f..3dc2416 100644 --- a/BUGs +++ b/BUGs @@ -1,3 +1,3 @@ -### Next: 82 +### Next: 83 BUG-60: entries per page in flat view will get how_many from each top-level dir in PATH (not a big issue, but it is a little misleading) -BUG-82: if you arrow next/prev fast enough we seem to break the current=int(lst[0]) -- what is in list at this point? +BUG-82: occasionally, the face_locn data in the DB is {} not [], not sure what sequence causes this diff --git a/TODO b/TODO index 611ee5e..14b28cf 100644 --- a/TODO +++ b/TODO @@ -1,27 +1,15 @@ ## GENERAL - * optimising for job scans... - - run_ai_on_path not finding previous job as jex path is path_prefix... - pa=# select * from jobextra where job_id = 45; - id | job_id | name | value - ----+--------+-------------+--------------------------- - 79 | 45 | person | all - 80 | 45 | ptype | Import - 83 | 45 | path_prefix | static/Import/new_img_dir - 84 | 45 | eid-0 | 2 - 85 | 45 | eid-1 | 31 - - BUT WHY 2 eids -- because we have photos and new_img_dir, interesting - that pp is the last one... need to do better with this on creation I think? - + *** could get better AI optim, by keeping track of just new files since scan (even if we did this from the DB), + then we could jsut feed those eid's explicitly into a 'run_ai_on_new_files' :) -- maybe particularly + if count('new files') < say 1000 do eids, otherwise do path AND no new refimgs * browser back/forward buttons dont work -- use POST -> redirect to GET * viewlist - - [DONE] can work out new view_eids server side, and pass them back as json data (fixes loss of fullscreen & back/fwd issues) - - [DONE] should really define the first/last of a GetEntries search and use definitive logic to show at start or end of entries (for next/prev buttons in viewer.html) - - [DONE] store "current", "first_eid", "last_eid" in pa_user_state - - use them on reload, e.g. if current set use it not view eid (should be okay with client-side skipping, and just make sure if it is viewlist we use next/prev logic not current - - need to set current/first_eid/last_eid on import/storage/bin as well as search + - can consider a POST every time we next/prev in viewer --> set only, to just update the OPT.current, every time you go back into the viewer, then it would go the last image viewed, rather than the first image on the last page you viewed... - can consider an optim-- new_view page makes calls to viewlist to ADD json data only, so only trigger a new "viewlist" if we dont have data for that part of the eids + - need some sort of clean-up of pa_user_state -- I spose its triggered by browser session, so maybe just after a week is lazy/good enough + -- SO TIMESTAMP pa_user_state entries!!! + job.py:@app.route("/jobs", methods=["GET", "POST"]) job.py:@app.route("/job/", methods=["GET","POST"]) -- these need to store 'job prefs' somewhere... (extras?) diff --git a/files.py b/files.py index 20e84c2..57427a1 100644 --- a/files.py +++ b/files.py @@ -174,6 +174,20 @@ def ClearJM_Message(id): db.session.commit() return +################################################################################ +# util function to just update the current/first/last positions needed for +# viewing / using pa_user_state DB table +################################################################################ +def UpdatePref( pref, OPT ): + if OPT.current>0: + pref.current=OPT.current + if OPT.first_eid>0: + pref.first_eid=OPT.first_eid + if OPT.last_eid>0: + pref.last_eid=OPT.last_eid + db.session.add(pref) + db.session.commit() + ################################################################################ # GetEntriesInFlatView: func. to retrieve DB entries appropriate for flat view ################################################################################ @@ -192,33 +206,20 @@ def GetEntriesInFlatView( OPT, prefix ): else: entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(OPT.offset).limit(OPT.how_many).all() last_entry=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name.desc()).limit(1) - print(f"GetEnt FLAT --> OPT={OPT}, len={len(entries)}") if OPT.first_eid == 0 and OPT.offset == 0 and len(entries): OPT.first_eid = entries[0].id - if OPT.last_eid == 0: - le=last_entry.all() - if len(le): - OPT.last_eid = le[0].id - pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first() - UpdatePref( pref, OPT ) + + # FIXME: would prefer to only get this if needed (e.g. when last_eid is 0), + # but with multiple dirs in say import path, then we only set last_eid on + # the first dir... need to to think this through better, pass param maybe? + le=last_entry.all() + if len(le): + OPT.last_eid = le[0].id + pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first() + UpdatePref( pref, OPT ) return entries -################################################################################ -# util function to just update the current/first/last positions needed for -# viewing / using pa_user_state DB table -################################################################################ -def UpdatePref( pref, OPT ): - if OPT.current>0: - pref.current=OPT.current - if OPT.first_eid>0: - pref.first_eid=OPT.first_eid - if OPT.last_eid>0: - pref.last_eid=OPT.last_eid - print( f"UpdatePref: c={pref.current}, f={pref.first_eid}, l={pref.last_eid}" ) - db.session.add(pref) - db.session.commit() - ################################################################################ # GetEntriesInFolderView: func. to retrieve DB entries appropriate for folder view # read inline comments to deal with variations / ordering... @@ -242,7 +243,7 @@ def GetEntriesInFolderView( OPT, prefix ): # this can occur if the path in settings does not exist as it wont be in # the DB if not dir: return entries - if OPT.noo == "Z to A" or "Newest": + if OPT.noo == "Z to A" or OPT.noo == "Newest": entries+= Entry.query.join(EntryDirLink).join(FileType).filter(EntryDirLink.dir_eid==dir.id).filter(FileType.name=='Directory').order_by(Entry.name.desc()).all() # just do A to Z / Oldest by default or if no valid option else: @@ -251,13 +252,25 @@ def GetEntriesInFolderView( OPT, prefix ): # add any files at the current CWD (based on dir_eid in DB) if OPT.noo == "Oldest": entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year,File.month,File.day,Entry.name).offset(OPT.offset).limit(OPT.how_many).all() + last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).limit(1) elif OPT.noo == "Newest": entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all() + last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year,File.month,File.day,Entry.name).limit(1) elif OPT.noo == "Z to A": entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name.desc()).offset(OPT.offset).limit(OPT.how_many).all() + last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name).limit(1) # just do A to Z by default or if no valid option else: entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name).offset(OPT.offset).limit(OPT.how_many).all() + last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name.desc()).limit(1) + if OPT.first_eid == 0 and OPT.offset == 0 and len(entries): + OPT.first_eid = entries[0].id + if OPT.last_eid == 0: + le=last_entry.all() + if len(le): + OPT.last_eid = le[0].id + pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first() + UpdatePref( pref, OPT ) return entries @@ -268,7 +281,6 @@ def GetEntriesInFolderView( OPT, prefix ): def GetEntries( OPT ): entries=[] if OPT.path_type == 'Search' or (OPT.path_type == 'View' and OPT.orig_ptype=='Search'): - print( f"getting entries: OPT={OPT}" ) search_term=OPT.orig_search_term if 'AI:' in search_term: search_term = search_term.replace('AI:','') @@ -320,7 +332,7 @@ def GetEntries( OPT ): # can only be 1 due to limit above for l in last_entry: OPT.last_eid = l.id - print( f"c={OPT.current}, f={OPT.first_eid}, l={OPT.last_eid} -- STORE THESE in pa_user_state" ) + # store first/last eid into prefs pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type,PA_UserState.orig_ptype==OPT.orig_ptype,PA_UserState.orig_search_term==search_term).first() UpdatePref( pref, OPT ) @@ -330,10 +342,6 @@ def GetEntries( OPT ): # import, storage, or bin images, reset OPT.path_type so that the paths array below works if 'View' in OPT.path_type: eid = OPT.url[6:] - print( f"we have a view, eid={eid}" ) - #e=Entry.query.get(eid) - #OPT.path_type=e.in_dir.in_path.type.name - print( f"pt={OPT.orig_ptype}, st={OPT.orig_search_term}" ) OPT.path_type= OPT.orig_ptype paths = [] @@ -367,10 +375,8 @@ def clear_jm_msg(id): @app.route("/ChangeFileOpts", methods=["POST"]) @login_required def ChangeFileOpts(): - # reset options based on form post, then redirect back to orig page + # reset options based on form post, then redirect back to orig page (with a GET to allow back button to work) OPT=States( request ) - for el in request.form: - print( f"{el}={request.form[el]}") return redirect( request.referrer ) ################################################################################ @@ -645,18 +651,22 @@ def viewlist(): OPT.current = int(lst[-1]) resp['current']=OPT.current + # OPT.first_eid can still be 0 IF we have gone past the first page, I could + # better set this in states rather than kludge this if... think about it if OPT.first_eid>0: resp['first_eid']=OPT.first_eid resp['last_eid']=OPT.last_eid resp['eids']=eids resp['offset']=OPT.offset - print( "DDP: SAVE PREF HERE TO GET NEW CURRENT AND FIX back button WITH view/XXX when you next/prev to different page" ) - + # save pref to keep the new current value, first/last pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.orig_ptype==OPT.orig_ptype,PA_UserState.view_eid==OPT.view_eid).first() UpdatePref( pref, OPT ) return resp +################################################################################ +# /view/id -> grabs data from DB and views it (GET) +################################################################################ @login_required @app.route("/view/", methods=["GET"]) def view(id): @@ -676,9 +686,9 @@ def view(id): eids=eids.rstrip(",") return render_template("viewer.html", current=int(id), eids=eids, objs=objs, OPT=OPT ) -################################################################################ -# /view/id -> grabs data from DB and views it -################################################################################ +################################################################################## +# /view/id -> grabs data from DB and views it (POST -> set state, redirect to GET) +################################################################################## @app.route("/view/", methods=["POST"]) @login_required def view_img_post(id): @@ -696,7 +706,6 @@ def view_img_post(id): def transform(): id = request.form['id'] amt = request.form['amt'] - print( f"transform called with id={id}, amt={amt}") jex=[] for el in request.form: @@ -793,7 +802,6 @@ def GetExistingPathsAsDiv(dt): ret='[ ' first_dir=1 for dir in dirs: - print( f"process dir matching for {new_dt} => {dir}") if not first_dir: ret +=", " bits=dir.rel_path.split('-') diff --git a/pa_job_manager.py b/pa_job_manager.py index 54d9ffe..2cc1acb 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -548,7 +548,7 @@ def JobsForPaths( parent_job, paths, ptype ): job3=Job(start_time=now, last_update=now, name="run_ai_on_path", state="New", wait_for=job1.id, pa_job_state="New", current_file_num=0 ) job3.extra.append( JobExtra( name="person", value="all" ) ) - job3.extra.append( JobExtra( name="ptype", value=ptype.name ) ) + job3.extra.append( JobExtra( name="path_type", value=ptype.id ) ) session.add(job3) session.commit() if parent_job: @@ -1284,8 +1284,8 @@ def find_last_successful_gfd_job(job): # find time of last run_ai_on_path job for this path #################################################################################################################################### def find_last_successful_ai_scan(job): - path=[jex.value for jex in job.extra if jex.name == "path"][0] - jobs=session.query(Job).join(JobExtra).filter(Job.name=="run_ai_on_path").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all() + path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0] + jobs=session.query(Job).join(JobExtra).filter(Job.name=="run_ai_on_path").filter(JobExtra.name=='path_type',JobExtra.value==path_type).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all() for j in jobs: return j.last_update.timestamp() return 0 @@ -1385,7 +1385,6 @@ def JobImportDir(job): dir.last_import_date = time.time() job.num_files=overall_file_cnt if found_new_files: - print("adding new_files jex" ) job.extra.append( JobExtra( name="new_files", value=found_new_files ) ) session.add(job) @@ -1396,16 +1395,17 @@ def JobImportDir(job): last_file_details=find_last_successful_gfd_job(job) last_ai_scan=find_last_successful_ai_scan(job) - print( f"last_scan={last_scan}" ) - print( f"last_file_details={last_file_details}" ) - print( f"last_ai_scan={last_ai_scan}" ) - for j in session.query(Job).filter(Job.wait_for==job.id).all(): if j.name == "getfiledetails" and last_file_details > last_scan: FinishJob(j, f"Job (#{j.id}) has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" ) + # scan found no new files and last ai scan was after the last file scan if j.name == "run_ai_on_path" and last_ai_scan > last_scan: newest_refimg = session.query(Refimg).order_by(Refimg.created_on.desc()).limit(1).all() - if newest_refimg and last_scan >= newest_refimg[0].created_on: + # IF we also have no new refimgs since last scan, then no need to run any AI again + if newest_refimg and newest_refimg[0].created_on < last_scan: + FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" ) + # IF we also have no new refimgs since last AI scan, then no need to run any AI again + elif newest_refimg and newest_refimg[0].created_on < last_ai_scan: FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" ) FinishJob(job, f"Finished Importing: {path} - Processed {overall_file_cnt} files, Found {found_new_files} new files, Removed {rm_cnt} file(s)") return @@ -1443,8 +1443,8 @@ def AddToJobImageCount(job, entry ): # so we can then just calls JobRunAIOn #################################################################################################################################### def JobRunAIOnPath(job): - which_ptype=[jex.value for jex in job.extra if jex.name == "ptype"][0] - paths=session.query(Path).join(PathType).filter(PathType.name==which_ptype).all() + path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0] + paths=session.query(Path).join(PathType).filter(PathType.id==path_type).all() path_cnt=0 for p in paths: d = session.query(Dir).join(PathDirLink).filter(PathDirLink.path_id==p.id).filter(Dir.rel_path=='').first() diff --git a/states.py b/states.py index ebce4c5..71832d1 100644 --- a/states.py +++ b/states.py @@ -47,14 +47,13 @@ class PA_UserState(db.Model): class States(PA): def __init__(self, request): self.path_type='' + self.orig_search_term = '' self.url = request.path self.view_eid = None self.current=0 self.first_eid=0 self.last_eid=0 - print( f"States() - path={request.path}, ref={request.referrer}" ) - # this is any next/prev or noo, grouping, etc. change (so use referrer to work out what to do with this) # because this can happen on a view, or files_up, etc. change this FIRST if 'ChangeFileOpts' in request.path: @@ -81,8 +80,6 @@ class States(PA): pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.view_eid==self.view_eid).first() self.url = pref.orig_url - print( f"NOW, url={self.url}" ) - if 'files_ip' in self.url or 'file_list_ip' in self.url: if self.path_type == "View": self.orig_ptype = 'Import' @@ -121,6 +118,8 @@ class States(PA): if self.path_type == 'View': pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.view_eid==self.view_eid).first() + # should find original path or search for this view (if not a search, search_term='') + orig_pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.orig_ptype,PA_UserState.orig_search_term==self.orig_search_term).first() elif self.path_type == 'Search': pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.orig_search_term==self.orig_search_term).first() else: @@ -140,7 +139,6 @@ class States(PA): self.orig_url = pref.orig_url self.view_eid = pref.view_eid self.current = pref.current - print( f"from existing pref, set self.first_eid to {pref.first_eid}" ) self.first_eid = pref.first_eid self.last_eid = pref.last_eid else: @@ -153,6 +151,8 @@ class States(PA): if self.path_type == "View": self.root='static/' + self.orig_ptype tmp=self.orig_ptype + self.first_eid=orig_pref.first_eid + self.last_eid=orig_pref.last_eid else: self.root='static/' + self.path_type tmp=self.path_type @@ -177,9 +177,6 @@ class States(PA): # the above are defaults, if we are here, then we have current values, use them instead if they are set -- AI: searches dont set them so then we use those in the DB first if request.method=="POST": - print("this was a POST, so use form vals to update PREF" ) - for el in request.form: - print( f"{el}={request.form[el]}" ) if 'noo' in request.form: self.noo=request.form['noo'] if 'how_many' in request.form: @@ -248,8 +245,6 @@ class States(PA): db.session.add(pref) db.session.commit() - print( f"saved pref={pref}" ) - return ################################################################################