clarified bug-82, fixed issue with AI scan optimising still scanning when it should not, finalised back button work for view/viewlist

BUGs

@@ -1,3 +1,3 @@
-### Next: 82
+### Next: 83
 BUG-60: entries per page in flat view will get how_many from each top-level dir in PATH (not a big issue, but it is a little misleading)
-BUG-82: if you arrow next/prev fast enough we seem to break the current=int(lst[0]) -- what is in list at this point?
+BUG-82: occasionally, the face_locn data in the DB is {} not [], not sure what sequence causes this

TODO

@@ -1,27 +1,15 @@
 ## GENERAL
-    * optimising for job scans...
+     *** could get better AI optim, by keeping track of just new files since scan (even if we did this from the DB), 
-        - run_ai_on_path not finding previous job as jex path is path_prefix...
+        then we could jsut feed those eid's explicitly into a 'run_ai_on_new_files' :) -- maybe particularly
-        pa=# select * from jobextra where job_id = 45;
+        if count('new files') < say 1000 do eids, otherwise do path AND no new refimgs
-         id | job_id |    name     |           value
-        ----+--------+-------------+---------------------------
-         79 |     45 | person      | all
-         80 |     45 | ptype       | Import
-         83 |     45 | path_prefix | static/Import/new_img_dir
-         84 |     45 | eid-0       | 2
-         85 |     45 | eid-1       | 31
-
-        BUT WHY 2 eids -- because we have photos and new_img_dir, interesting
-        that pp is the last one... need to do better with this on creation I think?
-
 
     * browser back/forward buttons dont work -- use POST -> redirect to GET
         * viewlist 
-            - [DONE] can work out new view_eids server side, and pass them back as json data (fixes loss of fullscreen & back/fwd issues)
+            - can consider a POST every time we next/prev in viewer --> set only, to just update the OPT.current, every time you go back into the viewer, then it would go the last image viewed, rather than the first image on the last page you viewed...
-            - [DONE] should really define the first/last of a GetEntries search and use definitive logic to show at start or end of entries (for next/prev buttons in viewer.html)
-            - [DONE] store "current", "first_eid", "last_eid" in pa_user_state 
-                - use them on reload, e.g. if current set use it not view eid (should be okay with client-side skipping, and just make sure if it is viewlist we use next/prev logic not current
-            - need to set current/first_eid/last_eid on import/storage/bin as well as search
             - can consider an optim-- new_view page makes calls to viewlist to ADD json data only, so only trigger a new "viewlist" if we dont have data for that part of the eids
+            - need some sort of clean-up of pa_user_state -- I spose its triggered by browser session, so maybe just after a week is lazy/good enough 
+                -- SO TIMESTAMP pa_user_state entries!!!
+
         job.py:@app.route("/jobs", methods=["GET", "POST"])
         job.py:@app.route("/job/<id>", methods=["GET","POST"])
             -- these need to store 'job prefs' somewhere... (extras?)

files.py

@@ -174,6 +174,20 @@ def ClearJM_Message(id):
     db.session.commit()
     return
 
+################################################################################
+# util function to just update the current/first/last positions needed for
+# viewing / using pa_user_state DB table
+################################################################################
+def UpdatePref( pref, OPT ):
+    if OPT.current>0:
+        pref.current=OPT.current
+    if OPT.first_eid>0:
+        pref.first_eid=OPT.first_eid
+    if OPT.last_eid>0:
+        pref.last_eid=OPT.last_eid
+    db.session.add(pref)
+    db.session.commit()
+
 ################################################################################
 # GetEntriesInFlatView:  func. to retrieve DB entries appropriate for flat view
 ################################################################################
@@ -192,33 +206,20 @@ def GetEntriesInFlatView( OPT, prefix ):
     else:
         entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
         last_entry=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name.desc()).limit(1)
-    print(f"GetEnt FLAT --> OPT={OPT}, len={len(entries)}")
     if OPT.first_eid == 0 and OPT.offset == 0 and len(entries):
         OPT.first_eid = entries[0].id
-    if OPT.last_eid == 0:
+    
-        le=last_entry.all()
+    # FIXME: would prefer to only get this if needed (e.g. when last_eid is 0),
-        if len(le):
+    # but with multiple dirs in say import path, then we only set last_eid on
-            OPT.last_eid = le[0].id
+    # the first dir... need to to think this through better, pass param maybe?
-            pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first()
+    le=last_entry.all()
-            UpdatePref( pref, OPT )
+    if len(le):
+        OPT.last_eid = le[0].id
+        pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first()
+        UpdatePref( pref, OPT )
 
     return entries
 
-################################################################################
-# util function to just update the current/first/last positions needed for
-# viewing / using pa_user_state DB table
-################################################################################
-def UpdatePref( pref, OPT ):
-    if OPT.current>0:
-        pref.current=OPT.current
-    if OPT.first_eid>0:
-        pref.first_eid=OPT.first_eid
-    if OPT.last_eid>0:
-        pref.last_eid=OPT.last_eid
-    print( f"UpdatePref: c={pref.current}, f={pref.first_eid}, l={pref.last_eid}" )
-    db.session.add(pref)
-    db.session.commit()
-
 ################################################################################
 # GetEntriesInFolderView: func. to retrieve DB entries appropriate for folder view
 # read inline comments to deal with variations / ordering...
@@ -242,7 +243,7 @@ def GetEntriesInFolderView( OPT, prefix ):
         # this can occur if the path in settings does not exist as it wont be in # the DB
         if not dir:
             return entries
-        if OPT.noo == "Z to A" or "Newest":
+        if OPT.noo == "Z to A" or OPT.noo == "Newest":
             entries+= Entry.query.join(EntryDirLink).join(FileType).filter(EntryDirLink.dir_eid==dir.id).filter(FileType.name=='Directory').order_by(Entry.name.desc()).all()
         # just do A to Z / Oldest by default or if no valid option
         else:
@@ -251,13 +252,25 @@ def GetEntriesInFolderView( OPT, prefix ):
     # add any files at the current CWD (based on dir_eid in DB)
     if OPT.noo == "Oldest":
         entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year,File.month,File.day,Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
+        last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).limit(1)
     elif OPT.noo == "Newest":
         entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
+        last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(File.year,File.month,File.day,Entry.name).limit(1)
     elif OPT.noo == "Z to A":
         entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name.desc()).offset(OPT.offset).limit(OPT.how_many).all()
+        last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name).limit(1)
     # just do A to Z by default or if no valid option
     else:
         entries+=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
+        last_entry=Entry.query.join(File).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir.id).order_by(Entry.name.desc()).limit(1)
+    if OPT.first_eid == 0 and OPT.offset == 0 and len(entries):
+        OPT.first_eid = entries[0].id
+    if OPT.last_eid == 0:
+        le=last_entry.all()
+        if len(le):
+            OPT.last_eid = le[0].id
+            pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type).first()
+            UpdatePref( pref, OPT )
 
     return entries
 
@@ -268,7 +281,6 @@ def GetEntriesInFolderView( OPT, prefix ):
 def GetEntries( OPT ):
     entries=[]
     if OPT.path_type == 'Search' or (OPT.path_type == 'View' and OPT.orig_ptype=='Search'):
-        print( f"getting entries: OPT={OPT}" )
         search_term=OPT.orig_search_term
         if 'AI:' in search_term:
             search_term = search_term.replace('AI:','')
@@ -320,7 +332,7 @@ def GetEntries( OPT ):
                 # can only be 1 due to limit above
                 for l in last_entry:
                     OPT.last_eid = l.id
-                print( f"c={OPT.current}, f={OPT.first_eid}, l={OPT.last_eid} -- STORE THESE in pa_user_state" )
+                # store first/last eid into prefs
                 pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==OPT.path_type,PA_UserState.orig_ptype==OPT.orig_ptype,PA_UserState.orig_search_term==search_term).first()
                 UpdatePref( pref, OPT )
 
@@ -330,10 +342,6 @@ def GetEntries( OPT ):
     # import, storage, or bin images, reset OPT.path_type so that the paths array below works
     if 'View' in OPT.path_type:
         eid = OPT.url[6:]
-        print( f"we have a view, eid={eid}" )
-        #e=Entry.query.get(eid)
-        #OPT.path_type=e.in_dir.in_path.type.name
-        print( f"pt={OPT.orig_ptype}, st={OPT.orig_search_term}" )
         OPT.path_type= OPT.orig_ptype
         
     paths = []
@@ -367,10 +375,8 @@ def clear_jm_msg(id):
 @app.route("/ChangeFileOpts", methods=["POST"])
 @login_required
 def ChangeFileOpts():
-    # reset options based on form post, then redirect back to orig page
+    # reset options based on form post, then redirect back to orig page (with a GET to allow back button to work)
     OPT=States( request )
-    for el in request.form:
-        print( f"{el}={request.form[el]}")
     return redirect( request.referrer )
     
 ################################################################################
@@ -645,18 +651,22 @@ def viewlist():
         OPT.current = int(lst[-1])
     
     resp['current']=OPT.current
+    # OPT.first_eid can still be 0 IF we have gone past the first page, I could
+    # better set this in states rather than kludge this if... think about it
     if OPT.first_eid>0:
         resp['first_eid']=OPT.first_eid
     resp['last_eid']=OPT.last_eid
     resp['eids']=eids
     resp['offset']=OPT.offset
-    print( "DDP: SAVE PREF HERE TO GET NEW CURRENT AND FIX back button WITH view/XXX when you next/prev to different page" )
+    # save pref to keep the new current value, first/last
-
     pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.orig_ptype==OPT.orig_ptype,PA_UserState.view_eid==OPT.view_eid).first()
     UpdatePref( pref, OPT )
 
     return resp
 
+################################################################################
+# /view/id -> grabs data from DB and views it (GET)
+################################################################################
 @login_required
 @app.route("/view/<id>", methods=["GET"])
 def view(id):
@@ -676,9 +686,9 @@ def view(id):
     eids=eids.rstrip(",")
     return render_template("viewer.html", current=int(id), eids=eids, objs=objs, OPT=OPT )
 
-################################################################################
+##################################################################################
-# /view/id -> grabs data from DB and views it
+# /view/id -> grabs data from DB and views it (POST -> set state, redirect to GET)
-################################################################################
+##################################################################################
 @app.route("/view/<id>", methods=["POST"])
 @login_required
 def view_img_post(id):
@@ -696,7 +706,6 @@ def view_img_post(id):
 def transform():
     id = request.form['id']
     amt = request.form['amt']
-    print( f"transform called with id={id}, amt={amt}")
 
     jex=[]
     for el in request.form:
@@ -793,7 +802,6 @@ def GetExistingPathsAsDiv(dt):
     ret='[ '
     first_dir=1
     for dir in dirs:
-        print( f"process dir matching for {new_dt} => {dir}")
         if not first_dir:
             ret +=", "
         bits=dir.rel_path.split('-')

pa_job_manager.py

@@ -548,7 +548,7 @@ def JobsForPaths( parent_job, paths, ptype ):
 
     job3=Job(start_time=now, last_update=now, name="run_ai_on_path", state="New", wait_for=job1.id, pa_job_state="New", current_file_num=0 )
     job3.extra.append( JobExtra( name="person", value="all" ) )
-    job3.extra.append( JobExtra( name="ptype", value=ptype.name ) )
+    job3.extra.append( JobExtra( name="path_type", value=ptype.id ) )
     session.add(job3)
     session.commit()
     if parent_job:
@@ -1284,8 +1284,8 @@ def find_last_successful_gfd_job(job):
 # find time of last run_ai_on_path job for this path
 ####################################################################################################################################
 def find_last_successful_ai_scan(job):
-    path=[jex.value for jex in job.extra if jex.name == "path"][0]
+    path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0]
-    jobs=session.query(Job).join(JobExtra).filter(Job.name=="run_ai_on_path").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
+    jobs=session.query(Job).join(JobExtra).filter(Job.name=="run_ai_on_path").filter(JobExtra.name=='path_type',JobExtra.value==path_type).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
     for j in jobs:
         return j.last_update.timestamp()
     return 0
@@ -1385,7 +1385,6 @@ def JobImportDir(job):
         dir.last_import_date = time.time()
     job.num_files=overall_file_cnt
     if found_new_files:
-        print("adding new_files jex" )
         job.extra.append( JobExtra( name="new_files", value=found_new_files ) )
         session.add(job)
 
@@ -1396,16 +1395,17 @@ def JobImportDir(job):
         last_file_details=find_last_successful_gfd_job(job)
         last_ai_scan=find_last_successful_ai_scan(job)
 
-        print( f"last_scan={last_scan}" )
-        print( f"last_file_details={last_file_details}" )
-        print( f"last_ai_scan={last_ai_scan}" )
-
         for j in session.query(Job).filter(Job.wait_for==job.id).all():
             if j.name == "getfiledetails" and last_file_details > last_scan:
                 FinishJob(j, f"Job (#{j.id}) has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
+            # scan found no new files and last ai scan was after the last file scan
             if j.name == "run_ai_on_path" and last_ai_scan > last_scan:
                 newest_refimg = session.query(Refimg).order_by(Refimg.created_on.desc()).limit(1).all()
-                if newest_refimg and last_scan >= newest_refimg[0].created_on:
+                # IF we also have no new refimgs since last scan, then no need to run any AI again
+                if newest_refimg and newest_refimg[0].created_on < last_scan:
+                    FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
+                # IF we also have no new refimgs since last AI scan, then no need to run any AI again
+                elif newest_refimg and newest_refimg[0].created_on < last_ai_scan:
                     FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
     FinishJob(job, f"Finished Importing: {path} - Processed {overall_file_cnt} files, Found {found_new_files} new files, Removed {rm_cnt} file(s)")
     return
@@ -1443,8 +1443,8 @@ def AddToJobImageCount(job, entry ):
 # so we can then just calls JobRunAIOn
 ####################################################################################################################################
 def JobRunAIOnPath(job):
-    which_ptype=[jex.value for jex in job.extra if jex.name == "ptype"][0]
+    path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0]
-    paths=session.query(Path).join(PathType).filter(PathType.name==which_ptype).all()
+    paths=session.query(Path).join(PathType).filter(PathType.id==path_type).all()
     path_cnt=0
     for p in paths:
         d = session.query(Dir).join(PathDirLink).filter(PathDirLink.path_id==p.id).filter(Dir.rel_path=='').first()

states.py

@@ -47,14 +47,13 @@ class PA_UserState(db.Model):
 class States(PA):
     def __init__(self, request):
         self.path_type=''
+        self.orig_search_term = ''
         self.url = request.path
         self.view_eid = None
         self.current=0
         self.first_eid=0
         self.last_eid=0
 
-        print( f"States() - path={request.path}, ref={request.referrer}" )
-
         # this is any next/prev or noo, grouping, etc. change (so use referrer to work out what to do with this)
         # because this can happen on a view, or files_up, etc. change this FIRST
         if 'ChangeFileOpts' in request.path:
@@ -81,8 +80,6 @@ class States(PA):
                 pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.view_eid==self.view_eid).first()
                 self.url = pref.orig_url
 
-        print( f"NOW, url={self.url}" )
-
         if 'files_ip' in self.url or 'file_list_ip' in self.url:
             if self.path_type == "View":
                 self.orig_ptype = 'Import'
@@ -121,6 +118,8 @@ class States(PA):
 
         if self.path_type == 'View':
             pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.view_eid==self.view_eid).first()
+            # should find original path or search for this view (if not a search, search_term='')
+            orig_pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.orig_ptype,PA_UserState.orig_search_term==self.orig_search_term).first()
         elif self.path_type == 'Search':
             pref=PA_UserState.query.filter(PA_UserState.pa_user_dn==current_user.dn,PA_UserState.path_type==self.path_type,PA_UserState.orig_search_term==self.orig_search_term).first()
         else:
@@ -140,7 +139,6 @@ class States(PA):
             self.orig_url = pref.orig_url
             self.view_eid = pref.view_eid
             self.current = pref.current
-            print( f"from existing pref, set self.first_eid to {pref.first_eid}" )
             self.first_eid = pref.first_eid
             self.last_eid = pref.last_eid
         else:
@@ -153,6 +151,8 @@ class States(PA):
             if self.path_type == "View":
                 self.root='static/' + self.orig_ptype
                 tmp=self.orig_ptype
+                self.first_eid=orig_pref.first_eid
+                self.last_eid=orig_pref.last_eid
             else:
                 self.root='static/' + self.path_type
                 tmp=self.path_type
@@ -177,9 +177,6 @@ class States(PA):
 
         # the above are defaults, if we are here, then we have current values, use them instead if they are set -- AI: searches dont set them so then we use those in the DB first
         if request.method=="POST":
-            print("this was a POST, so use form vals to update PREF" )
-            for el in request.form:
-                print( f"{el}={request.form[el]}" )
             if 'noo' in request.form:
                 self.noo=request.form['noo']
             if 'how_many' in request.form:
@@ -248,8 +245,6 @@ class States(PA):
         db.session.add(pref)
         db.session.commit()
 
-        print( f"saved pref={pref}" )
-
         return
 
 ################################################################################