partial implementation of first_eid, last_eid -- I think the vals work -- they do for searches anyway, but not stored in pa_user_state yet

BUGs

@@ -1,2 +1,3 @@
 ### Next: 82
 BUG-60: entries per page in flat view will get how_many from each top-level dir in PATH (not a big issue, but it is a little misleading)
+BUG-82: if you arrow next/prev fast enough we seem to break the current=int(lst[0]) -- what is in list at this point?

TODO

@@ -1,6 +1,24 @@
 ## GENERAL
+    * optimising for job scans...
+        - run_ai_on_path not finding previous job as jex path is path_prefix...
+        pa=# select * from jobextra where job_id = 45;
+         id | job_id |    name     |           value
+        ----+--------+-------------+---------------------------
+         79 |     45 | person      | all
+         80 |     45 | ptype       | Import
+         83 |     45 | path_prefix | static/Import/new_img_dir
+         84 |     45 | eid-0       | 2
+         85 |     45 | eid-1       | 31
+
+        BUT WHY 2 eids -- because we have photos and new_img_dir, interesting
+        that pp is the last one... need to do better with this on creation I think?
+
+
     * browser back/forward buttons dont work -- use POST -> redirect to GET
-        * viewlist can work out new view_eids server side, and pass them back as json data (fixes loss of fullscreen & back/fwd issues)
+        * viewlist 
+            - [DONE] can work out new view_eids server side, and pass them back as json data (fixes loss of fullscreen & back/fwd issues)
+            - [DONE] should really define the first/last of a GetEntries search and use definitive logic to show at start or end of entries (for next/prev buttons in viewer.html)
+            - need to keep "current", "first_eid", "last_eid" in pa_user_state to support back button / reloading view/<current> and better handling of next/prev
             - can consider an optim-- new_view page makes calls to viewlist to ADD json data only, so only trigger a new "viewlist" if we dont have data for that part of the eids
         job.py:@app.route("/jobs", methods=["GET", "POST"])
         job.py:@app.route("/job/<id>", methods=["GET","POST"])
@@ -8,15 +26,6 @@
         files.py:@app.route("/fix_dups", methods=["POST"])
             ???
 
-    * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach?
-
-    * refimg locns can lose an array idx of 0 always.
-
-    * search allow noo?
-
-    * optim to not run_ai_on_* for scan, needs to make sure last run_ai_on actually ran/worked - might have failed (or in my case was marked stale and I cancelled it)
-        -- also the case for get file details though, need to make sure last one was completed
-
     * [DONE] order/ find face with largest size and at least show that as unmatched
         - could also try to check it vs. other faces, if it matches more than say 10? we offer it up as a required ref img, then cut that face (with margin) out and use it is a new ref image / person
         * on viewer: allow face to be used to create person, add to existing person, and allow 'ignore', mark as 'not a face', etc. -> all into DB
@@ -26,6 +35,10 @@
                 https://stackoverflow.com/questions/31601393/create-context-menu-using-jquery-with-html-5-canvas
             - also allow joblog search from the viewer for that file...
 
+    * refimg locns can lose an array idx of 0 always.
+
+    * search allow noo?
+
     * delete folder
 
     * allow joblog search

files.py

@@ -244,10 +244,15 @@ def GetEntries( OPT ):
         if 'AI:' in search_term:
             search_term = search_term.replace('AI:','')
             all_entries = Entry.query.join(File).distinct().join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{search_term}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
+            if OPT.last_eid == 0:
+                last_entry=Entry.query.join(File).distinct().join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{search_term}%")).order_by(File.year,File.month,File.day,Entry.name.desc()).limit(1).all()
+                if len(last_entry):
+                    OPT.last_eid = last_entry[0].id
         else:
             file_data=Entry.query.join(File).filter(Entry.name.ilike(f"%{search_term}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
             dir_data=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.rel_path.ilike(f"%{search_term}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
             ai_data=Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{search_term}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(OPT.offset).limit(OPT.how_many).all()
+
             # remove any duplicates from combined data
             all_entries = []
             for f in file_data:
@@ -268,6 +273,26 @@ def GetEntries( OPT ):
                         break
                 if add_it:
                     all_entries.append(a)
+
+            # for all searches first_entry is worked out when first_eid not set yet & offset is 0 and we have some entries
+            if OPT.first_eid == 0 and OPT.offset == 0 and len(all_entries):
+                OPT.first_eid = all_entries[0].id
+            if OPT.last_eid == 0:
+                by_fname=  f"select e.id from entry e where e.name ilike '%%{search_term}%%'"
+                by_dirname=f"select e.id from entry e, entry_dir_link edl where edl.entry_id = e.id and edl.dir_eid in ( select d.eid from dir d where d.rel_path ilike '%%{search_term}%%' )"
+                by_ai     =f"select e.id from entry e, face_file_link ffl, face_refimg_link frl, person_refimg_link prl, person p where e.id =  ffl.file_eid and frl.face_id = ffl.face_id and frl.refimg_id = prl.refimg_id and prl.person_id = p.id and  p.tag ilike '%%{search_term}%%'"
+
+                sel_no_order=f"select e.*, f.* from entry e, file f where e.id=f.eid and e.id in ( {by_fname} union {by_dirname} union {by_ai} ) "
+                order_desc=f"f.year desc, f.month desc, f.day desc, e.name"
+                order_asc=f"f.year, f.month, f.day, e.name desc"
+
+                last_entry_sql= f"{sel_no_order} order by {order_asc} limit 1"
+                last_entry=db.engine.execute( last_entry_sql )
+                # can only be 1 due to limit above
+                for l in last_entry:
+                    OPT.last_eid = l.id
+                
+            print( f"f={OPT.first_eid}, l={OPT.last_eid} -- STORE THESE in pa_user_state" )
         return all_entries
 
     # if we are a view, then it will be of something else, e.g. a list of
@@ -535,13 +560,11 @@ def move_files():
     st.SetMessage( f"Created&nbsp;<a href=/job/{job.id}>Job #{job.id}</a>&nbsp;to move selected file(s)")
     return redirect("/jobs")
 
-################################################################################
-# /viewlist -> get new set of eids and set current to new img to view
-################################################################################
-@app.route("/viewlist", methods=["POST"])
 @login_required
+@app.route("/viewlist", methods=["POST"])
 def viewlist():
     OPT=States( request )
+    OPT.last_entry_in_db=0
     # Get next/prev set of data - e.g. if next set, then it will use orig_url
     # to go forward how_many from offset and then use viewer.html to show that
     # first obj of the new list of entries
@@ -556,39 +579,62 @@ def viewlist():
         OPT.last_entry_in_db=1
     objs = {}
     eids=""
+    resp={}
+    resp['objs']={}
     for e in entries:
         if not e.file_details:
-            print( f"seems {e.name} is not a file? -- {e.type}" )
             continue
-        objs[e.id]=e
-        # get new eids for viewer.html
         eids=eids+f"{e.id},"
+        resp['objs'][e.id]={}
+        resp['objs'][e.id]['url'] = e.FullPathOnFS()
+        resp['objs'][e.id]['name'] = e.name
+        resp['objs'][e.id]['type'] = e.type.name
+        if e.file_details.faces:
+            resp['objs'][e.id]['face_model'] = e.file_details.faces[0].facefile_lnk.model_used
+            resp['objs'][e.id]['faces'] = []
+
             # put locn data back into array format 
+            fid=0
             for face in e.file_details.faces:
                 face.locn = json.loads(face.locn)
+                fd= {}
+                fd['x'] = face.locn[3]
+                fd['y'] = face.locn[0]
+                fd['w'] = face.locn[1]-face.locn[3]
+                fd['h'] = face.locn[2]-face.locn[0]
+                if face.refimg:
+                    fd['who'] = face.refimg.person.tag
+                    fd['distance'] = round(face.refimg_lnk.face_distance,2)
+                resp['objs'][e.id]['faces'].append(fd)
+                fid+=1
+
     eids=eids.rstrip(",")
     lst = eids.split(',')
     if 'next' in request.form:
         current = int(lst[0])
     if 'prev' in request.form:
         current = int(lst[-1])
-    if hasattr( OPT, 'last_entry_in_db' ):
+    if OPT.last_entry_in_db:
         # force this back to the last image of the last page - its the last in the DB, so set OPT for it
         current = int(lst[-1])
         OPT.last_entry_in_db=current
     
-    return render_template("viewer.html", current=current, eids=eids, objs=objs, OPT=OPT )
+    resp['current']=current
+    resp['eids']=eids
+    resp['offset']=OPT.offset
+    resp['last_entry_in_db']=OPT.last_entry_in_db
+
+    return resp
 
 @login_required
 @app.route("/view/<id>", methods=["GET"])
 def view(id):
     OPT=States( request )
+    OPT.last_entry_in_db=0
     objs = {}
     entries=GetEntries( OPT )
     eids=""
     for e in entries:
-        print( f"id={e.id}, len(faces)={len(e.file_details.faces)}")
-        print(e.id)
         objs[e.id]=e
         eids += f"{e.id},"
         # if this is a dir, we wont view it with a click anyway, so move on...
@@ -596,7 +642,6 @@ def view(id):
             continue
         # put locn data back into array format 
         for face in e.file_details.faces:
-            print( f"face.locn before json: {face.locn}" )
             face.locn = json.loads(face.locn)
     eids=eids.rstrip(",")
     return render_template("viewer.html", current=int(id), eids=eids, objs=objs, OPT=OPT )

internal/js/view_support.js

@@ -52,7 +52,7 @@ function DrawImg()
     if( $('#fname_toggle').prop('checked' ) )
     {
         // reset fname for new image (if navigated left/right to get here)
-        $('#fname').html(PrettyFname(objs[current].name))
+        $('#fname').html(PrettyFname(objs[current].url))
         $('.figcaption').show()
     }
     else
@@ -60,7 +60,7 @@ function DrawImg()
 
     // if we have faces, the enable the toggles, otherwise disable them
     // and reset model select too
-    if( objs[current].faces.length )
+    if( objs[current].faces )
     {
         $('#faces').attr('disabled', false)
         $('#distance').attr('disabled', false)
@@ -75,7 +75,7 @@ function DrawImg()
     }
 
     // okay, we want faces drawn so lets do it
-    if( $('#faces').prop('checked') )
+    if( $('#faces').prop('checked') && objs[current].faces )
     {
         // draw rect on each face
         for( i=0; i<objs[current].faces.length; i++ )
@@ -157,7 +157,7 @@ function ViewImageOrVideo()
         $('#video_div').hide()
         if( $('#fname_toggle').prop('checked' ) )
             $('#img-cap').show()
-        $('#fname_i').html(PrettyFname(objs[current].name))
+        $('#fname_i').html(PrettyFname(objs[current].url))
         $('#figure').show()
         if( fullscreen )
             $('#canvas').get(0).requestFullscreen()
@@ -166,7 +166,7 @@ function ViewImageOrVideo()
     {
         $('#figure').hide()
         $('#video').prop('src', '../' + objs[current].url )
-        $('#fname_v').html(PrettyFname(objs[current].name))
+        $('#fname_v').html(PrettyFname(objs[current].url))
         if( $('#fname_toggle').prop('checked' ) )
             $('#img-cap').hide()
         ResizeVideo()

pa_job_manager.py

@@ -720,7 +720,7 @@ def HandleJobs(first_run=False):
             job.pa_job_state = 'Stale'
             session.add(job)
             AddLogForJob( job, "ERROR: Job has been marked stale as it did not complete" )
-            MessageToFE( job.id, "danger", f'Stale job, click&nbsp; <a href="javascript:document.body.innerHTML+=\'<form id=_fm method=POST action=/stale_jobs></form>\'; document.getElementById(\'_fm\').submit();">here</a>&nbsp;to restart or cancel' )
+            MessageToFE( job.id, "danger", f'Stale job, click&nbsp; <a href="javascript:document.body.innerHTML+=\'<form id=_fm method=GET action=/stale_jobs></form>\'; document.getElementById(\'_fm\').submit();">here</a>&nbsp;to restart or cancel' )
             session.commit()
             continue
         if job.pa_job_state == 'New':
@@ -1258,6 +1258,38 @@ def WithdrawDependantJobs( job, id, reason ):
     return
     
 
+####################################################################################################################################
+# next 3 funcs used to optimise whether to do dependant jobs (i.e. no new files, dont keep doing file details, ai scans
+# find last successful importdir job for this path
+####################################################################################################################################
+def find_last_time_new_files_found(job):
+    path=[jex.value for jex in job.extra if jex.name == "path"][0]
+    jobs = session.execute( f"select j.* from job j, jobextra jex1, jobextra jex2 where j.id = jex1.job_id and j.id = jex2.job_id and jex1.name ='path' and jex1.value = '{path}' and jex2.name = 'new_files'")
+
+    for j in jobs:
+        return j.last_update.timestamp()
+    return 0
+
+####################################################################################################################################
+# find time of last getfiledetails job for this path
+####################################################################################################################################
+def find_last_successful_gfd_job(job):
+    path=[jex.value for jex in job.extra if jex.name == "path"][0]
+    jobs=session.query(Job).join(JobExtra).filter(Job.name=="getfiledetails").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
+    for j in jobs:
+        return j.last_update.timestamp()
+    return 0
+
+####################################################################################################################################
+# find time of last run_ai_on_path job for this path
+####################################################################################################################################
+def find_last_successful_ai_scan(job):
+    path=[jex.value for jex in job.extra if jex.name == "path"][0]
+    jobs=session.query(Job).join(JobExtra).filter(Job.name=="run_ai_on_path").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
+    for j in jobs:
+        return j.last_update.timestamp()
+    return 0
+
 ####################################################################################################################################
 # JobImportDir(): job that scan import dir and processes entries in there - key function that uses os.walk() to traverse the 
 # file system and calls AddFile()/AddDir() as necessary
@@ -1352,16 +1384,28 @@ def JobImportDir(job):
         job.current_file_num += len(subdirs)
         dir.last_import_date = time.time()
     job.num_files=overall_file_cnt
+    if found_new_files:
+        print("adding new_files jex" )
+        job.extra.append( JobExtra( name="new_files", value=found_new_files ) )
+        session.add(job)
 
     rm_cnt=HandleAnyFSDeletions(job)
 
     if found_new_files == 0:
+        last_scan=find_last_time_new_files_found(job)
+        last_file_details=find_last_successful_gfd_job(job)
+        last_ai_scan=find_last_successful_ai_scan(job)
+
+        print( f"last_scan={last_scan}" )
+        print( f"last_file_details={last_file_details}" )
+        print( f"last_ai_scan={last_ai_scan}" )
+
         for j in session.query(Job).filter(Job.wait_for==job.id).all():
-            if j.name == "getfiledetails":
+            if j.name == "getfiledetails" and last_file_details > last_scan:
                 FinishJob(j, f"Job (#{j.id}) has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
-            if j.name == "run_ai_on_path":
+            if j.name == "run_ai_on_path" and last_ai_scan > last_scan:
                 newest_refimg = session.query(Refimg).order_by(Refimg.created_on.desc()).limit(1).all()
-                if newest_refimg and orig_last_import >= newest_refimg[0].created_on:
+                if newest_refimg and last_scan >= newest_refimg[0].created_on:
                     FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
     FinishJob(job, f"Finished Importing: {path} - Processed {overall_file_cnt} files, Found {found_new_files} new files, Removed {rm_cnt} file(s)")
     return

states.py

@@ -46,6 +46,8 @@ class States(PA):
         self.path_type=''
         self.url = request.path
         self.view_eid = None
+        self.first_eid = 0
+        self.last_eid = 0
 
         print( f"States() - path={request.path}, ref={request.referrer}" )
 

templates/viewer.html

@@ -26,11 +26,12 @@
     var current={{current}}
     var eids="{{eids}}"
     var eid_lst=eids.split(",")
+    var offset={{OPT.offset}}
+    var last_entry_in_db={{OPT.last_entry_in_db}}
 
     {% for id in objs %}
         e=new Object()
         e.url = "{{objs[id].FullPathOnFS()|safe}}"
-        e.name = e.url
         e.type = "{{objs[id].type.name}}"
         {% if objs[id].file_details.faces %}
             e.face_model="{{objs[id].file_details.faces[0].facefile_lnk.model_used}}"
@@ -75,20 +76,29 @@
 
     function CallViewListRoute(dir)
     {
-        s='<form id="_fmv" method="POST" action="/viewlist">'
+        data="eids="+$("#eids").val()
-        s+='<input type="hidden" name="eids" value="'+$("#eids").val() + '">'
+        data+="&cwd={{OPT.cwd}}"
-        s+='<input type="hidden" name="cwd" value="{{OPT.cwd}}">'
+        data+="&root={{OPT.root}}"
-        s+='<input type="hidden" name="root" value="{{OPT.root}}">'
+        data+="&orig_url={{OPT.orig_url}}"
-        s+='<input type="hidden" name="orig_url" value="{{OPT.orig_url}}">'
+        data+="&view_eid={{OPT.view_eid}}"
-        s+='<input type="hidden" name="view_eid" value="{{OPT.view_eid}}">'
+        // just to save this in pa_user_state
-        s+='<input type="hidden" name="fullscreen" value="' + fullscreen + '">'
+        data+="&fullscreen="+fullscreen
-        s+='<input type="hidden" name="' + dir + '" value="1">'
+        // direction (next/prev)
+        data+="&"+dir+ "=1"
         {% if search_term is defined %}
-            s+='<input type="hidden" name="search_term" value="{{search_term}}">'
+            data+="&search_term={{search_term}}"
         {% endif %}
-        s+='</form>'
+        $.ajax({ type: 'POST', data: data, url: '/viewlist', success: function(res){ 
-        $(s).appendTo('body')
+                console.log(res); 
-        $('#_fmv').submit();
+                current=res.current
+                eids=res.eids
+                objs=res.objs
+                eid_lst=eids.split(",")
+                offset=res.offset
+                last_entry_in_db=res.last_entry_in_db
+                ViewImageOrVideo()
+            }
+        })
     }
 
 </script>
@@ -109,7 +119,7 @@
                         prev=cidx-1
                         if( prev < 0 )
                         {
-                            if( {{OPT.offset}} )
+                            if( offset )
                             {
                                 CallViewListRoute('prev')
                                 return
@@ -160,13 +170,11 @@
 
                     <button title="Show next image" class="col-auto btn btn-outline-info px-2" style="padding: 10%" id="ra" 
                         onClick="
-                        {% if OPT.last_entry_in_db is defined %}
+                            if( current == last_entry_in_db )
-                            if( current == {{OPT.last_entry_in_db}} )
                             {
                                 $('#ra').attr('disabled', true )
                                 return
                             }
-                        {% endif %}
                             if( document.fullscreen == false )
                                 fullscreen = false
                             cidx = eid_lst.indexOf(current.toString())