fix up quick hack / partial code improvement for bug-140

BUGs

@@ -1,30 +1,9 @@
-### Next: 146
+### Next: 147
-BUG-145: when I selected a few files with ctrl-click, moved them, the end result was "no files in path"
+BUG-146: with an empty DB, I See 'No files in Path!' twice (for file*, except for files_rbp)
-BUG-143: if I skip next fast enough in files_ip, it can get the warning about changes to entry list... (would have though I disabled next page until loaded)
+BUG-118: can move files from Bin path, but it leaves the del_file entry for it - need to remove it
-BUG-142: after transforming, the face data is still in the old spots, really should delete it / make it recalc
+BUG-117: when search returns files that can be deleted and/or restored, the icon stays as delete and tries to delete!
-BUG-141: can currently try to flip a video (in a highlighted group)
-BUG-140: When db is restarted underneath PA, it crashes job mgr... It should just accept timeouts, and keep trying to reconnect every 2? mins
-
-BUG-100: I managed to get 2 photos matching mich in the NOT_WORKING photo (probably dif refimgs but same p.tag?)
-    = /photos/2012/20120414-damien/IMG_8467.JPG
 BUG-106: cant add trudy /pat? as refimgs via FaceDBox
     - seems the cropped trudy face is not sufficient to find a face, how odd...
       (it came from a face bbox, BUT, I have grown the face seln by 10%?)
-BUG-117: when search returns files that can be deleted and/or restored, the icon stays as delete and tries to delete!
+BUG-100: I managed to get 2 photos matching mich in the NOT_WORKING photo (probably dif refimgs but same p.tag?)
-BUG-118: can move files from Bin path, but it leaves the del_file entry for it - need to remove it
+    = /photos/2012/20120414-damien/IMG_8467.JPG
-BUG-119: "Uncaught (in promise) Error: A listener indicated an asynchronous
-            response by returning true, but the message channel closed before a response
-            was received"
-            investigate this (possible I'm calling check_for_jobs and maybe not doing the async right?)
-
-BUG-123: pa_job_manager crashed with timeout on connection (probably when I turned off traefik for a bit?).  Regardless, should be more fault tolerant --> maybe offer to restart pa_job_manager IF its crashed?
-        this definitely happened also, when I shutdown the DB back-end mid job, and it was able to be restarted, so could get f/e to at least suggest a restart of the contianer, or auto-restart job_mgr?
-
-BUG-125: when an image is highlighted, then post the contextmenu on a different image - the highlight does not move to the new image
-         and the selected menu function processes the original or the new depending on the way the code works.
-         There is a chance we need to change the document on click to a mouse down (or whatever the context menu 
-         uses for default), rather than just fix the highlight
-
-BUG-132: right arrow to go to next photo in viewer ALSO scrolls to the right, needs a return somewhere in the jscript
-BUG-134: when moving set of photos on page, then move another set of photos on page, the first set reappears. Could really delete them from the dom?
-BUG-137: after moving/refiling photos, the next shift-click is out of order (reload fixes it)

TODO

@@ -1,6 +1,3 @@
-? get rid of style and just use class -- think this should work, so change in 
-    templates/files.html for throbber, etc. and dont set style as much in view_support.js
-
 ### GENERAL
 * jobs for AI should show path name
 * rm dups job should show progress bar

internal/js/files_support.js

@@ -89,7 +89,7 @@ function GetExistingDirsAsDiv( dt, divname, ptype )
 
 // show the DBox for a move file, includes all thumbnails of selected files to move
 // and a pre-populated folder to move them into, with text field to add a suffix
-function MoveDBox(path_details)
+function MoveDBox()
 {
     $('#dbox-title').html('Move Selected File(s) to new directory in Storage Path')  
     div =`
@@ -99,7 +99,7 @@ function MoveDBox(path_details)
         <form id="mv_fm" class="form form-control-inline col-12">
         <input id="move_path_type" name="move_path_type" type="hidden"
         `
-    div += ' value="' + path_details[0].type.name + '"></input>'
+    div += ' value="' + move_paths[0].type.name + '"></input>'
     div+=GetSelnAsDiv()
     yr=$('.highlight').first().attr('yr')
     dt=$('.highlight').first().attr('date')
@@ -110,10 +110,10 @@ function MoveDBox(path_details)
         <alert class="alert alert-primary my-auto py-1">
         `
     // NB: alert-primary here is a hack to get the bg the same color as the alert primary by
-    div+= '<svg id="move_path_icon" width="20" height="20" fill="currentColor"><use xlink:href="' + path_details[0].icon_url + '"></svg>'
+    div+= '<svg id="move_path_icon" width="20" height="20" fill="currentColor"><use xlink:href="' + move_paths[0].icon_url + '"></svg>'
     div+= '<select id="rp_sel" name="rel_path" class="text-primary alert-primary py-1 border border-primary rounded" onChange="change_rp_sel()">'
-    for(p of path_details) {
+    for(p of move_paths) {
-        div+= `<option path_type="${p.type.name}" icon_url="${p.icon_url}" ${(p.type.name=='Storage') ? "selected" : ""}>${p.root_dir}</option>`
+        div+= `<option path_type="${p.type.name}" icon_url="${p.icon_url}">${p.root_dir}</option>`
     }
     div+= '</select>'
     div+=`
@@ -140,6 +140,13 @@ function MoveDBox(path_details)
         </div>
         </form>
     `
+    // force to Storage always - if in Import, liekly storing, if in Storage, likely moving, user can always override
+    div+=`
+        <script>
+            storage_rp = move_paths.find(item => item.type.name === "Storage")?.root_dir;
+            $('#rp_sel').val(storage_rp);change_rp_sel()
+        </script>
+        `
 
     $('#dbox-content').html(div)
     $('#dbox').modal('show')
@@ -194,7 +201,7 @@ function DelDBox(del_or_undel)
     else
     {
         which="restore"
-        col="sucess"
+        col="success"
     }
 
     document.ents_to_del=[]
@@ -752,6 +759,10 @@ function resetNextPrevButtons()
 // get list of eids for the next page, also make sure next/prev buttons make sense for page we are on
 function nextPage(successCallback)
 {
+     // start with disabling more next presses until we are ready to process them
+     $('.prev').prop('disabled', true).addClass('disabled');
+     $('.next').prop('disabled', true).addClass('disabled');
+
     // pageList[0] is the first entry on this page
     const currentPage=getPageNumberForId( pageList[0] )
     // should never happen / just return pageList unchanged
@@ -767,6 +778,10 @@ function nextPage(successCallback)
 // get list of eids for the prev page, also make sure next/prev buttons make sense for page we are on
 function prevPage(successCallback)
 {
+    // start with disabling more prev presses until we are ready to process them
+    $('.prev').prop('disabled', true).addClass('disabled');
+    $('.next').prop('disabled', true).addClass('disabled');
+
     // pageList[0] is the first entry on this page
     const currentPage=getPageNumberForId( pageList[0] )
     // should never happen / just return pageList unchanged
@@ -850,9 +865,9 @@ function handleMoveOrDeleteOrRestoreFileJobCompleted(job)
 {
     // this grabs the values from the object attributes of eid-0, eid-1, etc.
     const ids = job.extra.filter(item => item.name.startsWith("eid-")).map(item => item.value);
-
+    
     // find page number of first element to delete (this is the page we will return too)
-    pnum=getPageNumberForId( ids[0] )
+    pnum=getPageNumberForId( parseInt(ids[0]) )
 
     // remove amendment data
     for (const ent of ids)
@@ -897,6 +912,9 @@ function handleCheckAmendmentJobStatus(data)
             handleTransformImageJobCompleted(data.job, data.entry)
         else if ( data.job.name == 'delete_files' || data.job.name == 'restore_files' || data.job.name == 'move_files' )
             handleMoveOrDeleteOrRestoreFileJobCompleted(data.job)
+            // if we are viewing this file, then just go up / back,b/c this file is "gone" from this view
+            if( document.viewing )
+                goOutOfViewer()
     }
     else { setTimeout( function() { checkForAmendmentJobToComplete(data.job.id) }, 1000 ); }
 }
@@ -906,6 +924,12 @@ $.contextMenu({
     selector: '.entry',
     itemClickEvent: "click",
     build: function($triggerElement, e) {
+        // if we are not in the highlight set, then move the highlight to this element
+        if( ! $(e.currentTarget).is('.highlight') )
+        {
+            $('.highlight').removeClass('highlight');
+            $(e.currentTarget).addClass('highlight')
+        }
         // when right-clicking & no selection add one OR deal with ctrl/shift right-lick as it always changes seln
         if( NoSel() || e.ctrlKey || e.shiftKey )
         {
@@ -967,7 +991,7 @@ $.contextMenu({
             callback: function( key, options) {
                 if( key == "details" ) { DetailsDBox() }
                 if( key == "view" ) { startViewing( $(this).attr('id') ) }
-                if( key == "move" ) { MoveDBox(move_paths) }
+                if( key == "move" ) { MoveDBox() }
                 if( key == "del"  ) { DelDBox('Delete') }
                 if( key == "undel") { DelDBox('Restore') }
                 if( key == "r90"  ) { Transform(90) }

internal/js/files_transform.js

@@ -2,7 +2,6 @@
 // can only have 1 ammendment per image, its grayed out for other changes
 function removeAmendment( id )
 {
-    console.log( 'removing amendment for: ' + id )
     document.amendments=document.amendments.filter(obj => obj.eid !== id)
 }
 
@@ -11,11 +10,16 @@ function removeAmendment( id )
 function handleTransformImageJobCompleted(job, entry)
 {
     removeAmendment( entry.id )
-    // update viewer and files* views, in case we view/go up without a new page load
+    // update viewer  if we are viewing an image
-    // force reload with timestamped version of im.src
+    if( document.viewing )
-    im.src=im.src + '?t=' + new Date().getTime();
+    {
-    DrawImg()
+        // force reload with timestamped version of im.src
+        im.src=im.src + '?t=' + new Date().getTime();
+        DrawImg()
+    }
 
+    // ALWAYS update files* div as we could go back to this from a viewer, and
+    // the thumbnail needs the updated data
     idx = entryList.indexOf(entry.id)
     // replace data for this entry now its been transformed
     document.entries[idx]=entry

internal/js/view_support.js

@@ -80,16 +80,19 @@ function DrawImg()
     // -50 is a straight up hack, no idea why this works, but its good enough for me
     if (!Array.isArray(am))
     {
-        const style = 'position: absolute; left: 50%; top: 50%; transform: translate(-50%, -50%);';
+        $('#throbber').show()
-        $('#throbber').attr('style', style + ' height: 96px;');
+        $('#white-circle').show()
-        $('#white-circle').attr('style', style + ' height: 72px;');
         if(am.type.which == 'img' )
-            $('#inside-img').attr('style', style + ' height: 64px;').attr('src', '/internal/'+am.type.what );
+        {
+            $('#inside-img').attr('src', '/internal/'+am.type.what );
+            $('#inside-img').show()
+        }
         else
         {
-            $('#inside-icon').attr('style', `${style} color:${am.type.colour}; height: 64px;`)
+            $('#inside-icon').attr('style', `color:${am.type.colour};height:64px` )
             $('#inside-icon').attr('fill', am.type.colour )
             $('#inside-icon use').attr('xlink:href', `/internal/icons.svg#${am.type.what}`);
+            $('#inside-icon').show()
         }
     } else {
         $('#throbber').hide()

main.py

@@ -66,12 +66,19 @@ app.config['LDAP_USER_DN'] = 'ou=users'
 app.config['LDAP_GROUP_DN'] = 'ou=groups'
 app.config['LDAP_USER_RDN_ATTR'] = 'uid'
 app.config['LDAP_USER_LOGIN_ATTR'] = 'uid'
-app.config['LDAP_BIND_USER_DN'] = None
-app.config['LDAP_BIND_USER_PASSWORD'] = None
 app.config['LDAP_GROUP_OBJECT_FILTER'] = '(objectclass=posixGroup)'
 app.config['LDAP_BIND_USER_DN'] = None
 app.config['LDAP_BIND_USER_PASSWORD'] = None
 
+# stop db restarts from causing stales and client-side 'server errors' - its a
+# touch hacky, e.g. it issues a select 1 before EVERY request, likely should
+# ditch this and just have a short-lived pool, but need to work out if/where I
+# can catch the right exception myself and then dont need this, but for now...
+app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
+    "pool_pre_ping": True,
+    "pool_recycle": 280,  # Good practice to include this with pre-ping
+}
+
 
 db = SQLAlchemy(app)                           # create the (flask) sqlalchemy connection
 ma = Marshmallow(app)                          # set up Marshmallow - data marshalling / serialising

pa_job_manager.py

@@ -20,6 +20,7 @@ from sqlalchemy.orm import relationship
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm import scoped_session
+from contextlib import contextmanager
 
 ### LOCAL FILE IMPORTS ###
 from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE, SymlinkName, GenThumb, SECS_IN_A_DAY, PA_EXIF_ROTATER, PA
@@ -66,21 +67,41 @@ override_tbls={ "face_no_match_override", "face_force_match_override", "disconne
 # this is required to handle the duplicate processing code
 sys.setrecursionlimit(50000)
 
-# a Manager, which the Session will use for connection resources
-some_engine = create_engine(DB_URL)
 
-# create a configured "Session" class
+# 1. Add pool_pre_ping and pool_recycle here to handle db container disappearing underneath us
-#Session = sessionmaker(bind=some_engine)
+some_engine = create_engine(
+    DB_URL,
+    pool_pre_ping=True,    # check DB connection is still active before use
+    pool_recycle=300,      # churn connections regardless every 5 mins
+    pool_size=20,          # Parallel-ready base pool
+    max_overflow=10        # Burst capacity for high socket traffic
+)
 
-# create a Session
 session_factory = sessionmaker(bind=some_engine)
 Session = scoped_session(session_factory)
-session = Session()
+
+# HACK: need to remove this and use 'sess' as an actual param everywhere, butt here are 200+ so quick fix until retired
+session = Session
+
+# this is a way to handle a session failing
+@contextmanager
+def PA_db_session():
+    """Provide a transactional scope around a series of operations."""
+    # This creates a NEW session from the registry
+    s = Session()
+    try:
+        yield s
+        s.commit()
+    except Exception:
+        s.rollback()
+        raise
+    finally:
+        # This destroys the session and returns connection to pool
+        Session.remove()
 
 # this creates the Base (like db model in flask)
 Base = declarative_base()
 
-
 ################################################################################
 # Class describing PathType & in the database (via sqlalchemy)
 # series of pre-defined types of paths (import, storage, bin)
@@ -1897,6 +1918,11 @@ def JobTransformImage(job):
     amt=[jex.value for jex in job.extra if jex.name == "amt"][0]
     e=session.query(Entry).join(File).filter(Entry.id==id).first()
     PAprint( f"JobTransformImage: job={job.id}, id={id}, amt={amt}" )
+    # cant transfer non-image, but may get here if multi-select includes non-Image
+    if e.type.name != 'Image':
+        removeEntryAmendment( job, id )
+        FinishJob(job, "Cannot rotate file as it is not an Image","Failed")
+        return
 
     if amt == "fliph":
         AddLogForJob(job, f"INFO: Flipping {e.FullPathOnFS()} horizontally" )
@@ -1920,6 +1946,8 @@ def JobTransformImage(job):
     e.file_details.hash = md5( job, e )
     PAprint( f"JobTransformImage DONE thumb: job={job.id}, id={id}, amt={amt}" )
     session.add(e)
+    # any faces in this file are no longer valid, remove them
+    session.query(FaceFileLink).filter(FaceFileLink.file_eid==e.id).delete()
     removeEntryAmendment( job, id )
 
     FinishJob(job, "Finished Processesing image rotation/flip")
@@ -2744,7 +2772,13 @@ if __name__ == "__main__":
 
     InitialValidationChecks()
 
-    HandleJobs(True)
+    # Initial job run on startup (hence True in 1st param)
+    try:
+        with PA_db_session() as sess:
+            HandleJobs(True)
+    except Exception as e:
+        PAprint(f"ERROR: Initial job handle failed: {e}")
+
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         s.bind((PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT))
         # force timeout every 1 day so we can run scheduled jobs
@@ -2752,18 +2786,35 @@ if __name__ == "__main__":
         s.listen()
         while True:
             try:
+                # 1. Wait for connection
                 conn, addr = s.accept()
                 if DEBUG:
-                    PAprint( f"accept finished, tout={s.timeout}" )
+                    PAprint(f"Connection accepted from {addr}")
+
+                # 2. Process Jobs after a successful socket connection
+                with PA_db_session() as sess:
+                    HandleJobs(False)
+                    # Check for scheduled tasks as well
+                    if ScheduledJobs():
+                        HandleJobs(False)
 
             except socket.timeout:
                 if DEBUG:
-                    PAprint( f"timeout occurred, tout={s.timeout}" )
+                    PAprint("Socket timeout (Daily maintenance window) reached.")
-                if ScheduledJobs():
+                
-                    HandleJobs(False)
+                # 3. Process Scheduled Jobs during the timeout
+                try:
+                    with PA_db_session() as sess:
+                        if ScheduledJobs():
+                            HandleJobs(False)
+                except sqlalchemy.exc.OperationalError:
+                    PAprint("DB Connection lost during scheduled task window. Retrying next cycle.")
                 continue
-            else:
+
-                HandleJobs(False)
+            except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.InterfaceError) as e:
-                # in case we constantly have jobs running, the '1 day' last import might be missed, so check it after each job too
+                # This catches the DB container restart specifically
-                if ScheduledJobs():
+                PAprint(f"DATABASE ERROR: Connection lost. Retrying... {e}")
-                    HandleJobs(False)
+                time.sleep(5) # Brief pause before next socket listen
+                
+            except Exception as e:
+                PAprint(f"UNEXPECTED ERROR: {e}")

templates/files.html

@@ -140,10 +140,12 @@
                 <figure style="position: relative;" class="col col-auto border border-info rounded m-0 p-1" id="figure">
                     <canvas id="canvas"></canvas>
                     <!-- next 4 are placeholders and called on during amendments only in viewer code -->
-                    <img id="throbber" src="{{url_for('internal', filename='throbber.gif')}}?v={{js_vers[th]}}" style="display:none;">
+                    <img id="throbber" src="{{url_for('internal', filename='throbber.gif')}}?v={{js_vers[th]}}" style="display:none;height:96px"
-                    <img id="white-circle" src="{{url_for('internal', filename='white-circle.png')}}?v={{js_vers[th]}}" style="display:none;">
+                        class="position-absolute top-50 start-50 translate-middle">
-                    <img id="inside-img" style="display:none;">
+                    <img id="white-circle" src="{{url_for('internal', filename='white-circle.png')}}?v={{js_vers[th]}}" style="display:none;height:72px"
-                    <svg id="inside-icon" style="display:none;" fill="currentColor">
+                        class="position-absolute top-50 start-50 translate-middle">
+                    <img id="inside-img" style="display:none;height:64px" class="position-absolute top-50 start-50 translate-middle">
+                    <svg id="inside-icon" style="display:none;height:64px" class="position-absolute top-50 start-50 translate-middle">
                         <use xlink:href="{{url_for('internal', filename='icons.svg')}}?v={{js_vers['ic']}}#flip_v">
                         </use></svg>
                     <script>
@@ -274,10 +276,8 @@
     // force pageList to set pageList for & render the first page
     getPage(1,getPageFigures)
 
-    // FIXME: doco, but also gather all globals together, many make them all document. to be obviously global (and add fullscreen)
+    // gap is used to keep some space around video in viewer - tbh, not sure why anymore
     var gap=0.8
-    var grayscale=0
-    var throbber=0
 
     function PrettyFname(fname)
     {
@@ -327,5 +327,11 @@
         $('#viewer_del').on('click', function() { DelDBox('Restore') } )
     }
 
+    if( isMobile() )
+    {
+        $('#shift-key').css('visibility', 'visible');
+        $('#ctrl-key').css('visibility', 'visible');
+    }
+
 </script>
 {% endblock script_content %}