From 9bdd9d5b783356a2215a5148073accbde5608c9b Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 25 Oct 2025 18:21:28 +1100 Subject: [PATCH] revamp whole EA flow. Server created EAs when we do certain jobs (transform, delete_files, restore_files), then instead of faking amendments in the jscript, get job creation to return EA from job ORM, then check is now generic for end of any amendment job, and when it finishes, use that to clear our amendments in document, and redraw through normal UI code. No smarts in client, all driven by state from server, and if we reload a page mid jobs, it has required state, and because an amendment job is still progressing, it runs check code again --- TODO | 9 -- amend.py | 1 + files.py | 56 ++++++--- internal/js/files_support.js | 212 ++++++++++++++++++++++++--------- internal/js/files_transform.js | 98 ++++----------- job.py | 31 +++-- 6 files changed, 236 insertions(+), 171 deletions(-) diff --git a/TODO b/TODO index cdb651e..48f5e23 100644 --- a/TODO +++ b/TODO @@ -1,12 +1,3 @@ -* new viewing model (get ids of query on first load, then paginate only inside that known list) - - BUT, when we finish a delete, what do I do with pageList / entryList??? - - start by showing them as deleted (via amend) - - then on success, remove the ids from the *List arrays in js -- but do - this via repagination, invalidate page cache fully, then getPage(currentPage) - (e.g. assume 1, 2, 3 ... 40 in eList). delete 23, 24, - then reset lists to remove 23 and 24, pageList would then - get reset to page with: 21,22,25,26 ... 30, 31 - ? get rid of style and just use class -- think this should work, so change in templates/files.html for throbber, etc. and dont set style as much in view_support.js diff --git a/amend.py b/amend.py index c1538f6..ead68af 100644 --- a/amend.py +++ b/amend.py @@ -32,6 +32,7 @@ class EntryAmendment(PA,db.Model): job_id = db.Column(db.Integer, db.ForeignKey("job.id"), primary_key=True ) amend_type = db.Column(db.Integer, db.ForeignKey("amendment_type.id")) type = db.relationship("AmendmentType", backref="entry_amendment") + job = db.relationship("Job", back_populates="amendments") ################################################################################ diff --git a/files.py b/files.py index 4c41e57..d470549 100644 --- a/files.py +++ b/files.py @@ -5,6 +5,7 @@ from main import db, app, ma from sqlalchemy import Sequence, text, select, union, or_ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import joinedload +import numbers import os import glob import json @@ -266,6 +267,22 @@ class EntrySchema(ma.SQLAlchemyAutoSchema): def get_full_path(self, obj): return obj.FullPathOnFS() +class JobExtraSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = JobExtra + load_instance = True + name = ma.auto_field() + value = ma.auto_field() + +class JobSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = Job + load_instance = True + id = ma.auto_field() + name = ma.auto_field() + extra = ma.Nested(JobExtraSchema, many=True) + amendments = ma.Nested(EntryAmendmentSchema, many=True) + # global - this will be use more than once below, so do it once for efficiency entries_schema = EntrySchema(many=True) FOT_Schema = FaceOverrideTypeSchema(many=True) @@ -273,6 +290,8 @@ path_Schema = PathSchema(many=True) person_Schema = PersonSchema(many=True) et_schema = AmendmentTypeSchema(many=True) ea_schema = EntryAmendmentSchema(many=True) +job_schema = JobSchema(many=False) +job_schemas = JobSchema(many=True) ################################################################################ # /get_entries_by_ids -> route where we supply list of entry ids (for next/prev @@ -312,7 +331,7 @@ def process_ids(): ea = db.session.execute(stmt).unique().scalars().all() ea_data=ea_schema.dump(ea) - return jsonify(entries=entries_schema.dump(sorted_data), amend=ea_data) + return jsonify(entries=entries_schema.dump(sorted_data), amendments=ea_data) ################################################################################ @@ -629,7 +648,7 @@ def restore_files(): jex.append( JobExtra( name=f"{el}", value=str(request.form[el]) ) ) job=NewJob( name="restore_files", num_files=0, wait_for=None, jex=jex, desc="to restore selected file(s)" ) - return redirect("/jobs") + return jsonify( job=job_schema.dump(job) ) ################################################################################ # /delete_files -> create a job to delete files for the b/e to process @@ -642,7 +661,7 @@ def delete_files(): jex.append( JobExtra( name=f"{el}", value=str(request.form[el]) ) ) job=NewJob( name="delete_files", num_files=0, wait_for=None, jex=jex, desc="to delete selected file(s)" ) - return redirect("/jobs") + return jsonify( job=job_schema.dump(job) ) ################################################################################ # /move_files -> create a job to move files for the b/e to process @@ -685,7 +704,7 @@ def view(): # route called from front/end - if multiple images are being transformed, each transorm == a separate call # to this route (and therefore a separate transorm job. Each reponse allows the f/e to check the -# specific transorm job is finished (/check_transform_job) which will be called (say) every 1 sec. from f/e +# specific transorm job is finished (/check_amend_job_status) which will be called (say) every 1 sec. from f/e # with a spinning wheel, then when pa_job_mgr has finished it will return the transformed thumb @app.route("/transform", methods=["POST"]) @login_required @@ -698,27 +717,28 @@ def transform(): jex.append( JobExtra( name=f"{el}", value=str(request.form[el]) ) ) job=NewJob( name="transform_image", num_files=0, wait_for=None, jex=jex, desc="to transform selected file(s)" ) - return jsonify( job_id=job.id ) + return jsonify( job=job_schema.dump(job) ) ################################################################################ -# /check_transform_job -> URL that is called repeatedly by front-end waiting for the -# b/e to finish the transform job. Once done, the new / now -# transformed image's thumbnail is returned so the f/e can -# update with it +# /check_amend_job_status -> URL that is called repeatedly by front-end waiting +# for the b/e to finish the amendment job (delete/restore/move file). +# Once done, return "ok" ################################################################################ -@app.route("/check_transform_job", methods=["POST"]) +@app.route("/check_amend_job_status", methods=["POST"]) @login_required -def check_transform_job(): +def check_amend_job_status(): job_id = request.form['job_id'] - stmt=select(Job).where(Job.id==job_id) - job=db.session.execute(stmt).scalars().one_or_none() - j=jsonify( finished=False ) - if job.pa_job_state == 'Completed': - id=[jex.value for jex in job.extra if jex.name == "id"][0] - stmt=select(Entry).where(Entry.id==id) + stmt = select(Job).options(joinedload(Job.amendments)).where(Job.id == job_id) + job=db.session.execute(stmt).unique().scalars().first() + # FIXME: should validate job_id is real from UI + if job.name == 'transform_image': + eid=[jex.value for jex in job.extra if jex.name == "id"][0] + stmt=select(Entry).where(Entry.id==eid) ent=db.session.execute(stmt).scalars().all() ent_data=entries_schema.dump(ent) - j=jsonify( finished=True, entry=ent_data[0] ) + j=jsonify(finished=(job.pa_job_state == 'Completed'), job=job_schema.dump(job), entry=ent_data[0] ) + else: + j=jsonify(finished=(job.pa_job_state == 'Completed'), job=job_schema.dump(job)) return j ################################################################################ diff --git a/internal/js/files_support.js b/internal/js/files_support.js index 5f0db22..c6fcaf9 100644 --- a/internal/js/files_support.js +++ b/internal/js/files_support.js @@ -151,6 +151,37 @@ function MoveDBox(path_details) $("#suffix").keypress(function (e) { if (e.which == 13) { $("#move_submit").click(); return false; } } ) } +// This function is called anytime we have a job that returns amendments +// (visually we want to show this entry is being amended by a job) +// as we check for a job to end every second, we can call this multiple times +// during the runtime of a job, so only redraw/react to a new amendment +// NOTE: we update all views, as we might go into one via jscript before the job ends +function processAmendments( ams ) +{ + for (const am of ams) + { + // if we return anything here, we already have this amendment, so continue to next + if( document.amendments.filter(obj => obj.eid === am.eid).length > 0 ) + continue + + document.amendments.push(am) + + if( document.viewing && document.viewing.id == am.eid ) + { + im.src=im.src + '?t=' + new Date().getTime(); + DrawImg() + } + + // find where in the page this image is being viewed + idx = pageList.indexOf(am.eid) + // createFigureHtml uses matching document.amendments to show thobber, etc + html = createFigureHtml( document.entries[idx] ) + $('#'+am.eid).replaceWith( html ) + } +} + +// function to add data for document.amendment based on id and amt +// used when we transform several images in files_*, or single image in viewer // show the DBox for a delete/restore file, includes all thumbnails of selected files // with appropriate coloured button to Delete or Restore files` function DelDBox(del_or_undel) @@ -159,28 +190,36 @@ function DelDBox(del_or_undel) $('#dbox-title').html(del_or_undel+' Selected File(s)') div ='

' + del_or_undel + ' the following files?

' div+=GetSelnAsDiv() + if( del_or_undel == "Delete" ) + { + which="delete" + col="danger" + } + else + { + which="restore" + col="sucess" + } + + document.ents_to_del=[] + $('.highlight').each(function( cnt ) { document.ents_to_del[cnt]=parseInt($(this).attr('id')) } ) div+=`
- ` - div+=` - -
- ` - else - // just force page reload to / for now if restoring files from a search path -- a search (by name) - // would match the deleted/restored file, so it would be complex to clean up the UI (and can't reload, as DB won't be changed yet) - div+=` - '/restore_files', - success: function(data){ - if( $(location).attr('pathname').match('search') !== null || document.viewing ) { window.location='/' }; CheckForJobs() } }); return false" class="btn btn-outline-success col-2">Ok - - ` + + ` $('#dbox-content').html(div) $('#dbox').modal('show') } @@ -378,37 +417,37 @@ function createFigureHtml( obj ) // Image/Video/Unknown entry if (obj.type.name === "Image" || obj.type.name === "Video" || obj.type.name === "Unknown") { - const pathType = obj.in_dir.in_path.type.name; - const size = obj.file_details.size_mb; - const hash = obj.file_details.hash; - const inDir = `${obj.in_dir.in_path.path_prefix}/${obj.in_dir.rel_path}`; - const fname = obj.name; - const yr = obj.file_details.year; - const date = `${yr}${String(obj.file_details.month).padStart(2, '0')}${String(obj.file_details.day).padStart(2, '0')}`; - const prettyDate = `${obj.file_details.day}/${obj.file_details.month}/${obj.file_details.year}`; - const type = obj.type.name; + const pathType = obj.in_dir.in_path.type.name; + const size = obj.file_details.size_mb; + const hash = obj.file_details.hash; + const inDir = `${obj.in_dir.in_path.path_prefix}/${obj.in_dir.rel_path}`; + const fname = obj.name; + const yr = obj.file_details.year; + const date = `${yr}${String(obj.file_details.month).padStart(2, '0')}${String(obj.file_details.day).padStart(2, '0')}`; + const prettyDate = `${obj.file_details.day}/${obj.file_details.month}/${obj.file_details.year}`; + const type = obj.type.name; - // if amendment for this obj, do not add entry class - prevents highlighting - if( am ) { - ent="" - gs="style='filter: grayscale(100%);'" - am_html ='' - am_html +='' - if( am.type.which == 'icon' ) - am_html+=`` - else - am_html+=`` - } else { - ent="entry" - gs="" - am_html="" - } - html += ` -
- ${renderMedia(obj,gs,am_html)} -
`; + // if amendment for this obj, do not add entry class - prevents highlighting + if( am ) { + ent="" + gs="style='filter: grayscale(100%);'" + am_html ='' + am_html+='' + if( am.type.which == 'icon' ) + am_html+=`` + else + am_html+=`` + } else { + ent="entry" + gs="" + am_html="" + } + html += ` +
+ ${renderMedia(obj,gs,am_html)} + ` } // Directory entry else if (obj.type.name === "Directory" && OPT.folders) { @@ -422,7 +461,6 @@ function createFigureHtml( obj )
${obj.name}
-
`; html += ``; } @@ -434,7 +472,8 @@ function createFigureHtml( obj ) $('#${obj.id}').click( function(e) { DoSel(e, this ); SetButtonState(); return false; }); $('#${obj.id}').dblclick( function(e) { startViewing( $(this).attr('id') ) } ) } - ` + + ` return html } @@ -621,6 +660,7 @@ function getEntriesByIdSuccessHandler(res,pageNumber,successCallback,viewingIdx) document.entries=res; // cache this document.page[pageNumber]=res + // FIXME: I want to remove successCallback, instead: if viewing, or files_*, or file_list, then call relevant draw routine successCallback(res,viewingIdx) resetNextPrevButtons() // if search, disable folders @@ -663,11 +703,11 @@ function getPage(pageNumber, successCallback, viewingIdx=0) data: JSON.stringify(data), contentType: 'application/json', dataType: 'json', success: function(res) { - document.amendments=res.amend; - // this is only called when we are viewing a page in files/list view, so check for job(s) ending... - for (const tmp of document.amendments) { - CheckTransformJob(tmp.eid,tmp.job_id,handleTransformFiles) - } + document.amendments=res.amendments; + // only called when an amendment is pending & we are viewing a page in files/list view + // so check for amendment job(s) ending... + for (const tmp of document.amendments) + checkForAmendmentJobToComplete(tmp.job_id) getEntriesByIdSuccessHandler( res.entries, pageNumber, successCallback, viewingIdx ) }, error: function(xhr, status, error) { console.error("Error:", error); } }); @@ -811,6 +851,64 @@ function changeSize() $('.svg_cap').width(sz); } +// when a delete or restore files job has completed successfullly, then get ids +// find the page we are on, remove amendments & ids from entryList and re-get page +// which will reset pageList and the UI of images for that page +function handleDeleteOrRestoreFileJobCompleted(job) +{ + // this grabs the values from the object attributes of eid-0, eid-1, etc. + const ids = job.extra.map(item => item.value) + + // find page number of first element to delete (this is the page we will return too) + pnum=getPageNumberForId( ids[0] ) + + // remove amendment data + for (const ent of ids) + { + id=parseInt(ent) + removeAmendment( id ) + // remove the item in the entryList + index=entryList.indexOf(id); + if( index != -1 ) + entryList.splice(index, 1); // Remove the element + else + { + return; // have to get out of here, or calling getPage() below will loop forever + } + } + + // re-create pageList by reloading the page + getPage(pnum,getPageFigures) +} + +// POST to a check URL, that will tell us if the amendment job has completed, +// it also calls CheckForJobs() which will fix up the Active Jobs badge, +function checkForAmendmentJobToComplete(job_id) +{ + CheckForJobs() + $.ajax( { type: 'POST', data: '&job_id='+job_id, url: '/check_amend_job_status', + success: function(res) { handleCheckAmendmentJobStatus(res); } } ) +} + +// the status of a Amendment Job has been returned, finished is True/False +// if not finished try again in 1 second... If finished then invalidate page +// cache and based on job type call code correct func to update the UI appropriately +function handleCheckAmendmentJobStatus(data) +{ + if( data.finished ) + { + // invalidate the cache + document.page.length=0 + + // transforms contain the single transformed entry data for convenience + if( data.job.name == 'transform_image' ) + handleTransformImageJobCompleted(data.job, data.entry) + else if ( data.job.name == 'delete_files' || data.job.name == 'restore_files' ) + handleDeleteOrRestoreFileJobCompleted(data.job) + } + else { setTimeout( function() { checkForAmendmentJobToComplete(data.job.id) }, 1000 ); } +} + // different context menu on files $.contextMenu({ selector: '.entry', diff --git a/internal/js/files_transform.js b/internal/js/files_transform.js index a5b54eb..821e81a 100644 --- a/internal/js/files_transform.js +++ b/internal/js/files_transform.js @@ -2,75 +2,26 @@ // can only have 1 ammendment per image, its grayed out for other changes function removeAmendment( id ) { - document.amendments=document.amendments.filter(obj => obj.eid !== id) + console.log( 'removing amendment for: ' + id ) + document.amendments=document.amendments.filter(obj => obj.eid !== id) } -// POST to a check URL, that will tell us if the transformation has completed, -// if not, try again in 1 second... If it has finished then reset the thumbnail -// to full colour, put it back to being an entry and reset the thumbnail to the -// newly created one that was sent back in the response to the POST -function handleTransformFiles(data,id,job_id) +// If Transform job has finished then reset relevant document.entries +// with updated from DB, remove the amendment and redraw image +function handleTransformImageJobCompleted(job, entry) { - if( data.finished ) - { - id=parseInt(id) - idx = entryList.indexOf(id) - // replace data for this entry now its been transformed - document.entries[idx]=data.entry - // update cache too - // document.page[getPageNumberForId(id)][howFarIntoPageCache(id)]=data.entry - // FIXME: for now just invalidate whole cache - document.page.length=0 - removeAmendment( id ) - // redraw into figure html in dom - last={ 'printed': 'not required' } - html = createFigureHtml( data.entry, last, 9999 ) - $('#'+id).replaceWith( html ) - return false; - } - else - { - setTimeout( function() { CheckTransformJob(id,job_id,handleTransformFiles) }, 1000,id, job_id ); - } -} + removeAmendment( entry.id ) + // update viewer and files* views, in case we view/go up without a new page load + // force reload with timestamped version of im.src + im.src=im.src + '?t=' + new Date().getTime(); + DrawImg() -// POST to a check URL, that will tell us if the transformation has completed, -// if not, try again in 1 second... If it has finished then reset the image -// to full colour -function handleTransformViewing(data,id,job_id) -{ - if( data.finished ) - { - // stop throbber, remove grayscale & then force reload with timestamped version of im.src - im.src=im.src + '?t=' + new Date().getTime(); - removeAmendment( id ) - return false; - } - else - { - setTimeout( function() { CheckTransformJob(id,job_id,handleTransformViewing) }, 1000,id, job_id ); - } -} - -// POST to a check URL, that will tell us if the transformation has completed, -// if not, try again in 1 second... If it has finished then reset the thumbnail -// to full colour, put it back to being an entry and reset the thumbnail to the -// newly created one that was sent back in the response to the POST -function CheckTransformJob(id,job_id,successCallback) -{ - CheckForJobs() - $.ajax( { type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', - success: function(res) { successCallback(res,id,job_id); } } ) -} - -// function to add data for document.amendment based on id and amt -// used when we transform several images in files_*, or single image in viewer -function addTransformAmendment(id,amt) -{ - am={} - am.eid=parseInt(id) - am.type = document.amendTypes.filter(obj => obj.job_name === 'transform_image:'+amt )[0] - document.amendments.push(am) + idx = entryList.indexOf(entry.id) + // replace data for this entry now its been transformed + document.entries[idx]=entry + // redraw into figure html in dom + html = createFigureHtml( entry ) + $('#'+entry.id).replaceWith( html ) } // for each highlighted image, POST the transform with amt (90, 180, 270, @@ -81,15 +32,13 @@ function addTransformAmendment(id,amt) function Transform(amt) { // we are in the viewer with 1 image only... - if( document.viewing ) + if( $('#viewer_div').length && ! $('#viewer_div').hasClass('d-none') ) { post_data = '&amt='+amt+'&id='+document.viewing.id // POST /transform for image, grayscale the image, add throbber, & start checking for end of job $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data) { - addTransformAmendment(document.viewing.id, amt) - DrawImg(); - CheckTransformJob(document.viewing.id,data.job_id,handleTransformViewing); - return false; + processAmendments(data.job.amendments) + checkForAmendmentJobToComplete(data.job.id) } }) } else @@ -98,13 +47,8 @@ function Transform(amt) post_data = '&amt='+amt+'&id='+e.id // POST /transform for image, grayscale the thumbnail, add throbber, & start checking for end of job $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data){ - addTransformAmendment(e.id, amt) - last={ 'printed': 'not required' } - idx = pageList.indexOf(parseInt(e.id)) - html = createFigureHtml( document.entries[idx], last, 9999 ) - $('#'+e.id).replaceWith( html ) - CheckTransformJob(e.id,data.job_id,handleTransformFiles); - return false; + processAmendments(data.job.amendments) + checkForAmendmentJobToComplete(data.job.id) } }) } ) } diff --git a/job.py b/job.py index bdc2ce8..253200c 100644 --- a/job.py +++ b/job.py @@ -4,6 +4,7 @@ from flask import request, render_template, redirect, make_response, jsonify, ur from settings import Settings from main import db, app, ma from sqlalchemy import Sequence, func, select +from sqlalchemy.orm import joinedload from sqlalchemy.exc import SQLAlchemyError from datetime import datetime, timedelta import pytz @@ -58,10 +59,12 @@ class Job(db.Model): extra = db.relationship( "JobExtra") logs = db.relationship( "Joblog") + amendments = db.relationship("EntryAmendment", back_populates="job") def __repr__(self): return "".format(self.id, self.start_time, self.last_update, self.name, self.state, self.num_files, self.current_file_num, self.current_file, self.pa_job_state, self.wait_for, self.extra, self.logs) + ################################################################################ # Class describing PA_JobManager_Message and in the DB (via sqlalchemy) # the job manager can send a message back to the front end (this code) via the @@ -83,7 +86,7 @@ class PA_JobManager_Message(PA,db.Model): # Used in main html to show a red badge of # jobs to draw attention there are # active jobs being processed in the background ################################################################################ -def GetNumActiveJobs(): +def getNumActiveJobs(): ret=Job.query.filter(Job.pa_job_state != 'Completed').with_entities(func.count(Job.id).label('count') ).first() return ret[0] @@ -122,16 +125,16 @@ def NewJob(name, num_files="0", wait_for=None, jex=None, desc="No description pr if job.name == 'transform_image': id=[jex.value for jex in job.extra if jex.name == "id"][0] ea=EntryAmendment( eid=id, job_id=job.id, amend_type=at_id ) - print( f"just added an EA for eid={id}, j={job.id}" ) db.session.add(ea) - elif job.name == 'delete_files': + job.amendments.append(ea) + # FIXME: add ea to job.amend + elif job.name == 'delete_files' or job.name == 'restore_files': for j in jex: if 'eid-' in j.name: - ea=EntryAmendment( eid=j.value, amend_type=at_id ) + ea=EntryAmendment( eid=j.value, job_id=job.id, amend_type=at_id ) db.session.add(ea) - # need to return this to the f/e somehow - # this is for removes, really need to think about this more - #job.amendment=ea + # FIXME: add ea to job.amend + job.amendments.append(ea) SetFELog( message=f'Created Job #{job.id} to {desc}', level="success" ) WakePAJobManager(job.id) @@ -327,14 +330,22 @@ def joblog_search(): @app.route("/check_for_jobs", methods=["POST"]) @login_required def check_for_jobs(): - num=GetNumActiveJobs() + from files import job_schemas + + num=getNumActiveJobs() + messages = PA_JobManager_Message.query.all() sts=[] - for msg in PA_JobManager_Message.query.all(): + for msg in messages: u='' if 'Job #' not in msg.message and msg.job_id: u='Job #' + str(msg.job_id) + ': ' sts.append( { 'id': msg.id, 'message': u+msg.message, 'level': msg.level, 'job_id': msg.job_id, 'persistent': msg.persistent, 'cant_close': msg.cant_close } ) - return make_response( jsonify( num_active_jobs=num, sts=sts ) ) + + # get jobs mentioned in messages as we may need to process the by client for UI + job_list=[obj.job_id for obj in messages] + stmt = select(Job).options(joinedload(Job.amendments)).where(Job.id.in_(job_list)) + jobs=db.session.execute(stmt).unique().scalars().all() + return make_response( jsonify( num_active_jobs=num, sts=sts, jobs=job_schemas.dump(jobs) ) ) ############################################################################### # /clear_msg -> POST -> clears out a F/E message based on passed in