diff --git a/BUGs b/BUGs index 90293ca..4bd5201 100644 --- a/BUGs +++ b/BUGs @@ -1,15 +1,7 @@ -### Next: 141 +### Next: 143 +BUG-142: after transforming, the face data is still in the old spots, really should delete it / make it recalc +BUG-141: can currently try to flip a video (in a highlighted group) BUG-140: When db is restarted underneath PA, it crashes job mgr... It should just accept timeouts, and keep trying to reconnect every 2? mins -BUG-139: using any large entry list and going next a few times, ends say 4 pages of 50 into 4000 matches (entries from DB < 50)... - - confirmed this is when person has 2 or more refimgs: - - on page "2", we get 49 pulled back in the ORM instead of the 50 expected -- b/c I use that to indicate we must be at the end of the list if not 50 found - -- really, need to fix once and for all the eids / re-running query. - do GetEntries as we do now, once done however, get all entry ids. Stick those into the DB with a unique query-id and datestamp - new func to get all details needed for entries in an eid list (of 1-page) - show this page of entries - use current, full eidlist and to work our start/end of list (next/prev), disabling. - then client can keep current page of data, if you hit next/prev, use DB unique query id / full list and page of eids, and give full data for new page of entries - Implications though, are if a search is invalidated (maybe delete / move a photo), need to remove them from the list on the DB too OR let user know/decide to fix/wait. - BUG-100: I managed to get 2 photos matching mich in the NOT_WORKING photo (probably dif refimgs but same p.tag?) = /photos/2012/20120414-damien/IMG_8467.JPG @@ -31,11 +23,6 @@ BUG-125: when an image is highlighted, then post the contextmenu on a different There is a chance we need to change the document on click to a mouse down (or whatever the context menu uses for default), rather than just fix the highlight -BUG-130: moving files and then trying to go next page and it got confused... BUG-132: right arrow to go to next photo in viewer ALSO scrolls to the right, needs a return somewhere in the jscript -BUG-133: when rebuilding pa[dev], the first run fails to have symlinks to the right paths for Import/Storage, etc. a simple restart fixes - so potentially the intial copy or some other race condition? BUG-134: when moving set of photos on page, then move another set of photos on page, the first set reappears. Could really delete them from the dom? -BUG-135: failed to rotate: 2006/20061215-ITS-xmas-KP/DSC00582.JPG - not sure why && not repeatable, so its not the image, timing/race condition maybe? BUG-137: after moving/refiling photos, the next shift-click is out of order (reload fixes it) -BUG-138: Placeholder for all the ways we can get the front-end confused: - ---> JUST fix all these BUGs (relating to confused/lost state) by revisiting the overally complex way I remember state and my position in a list (probably FAR easier, to make an initial sql just save all eids, and then not try to recreate that list ever again and not care how I got into the list). Can attach a "running server-side sequence number", and if old sequence, and the original eid list results in a failure, then just pop up that the saved list is no longer valid, and ask user to re-do their search/list..." diff --git a/TODO b/TODO index 47bf247..cdb651e 100644 --- a/TODO +++ b/TODO @@ -1,18 +1,18 @@ -### major fix - go to everywhere I call GetEntries(), and redo the logic totally... - * client side: - * instead of removing deleted images from DOM, we should gray them out and put a big Del (red circle with line?) though it as overlay. - [DONE] * Create another table of entry_ammendments - note the deletions, rotations, flips of specific eids - then reproduce that on the client side visually as needed - [DONE] - at least grayed-out, to indicate a pending action is not complete. - - When job that flips, rotates, deletes completes then create an entry_amendment in the DB. - - Also hand fudge the jscript amendments for each job / next get_entry_by_id (if needed will also set amendments as needed) - - When job finishes, remove amendment from DB - - when job finishes, remove amendment from document.amendments - need to rework all the throbber stuff, I think it is probably better not to have a div I never use with the throbber in it, just add when I need it... - like in code for amendments. Also get rid of style and just use class +* new viewing model (get ids of query on first load, then paginate only inside that known list) + - BUT, when we finish a delete, what do I do with pageList / entryList??? + - start by showing them as deleted (via amend) + - then on success, remove the ids from the *List arrays in js -- but do + this via repagination, invalidate page cache fully, then getPage(currentPage) + (e.g. assume 1, 2, 3 ... 40 in eList). delete 23, 24, + then reset lists to remove 23 and 24, pageList would then + get reset to page with: 21,22,25,26 ... 30, 31 + +? get rid of style and just use class -- think this should work, so change in + templates/files.html for throbber, etc. and dont set style as much in view_support.js ### GENERAL - * jobs for AI should show path name - * rm dups job should show progress bar +* jobs for AI should show path name +* rm dups job should show progress bar * in viewer, there is no move button (maybe add one?) * think I killed pa_job_manager without passing an eid to a transform job, shouldn't crash - SHOULD JUST get AI to help clean-up and write defensive code here... diff --git a/internal/js/files_support.js b/internal/js/files_support.js index 485ecb2..5f0db22 100644 --- a/internal/js/files_support.js +++ b/internal/js/files_support.js @@ -94,7 +94,7 @@ function MoveOrDelCleanUpUI() // remove the images being moved (so UI immediately 'sees' the move) $("[name^=eid-]").each( function() { $('#'+$(this).attr('value')).remove() } ) // reorder the images via ecnt again, so future highlighting can work - document.mf_id=0; $('.figure').each( function() { $(this).attr('ecnt', document.mf_id ); document.mf_id++ } ) + // document.mf_id=0; $('.figure').each( function() { $(this).attr('ecnt', document.mf_id ); document.mf_id++ } ) $('#dbox').modal('hide') } @@ -354,34 +354,27 @@ function NoSel() { return true } +// quick wrapper to add a single
to the #figures div +function addFigure( obj ) +{ + html=createFigureHtml( obj ) + $('#figures').append( html ) +} + /** * Renders a group header or entry based on the object and options. * obj - The object containing file/directory details. - * last - Tracks the last printed group (e.g., { printed: null }). - * ecnt - Entry counter (e.g., { val: 0 }). * returns {string} - Generated HTML string. */ -function addFigure( obj, last, ecnt ) +function createFigureHtml( obj ) { - let html = ""; + // if am is null, no amendment for this obj, otherwise we have one + var am=null + for (const tmp of document.amendments) + if( tmp.eid == obj.id ) + am=tmp - // Grouping logic - if (OPT.grouping === "Day") { - if (last.printed !== obj.file_details.day) { - html += `
Day: ${obj.file_details.day} of ${obj.file_details.month}/${obj.file_details.year}
`; - last.printed = obj.file_details.day; - } - } else if (OPT.grouping === "Week") { - if (last.printed !== obj.file_details.woy) { - html += `
Week #: ${obj.file_details.woy} of ${obj.file_details.year}
`; - last.printed = obj.file_details.woy; - } - } else if (OPT.grouping === "Month") { - if (last.printed !== obj.file_details.month) { - html += `
Month: ${obj.file_details.month} of ${obj.file_details.year}
`; - last.printed = obj.file_details.month; - } - } + let html = ""; // Image/Video/Unknown entry if (obj.type.name === "Image" || obj.type.name === "Video" || obj.type.name === "Unknown") { @@ -395,13 +388,27 @@ function addFigure( obj, last, ecnt ) const prettyDate = `${obj.file_details.day}/${obj.file_details.month}/${obj.file_details.year}`; const type = obj.type.name; + // if amendment for this obj, do not add entry class - prevents highlighting + if( am ) { + ent="" + gs="style='filter: grayscale(100%);'" + am_html ='' + am_html +='' + if( am.type.which == 'icon' ) + am_html+=`` + else + am_html+=`` + } else { + ent="entry" + gs="" + am_html="" + } html += ` -
- ${renderMedia(obj)} -
- `; + ${renderMedia(obj,gs,am_html)} +
`; } // Directory entry else if (obj.type.name === "Directory" && OPT.folders) { @@ -410,7 +417,7 @@ function addFigure( obj, last, ecnt ) : obj.dir_details.in_path.path_prefix; html += ` -
+
@@ -419,66 +426,42 @@ function addFigure( obj, last, ecnt ) `; html += ``; } - - $('#figures').append( html ) - - // check if there is a pending amendment for this entry, if so mark it up - // (e.g. its being deleted, rotated, etc) - details in the am obj - for (const am of document.amendments) - { - if( am.eid == obj.id ) - { - $('#'+obj.id).find('img.thumb').attr('style', 'filter: grayscale(100%);' ) - $('#'+obj.id).removeClass('entry') - html='' - html+='' - if( am.type.which == 'icon' ) - html+=`` - else - html+=`` - $('#'+obj.id).find('a').append(html) + // moved the bindings to here as we need to reset them if we recreate this Figure (after a transform job) + html += `` + return html } // Helper function to render media (image/video/unknown) -function renderMedia(obj) { +function renderMedia(obj,gs,am_html) { const isImageOrUnknown = obj.type.name === "Image" || obj.type.name === "Unknown"; const isVideo = obj.type.name === "Video"; const path = `${obj.in_dir.in_path.path_prefix}/${obj.in_dir.rel_path}/${obj.name}`; const thumb = obj.file_details.thumbnail - ? `${obj.name}` + ? `${obj.name}` : ``; - let mediaHtml = `
${thumb}`; + let mediaHtml = `
${thumb}${am_html}`; - if (isImageOrUnknown) { - if (OPT.search_term) { - mediaHtml += ` -
- -
- `; - } - mediaHtml += ` - - `; - } else if (isVideo) { + if (isVideo) { mediaHtml += `
`; - if (OPT.search_term) { + } + if (OPT.search_term) { mediaHtml += `
`; - } } mediaHtml += `
`; @@ -527,7 +510,6 @@ function drawPageOfFigures() { $('#figures').empty() var last = { printed: null } - var ecnt=0 // something is up, let the user know if( document.alert ) @@ -557,30 +539,41 @@ function drawPageOfFigures() // with clas "back" this gets a different click handler which flags server to return data by 'going back/up' in dir tree // we give the server the id of the first item on the page so it can work out how to go back html=`
-
+
${back}
` - ecnt++ $('#figures').append(html) } for (const obj of document.entries) { - addFigure( obj, last, ecnt ) - ecnt++ + // Grouping logic + if (OPT.grouping === "Day") { + if (last.printed !== obj.file_details.day) { + $('#figures').append(`
Day: ${obj.file_details.day} of ${obj.file_details.month}/${obj.file_details.year}
` ); + last.printed = obj.file_details.day; + } + } else if (OPT.grouping === "Week") { + if (last.printed !== obj.file_details.woy) { + $('#figures').append(`
Week #: ${obj.file_details.woy} of ${obj.file_details.year}
` ); + last.printed = obj.file_details.woy; + } + } else if (OPT.grouping === "Month") { + if (last.printed !== obj.file_details.month) { + $('#figures').append(`
Month: ${obj.file_details.month} of ${obj.file_details.year}
` ); + last.printed = obj.file_details.month; + } + } + addFigure( obj ) } + $(".back").click( function(e) { getDirEntries(this.id,true) } ) if( document.entries.length == 0 ) if( OPT.search_term ) $('#figures').append( ` No matches for: '${OPT.search_term}'` ) else if( OPT.root_eid == 0 ) $('#figures').append( `No files in Path!` ) - $('.figure').click( function(e) { DoSel(e, this ); SetButtonState(); return false; }); - $('.figure').dblclick( function(e) { startViewing( $(this).attr('id') ) } ) - // for dir, getDirEntries 2nd param is back (or "up" a dir) - $(".dir").click( function(e) { document.back_id=this.id; getDirEntries(this.id,false) } ) - $(".back").click( function(e) { getDirEntries(this.id,true) } ) } // emtpy out file_list_div, and repopulate it with new page of content @@ -669,7 +662,14 @@ function getPage(pageNumber, successCallback, viewingIdx=0) type: 'POST', url: '/get_entries_by_ids', data: JSON.stringify(data), contentType: 'application/json', dataType: 'json', - success: function(res) { document.amendments=res.amend; getEntriesByIdSuccessHandler( res.entries, pageNumber, successCallback, viewingIdx ) }, + success: function(res) { + document.amendments=res.amend; + // this is only called when we are viewing a page in files/list view, so check for job(s) ending... + for (const tmp of document.amendments) { + CheckTransformJob(tmp.eid,tmp.job_id,handleTransformFiles) + } + getEntriesByIdSuccessHandler( res.entries, pageNumber, successCallback, viewingIdx ) + }, error: function(xhr, status, error) { console.error("Error:", error); } }); return } diff --git a/internal/js/files_transform.js b/internal/js/files_transform.js index 0d17b16..a5b54eb 100644 --- a/internal/js/files_transform.js +++ b/internal/js/files_transform.js @@ -1,11 +1,31 @@ +// This function will remove the matching amendment for this entry (id) +// can only have 1 ammendment per image, its grayed out for other changes +function removeAmendment( id ) +{ + document.amendments=document.amendments.filter(obj => obj.eid !== id) +} + +// POST to a check URL, that will tell us if the transformation has completed, +// if not, try again in 1 second... If it has finished then reset the thumbnail +// to full colour, put it back to being an entry and reset the thumbnail to the +// newly created one that was sent back in the response to the POST function handleTransformFiles(data,id,job_id) { if( data.finished ) { - $('#s'+id).hide() - $('#'+id).find('img.thumb').attr('style', 'filter: color(100%);' ); - $('#'+id).addClass('entry') - $('#'+id).find('.thumb').attr('src', 'data:image/jpeg;base64,'+data.thumbnail) + id=parseInt(id) + idx = entryList.indexOf(id) + // replace data for this entry now its been transformed + document.entries[idx]=data.entry + // update cache too + // document.page[getPageNumberForId(id)][howFarIntoPageCache(id)]=data.entry + // FIXME: for now just invalidate whole cache + document.page.length=0 + removeAmendment( id ) + // redraw into figure html in dom + last={ 'printed': 'not required' } + html = createFigureHtml( data.entry, last, 9999 ) + $('#'+id).replaceWith( html ) return false; } else @@ -15,17 +35,15 @@ function handleTransformFiles(data,id,job_id) } // POST to a check URL, that will tell us if the transformation has completed, -// if not, try again in 1 second... If it has finished then reset the thumbnail -// to full colour, put it back to being an entry and reset the thumbnail to the -// newly created one that was sent back in the response to the POST +// if not, try again in 1 second... If it has finished then reset the image +// to full colour function handleTransformViewing(data,id,job_id) { if( data.finished ) { // stop throbber, remove grayscale & then force reload with timestamped version of im.src - grayscale=0 - throbber=0 im.src=im.src + '?t=' + new Date().getTime(); + removeAmendment( id ) return false; } else @@ -41,7 +59,18 @@ function handleTransformViewing(data,id,job_id) function CheckTransformJob(id,job_id,successCallback) { CheckForJobs() - $.ajax( { type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', success: function(res) { successCallback(res,id,job_id); } } ) + $.ajax( { type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', + success: function(res) { successCallback(res,id,job_id); } } ) +} + +// function to add data for document.amendment based on id and amt +// used when we transform several images in files_*, or single image in viewer +function addTransformAmendment(id,amt) +{ + am={} + am.eid=parseInt(id) + am.type = document.amendTypes.filter(obj => obj.job_name === 'transform_image:'+amt )[0] + document.amendments.push(am) } // for each highlighted image, POST the transform with amt (90, 180, 270, @@ -55,16 +84,28 @@ function Transform(amt) if( document.viewing ) { post_data = '&amt='+amt+'&id='+document.viewing.id - // send /transform for this image, grayscale the thumbmail, add color spinning wheel overlay, and start checking for job end - $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data) { grayscale=1; throbber=1; DrawImg(); CheckTransformJob(document.viewing.id,data.job_id,handleTransformViewing); return false; } }) + // POST /transform for image, grayscale the image, add throbber, & start checking for end of job + $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data) { + addTransformAmendment(document.viewing.id, amt) + DrawImg(); + CheckTransformJob(document.viewing.id,data.job_id,handleTransformViewing); + return false; + } }) } else { - $('.highlight').each(function( id, e ) { + $('.highlight').each(function( cnt, e ) { post_data = '&amt='+amt+'&id='+e.id - // send /transform for this image, grayscale the thumbmail, add color spinning wheel overlay, and start checking for job end - $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data){ $('#'+e.id).find('img.thumb').attr('style', 'filter: grayscale(100%);' ); $('#'+e.id).removeClass('entry'); $('#s'+e.id).show(); CheckTransformJob(e.id,data.job_id,handleTransformFiles); return false; } }) + // POST /transform for image, grayscale the thumbnail, add throbber, & start checking for end of job + $.ajax({ type: 'POST', data: post_data, url: '/transform', success: function(data){ + addTransformAmendment(e.id, amt) + last={ 'printed': 'not required' } + idx = pageList.indexOf(parseInt(e.id)) + html = createFigureHtml( document.entries[idx], last, 9999 ) + $('#'+e.id).replaceWith( html ) + CheckTransformJob(e.id,data.job_id,handleTransformFiles); + return false; + } }) } ) } } - diff --git a/job.py b/job.py index 127c2a7..bdc2ce8 100644 --- a/job.py +++ b/job.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta import pytz import socket from shared import PA, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, NEWEST_LOG_LIMIT, OLDEST_LOG_LIMIT +from amend import EntryAmendment, inAmendmentTypes from flask_login import login_required, current_user from sqlalchemy.dialects.postgresql import INTERVAL from sqlalchemy.sql.functions import concat @@ -114,8 +115,25 @@ def NewJob(name, num_files="0", wait_for=None, jex=None, desc="No description pr db.session.add(job) db.session.commit() - SetFELog( message=f'Created Job #{job.id} to {desc}', level="success" ) + # if this job changes an eid we store that in DB and client shows until it finishes the job + at_id = inAmendmentTypes(job) + if at_id: + if job.name == 'transform_image': + id=[jex.value for jex in job.extra if jex.name == "id"][0] + ea=EntryAmendment( eid=id, job_id=job.id, amend_type=at_id ) + print( f"just added an EA for eid={id}, j={job.id}" ) + db.session.add(ea) + elif job.name == 'delete_files': + for j in jex: + if 'eid-' in j.name: + ea=EntryAmendment( eid=j.value, amend_type=at_id ) + db.session.add(ea) + # need to return this to the f/e somehow + # this is for removes, really need to think about this more + #job.amendment=ea + + SetFELog( message=f'Created Job #{job.id} to {desc}', level="success" ) WakePAJobManager(job.id) return job diff --git a/pa_job_manager.py b/pa_job_manager.py index 8be04d0..e3ee8a1 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -1,4 +1,3 @@ - # # This file controls the 'external' job control manager, that (periodically # # looks / somehow is pushed an event?) picks up new jobs, and processes them. @@ -15,7 +14,7 @@ ### SQLALCHEMY IMPORTS ### from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, String, Sequence, Float, ForeignKey, DateTime, LargeBinary, Boolean, func, text +from sqlalchemy import Column, Integer, String, Sequence, Float, ForeignKey, DateTime, LargeBinary, Boolean, func, text, select from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import relationship from sqlalchemy import create_engine @@ -23,7 +22,7 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import scoped_session ### LOCAL FILE IMPORTS ### -from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE, SymlinkName, GenThumb, SECS_IN_A_DAY, PA_EXIF_ROTATER +from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE, SymlinkName, GenThumb, SECS_IN_A_DAY, PA_EXIF_ROTATER, PA from datetime import datetime, timedelta, date ### PYTHON LIB IMPORTS ### @@ -46,6 +45,8 @@ import re import sys import ffmpeg import subprocess +# FIXME: remove this +import time # global debug setting @@ -512,6 +513,15 @@ class PA_JobManager_FE_Message(Base): def __repr__(self): return "
-
+
+ + + +