From 80ceb7aaed2daa44a4951078aa00719e00a829fc Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 15 Oct 2025 23:06:05 +1100 Subject: [PATCH] removed __repr__ from classes in files.py, and added in sqlalchemy class and marshmallow schemas for entry amendments, then load amendments on get_entry_by_id - so any page load (first or next/prev) will see amendments, we then display them into the files list and now add a white circle inside the throbber and overlay that with approrpiate icon/image - all of which is taken from amendment type and eid. tables.sql also updated to create the amendment data, tweaked icons.svg to remove hardcoded-colours for flip_[vh] --- TODO | 14 ++-- files.py | 72 ++++++++++++-------- internal/icons.svg | 6 +- internal/js/files_support.js | 124 +++++++++++++++++++++++++---------- tables.sql | 47 ++++++++----- 5 files changed, 175 insertions(+), 88 deletions(-) diff --git a/TODO b/TODO index 2326d68..47bf247 100644 --- a/TODO +++ b/TODO @@ -1,10 +1,14 @@ ### major fix - go to everywhere I call GetEntries(), and redo the logic totally... * client side: - * for real chance to stop confusion, instead of removing deleted images from DOM, we should gray them out and put a big Del (red circle with line?) though it as overlay. - * Create another table of entry_ammendments - note the deletions, rotations, flips of specific eids - then reproduce that on the client side visually as needed - - at least grayed-out, to indicate a pending action is not complete. - - When job that flips, rotates, deletes completes then lets update the query details (e.g. remove eids, or remove the ammendments) - - this actually is quite an improvement, if someone is deleting 2 as per above, I will see that as a pending change in my unrelated query, ditto flips, etc. + * instead of removing deleted images from DOM, we should gray them out and put a big Del (red circle with line?) though it as overlay. + [DONE] * Create another table of entry_ammendments - note the deletions, rotations, flips of specific eids - then reproduce that on the client side visually as needed + [DONE] - at least grayed-out, to indicate a pending action is not complete. + - When job that flips, rotates, deletes completes then create an entry_amendment in the DB. + - Also hand fudge the jscript amendments for each job / next get_entry_by_id (if needed will also set amendments as needed) + - When job finishes, remove amendment from DB + - when job finishes, remove amendment from document.amendments + need to rework all the throbber stuff, I think it is probably better not to have a div I never use with the throbber in it, just add when I need it... + like in code for amendments. Also get rid of style and just use class ### GENERAL * jobs for AI should show path name diff --git a/files.py b/files.py index 2e0d3d1..0b10e2b 100644 --- a/files.py +++ b/files.py @@ -31,7 +31,7 @@ from job import Job, JobExtra, Joblog, NewJob, SetFELog from path import PathType, Path from person import Refimg, Person, PersonRefimgLink from settings import Settings, SettingsIPath, SettingsSPath, SettingsRBPath -from shared import SymlinkName, ICON +from shared import SymlinkName, ICON, PA from dups import Duplicates from face import Face, FaceFileLink, FaceRefimgLink, FaceOverrideType, FaceNoMatchOverride, FaceForceMatchOverride @@ -41,41 +41,32 @@ from face import Face, FaceFileLink, FaceRefimgLink, FaceOverrideType, FaceNoMat # Class describing PathDirLink and in the DB (via sqlalchemy) # connects the entry (dir) with a path ################################################################################ -class PathDirLink(db.Model): +class PathDirLink(PA,db.Model): __tablename__ = "path_dir_link" path_id = db.Column(db.Integer, db.ForeignKey("path.id"), primary_key=True ) dir_eid = db.Column(db.Integer, db.ForeignKey("dir.eid"), primary_key=True ) - def __repr__(self): - return f"" - ################################################################################ # Class describing EntryDirLInk and in the DB (via sqlalchemy) # connects (many) entry contained in a directory (which is also an entry) ################################################################################ -class EntryDirLink(db.Model): +class EntryDirLink(PA,db.Model): __tablename__ = "entry_dir_link" entry_id = db.Column(db.Integer, db.ForeignKey("entry.id"), primary_key=True ) dir_eid = db.Column(db.Integer, db.ForeignKey("dir.eid"), primary_key=True ) - def __repr__(self): - return f"" - ################################################################################ # Class describing Dir and in the DB (via sqlalchemy) # rel_path: rest of dir after path, e.g. if path = /..../storage, then # rel_path could be 2021/20210101-new-years-day-pics # in_path: only in this structure, not DB, quick ref to the path this dir is in ################################################################################ -class Dir(db.Model): +class Dir(PA,db.Model): __tablename__ = "dir" eid = db.Column(db.Integer, db.ForeignKey("entry.id"), primary_key=True ) rel_path = db.Column(db.String, unique=True ) in_path = db.relationship("Path", secondary="path_dir_link", uselist=False) - def __repr__(self): - return f"" - ################################################################################ # Class describing Entry and in the DB (via sqlalchemy) # an entry is the common bits between files and dirs @@ -85,7 +76,7 @@ class Dir(db.Model): # in_dir - is the Dir that this entry is located in (convenience for class only) # FullPathOnFS(): method to get path on the FS for this Entry ################################################################################ -class Entry(db.Model): +class Entry(PA,db.Model): __tablename__ = "entry" id = db.Column(db.Integer, db.Sequence('file_id_seq'), primary_key=True ) name = db.Column(db.String, unique=False, nullable=False ) @@ -106,9 +97,6 @@ class Entry(db.Model): s=self.dir_details.in_path.path_prefix return s - def __repr__(self): - return f"" - ################################################################################ # Class describing FileType and in the DB (via sqlalchemy) # pre-defined list of file types (image, dir, etc.) ################################################################################ -class FileType(db.Model): +class FileType(PA,db.Model): __tablename__ = "file_type" id = db.Column(db.Integer, db.Sequence('file_type_id_seq'), primary_key=True ) name = db.Column(db.String, unique=True, nullable=False ) - def __repr__(self): - return f"" +class AmendmentType(PA,db.Model): + __tablename__ = "amendment_type" + id = db.Column(db.Integer, db.Sequence('file_type_id_seq'), primary_key=True ) + which = db.Column(db.String, nullable=False ) + what = db.Column(db.String, nullable=False ) + colour = db.Column(db.String, nullable=False ) + +class EntryAmendment(PA,db.Model): + __tablename__ = "entry_amendment" + eid = db.Column(db.Integer, db.ForeignKey("entry.id"), primary_key=True ) + amend_type = db.Column(db.Integer, db.ForeignKey("amendment_type.id")) + type = db.relationship("AmendmentType", backref="entry_amendment") ################################################################################ @@ -249,6 +244,18 @@ class FileSchema(ma.SQLAlchemyAutoSchema): load_instance = True faces = ma.Nested(FaceSchema,many=True,allow_none=True) +class AmendmentTypeSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = AmendmentType + load_instance = True + +class EntryAmendmentSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = EntryAmendment + load_instance = True + eid = ma.auto_field() + type = ma.Nested(AmendmentTypeSchema) + ################################################################################ # Schema for Entry so we can json for data to the client ################################################################################ @@ -309,7 +316,14 @@ def process_ids(): # Sort the entries according to the order of ids sorted_data = [entry_map[id_] for id_ in ids if id_ in entry_map] - return jsonify(entries_schema.dump(sorted_data)) + # get any pending entry amendments + stmt = select(EntryAmendment).join(AmendmentType) + ea = db.session.execute(stmt).unique().scalars().all() + ea_schema = EntryAmendmentSchema(many=True) + ea_data=ea_schema.dump(ea) + print( ea_data ) + + return jsonify(entries=entries_schema.dump(sorted_data), amend=ea_data) ################################################################################ @@ -328,21 +342,21 @@ def get_dir_entries(): # if we are going back, find the parent id and use that instead if back: # get parent of this dir, to go back - stmt=( select(EntryDirLink.dir_eid).filter(EntryDirLink.entry_id==dir_id) ) + stmt=select(EntryDirLink.dir_eid).filter(EntryDirLink.entry_id==dir_id) dir_id = db.session.execute(stmt).scalars().one_or_none() if not dir_id: # return valid as false, we need to let user know this is not an empty dir, it does not exist return jsonify( valid=False, entry_list=[] ) # Just double-check this is still in the DB, in case it got deleted since client made view - stmt=( select(Entry.id).where(Entry.id==dir_id) ) + stmt=select(Entry.id).where(Entry.id==dir_id) ent_id = db.session.execute(stmt).scalars().one_or_none() if not ent_id: # return valid as false, we need to let user know this is not an empty dir, it does not exist return jsonify( valid=False, entry_list=[] ) # get content of dir_id - stmt=( select(Entry.id).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir_id) ) + stmt=select(Entry.id).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir_id) stmt=stmt.order_by(*order_map.get(noo) ) return jsonify( valid=True, entry_list=db.session.execute(stmt).scalars().all() ) @@ -435,7 +449,7 @@ def GetQueryData( OPT ): if OPT.folders: # start folder view with only the root folder - stmt=( select(Entry.id).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir_id) ) + stmt=select(Entry.id).join(EntryDirLink).filter(EntryDirLink.dir_eid==dir_id) else: # get every File that is in the OPT.prefix Path stmt=( diff --git a/internal/icons.svg b/internal/icons.svg index a1b1e21..92a2f5b 100644 --- a/internal/icons.svg +++ b/internal/icons.svg @@ -161,13 +161,13 @@ c4.142,0,7.5-3.357,7.5-7.5S339.642,328,335.5,328z"/> - + - + - + diff --git a/internal/js/files_support.js b/internal/js/files_support.js index ce2ceb4..485ecb2 100644 --- a/internal/js/files_support.js +++ b/internal/js/files_support.js @@ -213,44 +213,82 @@ function DetailsDBox() } -// DoSel is called when a click event occurs, and sets the selection via adding +// DoSel is called when a click event occurs, and sets the selection via adding // 'highlight' to the class of the appropriate thumbnails // e == event (can see if shift/ctrl held down while left-clicking // el == element the click is on // this allows single-click to select, ctrl-click to (de)select 1 item, and -// shift-click to add all elements between highlighted area and clicked area, -// whether you click after highlight or before -function DoSel(e, el) -{ - if( e.ctrlKey || document.fake_ctrl === 1 ) - { - $(el).toggleClass('highlight') - if( document.fake_ctrl === 1 ) - document.fake_ctrl=0 - return - } - if( e.shiftKey || document.fake_shift === 1 ) - { - st=Number($('.highlight').first().attr('ecnt')) - end=Number($('.highlight').last().attr('ecnt')) - clicked=Number($(el).attr('ecnt')) - // if we shift-click first element, then st/end are NaN, so just highlightthe one clicked - if( isNaN(st) ) - { - $('.entry').slice( clicked, clicked+1 ).addClass('highlight') - return - } - if( clicked > end ) - $('.entry').slice( end, clicked+1 ).addClass('highlight') - else - $('.entry').slice( clicked, st ).addClass('highlight') +// shift-click to add all elements between highlighted area and clicked el, +// whether you click before highlight or after, or inside a gap and then back +// or forward to the closest higlighted entry - also, only works on entry class, +// so it ignores figures that we take entry off while we transform, etc it +function DoSel(e, el) { + const id = $(el).attr('id'); + const entries = $('.entry'); - if( document.fake_shift === 1 ) - document.fake_shift=0 - return + // Collect currently highlighted entries + const currentHighlights = $('.highlight'); + const highlighted = new Set(); + currentHighlights.each(function() { + highlighted.add($(this).attr('id')); + }); + + // Ctrl+click: toggle highlight for the clicked entry + if (e.ctrlKey || document.fake_ctrl === 1) { + $(el).toggleClass('highlight'); + if (highlighted.has(id)) { + highlighted.delete(id); + } else { + highlighted.add(id); + } + if (document.fake_ctrl === 1) { + document.fake_ctrl = 0; + } + return; + } + // Shift+click: select a range + else if (e.shiftKey || document.fake_shift === 1) { + if (currentHighlights.length === 0) { + // If no highlights, just highlight the clicked entry + $(el).addClass('highlight'); + highlighted.add(id); + } else { + // Find the nearest highlighted entry + const clickedIndex = entries.index($(el)); + let nearestHighlightIndex = -1; + let minDistance = Infinity; + + currentHighlights.each(function() { + const highlightIndex = entries.index($(this)); + const distance = Math.abs(highlightIndex - clickedIndex); + if (distance < minDistance) { + minDistance = distance; + nearestHighlightIndex = highlightIndex; + } + }); + + // Highlight the range between the nearest highlighted entry and the clicked entry + const from = Math.min(clickedIndex, nearestHighlightIndex); + const to = Math.max(clickedIndex, nearestHighlightIndex); + + for (let i = from; i <= to; i++) { + const entryId = entries.eq(i).attr('id'); + highlighted.add(entryId); + entries.eq(i).addClass('highlight'); + } + } + if (document.fake_shift === 1) { + document.fake_shift = 0; + } + return; + } + // Single click: clear all highlights and highlight the clicked entry + else { + $('.highlight').removeClass('highlight'); + highlighted.clear(); + $(el).addClass('highlight'); + highlighted.add(id); } - $('.highlight').removeClass('highlight') - $(el).addClass('highlight') } // if a selection exists, enable move & del/restore buttons otherwise disable them @@ -323,7 +361,7 @@ function NoSel() { * ecnt - Entry counter (e.g., { val: 0 }). * returns {string} - Generated HTML string. */ -function addFigure( obj, last, ecnt) +function addFigure( obj, last, ecnt ) { let html = ""; @@ -383,6 +421,24 @@ function addFigure( obj, last, ecnt) } $('#figures').append( html ) + + // check if there is a pending amendment for this entry, if so mark it up + // (e.g. its being deleted, rotated, etc) - details in the am obj + for (const am of document.amendments) + { + if( am.eid == obj.id ) + { + $('#'+obj.id).find('img.thumb').attr('style', 'filter: grayscale(100%);' ) + $('#'+obj.id).removeClass('entry') + html='' + html+='' + if( am.type.which == 'icon' ) + html+=`` + else + html+=`` + $('#'+obj.id).find('a').append(html) + } + } return } @@ -613,7 +669,7 @@ function getPage(pageNumber, successCallback, viewingIdx=0) type: 'POST', url: '/get_entries_by_ids', data: JSON.stringify(data), contentType: 'application/json', dataType: 'json', - success: function(res) { getEntriesByIdSuccessHandler( res, pageNumber, successCallback, viewingIdx ) }, + success: function(res) { document.amendments=res.amend; getEntriesByIdSuccessHandler( res.entries, pageNumber, successCallback, viewingIdx ) }, error: function(xhr, status, error) { console.error("Error:", error); } }); return } diff --git a/tables.sql b/tables.sql index 46f70c7..1f95653 100644 --- a/tables.sql +++ b/tables.sql @@ -1,4 +1,4 @@ -ALTER DATABASE pa SET TIMEZONE TO 'aUSTRALIA/vICTORIA'; +ALTER DATABASE pa SET TIMEZONE TO 'Australia/Victoria'; CREATE SEQUENCE pa_user_id_seq; CREATE SEQUENCE pa_user_state_id_seq; @@ -21,8 +21,8 @@ CREATE SEQUENCE query_id_seq; -- these are hard-coded at present, not sure I can reflexively find models from API? CREATE TABLE ai_model ( id INTEGER, name VARCHAR(24), description VARCHAR(80), CONSTRAINT pk_ai_model PRIMARY KEY(id) ); -INSERT INTO ai_model VALUES ( 1, 'HOG', 'NORMAL' ); -INSERT INTO ai_model VALUES ( 2, 'CNN', 'MORE ACCURATE / MUCH SLOWER' ); +INSERT INTO ai_model VALUES ( 1, 'hog', 'normal' ); +INSERT INTO ai_model VALUES ( 2, 'cnn', 'more accurate / much slower' ); CREATE TABLE settings( id INTEGER, @@ -122,10 +122,10 @@ CREATE TABLE face_refimg_link( face_id INTEGER, refimg_id INTEGER, face_distance CONSTRAINT fk_frl_refimg_id FOREIGN KEY (refimg_id) REFERENCES refimg(id) ); CREATE TABLE face_override_type ( id INTEGER, name VARCHAR UNIQUE, CONSTRAINT pk_face_override_type_id PRIMARY KEY(id) ); -INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'mANUAL MATCH TO EXISTING PERSON' ); -INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'nOT A FACE' ); -INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'tOO YOUNG' ); -INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'iGNORE FACE' ); +INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'Manual match to existing person' ); +INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'Not a face' ); +INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'Too young' ); +INSERT INTO face_override_type VALUES ( (SELECT NEXTVAL('face_override_type_id_seq')), 'Ignore face' ); -- keep non-redundant FACE because, when we rebuild data we may have a null FACE_ID, but still want to connect to this override -- from a previous AI pass... (would happen if we delete a file and then reimport/scan it), OR, more likely we change (say) a threshold, etc. @@ -165,20 +165,33 @@ CREATE TABLE joblog ( id INTEGER, job_id INTEGER, log_date TIMESTAMPTZ, log VARC CONSTRAINT pk_jl_id PRIMARY KEY(id), CONSTRAINT fk_jl_job_id FOREIGN KEY(job_id) REFERENCES job(id) ); CREATE TABLE pa_job_manager_fe_message ( id INTEGER, job_id INTEGER, level VARCHAR(16), message VARCHAR(8192), persistent BOOLEAN, cant_close BOOLEAN, - CONSTRAINT pa_job_manager_fe_acks_id PRIMARY KEY(id), + CONSTRAINT pk_pa_job_manager_fe_acks_id PRIMARY KEY(id), CONSTRAINT fk_pa_job_manager_fe_message_job_id FOREIGN KEY(job_id) REFERENCES job(id) ); +CREATE TABLE amendment_type ( id INTEGER, which VARCHAR(8), what VARCHAR(32), colour VARCHAR(32), + CONSTRAINT pk_amendment_type_id PRIMARY KEY(id) ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 1, 'icon', 'trash', 'red' ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 2, 'img', 'rot90.png', '#009EFF' ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 3, 'img', 'rot180.png', '#009EFF' ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 4, 'img', 'rot270.png', '#009EFF' ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 5, 'icon', 'flip_h', '#009EFF' ); +INSERT INTO amendment_type ( id, which, what, colour ) VALUES ( 6, 'icon', 'flip_v', '#009EFF' ); + +CREATE TABLE entry_amendment ( eid INTEGER, amend_type INTEGER, + CONSTRAINT pk_entry_amendment_eid_name PRIMARY KEY(eid,amend_type), + CONSTRAINT fk_entry_amendment_amendment_type FOREIGN KEY(amend_type) REFERENCES amendment_type(id) ); + -- default data for types of paths -INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'iMPORT' ); -INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'sTORAGE' ); -INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'bIN' ); -INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'mETADATA' ); +INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'Import' ); +INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'Storage' ); +INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'Bin' ); +INSERT INTO path_type VALUES ( (SELECT NEXTVAL('path_type_id_seq')), 'Metadata' ); -- default data for types of files -INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'iMAGE' ); -INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'vIDEO' ); -INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'dIRECTORY' ); -INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'uNKNOWN' ); +INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'Image' ); +INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'Video' ); +INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'Directory' ); +INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'Unknown' ); -- fake data only for making testing easier --INSERT INTO person VALUES ( (SELECT NEXTVAL('person_id_seq')), 'dad', 'Damien', 'De Paoli' ); @@ -186,7 +199,7 @@ INSERT INTO file_type VALUES ( (SELECT NEXTVAL('file_type_id_seq')), 'uNKNOWN' ) --INSERT INTO person VALUES ( (SELECT NEXTVAL('person_id_seq')), 'cam', 'Cameron', 'De Paoli' ); --INSERT INTO person VALUES ( (SELECT NEXTVAL('person_id_seq')), 'mich', 'Michelle', 'De Paoli' ); -- DEV(ddp): -INSERT INTO settings ( ID, BASE_PATH, IMPORT_PATH, STORAGE_PATH, RECYCLE_BIN_PATH, METADATA_PATH, AUTO_ROTATE, DEFAULT_REFIMG_MODEL, DEFAULT_SCAN_MODEL, DEFAULT_THRESHOLD, FACE_SIZE_LIMIT, SCHEDULED_IMPORT_SCAN, SCHEDULED_STORAGE_SCAN, SCHEDULED_BIN_CLEANUP, BIN_CLEANUP_FILE_AGE, JOB_ARCHIVE_AGE ) VALUES ( (SELECT NEXTVAL('settings_id_seq')), '/HOME/DDP/SRC/PHOTOASSISTANT/', 'IMAGES_TO_PROCESS/', 'PHOTOS/', '.PA_BIN/', '.PA_METADATA/', TRUE, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); +INSERT INTO settings ( id, base_path, import_path, storage_path, recycle_bin_path, metadata_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) VALUES ( (SELECT NEXTVAL('settings_id_seq')), '/home/ddp/src/photoassistant/', 'images_to_process/', 'photos/', '.pa_bin/', '.pa_metadata/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); -- DEV(cam): --INSERT INTO settings ( id, base_path, import_path, storage_path, recycle_bin_path, metadata_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) VALUES ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', '.pa_metadata/', TRUE, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); -- PROD: