From 6b7694f382f3807ee65da754bc668b664bc314d3 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 10 Jul 2022 15:21:31 +1000 Subject: [PATCH] first pass of consolidating search in DBox for existing person, and then using the results to add override force match to that person, and WORKING version of adding refimg to existing person too. Still does not kick off new AI scan at this point, and still need to re-format dbox to be easier to use and code for resetting DB contents, rescaning files from scratch and matching overrides back --- BUGs | 12 ++++- README | 2 +- TODO | 27 +++++------ internal/js/view_support.js | 83 ++++++++++++++++++++++----------- person.py | 92 +++++++++++++++++++++++++++++-------- 5 files changed, 152 insertions(+), 64 deletions(-) diff --git a/BUGs b/BUGs index 7001b5f..7e80a47 100644 --- a/BUGs +++ b/BUGs @@ -1,4 +1,4 @@ -### Next: 88 +### Next: 91 BUG-60: entries per page in flat view will get how_many from each top-level dir in PATH - causes viewer to get lost in eids for last_eid BUG-85: once we rebuild data from scratch, need to reset just clean out pa_user_state's BUG-87: using Person->show matches->then view an image shows bug in viewer with msising data in json -- that same image via a filename search works... @@ -17,3 +17,13 @@ BUG-87: using Person->show matches->then view an image shows bug in viewer with I deleted all faces with no locn, maybe it is a one-off with all the crap I did with the ORM / tmp_locn.... Will see if it comes back, this stays as a bug for now + +BUG-89: refimg face_locn also got transformed -- some sequence of adding/removing refimg from mich, caused Cams to fail??? + pa=# select id, fname, orig_w, orig_h, face_locn from refimg where id=1; + id | fname | orig_w | orig_h | face_locn + ----+----------+--------+--------+---------------------- + 1 | cam.jpg | 978 | 1348 | {514,869,1313,70} + +BUG-90: I added photo of mich as kid (ice-cream in paris) and then somehow it matches that photo to Cam's face, not the same img -- is this a bug in my code, or some weird quirk of face_recognition library??? + - either lib. is weird; OR + - I should get a lower score for mich in that image, and somehow my order/face matching is not doing the right thing diff --git a/README b/README index b3f2703..92a7ca7 100644 --- a/README +++ b/README @@ -72,7 +72,7 @@ To get back a 'working' but scanned set of data: # make a backup and store it in DB_BACKUP: sudo docker exec -it padb bash - root@2881f871e1c2:/# pg_dump --user=pa --password pa > /docker-entrypoint-initdb.d/tables.sql + root@2881f871e1c2:/# pg_dump --user=pa pa > /docker-entrypoint-initdb.d/tables.sql cp /srv/docker/container/padb/docker-entrypoint-initdb.d/tables.sql /home/ddp/src/photoassistant/DB_BACKUP/ mv /home/ddp/src/photoassistant/DB_BACKUP/tables.sql /home/ddp/src/photoassistant/DB_BACKUP/`date +%Y%m%d-tables.sql` gzip /home/ddp/src/photoassistant/DB_BACKUP/`date +%Y%m%d-tables.sql` diff --git a/TODO b/TODO index 1145a38..e0270ea 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,14 @@ ## GENERAL - * run_ai_on throws log line, for matching even if there are no faces, would be less noisy to not do that (or should say no faces?) * on viewer: - - allow face to be used to create person, add to existing person, and allow 'ignore', mark as 'not a face', etc - -> ignore/not a face/too young --> all need to go into DB so we can remember the 'override' when we re-ai-match - -> redraw 'ignore's as a greyed out box? - -> menu should only allow override IF we have put override on... - SO, override manual match, is awkward if somehow the file/face changes (e.g. we rescan a file for faces, do I delete override? if not and we rescan, there will he a new face id, how do I know which it connects with????) + - allow face to be used to: + - create person + [PARTIAL - person.py to go] - add to existing person + [DONE] - ignore/not a face/too young + [DONE] - redraw 'ignore's as a greyed out box? + [DONE] - menu should only allow override IF we have put override on... + --> need to test the 'override' when we re-ai-match (AFTER re-build from FS) + + * run_ai_on throws log line, for matching even if there are no faces, would be less noisy to not do that (or should say no faces?) * should allow right-click from View menu (particularly useful on search) to show other files around this one by date (maybe that folder or something?) @@ -16,8 +19,6 @@ then we could just feed those eid's explicitly into a 'run_ai_on_new_files' :) -- maybe particularly if count('new files') < say 1000 do eids, otherwise do path AND no new refimgs - * DECIDED TO DITCH multiple Dirs per Path, just adds complexity and not needed - will address BUG-60 - * does search of matching dirname give all entries of subdirs of subdirs, etc. (think not) -- maybe a TODO? * delete folder @@ -32,11 +33,8 @@ ??? * browser back/forward buttons dont work -- use POST -> redirect to GET - * viewlist - - can consider a POST every time we next/prev in viewer --> set only, to just update the OPT.current, every time you go back into the viewer, then it would go the last image viewed, rather than the first image on the last page you viewed... - - can consider an optim-- new_view page makes calls to viewlist to ADD json data only, so only trigger a new "viewlist" if we dont have data for that part of the eids - - need some sort of clean-up of pa_user_state -- I spose its triggered by browser session, so maybe just after a week is lazy/good enough - -- pa_user_state has last_used as a timestamp so can be used to delete old entries + - need some sort of clean-up of pa_user_state -- I spose its triggered by browser session, so maybe just after a week is lazy/good enough + -- pa_user_state has last_used as a timestamp so can be used to delete old entries GUI overhaul? * on a phone, the files.html page header is a mess "Oldest.." line is too large to fit on 1 line (make it a hamburger?) @@ -53,8 +51,7 @@ * get build process to create a random string for secret for PROD, otherwise use builtin for dev - * deal with changing/adding/removing a path in settings - -- consider this in light of only one dir per path + * deal with changing a path in settings * dup issues: * when we have lots of dups, sort the directories by alpha so its consistent when choosing diff --git a/internal/js/view_support.js b/internal/js/view_support.js index 30f1385..b7fe83b 100644 --- a/internal/js/view_support.js +++ b/internal/js/view_support.js @@ -263,24 +263,24 @@ $(document).ready( function() } ); // quick wrapper function to make calling this ajax code simpler in SearchForPerson -function OverrideForceMatch( person_id, face_id, face_pos, type_id ) +function OverrideForceMatch( person_id, key ) { // we have type_id passed in, so dig the NMO out, and use that below (its really just for name, but in case we change that in the DB) for( el in NMO ) { - if( NMO[el].type_id == type_id ) + if( NMO[el].type_id == item[key].type_id ) { fm_idx=el break } } - ofm='&person_id='+person_id+'&face_id='+face_id + ofm='&person_id='+person_id+'&face_id='+item[key].id $.ajax({ type: 'POST', data: ofm, url: '/override_force_match', success: function(data) { - objs[current].faces[face_pos].override={} - objs[current].faces[face_pos].override.who=data.person_tag - objs[current].faces[face_pos].override.distance='N/A' - objs[current].faces[face_pos].override.type_id=NMO[fm_idx].id - objs[current].faces[face_pos].override.type_name=NMO[fm_idx].name + objs[current].faces[item[key].which_face].override={} + objs[current].faces[item[key].which_face].override.who=data.person_tag + objs[current].faces[item[key].which_face].override.distance='N/A' + objs[current].faces[item[key].which_face].override.type_id=NMO[fm_idx].id + objs[current].faces[item[key].which_face].override.type_name=NMO[fm_idx].name $('#dbox').modal('hide') $('#faces').prop('checked',true) @@ -289,21 +289,42 @@ function OverrideForceMatch( person_id, face_id, face_pos, type_id ) } ) } +function AddRefImgTo( person_id, key ) +{ + d='&face_id='+item[key].id+'&person_id='+person_id+ + '&file_eid='+current+'&refimg_data='+item[key].refimg_data + console.log( d ) + $.ajax({ type: 'POST', data: d, url: '/add_refimg_to_person', + success: function(data) { + objs[current].faces[item[key].which_face].who=data.who + objs[current].faces[item[key].which_face].distance=data.distance + $('#dbox').modal('hide') + $('#faces').prop('checked',true) + DrawImg() + } + }) +} + // function to facilitate adding a face match override to this "found" person // uses Ajax to the f/e to get any person matching #stext's content (via any name/tag) // and displays results in #search_person_results -function SearchForPerson(face_id, face_pos, type_id) +function SearchForPerson(content, key, face_id, face_pos, type_id) { + console.log( 'type_id=' + item[key].type_id ) // make URI safe who = encodeURIComponent( $('#stext').val() ) // call ajax to find ppl $.ajax({ type: 'POST', data: null, url: '/find_persons/'+ who, success: function(data) { - content='Click one of the link(s) below to manually connect this face as once-off connection to the person:

' - for( var key in data ) { - var person = data[key]; - content+= ''+person.firstname+' '+person.surname+' ('+person.tag+')
' + for( var el in data ) { + var person = data[el]; + // NMO_1 is a non-match-override type_id==1 (or force match to existing person) + if( key == "NMO_1" ) + content+= '' + +person.firstname+' '+person.surname+' ('+person.tag+')
' + if( key == 'no_match_new_refimg' ) + content+= '' + +person.firstname+' '+person.surname+' ('+person.tag+')
' } $('#search_person_results').html( content ) } @@ -357,6 +378,23 @@ function AddNoMatchOverride(type_id, face_id, face_pos, type_id) } ) } +function AddSearch( content, key, face_pos ) +{ + html='
search for existing person:
' + html+=` +
+ +
+
+
+ ` + return html +} // function that is called when we click on a face in the viewer and we want to // potentially override the non-match / match... it shows the face, and then @@ -369,6 +407,7 @@ function FaceDBox(key, item) div+='
' $.ajax({ type: 'POST', data: null, url: '/get_face_from_image/'+item[key]['id'], success: function(img_data) { + item[key].refimg_data=img_data $('#face_img').html( '' ) } } ) @@ -394,7 +433,7 @@ function FaceDBox(key, item) } if ( key == 'no_match_new_refimg' ) { - div+='
search for existing person: NOT YET
' + div+=AddSearch( 'Click one of the link(s) below to add this face as a reference image to the person:

', key, face_pos ); } if ( key == 'wrong_person' ) { @@ -405,19 +444,7 @@ function FaceDBox(key, item) { if( item[key].name == 'Override: Manual match to existing person' ) { - div+='
search for existing person:
' - div+= - ` -
- -
-
-
- ` + div+=AddSearch( 'Click one of the link(s) below to manually connect this face as once-off connection to the person:

', key, face_pos ); } else { diff --git a/person.py b/person.py index 72cc5fd..d22b5c2 100644 --- a/person.py +++ b/person.py @@ -13,6 +13,10 @@ from face import Face, FaceRefimgLink, FaceOverrideType, FaceNoMatchOverride, Fa import os import json import time +from PIL import Image +import base64 +from io import BytesIO +import os.path # pylint: disable=no-member @@ -86,6 +90,32 @@ class PersonForm(FlaskForm): save = SubmitField('Save' ) delete = SubmitField('Delete' ) +################################################################################ +# Helper functions +# AddRefimgToPerson( filename, person ) +################################################################################ +def AddRefimgToPerson( filename, person ): + refimg = Refimg( fname=os.path.basename( filename ) ) + try: + refimg.thumbnail, refimg.orig_w, refimg.orig_h = GenThumb( filename ) + settings = Settings.query.first() + model=AIModel.query.get(settings.default_refimg_model) + refimg.face, face_locn = GenFace( filename, model=model.name ) + refimg.face_locn = json.dumps(face_locn) + refimg.model_used = settings.default_refimg_model + refimg.created_on = time.time() + os.remove(filename) + person.refimg.append(refimg) + db.session.add(person) + db.session.add(refimg) + db.session.commit() + st.SetMessage( f"Associated new Refimg ({refimg.fname}) with person: {person.tag}" ) + except SQLAlchemyError as e: + st.SetMessage( f"Failed to add Refimg: {e.orig}", "danger" ) + except Exception as e: + st.SetMessage( f"Failed to modify Refimg: {e}", "danger" ) + return + ################################################################################ # Routes for person data # @@ -197,33 +227,19 @@ def add_refimg(): person = Person.query.get(request.form['person_id']); if not person: raise Exception("could not find person to add reference image too!") - f=request.files['refimg_file'] - refimg = Refimg( fname=f.filename ) try: # save the actual uploaded image to reference_images/ + f=request.files['refimg_file'] fname=secure_filename(f.filename) if fname == "": raise Exception("invalid filename") fname = f"/tmp/{fname}" f.save( fname ) - refimg.thumbnail, refimg.orig_w, refimg.orig_h = GenThumb( fname ) - settings = Settings.query.first() - model=AIModel.query.get(settings.default_refimg_model) - refimg.face, face_locn = GenFace( fname, model=model.name ) - refimg.face_locn = json.dumps(face_locn) - refimg.model_used = settings.default_refimg_model - refimg.created_on = time.time() - os.remove(fname) - person.refimg.append(refimg) - db.session.add(person) - db.session.add(refimg) - db.session.commit() - st.SetMessage( f"Associated new Refimg ({refimg.fname}) with person: {person.tag}" ) - except SQLAlchemyError as e: - st.SetMessage( f"Failed to add Refimg: {e.orig}", "danger" ) except Exception as e: - st.SetMessage( f"Failed to modify Refimg: {e}", "danger" ) + st.SetMessage( f"Failed to load reference image: {e}", "danger" ) + + AddRefimgToPerson( fname, person ) return redirect( url_for( 'person', id=person.id) ) ################################################################################ @@ -245,6 +261,44 @@ def find_persons(who): return resp + + +################################################################################ +# /add_refimg_to_person/ -> POST +################################################################################ +@app.route("/add_refimg_to_person", methods=["POST"]) +@login_required +def add_refimg_to_person(): + resp={} + f = Face.query.get( request.form['face_id'] ) + p = Person.query.get( request.form['person_id'] ) + + file_eid = request.form['file_eid'] + refimg_data = request.form['refimg_data'] + + # undo the munging sending via http has done + refimg_data=refimg_data.replace(' ', '+' ) + + print( refimg_data ) + + # convert b64 encoded to a temp file to process... + bytes_decoded = base64.b64decode(refimg_data) + img = Image.open(BytesIO(bytes_decoded)) + out_jpg = img.convert("RGB") + # save file to /tmp/ + fname="/tmp/" + p.tag + '.jpg' + out_jpg.save(fname) + + # add this fname (of temp refimg) to person + AddRefimgToPerson( fname, p ) + + # DDP: + # need to create a new job to re-do AI now we have a new refimg in the mix + + resp['who']=p.tag + resp['distance']='0.0' + return resp + ################################################################################ # /override_force_match -> POST ################################################################################ @@ -254,7 +308,7 @@ def override_force_match(): person_id = request.form['person_id'] p = Person.query.get(person_id); if not p: - raise Exception("could not find person to add override too!") + raise Exception( f"could not find person (id={person_id}) to add override too!" ) face_id = request.form['face_id'] f = Face.query.get(face_id);