From 2357ee9a3da8889bac12882d88de82083b38ac42 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:44:41 +1000 Subject: [PATCH 01/67] FIXED: bug-39 - multiple logins --- BUGs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUGs b/BUGs index 76a52ec..cd93170 100644 --- a/BUGs +++ b/BUGs @@ -1 +1 @@ -### Next: 39 +### Next: 40 From 886776f737f33fc16556d8b19b14b99646d8bcde Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:46:26 +1000 Subject: [PATCH 02/67] Made saved users be in the DB, not in dict in memory of workers in gunicorn - otherwise we had BUG-39, and also added input validation to username to stop ldap injection on login form --- main.py | 32 ++++++++++++++++++-------------- tables.sql | 3 +++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/main.py b/main.py index 6bcfe6f..2ef3ff1 100644 --- a/main.py +++ b/main.py @@ -7,12 +7,10 @@ from wtforms import SubmitField, StringField, HiddenField, SelectField, IntegerF from flask_wtf import FlaskForm from status import st, Status from shared import CreateSelect, CreateFoldersSelect, LocationIcon, DB_URL -from flask_login import login_required, current_user - # for ldap auth from flask_ldap3_login import LDAP3LoginManager -from flask_login import LoginManager, login_user, UserMixin, current_user +from flask_login import LoginManager, login_user, login_required, UserMixin, current_user from flask_ldap3_login.forms import LDAPLoginForm import re @@ -49,11 +47,6 @@ login_manager = LoginManager(app) # Setup a Flask-Login Manager ldap_manager = LDAP3LoginManager(app) # Setup a LDAP3 Login Manager. login_manager.login_view = "login" # default login route, failed with url_for, so hard-coded -# Create a dictionary to store the users in when they authenticate -# This example stores users in memory. -users = {} - - ################################# Now, import non-book classes ################################### from settings import Settings @@ -63,6 +56,7 @@ from refimg import Refimg from job import Job, GetNumActiveJobs from ai import aistats from path import StoragePathNames +from user import PAUser ####################################### GLOBALS ####################################### # allow jinja2 to call these python functions directly @@ -98,9 +92,8 @@ class User(UserMixin): # returns None. @login_manager.user_loader def load_user(id): - if id in users: - return users[id] - return None + pau=PAUser.query.filter(PAUser.dn==id).first() + return pau # Declare The User Saver for Flask-Ldap3-Login # This method is called whenever a LDAPLoginForm() successfully validates. @@ -108,9 +101,14 @@ def load_user(id): # login controller. @ldap_manager.save_user def save_user(dn, username, data, memberships): - user = User(dn, username, data) - users[dn] = user - return user + pau=PAUser.query.filter(PAUser.dn==dn).first() + # if we already have a valid user/session, and say the web has restarted, just re-use it, dont make more users + if pau: + return pau + pau=PAUser(dn=dn) + db.session.add(pau) + db.session.commit() + return pau # default page, just the navbar @app.route("/", methods=["GET"]) @@ -129,9 +127,15 @@ def login(): form = LDAPLoginForm() form.submit.label.text="Login" + # the re matches on any special LDAP chars, we dont want someone + # ldap-injecting our username, so send them back to the login page instead + if request.method == 'POST' and re.search( r'[()\\*&!]', request.form['username']): + print( f"WARNING: Detected special LDAP chars in username: {request.form['username']}") + return redirect('/login') if form.validate_on_submit(): # Successfully logged in, We can now access the saved user object # via form.user. + print( f"form user = {form.user}" ) login_user(form.user, remember=True) # Tell flask-login to log them in. next = request.args.get("next") if next: diff --git a/tables.sql b/tables.sql index f82832f..07094d5 100644 --- a/tables.sql +++ b/tables.sql @@ -2,6 +2,8 @@ alter database PA set timezone to 'Australia/Victoria'; create table SETTINGS( ID integer, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, constraint PK_SETTINGS_ID primary key(ID) ); +create table PA_USER( ID integer, dn varchar, constraint PK_PA_USER_ID primary key(ID) ); + create table FILE_TYPE ( ID integer, NAME varchar(32) unique, constraint PK_FILE_TYPE_ID primary key(ID) ); create table PATH_TYPE ( ID integer, NAME varchar(16) unique, constraint PK_PATH_TYPE_ID primary key(ID) ); @@ -69,6 +71,7 @@ create table PA_JOB_MANAGER_FE_MESSAGE ( ID integer, JOB_ID integer, ALERT varch constraint PA_JOB_MANAGER_FE_ACKS_ID primary key(ID), constraint FK_PA_JOB_MANAGER_FE_MESSAGE_JOB_ID foreign key(JOB_ID) references JOB(ID) ); +create sequence PA_USER_ID_SEQ; create sequence PATH_ID_SEQ; create sequence PATH_TYPE_ID_SEQ; create sequence FILE_ID_SEQ; From f9d505a1b86a1ad803e65fdb3cd5f382bf054093 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:46:38 +1000 Subject: [PATCH 03/67] tweaked workers/threads, not sure what is best, but this will do --- wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrapper.sh b/wrapper.sh index 3316a7b..126def4 100755 --- a/wrapper.sh +++ b/wrapper.sh @@ -1,4 +1,4 @@ #!/bin/bash su mythtv -g mythtv -c "python3 /code/pa_job_manager.py" & -gunicorn --bind=0.0.0.0:443 --workers=8 --threads=8 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app +gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --preload From 848c2b61cd016217a12c2f02b8a5971ccb44a8f2 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:47:28 +1000 Subject: [PATCH 04/67] fixed login code --- TODO | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/TODO b/TODO index 5909535..6eeee30 100644 --- a/TODO +++ b/TODO @@ -1,14 +1,5 @@ ## GENERAL - * incorporate flask-login and flask-ldap3-login - https://flask-login.readthedocs.io/en/latest/ - https://flask-ldap3-login.readthedocs.io/en/latest/ - https://pythonhosted.org/Flask-Principal/ - # this is an example: - https://code.tutsplus.com/tutorials/flask-authentication-with-ldap--cms-23101 - * user management scope - - do I want admins only? I definitely * want a read-only / share (but to a subset potentially?) - (see point above for how to do all this) * fix up logging in general * comment your code * more OO goodness :) @@ -52,6 +43,7 @@ Admin -> delete old jobs / auto delete jobs older than ??? + -> do I want to have admin roles/users? ### UI ??? ipads can't do selections and contextMenus, do I want to re-factor to cater for this? From 798d548ee9da6a9333a8fc2895ac57a42209f420 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:48:05 +1000 Subject: [PATCH 05/67] Made saved users be in the DB --- user.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 user.py diff --git a/user.py b/user.py new file mode 100644 index 0000000..39e6fad --- /dev/null +++ b/user.py @@ -0,0 +1,21 @@ +from main import db +from sqlalchemy import Sequence +from flask_login import UserMixin +from sqlalchemy.exc import SQLAlchemyError +from status import st, Status + +# pylint: disable=no-member + +################################################################################ +# Class describing Person in the database, and via sqlalchemy, connected to the DB as well +################################################################################ +class PAUser(UserMixin,db.Model): + __tablename__ = "pa_user" + id = db.Column(db.Integer, db.Sequence('pa_user_id_seq'), primary_key=True) + dn = db.Column(db.String) + + def __repr__(self): + return self.dn + + def get_id(self): + return self.dn From ce296426abc208de311f91996c955af72ba379ed Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 16:48:15 +1000 Subject: [PATCH 06/67] login code --- templates/login.html | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 templates/login.html diff --git a/templates/login.html b/templates/login.html new file mode 100644 index 0000000..5b38214 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + +{% import "bootstrap/wtf.html" as wtf %} + + + + +
+{% if form.errors|length > 0 %} +
+ + {% set last_err = namespace(txt="") %} + {% for e in form.errors %} + {% if last_err.txt != form.errors[e] %} + {% set err = form.errors[e]|replace("['", "" ) %} + {% set err = err|replace("']", "" ) %} + {{err}} + {% set last_err.txt=form.errors[e] %} + {% endif %} + {% endfor %} + + +
+{% endif %} + +{#
#} + +
+

+ + + + +  Photo Assistant Login

+
+
+ + +
+
+ + +
+ +
+ {{ form.submit( class="form-control text-info") }} +
+ {{ form.hidden_tag() }} +
+
+
+ + + From 0c4da6e4afd39a03f5a6d7c3e25bd7ce0656fd0e Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 26 Jun 2021 17:33:21 +1000 Subject: [PATCH 07/67] updated TODO for set of next steps to start on AI for real --- TODO | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/TODO b/TODO index 6eeee30..6a42815 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,30 @@ ## GENERAL + * make code use FLASK_ENV var to use a pa-devdb equiv + * create a new table file_face_refimg_link: + file_id, face_enc, ref_img (can be null) + * need AI code to: + scan for unknown faces, instead of storing array of all faces in FILE->FACES use table above one row per FILE & FACE with refimg_link as null to start + * when we do ai matching, we find all refimg is null (for a specific file) and match that + * need pa_job_mgr AI jobs to have low-level functions: + FindUnknownFacesInFile() + MatchRefImgWithUnknownFace() + Then create wrapper funcs: + MatchPersonWithFile() + for each ref img for person + MatchRefImgWithUnknownFace() + + MatchPersonInDir() + for each file in Dir: + MatchPersonWithFile() + + * then in UI: + allow right-click folders in folder view (at least for AI scan) for all files in Dir + allow right-click AI + for all persons + for individuals + (and can be on an individual file or an individual dir, or selected files / selected dir) + * fix up logging in general * comment your code * more OO goodness :) From 1c2612e2cbc628d8405e02ebf945b6404af4e123 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 27 Jun 2021 13:11:49 +1000 Subject: [PATCH 08/67] spacing --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 2ef3ff1..5767c15 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,6 @@ login_manager = LoginManager(app) # Setup a Flask-Login Manager ldap_manager = LDAP3LoginManager(app) # Setup a LDAP3 Login Manager. login_manager.login_view = "login" # default login route, failed with url_for, so hard-coded - ################################# Now, import non-book classes ################################### from settings import Settings from files import Entry, GetJM_Message, ClearJM_Message From 4bda64ca17e0cf3447b688ebe30802ffca4d0fd6 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 27 Jun 2021 13:12:42 +1000 Subject: [PATCH 09/67] hide prod DB inside docker network so only paweb can get to prod, created a new DEV DB on port 65432 and if FLASK_ENV is development you get that one --- shared.py | 15 +++++++++------ wrapper.sh | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/shared.py b/shared.py index 69324d0..d7f065d 100644 --- a/shared.py +++ b/shared.py @@ -9,15 +9,18 @@ ICON["Import"]="fa-file-upload" ICON["Storage"]="fa-database" ICON["Bin"]="fa-trash-alt" -if hostname == PROD_HOST: - PA_JOB_MANAGER_HOST="192.168.0.2" - DB_URL = 'postgresql+psycopg2://pa:for_now_pa@192.168.0.2:55432/pa' -elif hostname == "lappy": +if hostname == "lappy": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@localhost:5432/pa' -else: +elif os.environ['FLASK_ENV'] == "development": PA_JOB_MANAGER_HOST="localhost" - DB_URL = 'postgresql+psycopg2://pa:for_now_pa@mara.ddp.net:55432/pa' + DB_URL = 'postgresql+psycopg2://pa:for_now_pa@mara.ddp.net:65432/pa' +elif os.environ['FLASK_ENV'] == "production": + PA_JOB_MANAGER_HOST="192.168.0.2" + DB_URL = 'postgresql+psycopg2://pa:for_now_pa@padb/pa' +else: + print( "ERROR: I do not know which environment (development, etc.) and which DB (on which host to use)" ) + exit( -1 ) PA_JOB_MANAGER_PORT=55430 diff --git a/wrapper.sh b/wrapper.sh index 126def4..bd19ce3 100755 --- a/wrapper.sh +++ b/wrapper.sh @@ -1,4 +1,4 @@ #!/bin/bash su mythtv -g mythtv -c "python3 /code/pa_job_manager.py" & -gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --preload +gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --env FLASK_ENV="production" From ade3745024e9054c6a2af91f6f6f56304e8a12f4 Mon Sep 17 00:00:00 2001 From: c-d-p Date: Sun, 27 Jun 2021 14:26:51 +1000 Subject: [PATCH 10/67] spacing --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 5767c15..8d8223b 100644 --- a/main.py +++ b/main.py @@ -20,8 +20,10 @@ import socket ####################################### Flask App globals ####################################### PROD_HOST="pa_web" + hostname = socket.gethostname() print( "Running on: {}".format( hostname) ) + app = Flask(__name__) ### what is this value? I gather I should change it? From 26ba27cc0d0d1a8c1f54051cee639d4af6f2b1c3 Mon Sep 17 00:00:00 2001 From: c-d-p Date: Sun, 27 Jun 2021 14:27:46 +1000 Subject: [PATCH 11/67] updated files.py and files.html to add a context menu submenu to look for faces in an image. Doesnt work yet, but the context menu works --- files.py | 4 +++- templates/files.html | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/files.py b/files.py index e6bdf90..288d936 100644 --- a/files.py +++ b/files.py @@ -269,6 +269,8 @@ def files_sp(): noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request ) entries=[] + people = Person.query.all() + # per storage path, add entries to view settings=Settings.query.first() paths = settings.storage_path.split("#") @@ -279,7 +281,7 @@ def files_sp(): else: entries+=GetEntriesInFlatView( cwd, prefix, noo, offset, how_many ) - return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) + return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root, people=people ) ################################################################################ diff --git a/templates/files.html b/templates/files.html index 874ebce..78c78c3 100644 --- a/templates/files.html +++ b/templates/files.html @@ -12,7 +12,6 @@ caption-side: bottom; } -
@@ -467,7 +466,16 @@ $.contextMenu({ details: { name: "Details..." }, view: { name: "View File" }, sep: "---", - move: { name: "Move selected file(s) to new storage folder" } + move: { name: "Move selected file(s) to new storage folder" }, + sep2: "---", + ai: { + name: "Scan file for faces", + items: { + {% for p in people %} + "ai-{{p.tag}}": {"name": "{{p.tag}}"}, + {% endfor %} + } + } } if( SelContainsBinAndNotBin() ) { item_list['both']= { name: 'Cannot delete and restore at same time', disabled: true } @@ -485,6 +493,7 @@ $.contextMenu({ if( key == "move" ) { MoveDBox() } if( key == "del" ) { DelDBox('Delete') } if( key == "undel" ) { DelDBox('Restore') } + if( key.startsWith("ai")) { console.log( key +'was chosen')} }, items: item_list }; From 68d1fcac60790ce0061e1aaddc95ea970490cf14 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 27 Jun 2021 14:30:33 +1000 Subject: [PATCH 12/67] fix BUG with non-existant path causing re-render of previous path --- files.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/files.py b/files.py index e6bdf90..2fb6389 100644 --- a/files.py +++ b/files.py @@ -252,6 +252,8 @@ def files_ip(): settings=Settings.query.first() paths = settings.import_path.split("#") for path in paths: + if not os.path.exists(path): + continue prefix = SymlinkName("Import",path,path+'/') if folders: entries+=GetEntriesInFolderView( cwd, prefix, noo, offset, how_many ) @@ -273,6 +275,8 @@ def files_sp(): settings=Settings.query.first() paths = settings.storage_path.split("#") for path in paths: + if not os.path.exists(path): + continue prefix = SymlinkName("Storage",path,path+'/') if folders: entries+=GetEntriesInFolderView( cwd, prefix, noo, offset, how_many ) @@ -295,6 +299,8 @@ def files_rbp(): settings=Settings.query.first() paths = settings.recycle_bin_path.split("#") for path in paths: + if not os.path.exists(path): + continue prefix = SymlinkName("Bin",path,path+'/') if folders: entries+=GetEntriesInFolderView( cwd, prefix, noo, offset, how_many ) From 76d278c37a467197b0127eaaefcc49a2b3ebda45 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 27 Jun 2021 14:31:26 +1000 Subject: [PATCH 13/67] instructions for upstream bootstrap improved, to note font awesome needs work as I think only I can download --- README | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README b/README index ca64e5a..cd9422f 100644 --- a/README +++ b/README @@ -22,10 +22,16 @@ pip packages: upstream packages... mkdir static/upstream cd static/upstream + mkdir bootstrap-4.6.0-dist + cd bootstrap-4.6.0-dist + mkdir css # for boostrap: wget https://cdn.jsdelivr.net/npm/bootstrap@4.6.0/dist/css/bootstrap.min.css - wget https://cdn.jsdelivr.net/npm/bootstrap@4.6.0/dist/js/bootstrap.bundle.min.js + + mkdir js + # to note we might need bootstrap.bundle.min.js if we use new features? + wget https://cdn.jsdelivr.net/npm/bootstrap@4.6.0/dist/js/bootstrap.min.js # for jquery https://code.jquery.com/jquery-3.6.0.min.js From 126b17aa332008567e3d72a9b2657323cecb85d7 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 27 Jun 2021 14:37:53 +1000 Subject: [PATCH 14/67] make prod pa_job_manager have FLASK_ENV of production and use localhost for job mgr host --- README | 2 +- TODO | 1 - shared.py | 2 +- wrapper.sh | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README b/README index cd9422f..f636a21 100644 --- a/README +++ b/README @@ -57,7 +57,7 @@ to run prod version of web server: gunicorn --bind="192.168.0.2:5000" --threads=2 --workers=2 main:app Also have to run the job manager for jobs to work: - python3 pa_job_manager.py + FLASK_ENV="development" python3 pa_job_manager.py To rebuild DB from scratch/empty data: diff --git a/TODO b/TODO index 6a42815..e1b2630 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,5 @@ ## GENERAL - * make code use FLASK_ENV var to use a pa-devdb equiv * create a new table file_face_refimg_link: file_id, face_enc, ref_img (can be null) * need AI code to: diff --git a/shared.py b/shared.py index d7f065d..e692310 100644 --- a/shared.py +++ b/shared.py @@ -16,7 +16,7 @@ elif os.environ['FLASK_ENV'] == "development": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@mara.ddp.net:65432/pa' elif os.environ['FLASK_ENV'] == "production": - PA_JOB_MANAGER_HOST="192.168.0.2" + PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@padb/pa' else: print( "ERROR: I do not know which environment (development, etc.) and which DB (on which host to use)" ) diff --git a/wrapper.sh b/wrapper.sh index bd19ce3..66ef496 100755 --- a/wrapper.sh +++ b/wrapper.sh @@ -1,4 +1,4 @@ #!/bin/bash -su mythtv -g mythtv -c "python3 /code/pa_job_manager.py" & +su mythtv -g mythtv -c "FLASK_ENV="production" python3 /code/pa_job_manager.py" & gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --env FLASK_ENV="production" From a6782c24ff16c890629dace2caff5692c3684975 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Mon, 28 Jun 2021 17:02:41 +1000 Subject: [PATCH 15/67] if you dont pass an FLASK_ENV, we now assume development --- shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared.py b/shared.py index e692310..8dbdc2c 100644 --- a/shared.py +++ b/shared.py @@ -12,7 +12,7 @@ ICON["Bin"]="fa-trash-alt" if hostname == "lappy": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@localhost:5432/pa' -elif os.environ['FLASK_ENV'] == "development": +elif 'FLASK_ENV' not in os.environ or os.environ['FLASK_ENV'] == "development": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@mara.ddp.net:65432/pa' elif os.environ['FLASK_ENV'] == "production": From 31db4fcca18952ac0177341c9efc6fbda2e51a82 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Mon, 28 Jun 2021 17:03:13 +1000 Subject: [PATCH 16/67] added in DB tables for new face DB structures/links --- pa_job_manager.py | 25 ++++++++++++++++++++++++- tables.sql | 13 +++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index e15f7b4..c4eb5d0 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -181,7 +181,6 @@ class DelFile(Base): def __repr__(self): return f"" - class FileType(Base): __tablename__ = "file_type" id = Column(Integer, Sequence('file_type_id_seq'), primary_key=True ) @@ -229,6 +228,30 @@ class Refimg(Base): def __repr__(self): return f"" +class Face(Base): + __tablename__ = "face" + id = Column(Integer, Sequence('face_id_seq'), primary_key=True ) + face = Column( LargeBinary ) + + def __repr__(self): + return f" Date: Mon, 28 Jun 2021 17:05:52 +1000 Subject: [PATCH 17/67] added in DB tables for new face DB structures/links --- pa_job_manager.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pa_job_manager.py b/pa_job_manager.py index c4eb5d0..dcd1349 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -1339,6 +1339,61 @@ def InitialValidationChecks(): FinishJob(job,"Finished Initial Validation Checks") return +#### CAM: New FACES/AI code + +def AddFaceToFile( face_data, file_eid ): + face = Face( face=face_data ) + session.add(face) + session.commit() + ffl = FaceFileLink( face_id=face.id, file_eid=file_eid ) + session.add(ffl) + session.commit() + return face + +def DelFacesForFile( eid ): + session.execute( f"delete from face where id in (select face_id from face_file_link where file_eid = {eid})" ) + session.commit() + return + +def MatchRefimgToFace( refimg_id, face_id ): + rfl = FaceRefimgLink( refimg_id = refimg_id, face_id = face_id ) + session.add(rfl) + session.commit() + return + +def UnmatchedFacesForFile( eid ): + rows = session.execute( f"select f.id, ffl.file_eid, frl.refimg_id from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where ffl.file_eid = {eid} and frl.refimg_id is null" ) + return rows + +### CAM: something like this -- HAVE NOT TRIED THIS IT WILL FAIL### +def ScanFileForPerson( eid, person_id, force=False ): + file_h = session.query(File).get( eid ) + # if we are forcing this, delete any old faces (this will also delete linked tables), and reset faces_created_on to None + if force: + DelFacesForFile( eid ) + file_h.faces_create_on = None + + # optimise: dont rescan if we already have faces (we are just going to try + # to match (maybe?) a refimg + if not file_h.faces_created_on: + # CAM: TODO: add Face Rec code to get unknown encodings + for face in unknown_encodings: + new_face = Face( face_data = face ) + session.add(new_face) + session.commit() + AddFaceToFile( new_face.id, eid ) + now=datetime.now(pytz.utc) + file_h.face_created_on = now + + ## now look for person + refimgs = session.query(Refimg).join(PersonRefimgLink).filter(PersonRefimgLink.person_id==person_id).all() + uf = UnmatchedFacesForFile( eid ) + for face in uf: + for r in refimgs: + # CAM: TODO: add Face rec code to see if there is match + if match: + MatchRefimgToFace( r.id, face.id ) + return if __name__ == "__main__": print("INFO: PA job manager starting - listening on {}:{}".format( PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT) ) From 4cb10c4a6bd26f63d925d309ba623c5043edbd6b Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Mon, 28 Jun 2021 18:52:05 +1000 Subject: [PATCH 18/67] started hooking up right-click menu for Dir and Files all the way through to calling the new ScanFileForPerson() - which is still incomplete but does use the new Faces DB linkages and functions --- TODO | 10 ++++----- ai.py | 22 +++++++++++++++++++ pa_job_manager.py | 51 +++++++++++++++++++++++++++++++++++++++++--- templates/files.html | 40 ++++++++++++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 10 deletions(-) diff --git a/TODO b/TODO index e1b2630..cc08613 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,11 @@ ## GENERAL - * create a new table file_face_refimg_link: - file_id, face_enc, ref_img (can be null) + * allow rotate of image (permanently on FS, so its right everywhere) + + * improve photo browser -> view file, rather than just allowing browser to show image + * need AI code to: - scan for unknown faces, instead of storing array of all faces in FILE->FACES use table above one row per FILE & FACE with refimg_link as null to start - * when we do ai matching, we find all refimg is null (for a specific file) and match that - * need pa_job_mgr AI jobs to have low-level functions: + - make use of new FACE structures/links ... (code for this is in pa_job_manager, just not being called/used as yet) && something like: FindUnknownFacesInFile() MatchRefImgWithUnknownFace() Then create wrapper funcs: diff --git a/ai.py b/ai.py index 29b3fe3..d551b9b 100644 --- a/ai.py +++ b/ai.py @@ -10,6 +10,9 @@ from person import Person, PersonRefimgLink from refimg import Refimg from flask_login import login_required, current_user +from job import Job, JobExtra, Joblog, NewJob + + # pylint: disable=no-member ################################################################################ @@ -28,3 +31,22 @@ def aistats(): last_fname = e.name entry['people'].append( { 'tag': p.tag } ) return render_template("aistats.html", page_title='Placeholder', entries=entries) + + +################################################################################ +# /run_ai_on -> CAM: needs more thought (what actual params, e.g list of file - +# tick, but which face or faces? are we forcing a re-finding of unknown faces +# or just looking for matches? (maybe in the long run there are different +# routes, not params - stuff we will work out as we go) +################################################################################ +@app.route("/run_ai_on", methods=["POST"]) +@login_required +def run_ai_on(): + jex=[] + for el in request.form: + jex.append( JobExtra( name=f"{el}", value=request.form[el] ) ) + print( f"would create new job with extras={jex}" ) + job=NewJob( "run_ai_on", 0, None, jex ) + st.SetAlert("success") + st.SetMessage( f"Created Job #{job.id} to Look for face(s) in selected file(s)") + return render_template("base.html") diff --git a/pa_job_manager.py b/pa_job_manager.py index dcd1349..b601662 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -137,9 +137,13 @@ class Entry(Base): in_dir = relationship ("Dir", secondary="entry_dir_link", uselist=False ) def FullPathOnFS(self): - s=self.in_dir.in_path.path_prefix + '/' - if len(self.in_dir.rel_path) > 0: - s += self.in_dir.rel_path + '/' + if self.in_dir: + s=self.in_dir.in_path.path_prefix + '/' + if len(self.in_dir.rel_path) > 0: + s += self.in_dir.rel_path + '/' + # this occurs when we have a dir that is the root of a path + else: + s=self.dir_details.in_path.path_prefix+'/' s += self.name return s @@ -431,6 +435,8 @@ def RunJob(job): JobRestoreFiles(job) elif job.name == "processai": JobProcessAI(job) + elif job.name == "run_ai_on": + JobRunAIOn(job) else: print("ERROR: Requested to process unknown job type: {}".format(job.name)) # okay, we finished a job, so check for any jobs that are dependant on this and run them... @@ -940,6 +946,44 @@ def JobProcessAI(job): FinishJob(job, "Finished Processesing AI") return +def WrapperForScanFileForPerson(job, entry): + which_person=[jex.value for jex in job.extra if jex.name == "person"][0] + if which_person == "all": + ppl=session.query(Person).all() + else: + ppl=session.query(Person).filter(Person.tag==which_person).all() + for person in ppl: + print( f"(wrapper) call == ScanFileForPerson( {entry.id}, {person.id}, force=False )" ) + return + +def JobRunAIOn(job): + AddLogForJob(job, f"INFO: Starting looking For faces in files job...") + which_person=[jex.value for jex in job.extra if jex.name == "person"][0] + if which_person == "all": + ppl=session.query(Person).all() + else: + ppl=session.query(Person).filter(Person.tag==which_person).all() + print( "JobRunAIOn() called" ) + for person in ppl: + print( f"person={person.tag}" ) + + for jex in job.extra: + if 'eid-' in jex.name: + entry=session.query(Entry).get(jex.value) + print( f'en={entry}' ) + if entry.type.name == 'Directory': + ProcessFilesInDir( job, entry, WrapperForScanFileForPerson ) + elif entry.type.name == 'Image': + which_file=session.query(Entry).join(File).filter(Entry.id==jex.value).first() + print( f"JobRunAIOn: file to process had id: {which_file.id}" ) + for person in ppl: + print( f"call == ScanFileForPerson( {which_file.id}, {person.id}, force=False )" ) + else: + AddLogForJob( job, f'Not processing Entry: {entry.name} - not an image' ) + print(" HARD EXITING to keep testing " ) + exit( -1 ) + FinishJob(job, "Finished Processesing AI") + return def GenHashAndThumb(job, e): # commit every 100 files to see progress being made but not hammer the database @@ -1056,6 +1100,7 @@ def compareAI(known_encoding, unknown_encoding): def ProcessFilesInDir(job, e, file_func): if DEBUG==1: + print( f"???? e={e}" ) print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") if e.type.name != 'Directory': file_func(job, e) diff --git a/templates/files.html b/templates/files.html index 78c78c3..2802bc4 100644 --- a/templates/files.html +++ b/templates/files.html @@ -222,7 +222,7 @@ {% endif %} {# if this dir is the toplevel of the cwd, show the folder icon #} {% if dirname| TopLevelFolderOf(cwd) %} -
+
{{obj.name}}
@@ -270,6 +270,13 @@ function GetSelnAsData() return to_del } +function RunAIOnSeln(person) +{ + post_data = GetSelnAsData() + post_data += '&person='+person.replace('ai-','') + $.ajax({ type: 'POST', data: post_data, url: '/run_ai_on', success: function(data){ window.location='/'; return false; } }) +} + function DelDBox(del_or_undel) { to_del = GetSelnAsData() @@ -457,6 +464,34 @@ function NoSel() { $('.figure').click( function(e) { DoSel(e, this ); SetButtonState(); return false; }); $(document).on('click', function(e) { $('.highlight').removeClass('highlight') ; SetButtonState() }); + +// different context menu on directory +$.contextMenu({ + selector: '.dir', + build: function($triggerElement, e){ + if( NoSel() ) + DoSel(e, e.currentTarget ) + item_list = { + ai: { + name: "Scan file for faces", + items: { + {% for p in people %} + "ai-{{p.tag}}": {"name": "{{p.tag}}"}, + {% endfor %} + "ai-all": {"name": "all"}, + } + } + } + return { + callback: function( key, options) { + if( key.startsWith("ai")) { RunAIOnSeln(key) } + }, + items: item_list + }; + } +}); + +// different context menu on files $.contextMenu({ selector: '.figure', build: function($triggerElement, e){ @@ -474,6 +509,7 @@ $.contextMenu({ {% for p in people %} "ai-{{p.tag}}": {"name": "{{p.tag}}"}, {% endfor %} + "ai-all": {"name": "all"}, } } } @@ -493,7 +529,7 @@ $.contextMenu({ if( key == "move" ) { MoveDBox() } if( key == "del" ) { DelDBox('Delete') } if( key == "undel" ) { DelDBox('Restore') } - if( key.startsWith("ai")) { console.log( key +'was chosen')} + if( key.startsWith("ai")) { RunAIOnSeln(key) } }, items: item_list }; From dcee8c96dd96185069dcdf0d78a55699b962e2eb Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 29 Jun 2021 00:43:51 +1000 Subject: [PATCH 19/67] minor updates --- README | 5 +++++ TODO | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README b/README index f636a21..0dbf533 100644 --- a/README +++ b/README @@ -1,5 +1,10 @@ In here we can put instructions on how to run this / any general info +to edit src: + +git.... +CAM: fill this in pls + ubuntu packages: sudo apt-get install -y mediainfo cmake python3-flask diff --git a/TODO b/TODO index cc08613..9274105 100644 --- a/TODO +++ b/TODO @@ -18,11 +18,11 @@ MatchPersonWithFile() * then in UI: - allow right-click folders in folder view (at least for AI scan) for all files in Dir - allow right-click AI - for all persons - for individuals - (and can be on an individual file or an individual dir, or selected files / selected dir) + allow right-click folders in folder view (at least for AI scan) for all files in Dir [DONE] + allow right-click AI [DONE] + for all persons [DONE] + for individuals [DONE] + (and can be on an individual file or an individual dir, or selected files / selected dir) [DONE] * fix up logging in general * comment your code From 643208f35f8906debb2659d56df20ae68894750d Mon Sep 17 00:00:00 2001 From: c-d-p Date: Tue, 29 Jun 2021 15:43:28 +1000 Subject: [PATCH 20/67] fixed the import path to be able to select all of the people under the ai context menu --- files.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/files.py b/files.py index abfd13b..a01e5dc 100644 --- a/files.py +++ b/files.py @@ -248,6 +248,9 @@ def files_ip(): noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request ) entries=[] + people = Person.query.all() + + # per import path, add entries to view settings=Settings.query.first() paths = settings.import_path.split("#") @@ -260,7 +263,7 @@ def files_ip(): else: entries+=GetEntriesInFlatView( cwd, prefix, noo, offset, how_many ) - return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) + return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root, people=people ) ################################################################################ # /files -> show thumbnail view of files from storage_path From 61b7a3e45789f29377aafb5a8b5b01cc436263b7 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 29 Jun 2021 15:46:02 +1000 Subject: [PATCH 21/67] updated DEV for Cams paths --- tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tables.sql b/tables.sql index 62faf60..7499cb6 100644 --- a/tables.sql +++ b/tables.sql @@ -123,6 +123,6 @@ insert into PERSON_REFIMG_LINK values ( 2, 2 ); insert into PERSON_REFIMG_LINK values ( 3, 3 ); insert into PERSON_REFIMG_LINK values ( 4, 4 ); -- DEV: -insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', '/home/ddp/src/photoassistant/storage/', '/home/ddp/src/photoassistant/.pa_bin/' ); +insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', '/home/ddp/src/photoassistant/storage/#c:/Users/cam/Desktop/code/python/photoassistant/storage/', '/home/ddp/src/photoassistant/.pa_bin/#c:/Users/cam/Desktop/code/python/photoassistant/.pa_bin/' ); -- PROD: --insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/Camera_uploads/', '/export/docker/storage/photos/', '/export/docker/storage/.pa_bin/' ); From 818fdaa685724ccbe26173d3b8856000d8d89b96 Mon Sep 17 00:00:00 2001 From: c-d-p Date: Tue, 29 Jun 2021 15:46:04 +1000 Subject: [PATCH 22/67] updated job manager to (theoretically) work for ai, but i havent tested the comparison code yet --- pa_job_manager.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index b601662..a55e997 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -952,8 +952,12 @@ def WrapperForScanFileForPerson(job, entry): ppl=session.query(Person).all() else: ppl=session.query(Person).filter(Person.tag==which_person).all() - for person in ppl: - print( f"(wrapper) call == ScanFileForPerson( {entry.id}, {person.id}, force=False )" ) + + if entry.type.name == 'Image': + print( f"JobRunAIOn: file to process had id: {entry.id}" ) + for person in ppl: + print( f"call == ScanFileForPerson( {entry.id}, {person.id}, force=False )" ) + ScanFileForPerson( entry, person.id, force=False) return def JobRunAIOn(job): @@ -970,7 +974,7 @@ def JobRunAIOn(job): for jex in job.extra: if 'eid-' in jex.name: entry=session.query(Entry).get(jex.value) - print( f'en={entry}' ) + print( f'en={entry.name}, {entry.type.name}' ) if entry.type.name == 'Directory': ProcessFilesInDir( job, entry, WrapperForScanFileForPerson ) elif entry.type.name == 'Image': @@ -978,10 +982,11 @@ def JobRunAIOn(job): print( f"JobRunAIOn: file to process had id: {which_file.id}" ) for person in ppl: print( f"call == ScanFileForPerson( {which_file.id}, {person.id}, force=False )" ) + ScanFileForPerson( which_file, person.id, force=False) else: AddLogForJob( job, f'Not processing Entry: {entry.name} - not an image' ) - print(" HARD EXITING to keep testing " ) - exit( -1 ) + #print(" HARD EXITING to keep testing " ) + #exit( -1 ) FinishJob(job, "Finished Processesing AI") return @@ -1011,7 +1016,7 @@ def ProcessAI(job, e): job.current_file_num+=1 return - file = e.FullPathOnFS() + file = e.FullPathOnFS() stat = os.stat(file) # find if file is newer than when we found faces before (fyi: first time faces_created_on == 0) if stat.st_ctime > e.file_details.faces_created_on: @@ -1101,7 +1106,7 @@ def compareAI(known_encoding, unknown_encoding): def ProcessFilesInDir(job, e, file_func): if DEBUG==1: print( f"???? e={e}" ) - print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") + # print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") if e.type.name != 'Directory': file_func(job, e) else: @@ -1411,32 +1416,39 @@ def UnmatchedFacesForFile( eid ): return rows ### CAM: something like this -- HAVE NOT TRIED THIS IT WILL FAIL### -def ScanFileForPerson( eid, person_id, force=False ): - file_h = session.query(File).get( eid ) +def ScanFileForPerson( e, person_id, force=False ): + file_h = session.query(File).get( e.id ) # if we are forcing this, delete any old faces (this will also delete linked tables), and reset faces_created_on to None if force: - DelFacesForFile( eid ) + DelFacesForFile( e.id ) file_h.faces_create_on = None # optimise: dont rescan if we already have faces (we are just going to try # to match (maybe?) a refimg if not file_h.faces_created_on: - # CAM: TODO: add Face Rec code to get unknown encodings + print("------------- CAM -----------:\n") + im = face_recognition.load_image_file(e.FullPathOnFS()) + print(im) + face_locations = face_recognition.face_locations(im) + print(f"FACE LOCATIONS: {face_locations}") + unknown_encodings = face_recognition.face_encodings(im, known_face_locations=face_locations) + print("AAAAAAAAAAAAAAAA " + str(len(unknown_encodings))) for face in unknown_encodings: new_face = Face( face_data = face ) session.add(new_face) session.commit() - AddFaceToFile( new_face.id, eid ) + AddFaceToFile( new_face.id, e.id ) now=datetime.now(pytz.utc) file_h.face_created_on = now ## now look for person refimgs = session.query(Refimg).join(PersonRefimgLink).filter(PersonRefimgLink.person_id==person_id).all() - uf = UnmatchedFacesForFile( eid ) + uf = UnmatchedFacesForFile( e.id ) for face in uf: for r in refimgs: - # CAM: TODO: add Face rec code to see if there is match + match = compareAI(r, uf) if match: + print(f'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA WE MATCHED: {r}, {uf}') MatchRefimgToFace( r.id, face.id ) return From 78713a6767d67a919cb1710432b8f41fd0824119 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 30 Jun 2021 14:28:15 +1000 Subject: [PATCH 23/67] updated stats to use new face tables and be more useful now amount of matches is in the thousands --- ai.py | 15 ++++----------- templates/aistats.html | 10 +++------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/ai.py b/ai.py index d551b9b..9112953 100644 --- a/ai.py +++ b/ai.py @@ -5,12 +5,13 @@ from main import db, app, ma from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError from status import st, Status -from files import Entry, File, FileRefimgLink +from files import Entry, File from person import Person, PersonRefimgLink from refimg import Refimg from flask_login import login_required, current_user from job import Job, JobExtra, Joblog, NewJob +from face import Face, FaceFileLink, FaceRefimgLink # pylint: disable=no-member @@ -21,16 +22,8 @@ from job import Job, JobExtra, Joblog, NewJob @app.route("/aistats", methods=["GET", "POST"]) @login_required def aistats(): - tmp=db.session.query(Entry,Person).join(File).join(FileRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(FileRefimgLink.matched==True).all() - entries=[] - last_fname="" - for e, p in tmp: - if last_fname != e.name: - entry = { 'name': e.name, 'people': [] } - entries.append( entry ) - last_fname = e.name - entry['people'].append( { 'tag': p.tag } ) - return render_template("aistats.html", page_title='Placeholder', entries=entries) + stats = db.session.execute( "select p.tag, count(f.id) from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag" ) + return render_template("aistats.html", page_title='AI Statistics', stats=stats) ################################################################################ diff --git a/templates/aistats.html b/templates/aistats.html index 0797cb6..5af9c62 100644 --- a/templates/aistats.html +++ b/templates/aistats.html @@ -3,13 +3,9 @@ {% block main_content %}

Basic AI stats

- - {% for e in entries %} - + + {% for s in stats %} + {% endfor %}
FileAI Matched people
{{e.name}} - {% for p in e.people %} - {{p.tag}} - {% endfor %} -
Person (tag)Number of files matched
{{s[0]}}{{s[1]}}
{% endblock main_content %} From ea663926f2174ec4cea908954079d4147e26c8ef Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 30 Jun 2021 14:28:43 +1000 Subject: [PATCH 24/67] created face.py so search / ai .py can use new face linking tables --- face.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 face.py diff --git a/face.py b/face.py new file mode 100644 index 0000000..48427b6 --- /dev/null +++ b/face.py @@ -0,0 +1,29 @@ +from main import db, app, ma +from sqlalchemy import Sequence +from sqlalchemy.exc import SQLAlchemyError + + +class Face(db.Model): + __tablename__ = "face" + id = db.Column(db.Integer, db.Sequence('face_id_seq'), primary_key=True ) + face = db.Column( db.LargeBinary ) + + def __repr__(self): + return f" Date: Wed, 30 Jun 2021 14:29:28 +1000 Subject: [PATCH 25/67] now using new face linking code, and working, removed many debugs, needs work (around log commits). Also put a quick hack to create Bin path on init, but need to rethink this bit --- pa_job_manager.py | 70 +++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index a55e997..af8470d 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -954,10 +954,8 @@ def WrapperForScanFileForPerson(job, entry): ppl=session.query(Person).filter(Person.tag==which_person).all() if entry.type.name == 'Image': - print( f"JobRunAIOn: file to process had id: {entry.id}" ) for person in ppl: - print( f"call == ScanFileForPerson( {entry.id}, {person.id}, force=False )" ) - ScanFileForPerson( entry, person.id, force=False) + ScanFileForPerson( job, entry, person.id, force=False) return def JobRunAIOn(job): @@ -967,26 +965,22 @@ def JobRunAIOn(job): ppl=session.query(Person).all() else: ppl=session.query(Person).filter(Person.tag==which_person).all() - print( "JobRunAIOn() called" ) + + # FIXME: probably shouldbe elsewhere, but this is optmised so if refimgs exist for ppl, then it wont regen them for person in ppl: - print( f"person={person.tag}" ) + generateKnownEncodings(person) for jex in job.extra: if 'eid-' in jex.name: entry=session.query(Entry).get(jex.value) - print( f'en={entry.name}, {entry.type.name}' ) if entry.type.name == 'Directory': ProcessFilesInDir( job, entry, WrapperForScanFileForPerson ) elif entry.type.name == 'Image': which_file=session.query(Entry).join(File).filter(Entry.id==jex.value).first() - print( f"JobRunAIOn: file to process had id: {which_file.id}" ) for person in ppl: - print( f"call == ScanFileForPerson( {which_file.id}, {person.id}, force=False )" ) - ScanFileForPerson( which_file, person.id, force=False) + ScanFileForPerson( job, which_file, person.id, force=False) else: AddLogForJob( job, f'Not processing Entry: {entry.name} - not an image' ) - #print(" HARD EXITING to keep testing " ) - #exit( -1 ) FinishJob(job, "Finished Processesing AI") return @@ -1093,6 +1087,7 @@ def generateKnownEncodings(person): img = face_recognition.load_image_file(file) location = face_recognition.face_locations(img) encodings = face_recognition.face_encodings(img, known_face_locations=location) + print(f"INFO: created encoding for refimg of {file}") refimg.encodings = encodings[0].tobytes() refimg.created_on = time.time() session.add(refimg) @@ -1105,8 +1100,7 @@ def compareAI(known_encoding, unknown_encoding): def ProcessFilesInDir(job, e, file_func): if DEBUG==1: - print( f"???? e={e}" ) - # print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") + print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") if e.type.name != 'Directory': file_func(job, e) else: @@ -1324,7 +1318,7 @@ def JobDeleteFiles(job): if 'eid-' in jex.name: del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() MoveFileToRecycleBin(job,del_me) - now=datetime.now(pytz.utc) + ynw=datetime.now(pytz.utc) next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) session.add(next_job) MessageToFE( job.id, "success", "Completed (delete of selected files)" ) @@ -1365,6 +1359,10 @@ def InitialValidationChecks(): break if not rbp_exists: AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); + else: + bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first() + if not bin_path: + ProcessRecycleBinDir(job) sp_exists=0 paths = settings.storage_path.split("#") for path in paths: @@ -1389,10 +1387,8 @@ def InitialValidationChecks(): FinishJob(job,"Finished Initial Validation Checks") return -#### CAM: New FACES/AI code - def AddFaceToFile( face_data, file_eid ): - face = Face( face=face_data ) + face = Face( face=face_data.tobytes() ) session.add(face) session.commit() ffl = FaceFileLink( face_id=face.id, file_eid=file_eid ) @@ -1412,46 +1408,50 @@ def MatchRefimgToFace( refimg_id, face_id ): return def UnmatchedFacesForFile( eid ): - rows = session.execute( f"select f.id, ffl.file_eid, frl.refimg_id from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where ffl.file_eid = {eid} and frl.refimg_id is null" ) + rows = session.execute( f"select f.* from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where ffl.file_eid = {eid} and frl.refimg_id is null" ) return rows -### CAM: something like this -- HAVE NOT TRIED THIS IT WILL FAIL### -def ScanFileForPerson( e, person_id, force=False ): +def ScanFileForPerson( job, e, person_id, force=False ): + AddLogForJob( job, f'INFO: Looking for person: {person_id} in file: {e.name}' ) file_h = session.query(File).get( e.id ) # if we are forcing this, delete any old faces (this will also delete linked tables), and reset faces_created_on to None if force: + AddLogForJob( job, f'INFO: force is true, so deleting old face information for {e.name}' ) DelFacesForFile( e.id ) - file_h.faces_create_on = None + file_h.faces_created_on = 0 # optimise: dont rescan if we already have faces (we are just going to try # to match (maybe?) a refimg - if not file_h.faces_created_on: - print("------------- CAM -----------:\n") + if file_h.faces_created_on == 0: + if DEBUG: + AddLogForJob( job, f"DEBUG: {e.name} is missing unknown faces, generating them" ) im = face_recognition.load_image_file(e.FullPathOnFS()) - print(im) face_locations = face_recognition.face_locations(im) - print(f"FACE LOCATIONS: {face_locations}") unknown_encodings = face_recognition.face_encodings(im, known_face_locations=face_locations) - print("AAAAAAAAAAAAAAAA " + str(len(unknown_encodings))) for face in unknown_encodings: - new_face = Face( face_data = face ) - session.add(new_face) - session.commit() - AddFaceToFile( new_face.id, e.id ) - now=datetime.now(pytz.utc) - file_h.face_created_on = now + AddFaceToFile( face, e.id ) + file_h.faces_created_on = time.time() + session.commit() ## now look for person refimgs = session.query(Refimg).join(PersonRefimgLink).filter(PersonRefimgLink.person_id==person_id).all() uf = UnmatchedFacesForFile( e.id ) + if DEBUG and not uf: + AddLogForJob( job, "DEBUG: {e.name} all faces already matched - finished" ) + for face in uf: for r in refimgs: - match = compareAI(r, uf) - if match: - print(f'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA WE MATCHED: {r}, {uf}') + unknown_face_data = numpy.frombuffer(face.face, dtype=numpy.float64) + refimg_face_data = numpy.frombuffer(r.encodings, dtype=numpy.float64) + match = compareAI(refimg_face_data, unknown_face_data) + if match[0]: + AddLogForJob(job, f'WE MATCHED: {r.fname} with file: {e.name} ') MatchRefimgToFace( r.id, face.id ) + # no need to keep looking for this face, we found it, go to next unknown face + break return + if __name__ == "__main__": print("INFO: PA job manager starting - listening on {}:{}".format( PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT) ) From b084f5d9515838967f9778a234a2b56a60d31ee9 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 30 Jun 2021 14:31:00 +1000 Subject: [PATCH 26/67] copy ref images over to make it easier for rebuilds --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index b66042e..ac549d7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,3 @@ new_img_dir static/Bin/* static/Import/* static/Storage/* -reference_images/* From 2b20160deb78d475d6ae7c0c4662692983ab5994 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 30 Jun 2021 14:31:12 +1000 Subject: [PATCH 27/67] search now uses new face linking tables --- files.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/files.py b/files.py index a01e5dc..cf0e00f 100644 --- a/files.py +++ b/files.py @@ -28,6 +28,7 @@ from refimg import Refimg from settings import Settings from shared import SymlinkName from dups import Duplicates +from face import Face, FaceFileLink, FaceRefimgLink # pylint: disable=no-member @@ -328,7 +329,8 @@ def search(): file_data=Entry.query.join(File).filter(Entry.name.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() dir_data=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.rel_path.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - ai_data=Entry.query.join(File).join(FileRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(FileRefimgLink.matched==True).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + ai_data=Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + print( Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(FileRefimgLink.matched==True).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many) ) all_entries = file_data + dir_data + ai_data From 1cfb07903bb0d125d852c04f8af74e480f1923b6 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 1 Jul 2021 21:54:26 +1000 Subject: [PATCH 28/67] slightly improve ai stats --- ai.py | 13 ++++++++++++- templates/aistats.html | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ai.py b/ai.py index 9112953..ea56faf 100644 --- a/ai.py +++ b/ai.py @@ -23,7 +23,18 @@ from face import Face, FaceFileLink, FaceRefimgLink @login_required def aistats(): stats = db.session.execute( "select p.tag, count(f.id) from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag" ) - return render_template("aistats.html", page_title='AI Statistics', stats=stats) + fstats={} + fstats['files_with_a_face'] = db.session.execute( "select count(distinct file_eid) as count from face_file_link" ).first()[0] + fstats['files_with_a_match'] = db.session.execute( "select count(distinct ffl.file_eid) as count from face_file_link ffl, face_refimg_link frl where frl.face_id = ffl.face_id" ).first()[0] + fstats['files_with_missing_matches'] = db.session.execute( "select count(distinct ffl.file_eid) from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where frl.refimg_id is null" ).first()[0] + + # files_with_no_matches? + + fstats['all_faces'] = db.session.execute( "select count(distinct face_id) as count from face_file_link" ).first()[0] + fstats['all_matched_faces'] = db.session.execute( "select count(distinct face_id) as count from face_refimg_link" ).first()[0] + fstats['all_unmatched_faces'] = db.session.execute( "select count(f.id) from face f left join face_refimg_link frl on f.id = frl.face_id where frl.refimg_id is null" ).first()[0] + + return render_template("aistats.html", page_title='AI Statistics', stats=stats, fstats=fstats ) ################################################################################ diff --git a/templates/aistats.html b/templates/aistats.html index 5af9c62..963c781 100644 --- a/templates/aistats.html +++ b/templates/aistats.html @@ -2,6 +2,16 @@ {% block main_content %}

Basic AI stats

+ + + + + + + + +
WhatAmount
Files with a face{{fstats['files_with_a_face']}}
Files with a matched face{{fstats['files_with_a_match']}}
Files with missing matches{{fstats['files_with_missing_matches']}}
All faces found{{fstats['all_faces']}}
All faces matched{{fstats['all_matched_faces']}}
All faces unmatched{{fstats['all_unmatched_faces']}}
+ {% for s in stats %} From afee30047173ec0e1e1e0e5d3ab8041c5ebddd8a Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 1 Jul 2021 21:54:52 +1000 Subject: [PATCH 29/67] quick hack to allow one time AI: at start of search to only call AI --- files.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/files.py b/files.py index cf0e00f..52de696 100644 --- a/files.py +++ b/files.py @@ -327,12 +327,16 @@ def search(): # always show flat results for search to start with folders=False - file_data=Entry.query.join(File).filter(Entry.name.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - dir_data=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.rel_path.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - ai_data=Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - print( Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(FileRefimgLink.matched==True).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many) ) - - all_entries = file_data + dir_data + ai_data + term=request.form['term'] + if 'AI:' in term: + term = term.replace('AI:','') + all_entries = Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{term}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + print( all_entries ) + else: + file_data=Entry.query.join(File).filter(Entry.name.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + dir_data=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.rel_path.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + ai_data=Entry.query.join(File).join(FaceFileLink).join(Face).join(FaceRefimgLink).join(Refimg).join(PersonRefimgLink).join(Person).filter(Person.tag.ilike(f"%{request.form['term']}%")).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + all_entries = file_data + dir_data + ai_data return render_template("files.html", page_title='View Files', search_term=request.form['term'], entry_data=all_entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) From d7e74ec53ade11c2d75365004000018798ad910b Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 1 Jul 2021 21:55:17 +1000 Subject: [PATCH 30/67] removed todo on doing AI, and added new ones --- TODO | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/TODO b/TODO index 9274105..f16d027 100644 --- a/TODO +++ b/TODO @@ -4,18 +4,10 @@ * improve photo browser -> view file, rather than just allowing browser to show image - * need AI code to: - - make use of new FACE structures/links ... (code for this is in pa_job_manager, just not being called/used as yet) && something like: - FindUnknownFacesInFile() - MatchRefImgWithUnknownFace() - Then create wrapper funcs: - MatchPersonWithFile() - for each ref img for person - MatchRefImgWithUnknownFace() - - MatchPersonInDir() - for each file in Dir: - MatchPersonWithFile() + * need AI job to: + log amount matched + also create refimg encoding when we load the refimg not per AI job + commit 100 logs on a job * then in UI: allow right-click folders in folder view (at least for AI scan) for all files in Dir [DONE] From 848cdeacc555870e80c204e4e9d36e34ba374e9a Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 1 Jul 2021 21:55:35 +1000 Subject: [PATCH 31/67] added some handy SQLs/commands for AI when logs suck --- README | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README b/README index 0dbf533..e4c4480 100644 --- a/README +++ b/README @@ -91,3 +91,20 @@ To get back a 'working' but scanned set of data: # gunzip -c /home/ddp/src/photoassistant/DB_BACKUP/20200126-all-imported-no-duplicates.sql.gz > /srv/docker/container/padb/docker-entrypoint-initdb.d/tables.sql ( cd /srv/docker/config/ ; sudo docker-compose stop padb ; yes | sudo docker-compose rm padb ; sudo rm -rf /srv/docker/container/padb/data/ ; sudo docker-compose up padb ) + + +HANDY SQLs/commands: +# long-running AI job (in this case #46), which is not committing joblog per file, and isnt tracking counts properly (temporary bug) + +sudo docker exec -it padb bash + echo 'select * from joblog where job_id = 46;' | psql --user=pa pa | grep 'ooking for' | awk '{ print $15 } ' | sort -u | wc -l + +# how many entries are in a path +sudo docker exec -it padb bash + psql --user=pa pa + select count(entry_id) from entry_dir_link where dir_eid in ( select distinct dir_eid from path_dir_link where path_id = 2 ); + +# how many Images are in a path +sudo docker exec -it padb bash + psql --user=pa pa + select count(distinct e.id) from entry e, entry_dir_link edl where e.type_id = 1 and e.id = edl.entry_id and edl.dir_eid in ( select distinct dir_eid from path_dir_link where path_id = 2 ); From f6a92d749f48d0813560444f183bac5fc197ba3f Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 1 Jul 2021 21:56:59 +1000 Subject: [PATCH 32/67] added really large amounts to dups to process at once, so we can still get them all on a page if we really want --- templates/dups.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/dups.html b/templates/dups.html index 434e116..e52c419 100644 --- a/templates/dups.html +++ b/templates/dups.html @@ -7,7 +7,7 @@
{% endfor %} + {% if log_cnt > logs|length %} + + + + + {% endif %}
Person (tag)Number of files matched
{{log.log_date|vicdate}}{{log.log|safe}}
Remaining logs truncated + +
@@ -73,7 +81,7 @@ {% endblock main_content %} {% block script_content %} From aa826f69333a77debfaedfc54c235018e598d208 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 3 Jul 2021 12:29:19 +1000 Subject: [PATCH 40/67] if > 100 logs, truncate them and add button to show all logs, and stop auto-refresh too --- TODO | 3 --- job.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/TODO b/TODO index d18ef70..f4750ad 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,5 @@ ## GENERAL - * trim joblog > (say) 100 ... [DONE] - * then click for rest - * refimgs need to be done via job_mgr: - have local FS chooser of ref img, do html file upload to b/e -> job mgr -> save it to reference_images// - remove refimg menu from top-level -> sub of Person (to view/mangage?) diff --git a/job.py b/job.py index 2978f1b..1d6b372 100644 --- a/job.py +++ b/job.py @@ -105,18 +105,24 @@ def jobs(): ############################################################################### # /job/ -> GET -> shows status/history of jobs ################################################################################ -@app.route("/job/", methods=["GET"]) +@app.route("/job/", methods=["GET","POST"]) @login_required def joblog(id): page_title='Show Job Details' joblog = Job.query.get(id) - logs=Joblog.query.filter(Joblog.job_id==id).order_by(Joblog.log_date).all() + log_cnt = db.session.execute( f"select count(id) from joblog where job_id = {id}" ).first()[0] + first_logs_only = True + if request.method == 'POST': + logs=Joblog.query.filter(Joblog.job_id==id).order_by(Joblog.log_date).all() + first_logs_only = False + else: + logs=Joblog.query.filter(Joblog.job_id==id).order_by(Joblog.log_date).limit(50).all() if joblog.pa_job_state == "Completed": duration=(joblog.last_update-joblog.start_time) else: duration=(datetime.now(pytz.utc)-joblog.start_time) duration= duration-timedelta(microseconds=duration.microseconds) - return render_template("joblog.html", job=joblog, logs=logs, duration=duration, page_title=page_title) + return render_template("joblog.html", job=joblog, logs=logs, log_cnt=log_cnt, duration=duration, page_title=page_title, first_logs_only=first_logs_only) ############################################################################### # /job/ -> GET -> shows status/history of jobs From a916fb8192dcd7a4e3a2b70e052c9de5550dcfe1 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 3 Jul 2021 12:30:01 +1000 Subject: [PATCH 41/67] minor update --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index f4750ad..579e6dc 100644 --- a/TODO +++ b/TODO @@ -10,7 +10,7 @@ - will require p.tag modification to move FS location - allow to also remove refimg from this view??? (simple X/bin icon overlay on thumb) - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI - (Fix BUG-40,41 & TODO: many ref imgs on create person should span multiple rows) + (Fix BUG-41 and bits of BUG-43 & TODO: many ref imgs on create person should span multiple rows) * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) From 518df7ee102d85937ea050a2cda5d433153897cc Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 3 Jul 2021 12:43:25 +1000 Subject: [PATCH 42/67] removed all remnants of FILE_REFIMG_LINK --- TODO | 13 ++++--------- files.py | 9 --------- pa_job_manager.py | 33 ++------------------------------- tables.sql | 5 ----- 4 files changed, 6 insertions(+), 54 deletions(-) diff --git a/TODO b/TODO index 579e6dc..32a4ddc 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,7 @@ ## GENERAL + * remove file_refimg_link stuff... + * refimgs need to be done via job_mgr: - have local FS chooser of ref img, do html file upload to b/e -> job mgr -> save it to reference_images// - remove refimg menu from top-level -> sub of Person (to view/mangage?) @@ -12,11 +14,11 @@ - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI (Fix BUG-41 and bits of BUG-43 & TODO: many ref imgs on create person should span multiple rows) - * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) - * need AI job to: log amount matched, amount comparing too -> count should actually be total files in 'entries' (as we can select random entries to check) + * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) + * allow rotate of image (permanently on FS, so its right everywhere) * improve photo browser -> view file, rather than just allowing browser to show image @@ -26,13 +28,6 @@ * more OO goodness :) ## DB - * Need to think about... - file (image) -> has X faces, Y matches - X == Y (optim: dont scan again) - say X-Y == 1, then to optimise, we need to only check the missing - face... at the moment, the DB structure is not that clever... - (file_refimg_link --> file_refimg_link needs a face_num?) - * Dir can have date in the DB, so we can do Oldest/Newest dirs in Folder view ### BACKEND diff --git a/files.py b/files.py index 52de696..4639f75 100644 --- a/files.py +++ b/files.py @@ -74,15 +74,6 @@ class Entry(db.Model): def __repr__(self): return "".format(self.id, self.name, self.type, self.dir_details, self.file_details, self.in_dir) -class FileRefimgLink(db.Model): - __tablename__ = "file_refimg_link" - file_id = db.Column(db.Integer, db.ForeignKey('file.eid'), unique=True, nullable=False, primary_key=True) - refimg_id = db.Column(db.Integer, db.ForeignKey('refimg.id'), unique=True, nullable=False, primary_key=True) - when_processed = db.Column(db.Float) - matched = db.Column(db.Boolean) - def __repr__(self): - return f"" -class FileRefimgLink(Base): - __tablename__ = "file_refimg_link" - file_id = Column(Integer, ForeignKey('file.eid'), unique=True, nullable=False, primary_key=True) - refimg_id = Column(Integer, ForeignKey('refimg.id'), unique=True, nullable=False, primary_key=True) - when_processed = Column(Float) - matched = Column(Boolean) - - def __repr__(self): - return f" Date: Sun, 4 Jul 2021 20:04:25 +1000 Subject: [PATCH 43/67] moved GenFace and GenThumb common code into shared, and hook it in both f/e and b/e where needed --- shared.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/shared.py b/shared.py index 8dbdc2c..d3499f3 100644 --- a/shared.py +++ b/shared.py @@ -1,5 +1,9 @@ import socket import os +import face_recognition +import io +import base64 +from PIL import Image, ImageOps hostname = socket.gethostname() PROD_HOST="pa_web" @@ -73,3 +77,33 @@ def SymlinkName(ptype, path, file): if symlink[-1] == '/': symlink=symlink[0:-1] return symlink + + +def GenThumb(fname): + print( f"GenThumb({fname})" ) + + try: + im_orig = Image.open(fname) + im = ImageOps.exif_transpose(im_orig) + bands = im.getbands() + if 'A' in bands: + im = im.convert('RGB') + im.thumbnail((THUMBSIZE,THUMBSIZE)) + img_bytearray = io.BytesIO() + im.save(img_bytearray, format='JPEG') + img_bytearray = img_bytearray.getvalue() + thumbnail = base64.b64encode(img_bytearray) + thumbnail = str(thumbnail)[2:-1] + return thumbnail + except Exception as e: + print( f"GenThumb failed: {e}") + return None + +def GenFace(fname): + img = face_recognition.load_image_file(fname) + location = face_recognition.face_locations(img) + encodings = face_recognition.face_encodings(img, known_face_locations=location) + if len(encodings): + return encodings[0].tobytes() + else: + return None From 388c3eed9b1c9f202de9660af971085d8b026834 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 4 Jul 2021 20:05:15 +1000 Subject: [PATCH 44/67] reference images are now added in person.py (and removed) as buttons on person page. Will need to re-think the whole of refimg.py --- templates/person.html | 59 ++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/templates/person.html b/templates/person.html index 727ca68..a46ec94 100644 --- a/templates/person.html +++ b/templates/person.html @@ -1,42 +1,67 @@ {% extends "base.html" %} {% block main_content %} -
-

{{page_title}}

-
- +
+

{{page_title}}

+ {% for field in form %} {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} {{field}}
{% elif field.type != 'SubmitField' %}
- {{ field.label( class="col-lg-2" ) }} + {{ field.label( class="col-lg-3" ) }} {{ field( class="form-control col" ) }}
{% endif %} {% endfor %}
-
Reference Images:
 -
- {% for ref_img in reference_imgs %} -
- {{ref_img.fname}} -
+
Reference Images:
 + {% for ref_img in person.refimg %} + {% set offset="" %} + {% if (loop.index % 10) == 0 %} + {% set offset= "offset-lg-3" %} + {% endif %} +
+ +
+
+
+
+ +
+
{{ref_img.fname}}
+
+
+
{% endfor %} -

- {{ form.submit( class="btn btn-primary offset-lg-2 col-lg-2" )}} + {{ form.save( id="save", class="btn btn-primary offset-lg-3 col-lg-2" )}} {% if 'Edit' in page_title %} {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} {% endif %}
+
+ + +
{% endblock main_content %} + +{% block script_content %} + +{% endblock script_content %} + From ddc9b18e3e0a65c8e245b54e9766d362bf11e7c9 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 4 Jul 2021 20:05:47 +1000 Subject: [PATCH 45/67] reference images now create thumb and face on first association, so have DB for that --- tables.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tables.sql b/tables.sql index 122b0c8..4df3903 100644 --- a/tables.sql +++ b/tables.sql @@ -40,8 +40,9 @@ create table ENTRY_DIR_LINK ( entry_id integer, dir_eid integer, create table PERSON ( ID integer, TAG varchar(48), FIRSTNAME varchar(48), SURNAME varchar(48), constraint PK_PERSON_ID primary key(ID) ); -create table REFIMG ( ID integer, FNAME varchar(256), ENCODINGS bytea, - CREATED_ON float, +create table REFIMG ( ID integer, FNAME varchar(256), FACE bytea, + ENCODINGS bytea, + CREATED_ON float, THUMBNAIL varchar, constraint PK_REFIMG_ID primary key(ID) ); create table FACE( ID integer, FACE bytea, constraint PK_FACE_ID primary key(ID) ); From d3df3ad754e1b04195993282feb4984d86ea7574 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 4 Jul 2021 20:06:04 +1000 Subject: [PATCH 46/67] make it clear if we are on DEV or PROD --- templates/base.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/base.html b/templates/base.html index 64e9bd6..533e674 100644 --- a/templates/base.html +++ b/templates/base.html @@ -49,7 +49,11 @@
-
- -
- Create Reference Image - View Reference Images -
-
From 95303fafac61f35e547d881d53ceb288852aa0e6 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Mon, 5 Jul 2021 16:53:00 +1000 Subject: [PATCH 56/67] fixed BUG-41 and 43, via new refimg -> person code changes --- TODO | 3 --- 1 file changed, 3 deletions(-) diff --git a/TODO b/TODO index 29bf882..8c17734 100644 --- a/TODO +++ b/TODO @@ -1,10 +1,7 @@ ## GENERAL * refimg - - remove refimg menu from top-level -> sub of Person (to view/mangage?) - -- at the moment refimg menu (view/create) needs thought, remove create totally, view might be okay? (BUT, might also cause issues) - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI - (Fix BUG-41 and bits of BUG-43 & TODO: many ref imgs on create person should span multiple rows) - store the face locations? (good for debugging later, and we calc them as part of GenFace anyway) ... (really, doing this properly, we shoudl keep the face locns for ALL images)... ---> may well mean in a week or so, we move to the new DB structure, and START FORM SCRATCH) From 3c56cac1ef199941a12c28d484b225f99647c5af Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 6 Jul 2021 17:40:53 +1000 Subject: [PATCH 57/67] fix up selections to work with ctrl/shift and right-click -- its a bit clunky but works, redid numbering to make that work and condensed down to 1 common code block for context menu for figures and dirs now --- templates/files.html | 96 +++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/templates/files.html b/templates/files.html index 2802bc4..8affac7 100644 --- a/templates/files.html +++ b/templates/files.html @@ -121,7 +121,7 @@ {# rare event of empty folder, still need to show back button #} {% if folders and entry_data|length == 0 %} {% if cwd != root %} -
+
@@ -139,7 +139,7 @@ {% for obj in entry_data %} {% if loop.index==1 and folders %} {% if cwd != root %} -
+
@@ -187,7 +187,7 @@ {% endif %} {% if obj.type.name != "Directory" %} {% if (not folders) or ((obj.in_dir.in_path.path_prefix+'/'+obj.in_dir.rel_path+'/'+obj.name) | TopLevelFolderOf(cwd)) %} -
+
{% if obj.type.name=="Image" %}
@@ -222,7 +222,7 @@ {% endif %} {# if this dir is the toplevel of the cwd, show the folder icon #} {% if dirname| TopLevelFolderOf(cwd) %} -
+
{{obj.name}}
@@ -404,19 +404,25 @@ function DoSel(e, el) } if( e.shiftKey ) { - st=Number($('.highlight').first().attr('img')) - end=Number($('.highlight').last().attr('img')) - clicked=Number($(el).attr('img')) + st=Number($('.highlight').first().attr('ecnt')) + end=Number($('.highlight').last().attr('ecnt')) + clicked=Number($(el).attr('ecnt')) + if( ! folders ) + { + st -= 1 + end -= 1 + clicked -= 1 + } // if we shift-click first element, then st/end are NaN, so just highlightthe one clicked if( isNaN(st) ) { - $('.figure').slice( clicked, clicked+1 ).addClass('highlight') + $('.entry').slice( clicked, clicked+1 ).addClass('highlight') return } if( clicked > end ) - $('.figure').slice( end, clicked+1 ).addClass('highlight') + $('.entry').slice( end, clicked+1 ).addClass('highlight') else - $('.figure').slice( clicked, st ).addClass('highlight') + $('.entry').slice( clicked, st ).addClass('highlight') return } $('.highlight').removeClass('highlight') @@ -435,6 +441,24 @@ function SetButtonState() { } } +function FiguresOrDirsOrBoth() { + var figure=false + var dir=false + $('.highlight').each(function( index ) { + if( $(this).hasClass('figure') ) { + figure=true + } + if( $(this).hasClass('dir') ) { + dir=true + } + } ) + if( figure & ! dir ) + return "figure" + if( ! figure & dir ) + return "dir" + return "both" +} + function SelContainsBinAndNotBin() { var bin=false var not_bin=false @@ -465,45 +489,25 @@ $('.figure').click( function(e) { DoSel(e, this ); SetButtonState(); return fals $(document).on('click', function(e) { $('.highlight').removeClass('highlight') ; SetButtonState() }); -// different context menu on directory -$.contextMenu({ - selector: '.dir', - build: function($triggerElement, e){ - if( NoSel() ) - DoSel(e, e.currentTarget ) - item_list = { - ai: { - name: "Scan file for faces", - items: { - {% for p in people %} - "ai-{{p.tag}}": {"name": "{{p.tag}}"}, - {% endfor %} - "ai-all": {"name": "all"}, - } - } - } - return { - callback: function( key, options) { - if( key.startsWith("ai")) { RunAIOnSeln(key) } - }, - items: item_list - }; - } -}); - // different context menu on files $.contextMenu({ - selector: '.figure', + selector: '.entry', build: function($triggerElement, e){ - if( NoSel() ) + // when right-clicking & no selection add one OR deal with ctrl/shift right-lick as it always changes seln + if( NoSel() || e.ctrlKey || e.shiftKey ) DoSel(e, e.currentTarget ) - item_list = { - details: { name: "Details..." }, - view: { name: "View File" }, - sep: "---", - move: { name: "Move selected file(s) to new storage folder" }, - sep2: "---", - ai: { + + if( FiguresOrDirsOrBoth() == "figure" ) + item_list = { + details: { name: "Details..." }, + view: { name: "View File" }, + sep: "---", + move: { name: "Move selected file(s) to new storage folder" }, + sep2: "---" } + else + item_list = {} + + item_list['ai'] = { name: "Scan file for faces", items: { {% for p in people %} @@ -512,7 +516,7 @@ $.contextMenu({ "ai-all": {"name": "all"}, } } - } + if( SelContainsBinAndNotBin() ) { item_list['both']= { name: 'Cannot delete and restore at same time', disabled: true } } else { From 17180d9f2442925183423411187c6b58a3627e93 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 6 Jul 2021 17:44:26 +1000 Subject: [PATCH 58/67] removed debugs --- person.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/person.py b/person.py index 02a7dde..4bf7d16 100644 --- a/person.py +++ b/person.py @@ -93,7 +93,6 @@ def new_person(): try: db.session.add(person) db.session.commit() - print(person) st.SetMessage( "Created new Person ({})".format(person.tag) ) return redirect( '/persons' ) except SQLAlchemyError as e: @@ -130,7 +129,6 @@ def person(id): new_refs=[] for ref_img in person.refimg: if "ref-img-id-{}".format(ref_img.id) in request.form: - print('{} was checked, id: {}'.format(ref_img.fname, ref_img.id)) new_refs.append(ref_img) person.refimg = new_refs db.session.add(person) @@ -143,7 +141,6 @@ def person(id): return render_template("person.html", form=form, page_title=page_title) else: person = Person.query.get(id) - print(person) form = PersonForm(request.values, obj=person) return render_template("person.html", person=person, form=form, page_title = page_title) @@ -155,7 +152,6 @@ def person(id): def add_refimg(): # now save into the DB person = Person.query.get(request.form['person_id']); - print( f"person={person}" ) if not person: raise Exception("could not find person to add reference image too!") f=request.files['refimg_file'] @@ -163,7 +159,6 @@ def add_refimg(): try: # save the actual uploaded image to reference_images/ fname=secure_filename(f.filename) - print( f"fname={fname}" ) if fname == "": raise Exception("invalid filename") From 8b857bf9946e7ef640ed9b60f93af11a38862f1f Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 6 Jul 2021 17:45:15 +1000 Subject: [PATCH 59/67] Fixed: BUG-40: RunAI file counts broken --- BUGs | 3 --- pa_job_manager.py | 47 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/BUGs b/BUGs index 0b3fafb..af06ee4 100644 --- a/BUGs +++ b/BUGs @@ -1,4 +1 @@ ### Next: 44 -BUG-40: RunAI file counts are broken & current state is new when it should be in progress -BUG-41: Ref img had wrong perms and when copied over for docker failed -BUG-43: creating person and selecting the ref img, does not actually keep the ref image & if r.encodings is null, we sail on with the work - should just error out/stop the AI job diff --git a/pa_job_manager.py b/pa_job_manager.py index 3d8b274..b5438ad 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -925,11 +925,11 @@ def JobImportDir(job): FinishJob(job, f"Finished Importing: {path} - Processed {overall_file_cnt} files, Removed {rm_cnt} file(s)") return -def RunFuncOnFilesInPath( job, path, file_func ): +def RunFuncOnFilesInPath( job, path, file_func, count_dirs ): d = session.query(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==path).filter(Dir.rel_path=='').first() files = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==d.eid).all() for e in files: - ProcessFilesInDir(job, e, file_func) + ProcessFilesInDir(job, e, file_func, count_dirs) return @@ -944,7 +944,7 @@ def JobProcessAI(job): for person in people: generateKnownEncodings(person) - RunFuncOnFilesInPath( job, path, ProcessAI ) + RunFuncOnFilesInPath( job, path, ProcessAI, True ) FinishJob(job, "Finished Processesing AI") return @@ -961,8 +961,16 @@ def WrapperForScanFileForPerson(job, entry): AddLogForJob( job, f'INFO: processing File: {entry.name}' ) for person in ppl: ScanFileForPerson( job, entry, person.id, force=False) + # processed this file, add 1 to count + job.current_file_num+=1 return +def AddToJobImageCount(job, entry ): + if entry.type.name == 'Image': + job.num_files += 1 + return + + def JobRunAIOn(job): AddLogForJob(job, f"INFO: Starting looking For faces in files job...") which_person=[jex.value for jex in job.extra if jex.name == "person"][0] @@ -971,21 +979,36 @@ def JobRunAIOn(job): else: ppl=session.query(Person).filter(Person.tag==which_person).all() - # FIXME: probably should be elsewhere, but this is optmised so if refimgs exist for ppl, then it wont regen them -# for person in ppl: -# generateKnownEncodings(person) + # start by working out how many images in this selection we will need face match on + job.num_files = 0 + for jex in job.extra: + if 'eid-' in jex.name: + entry=session.query(Entry).get(jex.value) + if entry.type.name == 'Directory': + # False in last param says, dont count dirs (we won't AI a dir entry itself) + ProcessFilesInDir( job, entry, AddToJobImageCount, False ) + elif entry.type.name == 'Image': + job.num_files += 1 + # update job, so file count UI progress bar will work + # remember that ProcessFilesInDir updates the current_file_num so zero it out so we can start again + job.current_file_num = 0 + print( f"about to commit: {job}" ) + session.commit() for jex in job.extra: if 'eid-' in jex.name: entry=session.query(Entry).get(jex.value) if entry.type.name == 'Directory': - ProcessFilesInDir( job, entry, WrapperForScanFileForPerson ) + # False in last param says, dont count dirs (we won't AI a dir entry itself) + ProcessFilesInDir( job, entry, WrapperForScanFileForPerson, False ) elif entry.type.name == 'Image': which_file=session.query(Entry).join(File).filter(Entry.id==jex.value).first() if DEBUG: AddLogForJob( job, f'INFO: processing File: {entry.name}' ) for person in ppl: ScanFileForPerson( job, which_file, person.id, force=False) + # processed this file, add 1 to count + job.current_file_num+=1 else: AddLogForJob( job, f'Not processing Entry: {entry.name} - not an image' ) FinishJob(job, "Finished Processesing AI") @@ -1087,17 +1110,19 @@ def compareAI(known_encoding, unknown_encoding): return results -def ProcessFilesInDir(job, e, file_func): +def ProcessFilesInDir(job, e, file_func, count_dirs): if DEBUG==1: print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") if e.type.name != 'Directory': file_func(job, e) else: d=session.query(Dir).filter(Dir.eid==e.id).first() - job.current_file_num+=1 + if count_dirs: + print( f"{e.name} about to add 1 to cfn for a dir, {job.current_file_num}" ) + job.current_file_num+=1 files = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==d.eid).all() for sub in files: - ProcessFilesInDir(job, sub, file_func) + ProcessFilesInDir(job, sub, file_func, count_dirs) return def JobGetFileDetails(job): @@ -1110,7 +1135,7 @@ def JobGetFileDetails(job): job.current_file_num = 0 job.num_files = p.num_files session.commit() - RunFuncOnFilesInPath( job, path_prefix, GenHashAndThumb ) + RunFuncOnFilesInPath( job, path_prefix, GenHashAndThumb, True ) FinishJob(job, "File Details job finished") session.commit() return From fbd9e72f285e91a035fc6647662cc33555a151b8 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 7 Jul 2021 17:42:00 +1000 Subject: [PATCH 60/67] updated TODO to remove AI job count issues, and added optimised AI searching code to reduce DB queries on recursion into dirs --- TODO | 16 +++++++--------- pa_job_manager.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/TODO b/TODO index 8c17734..bfe1bf7 100644 --- a/TODO +++ b/TODO @@ -1,20 +1,22 @@ ## GENERAL + * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) + * refimg - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI - store the face locations? (good for debugging later, and we calc them as part of GenFace anyway) ... (really, doing this properly, we shoudl keep the face locns for ALL images)... ---> may well mean in a week or so, we move to the new DB structure, and START FORM SCRATCH) - * need AI job to: - log amount matched, amount comparing too -> count should actually be total files in 'entries' (as we can select random entries to check) - - * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) - * allow rotate of image (permanently on FS, so its right everywhere) * improve photo browser -> view file, rather than just allowing browser to show image + * allow for threshold/settings to be tweaked from the GUI + - it would be good to then say, just run the scanner against this image or maybe this DIR, to see how it IDs ppl + ---> settings for default value + ---> override table to do per file combos? + * fix up logging in general * comment your code * more OO goodness :) @@ -76,10 +78,6 @@ https://www.highcharts.com/demo/heatmap https://www.highcharts.com/demo/packed-bubble-split -### AI - * allow for threshold/settings to be tweaked from the GUI - - it would be good to then say, just run the scanner against this image or maybe this DIR, to see how it IDs ppl - ### SORTER * exif processing? diff --git a/pa_job_manager.py b/pa_job_manager.py index b5438ad..c0d7421 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -951,16 +951,12 @@ def JobProcessAI(job): def WrapperForScanFileForPerson(job, entry): which_person=[jex.value for jex in job.extra if jex.name == "person"][0] - if which_person == "all": - ppl=session.query(Person).all() - else: - ppl=session.query(Person).filter(Person.tag==which_person).all() if entry.type.name == 'Image': if DEBUG: AddLogForJob( job, f'INFO: processing File: {entry.name}' ) - for person in ppl: - ScanFileForPerson( job, entry, person.id, force=False) + for pid in job.ppl: + ScanFileForPerson( job, entry, pid, force=False) # processed this file, add 1 to count job.current_file_num+=1 return @@ -992,9 +988,14 @@ def JobRunAIOn(job): # update job, so file count UI progress bar will work # remember that ProcessFilesInDir updates the current_file_num so zero it out so we can start again job.current_file_num = 0 - print( f"about to commit: {job}" ) session.commit() + ppl_lst=[] + for person in ppl: + ppl_lst.append(person.id) + + job.ppl = ppl_lst + for jex in job.extra: if 'eid-' in jex.name: entry=session.query(Entry).get(jex.value) @@ -1430,7 +1431,6 @@ def UnmatchedFacesForFile( eid ): return rows def ScanFileForPerson( job, e, person_id, force=False ): -# AddLogForJob( job, f'INFO: Looking for person: {person_id} in file: {e.name}' ) file_h = session.query(File).get( e.id ) # if we are forcing this, delete any old faces (this will also delete linked tables), and reset faces_created_on to None if force: From 3dbcf68dadaa56cfc4058a6bb6bfdd5320c3b2a9 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 7 Jul 2021 18:05:24 +1000 Subject: [PATCH 61/67] force session commit in AddLogForJob() if no recent commit - 5 seconds --- TODO | 2 -- pa_job_manager.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/TODO b/TODO index bfe1bf7..ffda7fe 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,5 @@ ## GENERAL - * with any job, count logs, then commit per 100 log lines of a job (and then ditch the commit in import dir for this) - * refimg - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI - store the face locations? (good for debugging later, and we calc them as part of GenFace anyway) ... diff --git a/pa_job_manager.py b/pa_job_manager.py index c0d7421..751d01b 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -409,7 +409,16 @@ def AddLogForJob(job, message): session.add(log) # some logs have DEBUG: in front, so clean that up message = message.replace("DEBUG:", "" ) - print( f"DEBUG: {message}" ) + # if its been more than 5 seconds since our last log, then commit to the DB to show some progress + if hasattr(job, 'last_commit'): + if (now - job.last_commit).seconds > 5: + job.last_commmit=now + print( "DELME: we have taken longer than 5 seconds since last commit so do it") + session.commit() + else: + job.last_commit = now + if DEBUG: + print( f"DEBUG: {message}" ) return def RunJob(job): @@ -1119,7 +1128,6 @@ def ProcessFilesInDir(job, e, file_func, count_dirs): else: d=session.query(Dir).filter(Dir.eid==e.id).first() if count_dirs: - print( f"{e.name} about to add 1 to cfn for a dir, {job.current_file_num}" ) job.current_file_num+=1 files = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==d.eid).all() for sub in files: From cd6a42088d19ca7a89948e89ff24fe17908dec02 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:29:38 +1000 Subject: [PATCH 62/67] remove old refimg.encodings generation, we now do this when we make a person/refimg, so its always done before an AI job needs to use it --- pa_job_manager.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index 751d01b..9c68b11 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -216,13 +216,15 @@ class Refimg(Base): __tablename__ = "refimg" id = Column(Integer, Sequence('refimg_id_seq'), primary_key=True ) fname = Column(String(256), unique=True, nullable=False) - face = Column(LargeBinary) + face = Column(LargeBinary, unique=True, nullable=False) thumbnail = Column(String, unique=False, nullable=True) - encodings = Column(LargeBinary) created_on = Column(Float) + orig_w = Column(Integer) + orig_h = Column(Integer) + face_locn = Column(String) def __repr__(self): - return f"" + return f"" class Face(Base): __tablename__ = "face" @@ -949,10 +951,6 @@ def JobProcessAI(job): p = session.query(Path).filter(Path.path_prefix==path).first() job.num_files=p.num_files - people = session.query(Person).all() - for person in people: - generateKnownEncodings(person) - RunFuncOnFilesInPath( job, path, ProcessAI, True ) FinishJob(job, "Finished Processesing AI") @@ -1099,22 +1097,6 @@ def generateUnknownEncodings(im): return unknown_encodings -def generateKnownEncodings(person): - for refimg in person.refimg: - file = 'reference_images/'+refimg.fname - stat = os.stat(file) - if refimg.created_on and stat.st_ctime < refimg.created_on: - print("OPTIM: skipping re-creating encoding for refimg because file has not changed") - continue - img = face_recognition.load_image_file(file) - location = face_recognition.face_locations(img) - encodings = face_recognition.face_encodings(img, known_face_locations=location) - print(f"INFO: created encoding for refimg of {file}") - refimg.encodings = encodings[0].tobytes() - refimg.created_on = time.time() - session.add(refimg) - session.commit() - def compareAI(known_encoding, unknown_encoding): results = face_recognition.compare_faces([known_encoding], unknown_encoding, tolerance=0.55) return results From 32209d9ba649f961751a228aad016306d81e3b0a Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:30:01 +1000 Subject: [PATCH 63/67] added tiny thumbnails to person list --- templates/persons.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/templates/persons.html b/templates/persons.html index e68328e..d47c11a 100644 --- a/templates/persons.html +++ b/templates/persons.html @@ -4,12 +4,18 @@

Show All People

- + {% for person in persons %} - + + + {% endfor %}
TagFirstname(s)Surname
TagFirstname(s)Surname
{{person.tag}}{{person.firstname}}{{person.surname}}
{{person.surname}} + {% for refimg in person.refimg %} + + {% endfor %} +
From 955df598eee6701c629a747bbc514a22765254c2 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:30:57 +1000 Subject: [PATCH 64/67] remove encodings, add face, orig_w, orig_h, face_locn to refimg table AND remove DEV refimg data, it needs to be done in the UI now --- tables.sql | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tables.sql b/tables.sql index 4df3903..148c9dd 100644 --- a/tables.sql +++ b/tables.sql @@ -40,9 +40,7 @@ create table ENTRY_DIR_LINK ( entry_id integer, dir_eid integer, create table PERSON ( ID integer, TAG varchar(48), FIRSTNAME varchar(48), SURNAME varchar(48), constraint PK_PERSON_ID primary key(ID) ); -create table REFIMG ( ID integer, FNAME varchar(256), FACE bytea, - ENCODINGS bytea, - CREATED_ON float, THUMBNAIL varchar, +create table REFIMG ( ID integer, FNAME varchar(128), FACE bytea, ORIG_W integer, ORIG_H integer, FACE_LOCN varchar(32), CREATED_ON float, THUMBNAIL varchar, constraint PK_REFIMG_ID primary key(ID) ); create table FACE( ID integer, FACE bytea, constraint PK_FACE_ID primary key(ID) ); @@ -110,14 +108,6 @@ insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'dad', 'Damien', insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mum', 'Mandy', 'De Paoli' ); insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' ); insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' ); -insert into REFIMG values ( (select nextval('REFIMG_ID_SEQ')), 'dad.jpg'); -insert into REFIMG values ( (select nextval('REFIMG_ID_SEQ')), 'mum.jpg'); -insert into REFIMG values ( (select nextval('REFIMG_ID_SEQ')), 'cam.jpg'); -insert into REFIMG values ( (select nextval('REFIMG_ID_SEQ')), 'mich.jpg'); -insert into PERSON_REFIMG_LINK values ( 1, 1 ); -insert into PERSON_REFIMG_LINK values ( 2, 2 ); -insert into PERSON_REFIMG_LINK values ( 3, 3 ); -insert into PERSON_REFIMG_LINK values ( 4, 4 ); -- DEV: insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', '/home/ddp/src/photoassistant/storage/#c:/Users/cam/Desktop/code/python/photoassistant/storage/', '/home/ddp/src/photoassistant/.pa_bin/#c:/Users/cam/Desktop/code/python/photoassistant/.pa_bin/' ); -- PROD: From e622774c0f56cb17c4f256e6d8895e12800f76bb Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:31:37 +1000 Subject: [PATCH 65/67] made GenThumb return thumb and orig_w, orig_h - not sure this stays, but for optimising it prob. will -- its not being used in pa_job_mgr yet, and that will see the final version of this --- shared.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shared.py b/shared.py index d3499f3..0e36a04 100644 --- a/shared.py +++ b/shared.py @@ -88,22 +88,23 @@ def GenThumb(fname): bands = im.getbands() if 'A' in bands: im = im.convert('RGB') + orig_w, orig_h = im.size im.thumbnail((THUMBSIZE,THUMBSIZE)) img_bytearray = io.BytesIO() im.save(img_bytearray, format='JPEG') img_bytearray = img_bytearray.getvalue() thumbnail = base64.b64encode(img_bytearray) thumbnail = str(thumbnail)[2:-1] - return thumbnail + return thumbnail, orig_w, orig_h except Exception as e: print( f"GenThumb failed: {e}") - return None + return None, None, None def GenFace(fname): img = face_recognition.load_image_file(fname) location = face_recognition.face_locations(img) encodings = face_recognition.face_encodings(img, known_face_locations=location) if len(encodings): - return encodings[0].tobytes() + return encodings[0].tobytes(), location else: - return None + return None, None From ce81898a0716b194f0fb403be56e6cfda997239e Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:32:03 +1000 Subject: [PATCH 66/67] another useful sql for finding usable list of filenames that equate to a search --- README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README b/README index e4c4480..a4acbfe 100644 --- a/README +++ b/README @@ -108,3 +108,8 @@ sudo docker exec -it padb bash sudo docker exec -it padb bash psql --user=pa pa select count(distinct e.id) from entry e, entry_dir_link edl where e.type_id = 1 and e.id = edl.entry_id and edl.dir_eid in ( select distinct dir_eid from path_dir_link where path_id = 2 ); + + +# get abs filenames of matching files (for liz person.tag, but could easily add +# d.rel_path like 'liz' too : +select '"'||replace(replace(p.path_prefix,'static/Storage/',''),'static/Import/', '')||'/'||d.rel_path||'/'||e.name||'"' from entry e, entry_dir_link edl, path_dir_link pdl, path p, dir d where e.id = edl.entry_id and edl.dir_eid = pdl.dir_eid and pdl.path_id = p.id and d.eid = edl.dir_eid and e.id in ( select e.id from entry e, face_file_link ffl, face_refimg_link frl, person_refimg_link prl, person p where e.id = ffl.file_eid and ffl.face_id = frl.face_id and frl.refimg_id = prl.refimg_id and prl.person_id = p.id and p.tag = 'liz' ); From f394e39c2fec912167e83bc33cbb366cd442293e Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sun, 11 Jul 2021 22:35:59 +1000 Subject: [PATCH 67/67] refimgs now contain face, orig_w, orig_h and face_locns. This is done via json.* to allow arrays to be saved/loaded back into face_locn - not useful for refimg as there has to be only 1, but tested for images where there will be many faces. This commit has a fair few changes. So overall, no more refimg menus/creation. You now create a person (the add button is hidden until you save), when you save you go back to the person you created rather than the list of persons. From there you can click add ref img, and it will create a thumbnail, and draw a green box around the face locations based on the data. Persons can have many refimgs, and they will all work the same, be formatted prettily no matter how many you have. Each refimg "tab" not only has the thumbnail, but also a red X click to delete button that will remove all refimg data and connection to the person table too. This all works/is tested. --- TODO | 22 +++--- person.py | 15 +++- templates/person.html | 165 +++++++++++++++++++++++++++--------------- 3 files changed, 131 insertions(+), 71 deletions(-) diff --git a/TODO b/TODO index ffda7fe..10e373e 100644 --- a/TODO +++ b/TODO @@ -1,20 +1,21 @@ ## GENERAL - * refimg - - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI - - store the face locations? (good for debugging later, and we calc them as part of GenFace anyway) ... - (really, doing this properly, we shoudl keep the face locns for ALL images)... - ---> may well mean in a week or so, we move to the new DB structure, and START FORM SCRATCH) - * allow rotate of image (permanently on FS, so its right everywhere) * improve photo browser -> view file, rather than just allowing browser to show image + * face locations: + START FORM SCRATCH so all images have face_locn data + right now GenThumb is in shared, and does width, height as well --> in person.py BUT need this for pa_job_manager + * allow for threshold/settings to be tweaked from the GUI - it would be good to then say, just run the scanner against this image or maybe this DIR, to see how it IDs ppl ---> settings for default value ---> override table to do per file combos? + * refimg + - remove AI menu from top-level -> make a sub-of Person, and just have Match or AI + * fix up logging in general * comment your code * more OO goodness :) @@ -33,9 +34,6 @@ *** Need to use thread-safe sessions per Thread, half-assed version did not work - - would it be quicker/smarter to use md5 hash matching on import (and if - so, not re-do face* ) ??? - need a manual button to restart a job in the GUI, (based on file-level optims, just run the job as new and it will optim over already done parts and continue) @@ -67,7 +65,6 @@ need to copy into here the jquery/fa files so we don't need internet to function - for that matter run lightspeed against all this - timelineview? (I think maybe sunburst for large amounts of files, then maybe something more timeline-series for drilling in?) (vertical timeline, date has thumbnails (small) horizontally along a page, etc.? @@ -81,3 +78,8 @@ * exif processing? * location stuff - test a new photo from my camera out -- image is in dir, need to look at exifread output + +### FUTURE: + * can emby use nfo for images (for AI/tags?) + -NO sadly + diff --git a/person.py b/person.py index 4bf7d16..369d1d2 100644 --- a/person.py +++ b/person.py @@ -10,6 +10,7 @@ from werkzeug import secure_filename from shared import GenFace, GenThumb from face import FaceRefimgLink import os +import json # pylint: disable=no-member @@ -21,6 +22,9 @@ class Refimg(db.Model): id = db.Column(db.Integer, db.Sequence('refimg_id_seq'), primary_key=True ) fname = db.Column(db.String(256), unique=True, nullable=False) face = db.Column(db.LargeBinary, unique=True, nullable=False) + orig_w = db.Column(db.Integer) + orig_h = db.Column(db.Integer) + face_locn = db.Column(db.String) thumbnail = db.Column(db.String, unique=True, nullable=False) created_on = db.Column(db.Float) @@ -41,7 +45,7 @@ class Person(db.Model): tag = db.Column(db.String(48), unique=False, nullable=False) surname = db.Column(db.String(48), unique=False, nullable=False) firstname = db.Column(db.String(48), unique=False, nullable=False) - refimg = db.relationship('Refimg', secondary=PersonRefimgLink.__table__) + refimg = db.relationship('Refimg', secondary=PersonRefimgLink.__table__, order_by=Refimg.id) def __repr__(self): return "".format(self.tag,self.firstname, self.surname, self.refimg) @@ -94,7 +98,7 @@ def new_person(): db.session.add(person) db.session.commit() st.SetMessage( "Created new Person ({})".format(person.tag) ) - return redirect( '/persons' ) + return redirect( f'/person/{person.id}' ) except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to add Person: {}".format(e.orig) ) @@ -141,6 +145,8 @@ def person(id): return render_template("person.html", form=form, page_title=page_title) else: person = Person.query.get(id) + for r in person.refimg: + r.face_locn=json.loads(r.face_locn) form = PersonForm(request.values, obj=person) return render_template("person.html", person=person, form=form, page_title = page_title) @@ -164,8 +170,9 @@ def add_refimg(): fname = f"/tmp/{fname}" f.save( fname ) - refimg.thumbnail = GenThumb( fname ) - refimg.face = GenFace( fname ) + refimg.thumbnail, refimg.orig_w, refimg.orig_h = GenThumb( fname ) + refimg.face, face_locn = GenFace( fname ) + refimg.face_locn = json.dumps(face_locn) os.remove(fname) person.refimg.append(refimg) db.session.add(person) diff --git a/templates/person.html b/templates/person.html index a46ec94..827f052 100644 --- a/templates/person.html +++ b/templates/person.html @@ -1,67 +1,118 @@ {% extends "base.html" %} {% block main_content %} -
-

{{page_title}}

- - {% for field in form %} - {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} - {{field}}
- {% elif field.type != 'SubmitField' %} -
- {{ field.label( class="col-lg-3" ) }} - {{ field( class="form-control col" ) }} -
- {% endif %} - {% endfor %} -
-
Reference Images:
 - {% for ref_img in person.refimg %} - {% set offset="" %} - {% if (loop.index % 10) == 0 %} - {% set offset= "offset-lg-3" %} - {% endif %} -
- -
-
-
-
- -
-
{{ref_img.fname}}
-
-
-
- {% endfor %} -
-
-
-
-
- {{ form.save( id="save", class="btn btn-primary offset-lg-3 col-lg-2" )}} - {% if 'Edit' in page_title %} - {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} - {% endif %} -
- -
- - -
-
-
+ + +
+

{{page_title}}

+
+ {% for field in form %} + {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} + {{field}}
+ {% elif field.type != 'SubmitField' %} +
+ {{ field.label( class="col-lg-3" ) }} + {{ field( class="form-control col" ) }} +
+ {% endif %} + {% endfor %} +
+
Reference Images:
 + {% for refimg in person.refimg %} + {% set offset="" %} + {% if (loop.index % 10) == 0 %} + {% set offset= "offset-lg-3" %} + {% endif %} +
+
+ +
+
+ + +
+ +
+
{{refimg.fname}}
+
+
+
+
+ {% endfor %} +
+
+
+
+
+ {{ form.save( id="save", class="btn btn-primary offset-lg-3 col-lg-2" )}} + {% if 'Edit' in page_title %} + {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} + {% endif %} +
+
+ {% if person.id %} +
+ + +
+ {% endif %} +
{% endblock main_content %} {% block script_content %} {% endblock script_content %} -