big change to get metadata working fully in DB and on Filesystem, and recover from most common scenarios, improved GUI as well for allowing an immediate search after adding refimg as well

This commit is contained in:
2022-08-01 23:44:38 +10:00
parent 391b61f3c4
commit a8af00fe66
13 changed files with 523 additions and 92 deletions

58
TODO
View File

@@ -1,14 +1,47 @@
## GENERAL
* need force scan on a file as an option in GUI (to test below)
## MIGRATION/NEXT Production build:
drop table FACE_NO_MATCH_OVERRIDE;
drop table FACE_FORCE_MATCH_OVERRIDE;
drop table DISCONNECTED_NO_MATCH_OVERRIDE;
drop table DISCONNECTED_FORCE_MATCH_OVERRIDE;
* keep overrides across 'deletes/rescans'
[DONE] - when we delete/force a full scan then move overrides into disconnected* tables
[PARTIAL] - when an individual face is deleted - need to keep any associated override
code is [DONE]
TEST! (no way to force a delete via gui as yet)
- when we scan a new face, we need to see if there is a matching override, if so, add override back & delete disc*
- TEST (add an override, delete refimg, re-add refimg & re-scan)
- TEST (forcescan job)
create table FACE_NO_MATCH_OVERRIDE ( ID integer, FACE_ID integer, TYPE_ID integer,
constraint FK_FNMO_FACE_ID foreign key (FACE_ID) references FACE(ID),
constraint FK_FNMO_TYPE foreign key (TYPE_ID) references FACE_OVERRIDE_TYPE(ID),
constraint PK_FNMO_ID primary key(ID) );
create table FACE_FORCE_MATCH_OVERRIDE ( ID integer, FACE_ID integer, PERSON_ID integer, constraint PK_FACE_FORCE_MATCH_OVERRIDE_ID primary key(ID) );
create table DISCONNECTED_NO_MATCH_OVERRIDE ( FACE bytea, TYPE_ID integer,
constraint FK_DNMO_TYPE_ID foreign key (TYPE_ID) references FACE_OVERRIDE_TYPE(ID),
constraint PK_DNMO_FACE primary key (FACE) );
create table DISCONNECTED_FORCE_MATCH_OVERRIDE ( FACE bytea, PERSON_ID integer,
constraint FK_DFMO_PERSON_ID foreign key (PERSON_ID) references PERSON(ID),
constraint PK_DFMO_FACE primary key (FACE) );
drop table SETTINGS
create table SETTINGS(
ID integer,
BASE_PATH varchar, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, METADATA_PATH varchar,
AUTO_ROTATE Boolean,
DEFAULT_REFIMG_MODEL integer, DEFAULT_SCAN_MODEL integer, DEFAULT_THRESHOLD float,
FACE_SIZE_LIMIT integer,
SCHEDULED_IMPORT_SCAN integer, SCHEDULED_STORAGE_SCAN integer,
SCHEDULED_BIN_CLEANUP integer, BIN_CLEANUP_FILE_AGE integer,
JOB_ARCHIVE_AGE integer,
constraint PK_SETTINGS_ID primary key(ID),
constraint FK_DEFAULT_REFIMG_MODEL foreign key (DEFAULT_REFIMG_MODEL) references AI_MODEL(ID),
constraint FK_DEFAULT_SCAN_MODEL foreign key (DEFAULT_SCAN_MODEL) references AI_MODEL(ID) );
insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, metadata_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', '.pa_metadata/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 4 );
## then docker-compose build...
## GENERAL
* put try: around any os.remove, etc.
* remove Paths from SettingsIPath, etc.
* should I change the rotation code to use that jpeg util to reduce/remove compression loss?
@@ -72,6 +105,8 @@
* think about security - in job_mgr anywhere I can os.replace/remove NEED to protect, etc
* real first-run, 'no or empty settings' -- need to think this through
## DB
* Dir can have date in the DB, so we can do Oldest/Newest dirs in Folder view
@@ -86,8 +121,7 @@
(only show in DEV for now)
### AI
* faces per file (need a threshold for too many? OR
* consider size of bbox of face / 'high-quality' faces -- if face is too small in image, dont match it
* faces per file - need a threshold for too many?
### UI
* viewer needs to allow toggle to scan_model (and prob. right-click on file... AI (with CNN) AI (with hog)