first pass at keeping overrides on face delete - should at least stop code crashing if you try to delete faces from DB and they still had a matching override

2022-07-28 21:23:34 +10:00
parent 96810fa1e3
commit 391b61f3c4
5 changed files with 135 additions and 16 deletions
--- a/20
+++ b/20
@@ -1,14 +1,14 @@
 ## GENERAL
-    * on viewer: 
-        - allow face to be used to:
-            [DONE] - create person
-            [DONE] - add to existing person
-            [DONE]   --> can choose to do AI search (for these 2 options)
-            [DONE] - ignore/not a face/too young 
-            [DONE] - redraw 'ignore's as a greyed out box?
-            [DONE] - menu should only allow override IF we have put override on...
-            all NMO's need to handle delete data and rebuild / allow recreation of content form FS (not just test, it causes a bug now / db constraint violation)
-                --> need to test the 'override' when we re-ai-match (AFTER re-build from FS)
+	* need force scan on a file as an option in GUI (to test below)
+
+	* keep overrides across 'deletes/rescans'
+		[DONE] - when we delete/force a full scan then move overrides into disconnected* tables
+		[PARTIAL] - when an individual face is deleted - need to keep any associated override
+				code is [DONE]
+				TEST! (no way to force a delete via gui as yet)
+		- when we scan a new face, we need to see if there is a matching override, if so, add override back & delete disc*
+		- TEST (add an override, delete refimg, re-add refimg & re-scan)
+		- TEST (forcescan job)

    * should I change the rotation code to use that jpeg util to reduce/remove compression loss?

--- a/face.py
+++ b/face.py
@@ -79,7 +79,6 @@ class FaceNoMatchOverride(db.Model):
    face_id = db.Column(db.Integer, db.ForeignKey("face.id"), primary_key=True )
    type_id = db.Column(db.Integer, db.ForeignKey("face_override_type.id"))
    type = db.relationship("FaceOverrideType")
-    face = db.Column( db.LargeBinary )

    def __repr__(self):
        return f"<id: {self.id}, face_id={self.face_id}, type: {self.type}>"
@@ -89,7 +88,6 @@ class FaceManualOverride(db.Model):
    __tablename__ = "face_manual_override"
    id = db.Column(db.Integer, db.Sequence('face_override_id_seq'), primary_key=True )
    face_id = db.Column(db.Integer, db.ForeignKey("face.id"), primary_key=True )
-    face = db.Column( db.LargeBinary )
    person_id = db.Column(db.Integer, db.ForeignKey("person.id"), primary_key=True )
    person = db.relationship("Person")

--- a/pa_job_manager.py
+++ b/pa_job_manager.py
@@ -1,4 +1,4 @@
-###
+#
 #
 # This file controls the 'external' job control manager, that (periodically #
 # looks / somehow is pushed an event?) picks up new jobs, and processes them.
@@ -360,6 +360,79 @@ class FaceRefimgLink(Base):
    def __repr__(self):
        return f"<face_id: {self.face_id}, refimg_id={self.refimg_id}"

+
+################################################################################
+# Class describing types of non-match overrides for faces
+################################################################################
+class FaceOverrideType(Base):
+    __tablename__ = "face_override_type"
+    id = Column(Integer, Sequence('face_override_type_id_seq'), primary_key=True )
+    name = Column( String )
+
+    def __repr__(self):
+        return f"<id: {self.id}, name={self.name}>"
+
+################################################################################
+# Class containing which faces are not ever allowed to match (and the
+# type/reason why)
+################################################################################
+class FaceNoMatchOverride(Base):
+    __tablename__ = "face_no_match_override"
+    id = Column(Integer, Sequence('face_override_id_seq'), primary_key=True )
+    face_id = Column(Integer, ForeignKey("face.id"), primary_key=True )
+    type_id = Column(Integer, ForeignKey("face_override_type.id"))
+    type = relationship("FaceOverrideType")
+
+    def __repr__(self):
+        return f"<id: {self.id}, face_id={self.face_id}, type: {self.type}>"
+
+
+################################################################################
+# Class containing a manual / forced match of a face in a file to a person
+################################################################################
+class FaceManualOverride(Base):
+    __tablename__ = "face_manual_override"
+    id = Column(Integer, Sequence('face_override_id_seq'), primary_key=True )
+    face_id = Column(Integer, ForeignKey("face.id"), primary_key=True )
+    person_id = Column(Integer, ForeignKey("person.id"), primary_key=True )
+    person = relationship("Person")
+
+    def __repr__(self):
+        return f"<id: {self.id}, face_id={self.face_id}, person_id={self.person_id}>"
+
+
+################################################################################
+# Class describing DisconnectedNoMatchOverride in the database and DB via 
+# sqlalchemy - Used when a face with an override is deleted from the DB to keep
+# the raw data so that we can reconnect the override if we ever scan that same
+# file/face again (think delete/undelete file, rebuild DB from file sys/from
+# scratch, etc)
+# used specifically for a face that should not ever be a match
+################################################################################
+class DisconnectedNoMatchOverride(Base):
+    __tablename__ = "disconnected_no_match_override"
+    face = Column( LargeBinary, primary_key=True )
+    type_id = Column(Integer, ForeignKey("face_override_type.id"))
+
+    def __repr__(self):
+        return f"<face: {self.face}, type_id={self.type_id}"
+
+################################################################################
+# Class describing DisconnectedManualOverride in the database and DB via 
+# sqlalchemy - Used when a face with an override is deleted from the DB to keep
+# the raw data so that we can reconnect the override if we ever scan that same
+# file/face again (think delete/undelete file, rebuild DB from file sys/from
+# scratch, etc)
+# used specifically for a match that was forced between a face and a person
+################################################################################
+class DisconnectedManualOverride(Base):
+    __tablename__ = "disconnected_manual_override"
+    face = Column( LargeBinary, primary_key=True )
+    person_id = Column(Integer, ForeignKey('person.id'))
+
+    def __repr__(self):
+        return f"<face: {self.face}, person_id={self.person_id}"
+
 ################################################################################
 # Class describing logs for each job and via sqlalchemy, connected to the DB as well
 ################################################################################
@@ -828,12 +901,36 @@ def JobScanStorageDir(job):
    MessageToFE( job.id, "success", "Completed (scan for new files)" )
    return

+##############################################################################
+# All face Overrides should not just be deleted, they should be disconnected
+# from the file (in face_file_link), but instead keep the raw face data so
+# that a face that is found in a future scan can still keep the override
+# connection
+##############################################################################
+def DisconnectAllOverrides():
+    overrides=session.query(FaceNoMatchOverride).all()
+    for o in overrides:
+        f=session.query(Face).get(o.face_id)
+        session.add( DisconnectedNoMatchOverride( face=f.face, type_id=o.type_id ) )
+    overrides=session.query(FaceNoMatchOverride).delete()
+
+    overrides=session.query(FaceManualOverride).all()
+    for o in overrides:
+        f=session.query(Face).get(o.face_id)
+        session.add( DisconnectedManualOverride( face=f.face, person_id=o.person_id ) )
+    overrides=session.query(FaceManualOverride).delete()
+    session.commit()
+
+    return
+       
+
 ##############################################################################
 # JobForceScan(): start and process the job to force scanning now - so delete
 # all attached data in DB, then scan import and storage paths
 ##############################################################################
 def JobForceScan(job):
    JobProgressState( job, "In Progress" )
+    DisconnectAllOverrides()
    session.query(PA_UserState).delete()
    session.query(FaceFileLink).delete()
    session.query(FaceRefimgLink).delete()
@@ -1984,7 +2081,23 @@ def DelMatchesForFile( job, ent ):
 # DelFacesForFile(): quick func to delete any faces associated with the specified file
 ####################################################################################################################################
 def DelFacesForFile( eid ):
+    ffl=session.query(FaceFileLink).filter(FaceFileLink.file_eid==eid).all()
+
+    for link in ffl:
+        # find any manaul overrides on this face (before we delete it, and put them into the disc* table)
+        o=session.query(FaceManualOverride).filter(FaceManualOverride.face_id==link.face_id).one()
+        if o:
+            f=session.query(Face).get(link.face_id)
+            session.add( DisconnectedManualOverride( face=f.face, person_id=o.person_id ) )
+
+        # find any no-match overrides on this face (before we delete it, and put them into the disc* table)
+        o=session.query(FaceNoMatchOverride).filter(FaceNoMatchOverride.face_id==link.face_id).one()
+        if o:
+            f=session.query(Face).get(link.face_id)
+            session.add( DisconnectedNoMatchOverride( face=f.face, type_id=o.type_id ) )
+
    session.execute( f"delete from face where id in (select face_id from face_file_link where file_eid = {eid})" )
+
    session.commit()
    return

--- a/person.py
+++ b/person.py
@@ -348,7 +348,7 @@ def override_force_match():
    if not f:
        raise Exception("could not find face to add override for!")

-    mo = FaceManualOverride( face_id=f.id, face=f.face, person_id=p.id )
+    mo = FaceManualOverride( face_id=f.id, person_id=p.id )
    db.session.add( mo )
    db.session.commit()

@@ -410,7 +410,7 @@ def add_no_match_override():
    if not t:
        raise Exception("could not find override_type to add override for!")

-    nmo = FaceNoMatchOverride( face_id=f.id, face=f.face, type_id=t.id )
+    nmo = FaceNoMatchOverride( face_id=f.id, type_id=t.id )
    db.session.add( nmo )
    db.session.commit()

--- a/tables.sql
+++ b/tables.sql
@@ -129,7 +129,15 @@ create table FACE_NO_MATCH_OVERRIDE ( ID integer, FACE_ID integer, TYPE_ID integ
    constraint PK_FNMO_ID primary key(ID) );

 -- manual match goes to person not refimg, so on search, etc. we deal with this anomaly (via sql not ORM)
-create table FACE_MANUAL_OVERRIDE ( ID integer, FACE_ID integer, PERSON_ID integer, FACE bytea, constraint PK_FACE_MANUAL_OVERRIDE_ID primary key(ID) );
+create table FACE_MANUAL_OVERRIDE ( ID integer, FACE_ID integer, PERSON_ID integer, constraint PK_FACE_MANUAL_OVERRIDE_ID primary key(ID) );
+
+create table DISCONNECTED_NO_MATCH_OVERRIDE ( FACE bytea, TYPE_ID integer, 
+	constraint FK_DNMO_TYPE_ID foreign key (TYPE_ID) references FACE_OVERRIDE_TYPE(ID),
+	constraint PK_DNMO_FACE primary key (FACE) );
+
+create table DISCONNECTED_MANUAL_OVERRIDE ( FACE bytea, PERSON_ID integer, 
+	constraint FK_DMO_PERSON_ID foreign key (PERSON_ID) references PERSON(ID),
+	constraint PK_DMO_FACE primary key (FACE) );

 create table PERSON_REFIMG_LINK ( PERSON_ID integer, REFIMG_ID integer,
    constraint PK_PRL primary key(PERSON_ID, REFIMG_ID),