diff --git a/pa_job_manager.py b/pa_job_manager.py index a55e997..af8470d 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -954,10 +954,8 @@ def WrapperForScanFileForPerson(job, entry): ppl=session.query(Person).filter(Person.tag==which_person).all() if entry.type.name == 'Image': - print( f"JobRunAIOn: file to process had id: {entry.id}" ) for person in ppl: - print( f"call == ScanFileForPerson( {entry.id}, {person.id}, force=False )" ) - ScanFileForPerson( entry, person.id, force=False) + ScanFileForPerson( job, entry, person.id, force=False) return def JobRunAIOn(job): @@ -967,26 +965,22 @@ def JobRunAIOn(job): ppl=session.query(Person).all() else: ppl=session.query(Person).filter(Person.tag==which_person).all() - print( "JobRunAIOn() called" ) + + # FIXME: probably shouldbe elsewhere, but this is optmised so if refimgs exist for ppl, then it wont regen them for person in ppl: - print( f"person={person.tag}" ) + generateKnownEncodings(person) for jex in job.extra: if 'eid-' in jex.name: entry=session.query(Entry).get(jex.value) - print( f'en={entry.name}, {entry.type.name}' ) if entry.type.name == 'Directory': ProcessFilesInDir( job, entry, WrapperForScanFileForPerson ) elif entry.type.name == 'Image': which_file=session.query(Entry).join(File).filter(Entry.id==jex.value).first() - print( f"JobRunAIOn: file to process had id: {which_file.id}" ) for person in ppl: - print( f"call == ScanFileForPerson( {which_file.id}, {person.id}, force=False )" ) - ScanFileForPerson( which_file, person.id, force=False) + ScanFileForPerson( job, which_file, person.id, force=False) else: AddLogForJob( job, f'Not processing Entry: {entry.name} - not an image' ) - #print(" HARD EXITING to keep testing " ) - #exit( -1 ) FinishJob(job, "Finished Processesing AI") return @@ -1093,6 +1087,7 @@ def generateKnownEncodings(person): img = face_recognition.load_image_file(file) location = face_recognition.face_locations(img) encodings = face_recognition.face_encodings(img, known_face_locations=location) + print(f"INFO: created encoding for refimg of {file}") refimg.encodings = encodings[0].tobytes() refimg.created_on = time.time() session.add(refimg) @@ -1105,8 +1100,7 @@ def compareAI(known_encoding, unknown_encoding): def ProcessFilesInDir(job, e, file_func): if DEBUG==1: - print( f"???? e={e}" ) - # print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") + print( f"DEBUG: ProcessFilesInDir: {e.FullPathOnFS()}") if e.type.name != 'Directory': file_func(job, e) else: @@ -1324,7 +1318,7 @@ def JobDeleteFiles(job): if 'eid-' in jex.name: del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() MoveFileToRecycleBin(job,del_me) - now=datetime.now(pytz.utc) + ynw=datetime.now(pytz.utc) next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) session.add(next_job) MessageToFE( job.id, "success", "Completed (delete of selected files)" ) @@ -1365,6 +1359,10 @@ def InitialValidationChecks(): break if not rbp_exists: AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); + else: + bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first() + if not bin_path: + ProcessRecycleBinDir(job) sp_exists=0 paths = settings.storage_path.split("#") for path in paths: @@ -1389,10 +1387,8 @@ def InitialValidationChecks(): FinishJob(job,"Finished Initial Validation Checks") return -#### CAM: New FACES/AI code - def AddFaceToFile( face_data, file_eid ): - face = Face( face=face_data ) + face = Face( face=face_data.tobytes() ) session.add(face) session.commit() ffl = FaceFileLink( face_id=face.id, file_eid=file_eid ) @@ -1412,46 +1408,50 @@ def MatchRefimgToFace( refimg_id, face_id ): return def UnmatchedFacesForFile( eid ): - rows = session.execute( f"select f.id, ffl.file_eid, frl.refimg_id from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where ffl.file_eid = {eid} and frl.refimg_id is null" ) + rows = session.execute( f"select f.* from face f left join face_refimg_link frl on f.id = frl.face_id join face_file_link ffl on f.id = ffl.face_id where ffl.file_eid = {eid} and frl.refimg_id is null" ) return rows -### CAM: something like this -- HAVE NOT TRIED THIS IT WILL FAIL### -def ScanFileForPerson( e, person_id, force=False ): +def ScanFileForPerson( job, e, person_id, force=False ): + AddLogForJob( job, f'INFO: Looking for person: {person_id} in file: {e.name}' ) file_h = session.query(File).get( e.id ) # if we are forcing this, delete any old faces (this will also delete linked tables), and reset faces_created_on to None if force: + AddLogForJob( job, f'INFO: force is true, so deleting old face information for {e.name}' ) DelFacesForFile( e.id ) - file_h.faces_create_on = None + file_h.faces_created_on = 0 # optimise: dont rescan if we already have faces (we are just going to try # to match (maybe?) a refimg - if not file_h.faces_created_on: - print("------------- CAM -----------:\n") + if file_h.faces_created_on == 0: + if DEBUG: + AddLogForJob( job, f"DEBUG: {e.name} is missing unknown faces, generating them" ) im = face_recognition.load_image_file(e.FullPathOnFS()) - print(im) face_locations = face_recognition.face_locations(im) - print(f"FACE LOCATIONS: {face_locations}") unknown_encodings = face_recognition.face_encodings(im, known_face_locations=face_locations) - print("AAAAAAAAAAAAAAAA " + str(len(unknown_encodings))) for face in unknown_encodings: - new_face = Face( face_data = face ) - session.add(new_face) - session.commit() - AddFaceToFile( new_face.id, e.id ) - now=datetime.now(pytz.utc) - file_h.face_created_on = now + AddFaceToFile( face, e.id ) + file_h.faces_created_on = time.time() + session.commit() ## now look for person refimgs = session.query(Refimg).join(PersonRefimgLink).filter(PersonRefimgLink.person_id==person_id).all() uf = UnmatchedFacesForFile( e.id ) + if DEBUG and not uf: + AddLogForJob( job, "DEBUG: {e.name} all faces already matched - finished" ) + for face in uf: for r in refimgs: - match = compareAI(r, uf) - if match: - print(f'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA WE MATCHED: {r}, {uf}') + unknown_face_data = numpy.frombuffer(face.face, dtype=numpy.float64) + refimg_face_data = numpy.frombuffer(r.encodings, dtype=numpy.float64) + match = compareAI(refimg_face_data, unknown_face_data) + if match[0]: + AddLogForJob(job, f'WE MATCHED: {r.fname} with file: {e.name} ') MatchRefimgToFace( r.id, face.id ) + # no need to keep looking for this face, we found it, go to next unknown face + break return + if __name__ == "__main__": print("INFO: PA job manager starting - listening on {}:{}".format( PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT) )