diff --git a/BUGs b/BUGs index eea6205..f4baa85 100644 --- a/BUGs +++ b/BUGs @@ -1,4 +1 @@ ### Next: 100 -BUG-97: if restart a job, we reset start time too (debatable) - - prob. better making a job have anull start time rather than now() as default, then if its null set it when we run the job - (this would set it at actual run time not create time, and if we restart, it would leave orig start time correct) diff --git a/job.py b/job.py index ca292d8..884be27 100644 --- a/job.py +++ b/job.py @@ -272,4 +272,7 @@ def joblog_search(): ################################################################################ @app.template_filter('vicdate') def _jinja2_filter_datetime(date, fmt=None): - return date.strftime("%d/%m/%Y %I:%M:%S %p") + if date: + return date.strftime("%d/%m/%Y %I:%M:%S %p") + else: + return "N/A" diff --git a/pa_job_manager.py b/pa_job_manager.py index 745453d..1b6076f 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -455,6 +455,25 @@ class PA_UserState(Base): return f"" +############################################################################## +# NewJob(): convenience function to create a job, appropriately +############################################################################## +def NewJob(name, num_files=0, wait_for=None, jex=None, parent_job=None ): + job=Job( name=name, current_file_num=0, current_file='', num_files=num_files, + wait_for=wait_for, state="New", pa_job_state="New", start_time=None, last_update=None) + if jex != None: + for e in jex: + job.extra.append(e) + + session.add(job) + session.commit() + if parent_job: + str=f"adding job id={job.id} {job.name}" + if job.wait_for: + str+=f" (wait for: {job.wait_for})" + AddLogForJob(parent_job, str ) + return job + ############################################################################## # MessageToFE(): sends a specific alert/messasge for a given job via the DB to # the front end @@ -563,36 +582,25 @@ def JobsForPaths( parent_job, paths, ptype ): if p: cfn=p.num_files - job1=Job(start_time=now, last_update=now, name="importdir", state="New", wait_for=None, pa_job_state="New", current_file_num=0, num_files=cfn ) - job1.extra.append( JobExtra( name="path", value=path ) ) - job1.extra.append( JobExtra( name="path_type", value=ptype.id ) ) - session.add(job1) - session.commit() - if parent_job: - AddLogForJob(parent_job, f"adding job id={job1.id} {job1.name}") - # force commit to make job.id be valid in use of wait_for later - session.commit() - job2=Job(start_time=now, last_update=now, name="getfiledetails", state="New", wait_for=job1.id, pa_job_state="New", current_file_num=0 ) - job2.extra.append( JobExtra( name="path", value=path ) ) - session.add(job2) - session.commit() - if parent_job: - AddLogForJob(parent_job, f"adding job id={job2.id} {job2.name} (wait for: {job2.wait_for})") + jex=[] + jex.append( JobExtra( name="path", value=path ) ) + jex.append( JobExtra( name="path_type", value=ptype.id ) ) + job1=NewJob( "importdir", cfn, None, jex, parent_job ) - job3=Job(start_time=now, last_update=now, name="run_ai_on_path", state="New", wait_for=job1.id, pa_job_state="New", current_file_num=0 ) - job3.extra.append( JobExtra( name="person", value="all" ) ) - job3.extra.append( JobExtra( name="path_type", value=ptype.id ) ) - session.add(job3) - session.commit() - if parent_job: - AddLogForJob(parent_job, f"adding job id={job3.id} {job3.name} (wait for: {job3.wait_for})") + jex=[] + jex.append( JobExtra( name="path", value=path ) ) + job2=NewJob("getfiledetails", 0, job1.id, jex, parent_job ) + + # can start straight after importdir - job1, does not need details + jex=[] + jex.append( JobExtra( name="person", value="all" ) ) + jex.append( JobExtra( name="path_type", value=ptype.id ) ) + job3=NewJob("run_ai_on_path", 0, job1.id, jex, parent_job ) + + # careful here, wait for getfiledetails (job2), the ai job cannot cause a dup + # but it can fail - in which case the checkdup will be withdrawn + job4=NewJob( "checkdups", 0, job2.id, None, parent_job ) - # careful here, wait for getfiledetails, the ai job cannot cause a dup, but it can fail - in which case the checkdup will be withdrawn - job4=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=job2.id, pa_job_state="New", current_file_num=0 ) - session.add(job4) - session.commit() - if parent_job: - AddLogForJob(parent_job, f"adding job id={job4.id} {job4.name} (wait for: {job4.wait_for})" ) HandleJobs(False) return @@ -682,7 +690,9 @@ def AddLogForJob(job, message): ############################################################################## def RunJob(job): # session = Session() - job.start_time=datetime.now(pytz.utc) + # only update start_time if we have never set it - stops restarts resetting start_time + if not job.start_time: + job.start_time=datetime.now(pytz.utc) if job.name =="scannow": JobScanNow(job) elif job.name =="forcescan": @@ -726,6 +736,8 @@ def RunJob(job): def FinishJob(job, last_log, state="Completed", pa_job_state="Completed"): job.state=state job.pa_job_state=pa_job_state + if not job.start_time: + job.start_time=datetime.now(pytz.utc) job.last_update=datetime.now(pytz.utc) AddLogForJob(job, last_log) if job.state=="Failed": @@ -1816,10 +1828,7 @@ def JobRemoveDups(job): FinishJob(job, f"Finished removing {dup_cnt} duplicate files" ) # Need to put another checkdups job in now to force / validate we have no dups - now=datetime.now(pytz.utc) - next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(next_job) - session.commit() + next_job=NewJob( "checkdups" ) AddLogForJob(job, "adding job id={} {} to confirm there are no more duplicates".format( next_job.id, next_job.id, next_job.name ) ) return @@ -1849,9 +1858,7 @@ def JobMoveFiles(job): if 'eid-' in jex.name: move_me=session.query(Entry).get(jex.value) MoveEntriesToOtherFolder( job, move_me, dst_storage_path, f"{prefix}{suffix}" ) - now=datetime.now(pytz.utc) - next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(next_job) + next_job=NewJob( "checkdups" ) MessageToFE( job.id, "success", "Completed (move of selected files)" ) FinishJob(job, f"Finished move selected file(s)") return @@ -1866,9 +1873,7 @@ def JobDeleteFiles(job): if 'eid-' in jex.name: del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() MoveFileToRecycleBin(job,del_me) - now=datetime.now(pytz.utc) - next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(next_job) + next_job=NewJob( "checkdups" ) MessageToFE( job.id, "success", "Completed (delete of selected files)" ) FinishJob(job, f"Finished deleting selected file(s)") return @@ -1883,9 +1888,7 @@ def JobRestoreFiles(job): if 'eid-' in jex.name: restore_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() RestoreFile(job,restore_me) - now=datetime.now(pytz.utc) - next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(next_job) + next_job=NewJob( "checkdups" ) MessageToFE( job.id, "success", "Completed (restore of selected files)" ) FinishJob(job, f"Finished restoring selected file(s)") return @@ -1897,8 +1900,7 @@ def JobRestoreFiles(job): #################################################################################################################################### def InitialValidationChecks(): now=datetime.now(pytz.utc) - job=Job(start_time=now, last_update=now, name="init", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(job) + job=NewJob( "init" ) settings = session.query(Settings).first() AddLogForJob(job, f"INFO: Starting Initial Validation checks...") JobProgressState( job, "In Progress" ) @@ -1913,9 +1915,6 @@ def InitialValidationChecks(): AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" ) # create symlink and Path/Dir if needed ProcessRecycleBinDir(job) -# bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first() -# if not bin_path: -# ProcessRecycleBinDir(job) else: AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); sp_exists=0 @@ -2143,8 +2142,7 @@ def CheckAndRunBinClean(): now=datetime.now(pytz.utc) if not j or (now-j.last_update).days >= settings.scheduled_bin_cleanup: print( f"INFO: Should force clean up bin path, del files older than {settings.bin_cleanup_file_age} days old" ) - job=Job(start_time=now, last_update=now, name="clean_bin", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(job) + job=NewJob( "clean_bin" ) created_jobs=True return created_jobs @@ -2164,13 +2162,11 @@ def ScheduledJobs(): now=datetime.now(pytz.utc) if ndays_since_last_im_scan >= settings.scheduled_import_scan: print( f"INFO: Time to force an import scan, last scan was {ndays_since_last_im_scan} days ago" ) - job=Job(start_time=now, last_update=now, name="scannow", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(job) + job=NewJob( "scannow" ) created_jobs=True if ndays_since_last_st_scan >= settings.scheduled_storage_scan: print( f"INFO: Time to force a storage scan, last scan was {ndays_since_last_st_scan}" ) - job=Job(start_time=now, last_update=now, name="scan_sp", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) - session.add(job) + job=NewJob( "scan_sp" ) created_jobs=True if CheckAndRunBinClean(): created_jobs=True diff --git a/tables.sql b/tables.sql index c3c0e27..ab19be6 100644 --- a/tables.sql +++ b/tables.sql @@ -170,8 +170,8 @@ insert into FILE_TYPE values ( (select nextval('FILE_TYPE_ID_SEQ')), 'Unknown' ) --insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' ); --insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' ); -- DEV(ddp): ---insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/', 'photos/', '.pa_bin/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); +insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/', 'photos/', '.pa_bin/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); -- DEV(cam): --insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'c:\images_to_process', 'photos/', '.pa_bin/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 3 ); -- PROD: -insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 4 ); +-- insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, auto_rotate, default_refimg_model, default_scan_model, default_threshold, face_size_limit, scheduled_import_scan, scheduled_storage_scan, scheduled_bin_cleanup, bin_cleanup_file_age, job_archive_age ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', true, 1, 1, '0.55', 43, 1, 1, 7, 30, 4 );