diff --git a/BUGs b/BUGs index 9be332f..02516cb 100644 --- a/BUGs +++ b/BUGs @@ -1,3 +1,13 @@ ### Next: 49 BUG-45: when deleting, the .pa_bin path has c:\ ... in it, not the real path - -- due to multiple Bin paths, really only should have one... Prob. best to re-jig Setting to have Base path (diff for me and Cam), then only 1 Bin path of base, can allow multiple import / storage as it works anyway + --> due to multiple Bin paths, really only should have one... Prob. best to re-jig Setting to have Base path (diff for me and Cam), then only 1 Bin path of base, can allow multiple import / storage as it works anyway + --> deletions are causing odd crash out for 'now'? + File "pa_job_manager.py", line 1781, in + HandleJobs() + File "pa_job_manager.py", line 691, in HandleJobs + RunJob(job) + File "pa_job_manager.py", line 625, in RunJob + JobDeleteFiles(job) + File "pa_job_manager.py", line 1575, in JobDeleteFiles + next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) + NameError: name 'now' is not defined diff --git a/TODO b/TODO index 90d98c4..451bd3b 100644 --- a/TODO +++ b/TODO @@ -10,6 +10,7 @@ * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach? * from menu, we could try to get smart/fancy... say find face with largest size, check it vs. other faces, if it matches more than say 10? we offer it up as a required ref img, then cut that face (with margin) out and use it is a new ref image / person + - read that guys face matching / clustering / nearest neighbour examples, for a whole new AI capability * fix up logging in general * comment your code diff --git a/options.py b/options.py index 2cf3022..afc6f24 100644 --- a/options.py +++ b/options.py @@ -1,4 +1,4 @@ -from settings import Settings +from settings import Settings, SettingsRBPath, SettingsIPath, SettingsSPath from shared import PA ################################################################################ @@ -23,15 +23,16 @@ class Options(PA): if 'files_sp' in url: self.noo="A to Z" self.path_type = 'Storage' - self.paths = settings.storage_path.split("#") + self.paths = SettingsSPath() elif 'files_rbp' in url: self.path_type = 'Bin' - self.paths = settings.recycle_bin_path.split("#") + self.paths = [] + self.paths.append(SettingsRBPath()) else: self.folders=False self.path_type = 'Import' self.cwd='static/Import' - self.paths = settings.import_path.split("#") + self.paths = SettingsIPath() self.cwd='static/' + self.path_type self.root=self.cwd diff --git a/pa_job_manager.py b/pa_job_manager.py index eae082e..c629fe4 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -231,6 +231,7 @@ class FileType(Base): class Settings(Base): __tablename__ = "settings" id = Column(Integer, Sequence('settings_id_seq'), primary_key=True ) + base_path = Column(String) import_path = Column(String) storage_path = Column(String) recycle_bin_path = Column(String) @@ -239,7 +240,7 @@ class Settings(Base): default_threshold = Column(Integer) def __repr__(self): - return f"" + return f"" ################################################################################ # Class describing Person to Refimg link in DB via sqlalchemy @@ -421,50 +422,89 @@ def MessageToFE( job_id, alert, message ): session.commit() return msg.id +############################################################################## +# SettingsRBPath(): return modified array of paths (take each path in +# recycle_bin_path and add base_path if needed) +############################################################################## +def SettingsRBPath(): + settings = session.query(Settings).first() + if settings == None: + raise Exception("Cannot create file data with no settings / recycle bin path is missing") + # path setting is an absolute path, just use it, otherwise prepend base_path first + if settings.recycle_bin_path[0] == '/': + path = settings.recycle_bin_path + else: + path = settings.base_path+settings.recycle_bin_path + return path + ############################################################################## # ProcessRecycleBinDir(): create Path/symlink if needed (func called once on # startup) ############################################################################## def ProcessRecycleBinDir(job): - settings = session.query(Settings).first() - if settings == None: - raise Exception("Cannot create file data with no settings / recycle bin path is missing") + path = SettingsRBPath() + print( f"here1: {path}" ) + if not os.path.exists( path ): + AddLogForJob( job, f"Not Importing {path} -- Path does not exist" ) + return + ptype = session.query(PathType).filter(PathType.name=='Bin').first() - paths = settings.recycle_bin_path.split("#") - - for path in paths: - if not os.path.exists( path ): - AddLogForJob( job, f"Not Importing {path} -- Path does not exist" ) - continue - symlink=SymlinkName(ptype.name, path, path) - # create the Path (and Dir objects for the Bin) - AddPath( job, symlink, ptype.id ) - + # check/create if needed + symlink=CreateSymlink(job,ptype.id,path) + print( f"here2: {path}, s={symlink}" ) + # create the Path (and Dir objects for the Bin) + AddPath( job, symlink, ptype.id ) session.commit() return +############################################################################## +# SettingsSPath(): return modified array of paths (take each path in +# storage_path and add base_path if needed) +############################################################################## +def SettingsSPath(): + paths=[] + settings = session.query(Settings).first() + if settings == None: + raise Exception("Cannot create file data with no settings / storage path is missing") + for p in settings.storage_path.split("#"): + if p[0] == '/': + paths.append(p) + else: + paths.append(settings.base_path+p) + return paths + ############################################################################## # ProcessStorageDirs(): wrapper func to call passed in job for each # storage path defined in Settings - called via scan storage job ############################################################################## def ProcessStorageDirs(parent_job): - settings = session.query(Settings).first() - if settings == None: - raise Exception("Cannot create file data with no settings / storage path is missing") - paths = settings.storage_path.split("#") + paths = SettingsSPath() ptype = session.query(PathType).filter(PathType.name=='Storage').first() JobsForPaths( parent_job, paths, ptype ) return +############################################################################## +# SettingsIPath(): return modified array of paths (take each path in +# import_path and add base_path if needed) +############################################################################## +def SettingsIPath(): + paths=[] + settings = session.query(Settings).first() + if settings == None: + raise Exception("Cannot create file data with no settings / import path is missing") + for p in settings.import_path.split("#"): + if p[0] == '/': + paths.append(p) + else: + paths.append(settings.base_path+p) + return paths + ############################################################################## # ProcessImportDirs(): wrapper func to call passed in job for each # storage path defined in Settings - called via scan import job ############################################################################## def ProcessImportDirs(parent_job): - settings = session.query(Settings).first() - if settings == None: - raise Exception("Cannot create file data with no settings / import path is missing") - paths = settings.import_path.split("#") + paths = SettingsIPath() ptype = session.query(PathType).filter(PathType.name=='Import').first() JobsForPaths( parent_job, paths, ptype ) return @@ -720,7 +760,7 @@ def JobForceScan(job): ############################################################################## # CreateSymlink(): to serve static content of the images, we create a symlink -# from inside the static subdir of each import_path that exists +# from inside the static subdir of each path that exists ############################################################################## def CreateSymlink(job,ptype,path): path_type = session.query(PathType).get(ptype) @@ -770,7 +810,7 @@ def AddDir(job, dirname, in_dir, rel_path, in_path ): dtype=session.query(FileType).filter(FileType.name=='Directory').first() e=Entry( name=dirname, type=dtype, exists_on_fs=True ) e.dir_details=dir - # no in_dir occurs when we Add the actual Dir for the import_path (top of the tree) + # no in_dir occurs when we Add the actual Dir for the Path (top of the tree) if in_dir: e.in_dir=in_dir if DEBUG==1: @@ -924,7 +964,7 @@ def RestoreFile(job,restore_me): def MoveFileToRecycleBin(job,del_me): try: settings = session.query(Settings).first() - dst_dir=settings.recycle_bin_path + '/' + del_me.in_dir.in_path.path_prefix.replace('static/','') + '/' + del_me.in_dir.rel_path + '/' + dst_dir= SettingsRBPath() + '/' + del_me.in_dir.in_path.path_prefix.replace('static/','') + '/' + del_me.in_dir.rel_path + '/' os.makedirs( dst_dir,mode=0o777, exist_ok=True ) src=del_me.FullPathOnFS() dst=dst_dir + '/' + del_me.name @@ -1531,7 +1571,7 @@ def JobDeleteFiles(job): if 'eid-' in jex.name: del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() MoveFileToRecycleBin(job,del_me) - ynw=datetime.now(pytz.utc) + now=datetime.now(pytz.utc) next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) session.add(next_job) MessageToFE( job.id, "success", "Completed (delete of selected files)" ) @@ -1564,28 +1604,25 @@ def InitialValidationChecks(): job=Job(start_time=now, last_update=now, name="init", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) session.add(job) settings = session.query(Settings).first() - rbp_exists=0 - paths = settings.recycle_bin_path.split("#") AddLogForJob(job, f"INFO: Starting Initial Validation checks...") - for path in paths: - if os.path.exists(path): - rbp_exists=1 - ptype = session.query(PathType).filter(PathType.name=='Bin').first().id - symlink=CreateSymlink(job,ptype,path) - root, dirs, files = next(os.walk(path)) - if len(dirs) + len(files) > 0: - AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" ) - AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one") - AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" ) - break - if not rbp_exists: - AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); + path=SettingsRBPath() + rbp_exists=0 + if os.path.exists(path): + rbp_exists=1 + root, dirs, files = next(os.walk(path)) + if len(dirs) + len(files) > 0: + AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" ) + AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one") + AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" ) + # create symlink and Path/Dir if needed + ProcessRecycleBinDir(job) +# bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first() +# if not bin_path: +# ProcessRecycleBinDir(job) else: - bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first() - if not bin_path: - ProcessRecycleBinDir(job) + AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); sp_exists=0 - paths = settings.storage_path.split("#") + paths = SettingsSPath() for path in paths: if os.path.exists(path): sp_exists=1 @@ -1594,7 +1631,7 @@ def InitialValidationChecks(): if not sp_exists: AddLogForJob(job, "ERROR: None of the storage paths in the settings exist - Please fix now"); ip_exists=0 - paths = settings.import_path.split("#") + paths = SettingsIPath() for path in paths: if os.path.exists(path): ip_exists=1 diff --git a/settings.py b/settings.py index 1a0590a..d7370f1 100644 --- a/settings.py +++ b/settings.py @@ -26,6 +26,7 @@ class AIModel(db.Model): ################################################################################ class Settings(db.Model): id = db.Column(db.Integer, db.Sequence('settings_id_seq'), primary_key=True ) + base_path = db.Column(db.String) import_path = db.Column(db.String) storage_path = db.Column(db.String) recycle_bin_path = db.Column(db.String) @@ -52,6 +53,7 @@ settings_schema = SettingsSchema(many=True) ################################################################################ class SettingsForm(FlaskForm): id = HiddenField() + base_path = StringField('Base Path to use below (optional):', [validators.DataRequired()]) import_path = StringField('Path(s) to import from:', [validators.DataRequired()]) storage_path = StringField('Path to store sorted images to:', [validators.DataRequired()]) recycle_bin_path = StringField('Path to temporarily store deleted images in:', [validators.DataRequired()]) @@ -92,3 +94,53 @@ def settings(): else: form = SettingsForm( obj=Settings.query.first() ) return render_template("settings.html", form=form, page_title = page_title) + +############################################################################## +# SettingsRBPath(): return modified array of paths (take each path in +# recycle_bin_path and add base_path if needed) +############################################################################## +def SettingsRBPath(): + settings = Settings.query.first() + if settings == None: + print("Cannot create file data with no settings / recycle bin path is missing") + return + # path setting is an absolute path, just use it, otherwise prepend base_path first + if settings.recycle_bin_path[0] == '/': + path = settings.recycle_bin_path + else: + path = settings.base_path+settings.recycle_bin_path + return path + +############################################################################## +# SettingsSPath(): return modified array of paths (take each path in +# storage_path and add base_path if needed) +############################################################################## +def SettingsSPath(): + paths=[] + settings = Settings.query.first() + if settings == None: + print("Cannot create file data with no settings / storage path is missing") + return + for p in settings.storage_path.split("#"): + if p[0] == '/': + paths.append(p) + else: + paths.append(settings.base_path+p) + return paths + +############################################################################## +# SettingsIPath(): return modified array of paths (take each path in +# import_path and add base_path if needed) +############################################################################## +def SettingsIPath(): + paths=[] + settings = Settings.query.first() + if settings == None: + print ("Cannot create file data with no settings / import path is missing") + return + for p in settings.import_path.split("#"): + if p[0] == '/': + paths.append(p) + else: + paths.append(settings.base_path+p) + return paths diff --git a/tables.sql b/tables.sql index 9681a64..49d8b22 100644 --- a/tables.sql +++ b/tables.sql @@ -6,7 +6,7 @@ insert into AI_MODEL values ( 1, 'hog', 'normal' ); insert into AI_MODEL values ( 2, 'cnn', 'more accurate / much slower' ); create table SETTINGS( - ID integer, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, + ID integer, BASE_PATH varchar, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, DEFAULT_REFIMG_MODEL integer, DEFAULT_SCAN_MODEL integer, DEFAULT_THRESHOLD float, constraint PK_SETTINGS_ID primary key(ID), constraint FK_DEFAULT_REFIMG_MODEL foreign key (DEFAULT_REFIMG_MODEL) references AI_MODEL(ID), @@ -120,7 +120,9 @@ insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'dad', 'Damien', insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mum', 'Mandy', 'De Paoli' ); insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' ); insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' ); --- DEV: -insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', '/home/ddp/src/photoassistant/storage/#c:/Users/cam/Desktop/code/python/photoassistant/storage/', '/home/ddp/src/photoassistant/.pa_bin/', 2, 1, '0.55' ); +-- DEV(ddp): +insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#photos/#new_img_dir/', 'storage/', '.pa_bin/', 2, 1, '0.55' ); +-- DEV(cam): +--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'images_to_process#photos#new_img_dir/', 'storage/', '.pa_bin/', 2, 1, '0.55' ); -- PROD: ---insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/Camera_uploads/', '/export/docker/storage/photos/', '/export/docker/storage/.pa_bin/', 2, 1, '0.55' ); +--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55' );