use new base_path Setting, but have not tested use of absolute paths instead of relative paths, also need better tooltips for the paths -- AND, still have odd trailing slash due to SymLinkName, etc. being too complex

BUGs

@@ -1,3 +1,13 @@
 ### Next: 49
 BUG-45: when deleting, the .pa_bin path has c:\ ... in it, not the real path
-        -- due to multiple Bin paths, really only should have one... Prob. best to re-jig Setting to have Base path (diff for me and Cam), then only 1 Bin path of base, can allow multiple import / storage as it works anyway
+      --> due to multiple Bin paths, really only should have one... Prob. best to re-jig Setting to have Base path (diff for me and Cam), then only 1 Bin path of base, can allow multiple import / storage as it works anyway
+      --> deletions are causing odd crash out for 'now'?
+          File "pa_job_manager.py", line 1781, in <module>
+            HandleJobs()
+          File "pa_job_manager.py", line 691, in HandleJobs
+            RunJob(job)
+          File "pa_job_manager.py", line 625, in RunJob
+            JobDeleteFiles(job)
+          File "pa_job_manager.py", line 1575, in JobDeleteFiles
+            next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 )
+        NameError: name 'now' is not defined

TODO

@@ -10,6 +10,7 @@
 
     * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach?
     * from menu, we could try to get smart/fancy... say find face with largest size, check it vs. other faces, if it matches more than say 10? we offer it up as a required ref img, then cut that face (with margin) out and use it is a new ref image / person
+        - read that guys face matching / clustering / nearest neighbour examples, for a whole new AI capability
 
     * fix up logging in general
     * comment your code

options.py

@@ -1,4 +1,4 @@
-from settings import Settings
+from settings import Settings, SettingsRBPath, SettingsIPath, SettingsSPath
 from shared import PA
 
 ################################################################################
@@ -23,15 +23,16 @@ class Options(PA):
         if 'files_sp' in url:
             self.noo="A to Z"
             self.path_type = 'Storage'
-            self.paths = settings.storage_path.split("#")
+            self.paths = SettingsSPath()
         elif 'files_rbp' in url:
             self.path_type = 'Bin'
-            self.paths = settings.recycle_bin_path.split("#")
+            self.paths = []
+            self.paths.append(SettingsRBPath())
         else:
             self.folders=False
             self.path_type = 'Import'
             self.cwd='static/Import'
-            self.paths = settings.import_path.split("#")
+            self.paths = SettingsIPath()
 
         self.cwd='static/' + self.path_type
         self.root=self.cwd

pa_job_manager.py

@@ -231,6 +231,7 @@ class FileType(Base):
 class Settings(Base):
     __tablename__ = "settings"
     id = Column(Integer, Sequence('settings_id_seq'), primary_key=True )
+    base_path = Column(String)
     import_path = Column(String)
     storage_path = Column(String)
     recycle_bin_path = Column(String)
@@ -239,7 +240,7 @@ class Settings(Base):
     default_threshold = Column(Integer)
 
     def __repr__(self):
-        return f"<id: {self.id}, import_path: {self.import_path}, recycle_bin_path: {self.recycle_bin_path}, default_refimg_model: {self.default_refimg_model}, default_scan_model: {self.default_scan_model}, default_threshold: {self.default_threshold}>"
+        return f"<id: {self.id}, base_path: {self.base_path}, import_path: {self.import_path}, storage_path: {self.storage_path}, recycle_bin_path: {self.recycle_bin_path}, default_refimg_model: {self.default_refimg_model}, default_scan_model: {self.default_scan_model}, default_threshold: {self.default_threshold}>"
         
 ################################################################################
 # Class describing Person to Refimg link in DB via sqlalchemy
@@ -421,50 +422,89 @@ def MessageToFE( job_id, alert, message ):
     session.commit()
     return msg.id
 
+##############################################################################
+# SettingsRBPath(): return modified array of paths (take each path in 
+# recycle_bin_path and add base_path if needed)
+##############################################################################
+def SettingsRBPath():
+    settings = session.query(Settings).first()
+    if settings == None:
+        raise Exception("Cannot create file data with no settings / recycle bin path is missing")
+    # path setting is an absolute path, just use it, otherwise prepend base_path first
+    if settings.recycle_bin_path[0] == '/':
+        path = settings.recycle_bin_path
+    else:
+        path = settings.base_path+settings.recycle_bin_path
+    return path
+
 ##############################################################################
 # ProcessRecycleBinDir(): create Path/symlink if needed (func called once on
 # startup)
 ##############################################################################
 def ProcessRecycleBinDir(job):
-    settings = session.query(Settings).first()
+    path = SettingsRBPath()
-    if settings == None:
+    print( f"here1: {path}" )
-        raise Exception("Cannot create file data with no settings / recycle bin path is missing")
+    if not os.path.exists( path ):
+        AddLogForJob( job, f"Not Importing {path} -- Path does not exist" )
+        return
+
     ptype = session.query(PathType).filter(PathType.name=='Bin').first()
-    paths = settings.recycle_bin_path.split("#")
+    # check/create if needed
-
+    symlink=CreateSymlink(job,ptype.id,path)
-    for path in paths:
+    print( f"here2: {path}, s={symlink}" )
-        if not os.path.exists( path ):
+    # create the Path (and Dir objects for the Bin)
-            AddLogForJob( job, f"Not Importing {path} -- Path does not exist" )
+    AddPath( job, symlink, ptype.id )
-            continue
-        symlink=SymlinkName(ptype.name, path, path)
-        # create the Path (and Dir objects for the Bin)
-        AddPath( job, symlink, ptype.id )
-
     session.commit()
     return
 
+##############################################################################
+# SettingsSPath(): return modified array of paths (take each path in 
+# storage_path and add base_path if needed)
+##############################################################################
+def SettingsSPath():
+    paths=[]
+    settings = session.query(Settings).first()
+    if settings == None:
+        raise Exception("Cannot create file data with no settings / storage path is missing")
+    for p in settings.storage_path.split("#"):
+        if p[0] == '/':
+            paths.append(p)
+        else:
+            paths.append(settings.base_path+p)
+    return paths
+
 ##############################################################################
 # ProcessStorageDirs(): wrapper func to call passed in job for each 
 # storage path defined in Settings - called via scan storage job
 ##############################################################################
 def ProcessStorageDirs(parent_job):
-    settings = session.query(Settings).first()
+    paths = SettingsSPath()
-    if settings == None:
-        raise Exception("Cannot create file data with no settings / storage path is missing")
-    paths = settings.storage_path.split("#")
     ptype = session.query(PathType).filter(PathType.name=='Storage').first()
     JobsForPaths( parent_job, paths, ptype )
     return
 
+##############################################################################
+# SettingsIPath(): return modified array of paths (take each path in 
+# import_path and add base_path if needed)
+##############################################################################
+def SettingsIPath():
+    paths=[]
+    settings = session.query(Settings).first()
+    if settings == None:
+        raise Exception("Cannot create file data with no settings / import path is missing")
+    for p in settings.import_path.split("#"):
+        if p[0] == '/':
+            paths.append(p)
+        else:
+            paths.append(settings.base_path+p)
+    return paths
+
 ##############################################################################
 # ProcessImportDirs(): wrapper func to call passed in job for each 
 # storage path defined in Settings - called via scan import job
 ##############################################################################
 def ProcessImportDirs(parent_job):
-    settings = session.query(Settings).first()
+    paths = SettingsIPath()
-    if settings == None:
-        raise Exception("Cannot create file data with no settings / import path is missing")
-    paths = settings.import_path.split("#")
     ptype = session.query(PathType).filter(PathType.name=='Import').first()
     JobsForPaths( parent_job, paths, ptype )
     return
@@ -720,7 +760,7 @@ def JobForceScan(job):
 
 ##############################################################################
 # CreateSymlink(): to serve static content of the images, we create a symlink 
-# from inside the static subdir of each import_path that exists
+# from inside the static subdir of each path that exists
 ##############################################################################
 def CreateSymlink(job,ptype,path):
     path_type = session.query(PathType).get(ptype)
@@ -770,7 +810,7 @@ def AddDir(job, dirname, in_dir, rel_path, in_path ):
     dtype=session.query(FileType).filter(FileType.name=='Directory').first()
     e=Entry( name=dirname, type=dtype, exists_on_fs=True )
     e.dir_details=dir
-    # no in_dir occurs when we Add the actual Dir for the import_path (top of the tree)
+    # no in_dir occurs when we Add the actual Dir for the Path (top of the tree)
     if in_dir:
         e.in_dir=in_dir
     if DEBUG==1:
@@ -924,7 +964,7 @@ def RestoreFile(job,restore_me):
 def MoveFileToRecycleBin(job,del_me):
     try:
         settings = session.query(Settings).first()
-        dst_dir=settings.recycle_bin_path + '/' + del_me.in_dir.in_path.path_prefix.replace('static/','') + '/' + del_me.in_dir.rel_path + '/'
+        dst_dir= SettingsRBPath() + '/' + del_me.in_dir.in_path.path_prefix.replace('static/','') + '/' + del_me.in_dir.rel_path + '/'
         os.makedirs( dst_dir,mode=0o777, exist_ok=True )
         src=del_me.FullPathOnFS()
         dst=dst_dir + '/' + del_me.name
@@ -1531,7 +1571,7 @@ def JobDeleteFiles(job):
         if 'eid-' in jex.name:
             del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
             MoveFileToRecycleBin(job,del_me)
-    ynw=datetime.now(pytz.utc)
+    now=datetime.now(pytz.utc)
     next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 )
     session.add(next_job)
     MessageToFE( job.id, "success", "Completed (delete of selected files)" )
@@ -1564,28 +1604,25 @@ def InitialValidationChecks():
     job=Job(start_time=now, last_update=now, name="init", state="New", wait_for=None, pa_job_state="New", current_file_num=0 )
     session.add(job)
     settings = session.query(Settings).first()
-    rbp_exists=0
-    paths = settings.recycle_bin_path.split("#")
     AddLogForJob(job, f"INFO: Starting Initial Validation checks...")
-    for path in paths:
+    path=SettingsRBPath()
-        if os.path.exists(path):
+    rbp_exists=0
-            rbp_exists=1
+    if os.path.exists(path):
-            ptype = session.query(PathType).filter(PathType.name=='Bin').first().id
+        rbp_exists=1
-            symlink=CreateSymlink(job,ptype,path)
+        root, dirs, files = next(os.walk(path))
-            root, dirs, files = next(os.walk(path))
+        if len(dirs) + len(files) > 0:
-            if len(dirs) + len(files) > 0:
+            AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" )
-                AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" )
+            AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one")
-                AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one")
+            AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" )
-                AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" )
+        # create symlink and Path/Dir if needed
-            break
+        ProcessRecycleBinDir(job)
-    if not rbp_exists:
+#        bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first()
-        AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now");
+#        if not bin_path:
+#            ProcessRecycleBinDir(job)
     else:
-        bin_path=session.query(Path).join(PathType).filter(PathType.name=='Bin').first()
+        AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now");
-        if not bin_path:
-            ProcessRecycleBinDir(job)
     sp_exists=0
-    paths = settings.storage_path.split("#")
+    paths = SettingsSPath()
     for path in paths:
         if os.path.exists(path):
             sp_exists=1
@@ -1594,7 +1631,7 @@ def InitialValidationChecks():
     if not sp_exists:
         AddLogForJob(job, "ERROR: None of the storage paths in the settings exist - Please fix now");
     ip_exists=0
-    paths = settings.import_path.split("#")
+    paths = SettingsIPath()
     for path in paths:
         if os.path.exists(path):
             ip_exists=1

settings.py

@@ -26,6 +26,7 @@ class AIModel(db.Model):
 ################################################################################
 class Settings(db.Model):
     id = db.Column(db.Integer, db.Sequence('settings_id_seq'), primary_key=True )
+    base_path = db.Column(db.String)
     import_path = db.Column(db.String)
     storage_path = db.Column(db.String)
     recycle_bin_path = db.Column(db.String)
@@ -52,6 +53,7 @@ settings_schema = SettingsSchema(many=True)
 ################################################################################
 class SettingsForm(FlaskForm):
     id = HiddenField()
+    base_path = StringField('Base Path to use below (optional):', [validators.DataRequired()])
     import_path = StringField('Path(s) to import from:', [validators.DataRequired()])
     storage_path = StringField('Path to store sorted images to:', [validators.DataRequired()])
     recycle_bin_path = StringField('Path to temporarily store deleted images in:', [validators.DataRequired()])
@@ -92,3 +94,53 @@ def settings():
     else:
         form = SettingsForm( obj=Settings.query.first() )
         return render_template("settings.html", form=form, page_title = page_title)
+
+##############################################################################
+# SettingsRBPath(): return modified array of paths (take each path in
+# recycle_bin_path and add base_path if needed)
+##############################################################################
+def SettingsRBPath():
+    settings = Settings.query.first()
+    if settings == None:
+        print("Cannot create file data with no settings / recycle bin path is missing")
+        return
+    # path setting is an absolute path, just use it, otherwise prepend base_path first
+    if settings.recycle_bin_path[0] == '/':
+        path = settings.recycle_bin_path
+    else:
+        path = settings.base_path+settings.recycle_bin_path
+    return path
+
+##############################################################################
+# SettingsSPath(): return modified array of paths (take each path in
+# storage_path and add base_path if needed)
+##############################################################################
+def SettingsSPath():
+    paths=[]
+    settings = Settings.query.first()
+    if settings == None:
+        print("Cannot create file data with no settings / storage path is missing")
+        return
+    for p in settings.storage_path.split("#"):
+        if p[0] == '/':
+            paths.append(p)
+        else:
+            paths.append(settings.base_path+p)
+    return paths
+
+##############################################################################
+# SettingsIPath(): return modified array of paths (take each path in
+# import_path and add base_path if needed)
+##############################################################################
+def SettingsIPath():
+    paths=[]
+    settings = Settings.query.first()
+    if settings == None:
+        print ("Cannot create file data with no settings / import path is missing")
+        return
+    for p in settings.import_path.split("#"):
+        if p[0] == '/':
+            paths.append(p)
+        else:
+            paths.append(settings.base_path+p)
+    return paths

tables.sql

@@ -6,7 +6,7 @@ insert into AI_MODEL values ( 1, 'hog', 'normal' );
 insert into AI_MODEL values ( 2, 'cnn', 'more accurate / much slower' );
 
 create table SETTINGS( 
-    ID integer, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, 
+    ID integer, BASE_PATH varchar, IMPORT_PATH varchar, STORAGE_PATH varchar, RECYCLE_BIN_PATH varchar, 
     DEFAULT_REFIMG_MODEL integer, DEFAULT_SCAN_MODEL integer, DEFAULT_THRESHOLD float,
     constraint PK_SETTINGS_ID primary key(ID),
     constraint FK_DEFAULT_REFIMG_MODEL foreign key (DEFAULT_REFIMG_MODEL) references AI_MODEL(ID),
@@ -120,7 +120,9 @@ insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'dad', 'Damien',
 insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mum', 'Mandy', 'De Paoli' );
 insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'cam', 'Cameron', 'De Paoli' );
 insert into PERSON values ( (select nextval('PERSON_ID_SEQ')), 'mich', 'Michelle', 'De Paoli' );
--- DEV:
+-- DEV(ddp):
-insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', '/home/ddp/src/photoassistant/storage/#c:/Users/cam/Desktop/code/python/photoassistant/storage/', '/home/ddp/src/photoassistant/.pa_bin/', 2, 1, '0.55' );
+insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/home/ddp/src/photoassistant/', 'images_to_process/#photos/#new_img_dir/', 'storage/', '.pa_bin/', 2, 1, '0.55' );
+-- DEV(cam):
+--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), 'c:/Users/cam/Desktop/code/python/photoassistant/', 'images_to_process#photos#new_img_dir/', 'storage/', '.pa_bin/', 2, 1, '0.55' );
 -- PROD:
---insert into SETTINGS ( id, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/Camera_uploads/', '/export/docker/storage/photos/', '/export/docker/storage/.pa_bin/', 2, 1, '0.55' );
+--insert into SETTINGS ( id, base_path, import_path, storage_path, recycle_bin_path, default_refimg_model, default_scan_model, default_threshold ) values ( (select nextval('SETTINGS_ID_SEQ')), '/export/docker/storage/', 'Camera_uploads/', 'photos/', '.pa_bin/', 2, 1, '0.55' );