diff --git a/pa_job_manager.py b/pa_job_manager.py index 29f2630..c77ff32 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -571,13 +571,15 @@ def MessageToFE( job_id, alert, message ): def SettingsRBPath(): settings = session.query(Settings).first() if settings == None: - raise Exception("Cannot create file data with no settings / recycle bin path is missing") + print("Cannot create file data with no settings / recycle bin path is missing") + return # path setting is an absolute path, just use it, otherwise prepend base_path first - if settings.recycle_bin_path[0] == '/': - path = settings.recycle_bin_path + p=settings.recycle_bin_path + if p == '/': + return p else: - path = settings.base_path+settings.recycle_bin_path - return path + return settings.base_path+p + ############################################################################## # ProcessRecycleBinDir(): create Path/symlink if needed (func called once on @@ -601,42 +603,41 @@ def ProcessRecycleBinDir(job): # storage_path and add base_path if needed) ############################################################################## def SettingsSPath(): - paths=[] settings = session.query(Settings).first() - if settings == None: - raise Exception("Cannot create file data with no settings / storage path is missing") - for p in settings.storage_path.split("#"): - if p[0] == '/': - paths.append(p) - else: - paths.append(settings.base_path+p) - return paths + if settings == None or settings.storage_path == "": + print("Cannot create file data with no settings / storage path is missing") + return + p=settings.storage_path + if p[0] == '/': + return p + else: + return settings.base_path+p ############################################################################## # ProcessStorageDirs(): wrapper func to call passed in job for each # storage path defined in Settings - called via scan storage job ############################################################################## def ProcessStorageDirs(parent_job): - paths = SettingsSPath() + path = SettingsSPath() ptype = session.query(PathType).filter(PathType.name=='Storage').first() - JobsForPaths( parent_job, paths, ptype ) + JobsForPath( parent_job, path, ptype ) return + ############################################################################## -# SettingsIPath(): return modified array of paths (take each path in -# import_path and add base_path if needed) +# SettingsIPath(): return import path (abs or add base_path if needed) ############################################################################## def SettingsIPath(): paths=[] settings = session.query(Settings).first() - if settings == None: - raise Exception("Cannot create file data with no settings / import path is missing") - for p in settings.import_path.split("#"): - if p[0] == '/': - paths.append(p) - else: - paths.append(settings.base_path+p) - return paths + if settings == None or settings.import_path == "": + print("Cannot create file data with no settings / import path is missing") + return + p=settings.import_path + if p[0] == '/': + return p + else: + return settings.base_path+p ############################################################################## @@ -659,35 +660,35 @@ def SettingsMPath(): # storage path defined in Settings - called via scan import job ############################################################################## def ProcessImportDirs(parent_job): - paths = SettingsIPath() + path = SettingsIPath() ptype = session.query(PathType).filter(PathType.name=='Import').first() - JobsForPaths( parent_job, paths, ptype ) + JobsForPath( parent_job, path, ptype ) return ############################################################################## -# JobsForPaths(): wrapper func to create jobs for passed in parent_job for -# each path passed in, with path_type passed in - used by -# Process{Storage|Import}Dirs() above +# JobsForPath(): wrapper func to create jobs for passed in parent_job for +# path, with path_type passed in - used by Process{Storage|Import}Dirs() above ############################################################################## -def JobsForPaths( parent_job, paths, ptype ): - now=datetime.now(pytz.utc) - # make new set of Jobs per path... HandleJobs will make them run later - for path in paths: - p=session.query(Path).filter(Path.path_prefix==SymlinkName(ptype.name,path,path+'/')).first() - cfn=0 - if p: - cfn=p.num_files - - jex=[] - jex.append( JobExtra( name="path", value=path ) ) - jex.append( JobExtra( name="path_type", value=ptype.id ) ) - job1=NewJob( "importdir", cfn, None, jex, parent_job ) +def JobsForPath( parent_job, path, ptype ): - jex=[] - jex.append( JobExtra( name="path", value=path ) ) - job2=NewJob("getfiledetails", 0, job1.id, jex, parent_job ) + # try to find top of 'path' and use num_files for job later on + cfn=0 + p=session.query(Path).filter(Path.path_prefix==SymlinkName(ptype.name,path,path+'/')).first() + if p: + cfn=p.num_files + + # start with import dir + jex=[] + jex.append( JobExtra( name="path", value=path ) ) + jex.append( JobExtra( name="path_type", value=ptype.id ) ) + job1=NewJob( "importdir", cfn, None, jex, parent_job ) - # can start straight after importdir - job1, does not need details + # then get file details (hash/thumbs) + jex=[] + jex.append( JobExtra( name="path", value=path ) ) + job2=NewJob("getfiledetails", 0, job1.id, jex, parent_job ) + + # can start straight after importdir - job1, does not need details (job2) jex=[] jex.append( JobExtra( name="person", value="all" ) ) jex.append( JobExtra( name="path_type", value=ptype.id ) ) @@ -697,6 +698,7 @@ def JobsForPaths( parent_job, paths, ptype ): # but it can fail - in which case the checkdup will be withdrawn job4=NewJob( "checkdups", 0, job2.id, None, parent_job ) + # okay, now process all the new jobs HandleJobs(False) return @@ -2331,23 +2333,21 @@ def InitialValidationChecks(): else: AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now"); sp_exists=0 - paths = SettingsSPath() - for path in paths: - if os.path.exists(path): - sp_exists=1 - ptype = session.query(PathType).filter(PathType.name=='Storage').first().id - symlink=CreateSymlink(job,ptype,path) + path = SettingsSPath() + if os.path.exists(path): + sp_exists=1 + ptype = session.query(PathType).filter(PathType.name=='Storage').first().id + symlink=CreateSymlink(job,ptype,path) if not sp_exists: - AddLogForJob(job, "ERROR: None of the storage paths in the settings exist - Please fix now"); + AddLogForJob(job, "ERROR: the storage path in the settings does not exist - Please fix now"); ip_exists=0 - paths = SettingsIPath() - for path in paths: - if os.path.exists(path): - ip_exists=1 - ptype = session.query(PathType).filter(PathType.name=='Import').first().id - symlink=CreateSymlink(job,ptype,path) + path = SettingsIPath() + if os.path.exists(path): + ip_exists=1 + ptype = session.query(PathType).filter(PathType.name=='Import').first().id + symlink=CreateSymlink(job,ptype,path) if not ip_exists: - AddLogForJob(job, "ERROR: None of the import paths in the settings exist - Please fix now"); + AddLogForJob(job, "ERROR: the import path in the settings does not exist - Please fix now"); path=SettingsMPath() mp_exists=0