From add702a1e2f8d9f836f48424dc88a96091c395a3 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Thu, 24 Jun 2021 17:34:06 +1000 Subject: [PATCH] code for making a new folder and moving content to it --- pa_job_manager.py | 67 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index c032e6e..e15f7b4 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -298,7 +298,7 @@ def ProcessRecycleBinDir(parent_job): if settings == None: raise Exception("Cannot create file data with no settings / recycle bin path is missing") paths = settings.recycle_bin_path.split("#") - ptype = session.query(PathType).filter(PathType.name=='Bin').first().id + ptype = session.query(PathType).filter(PathType.name=='Bin').first() JobsForPaths( parent_job, paths, ptype ) return @@ -307,7 +307,7 @@ def ProcessStorageDirs(parent_job): if settings == None: raise Exception("Cannot create file data with no settings / storage path is missing") paths = settings.storage_path.split("#") - ptype = session.query(PathType).filter(PathType.name=='Storage').first().id + ptype = session.query(PathType).filter(PathType.name=='Storage').first() JobsForPaths( parent_job, paths, ptype ) return @@ -316,22 +316,21 @@ def ProcessImportDirs(parent_job): if settings == None: raise Exception("Cannot create file data with no settings / import path is missing") paths = settings.import_path.split("#") - ptype = session.query(PathType).filter(PathType.name=='Import').first().id + ptype = session.query(PathType).filter(PathType.name=='Import').first() JobsForPaths( parent_job, paths, ptype ) return def JobsForPaths( parent_job, paths, ptype ): now=datetime.now(pytz.utc) # make new set of Jobs per path... HandleJobs will make them run later - path_type = session.query(PathType).get(ptype) for path in paths: - p=session.query(Path).filter(Path.path_prefix==SymlinkName(path_type.name,path,path+'/')).first() + p=session.query(Path).filter(Path.path_prefix==SymlinkName(ptype.name,path,path+'/')).first() cfn=0 if p: cfn=p.num_files jex=JobExtra( name="path", value=path ) - jex2=JobExtra( name="path_type", value=ptype ) + jex2=JobExtra( name="path_type", value=ptype.id ) job=Job(start_time=now, last_update=now, name="importdir", state="New", wait_for=None, pa_job_state="New", current_file_num=0, num_files=cfn ) job.extra.append(jex) job.extra.append(jex2) @@ -705,6 +704,46 @@ def MoveFileToRecycleBin(job,del_me): AddLogForJob(job, f"Deleted file: {del_me.name} - (moved to {os.path.dirname(del_me.FullPathOnFS())})" ) return +# Function that moves a file into a new folder in the storage path - if needed it makes the folder on the FS, +# moves the file into the folder on the FS and then changes the database path to the relevant Storage path +def MoveFileToNewFolderInStorage(job,move_me, dst_storage_path, dst_rel_path): + print( f"MoveFileToNewFolderInStorage: {move_me} to {dst_storage_path} in new? folder: {dst_storage_path}") + try: + dst_dir=dst_storage_path.path_prefix + '/' + dst_rel_path + print( f"would make dir: {dst_dir}" ) + os.makedirs( dst_dir,mode=0o777, exist_ok=True ) + src=move_me.FullPathOnFS() + dst=dst_dir + '/' + move_me.name + os.replace( src, dst ) + print( f"would mv {src} {dst}" ) + except Exception as e: + print( f"ERROR: Failed to move file to new location on filesystem - which={src}, location={dir}, err: {e}") + + # need these for AddDir calls below to work + parent_dir=session.query(Dir).join(PathDirLink).filter(PathDirLink.path_id==dst_storage_path.id).first() + + # remove static from rel path, as it will move to static/Bin anyway... +# new_rel_path=del_me.in_dir.in_path.path_prefix.replace('static/','') + # if there is a relative path on this dir, add it to the new_rel_path as there is only ever 1 Bin path +# if len(del_me.in_dir.rel_path): +# new_rel_path += '/' + del_me.in_dir.rel_path + + # okay, go through new relative path and AddDir any missing subdirs of this + # path (think Import/Dir1/Dir2) which b/c we have bin_path in AddDir will + # create Bin/Import, Bin/Import/Dir1, Bin/Import/Dir1/Dir2 + part_rel_path="" + for dirname in dst_rel_path.split("/"): + part_rel_path += f"{dirname}" + print( f"Should make a Dir in the DB for {dirname} with parent: {parent_dir}, prp={part_rel_path} in storage path" ) + new_dir=AddDir( job, dirname, parent_dir, part_rel_path, dst_storage_path ) + parent_dir=new_dir + part_rel_path += "/" + print( f"now should change {move_me} in_dir to {new_dir} created above in {dst_storage_path}" ) + move_me.in_dir = new_dir + move_me.in_path = dst_storage_path + print( f"DONE change of {move_me} in_dir to {new_dir} created above" ) + AddLogForJob(job, f"{move_me.name} - (moved to {os.path.dirname(move_me.FullPathOnFS())})" ) + return # Convenience function to remove a dir from the database - and its associated links def RemoveDirFromDB(id): @@ -1186,19 +1225,19 @@ def RemoveDups(job): AddLogForJob(job, "adding job id={} {} to confirm there are no more duplicates".format( next_job.id, next_job.id, next_job.name ) ) return -def MoveFileToStorage(job, move_me, dst_dir): - AddLogForJob(job, f"TEST: Moving {move_me.name} to {dst_dir} in storage path" ) - return - def JobMoveFiles(job): AddLogForJob(job, f"INFO: Starting Move Files job...") AddLogForJob(job, f"INFO: NOT PROCESSING THIS - TESTING...") prefix=[jex.value for jex in job.extra if jex.name == "prefix"][0] suffix=[jex.value for jex in job.extra if jex.name == "suffix"][0] + storage_rp=[jex.value for jex in job.extra if jex.name == "storage_rp"][0] + print( f"storage_rp={storage_rp}" ) + dst_storage_path = session.query(Path).filter(Path.path_prefix=='static/Storage/'+ storage_rp).first() + print( f"dsp={dst_storage_path}" ) for jex in job.extra: if 'eid-' in jex.name: move_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first() - MoveFileToStorage(job,move_me, f"{prefix}{suffix}" ) + MoveFileToNewFolderInStorage(job, move_me, dst_storage_path, f"{prefix}{suffix}" ) now=datetime.now(pytz.utc) next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 ) session.add(next_job) @@ -1247,9 +1286,9 @@ def InitialValidationChecks(): symlink=CreateSymlink(job,ptype,path) path, dirs, files = next(os.walk(path)) if len(dirs) + len(files) > 0: - AddLogForJob(job, "NFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" ) - AddLogForJob(job, "ODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one") - AddLogForJob(job, "ARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" ) + AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" ) + AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one") + AddLogForJob(job, "WARNING: IF the files in the bin are in the DB (succeeded from GUI deletes) then this is okay, otherwise you should delete contents form the recycle bin and restart the job manager)" ) break if not rbp_exists: AddLogForJob(job, "ERROR: The bin path in settings does not exist - Please fix now");