From b1917855e2ee22f4e2e0c138657adab3faefef9c Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 8 Jan 2022 11:24:53 +1100 Subject: [PATCH] code clean up, created CreateFSLocation() and refactored code to use it --- pa_job_manager.py | 83 +++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/pa_job_manager.py b/pa_job_manager.py index e1a04c6..f99890e 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -944,21 +944,21 @@ def RestoreFile(job,restore_me): if len(new_rel_path) > 0 and new_rel_path[0] == '/': new_rel_path=new_rel_path[1:] - # path (think Import/Dir1/Dir2) which b/c we have orig_path in AddDir will - # create static/Import, static/Import/Dir1, static/Import/Dir1/Dir2 - part_rel_path="" - for dirname in new_rel_path.split("/"): - part_rel_path += f"{dirname}" - new_dir=AddDir( job, dirname, parent_dir, part_rel_path, orig_path ) - parent_dir=new_dir - part_rel_path += "/" + # okay, go through new relative path and AddDir any missing subdirs of this + new_dir=CreateFSLocation( job, orig_path, new_rel_path ) + + # keep orig parent dir, in recycle bin, if this restore moves last file out, then its now empty and can be deleted + orig_parent_dir_e = session.query(Entry).get(restore_me.in_dir.eid) + # reset restored file into its new dir restore_me.in_dir = new_dir AddLogForJob(job, f"Restored file: {restore_me.name} to {os.path.dirname(restore_me.FullPathOnFS())}" ) - ### when restoring, an original dir tree might have been removed, so need make it (if needed) - os.makedirs( os.path.dirname(restore_me.FullPathOnFS()),mode=0o777, exist_ok=True ) + # remove DelFile entry for this restored file session.query(DelFile).filter(DelFile.file_eid==restore_me.id).delete() + + # remove if now empty + CleanUpDirInDB(job, orig_parent_dir_e) session.commit() return @@ -995,20 +995,13 @@ def MoveFileToRecycleBin(job,del_me): new_rel_path += '/' + del_me.in_dir.rel_path # okay, go through new relative path and AddDir any missing subdirs of this - # path (think Import/Dir1/Dir2) which b/c we have bin_path in AddDir will - # create Bin/Import, Bin/Import/Dir1, Bin/Import/Dir1/Dir2 - part_rel_path="" - for dirname in new_rel_path.split("/"): - part_rel_path += f"{dirname}" - new_dir=AddDir( job, dirname, parent_dir, part_rel_path, bin_path ) - parent_dir=new_dir - part_rel_path += "/" + new_dir=CreateFSLocation( job, bin_path, new_rel_path ) # keep original parent_dir Entry before we 'move' it, as we need to see if its now empty, if so del it - parent_dir_e = session.query(Entry).get(del_me.in_dir.eid) + orig_parent_dir_e = session.query(Entry).get(del_me.in_dir.eid) del_me.in_dir = new_dir AddLogForJob(job, f"Deleted file: {del_me.name} - (moved to {os.path.dirname(del_me.FullPathOnFS())})" ) - CleanUpDirInDB(job, parent_dir_e) + CleanUpDirInDB(job, orig_parent_dir_e) return @@ -1053,16 +1046,11 @@ def MoveEntriesToOtherFolder(job, move_me, dst_storage_path, dst_rel_path): # we use the new path to this new Dir with the full location (the old dir is put into the new location) ResetAnySubdirPaths( move_me, dst_storage_path, move_me.dir_details.rel_path ) else: - # scen 3 -- effectively renaming a dir - # we have a new path (dst_rel_path)... The last component of that is what we will rename move_me to. - # however, it couild be a path of existing/new or a combo up to the rename part, so use AddDir to find/create - # as needed to then "move" (rename) move_me to into the full dst_rel_path - part_rel_path="" - for dirname in os.path.dirname(dst_rel_path).split("/"): - part_rel_path += f"{dirname}" - parent_dir=AddDir( job, dirname, parent_dir, part_rel_path, dst_storage_path ) - part_rel_path += "/" - move_me.in_dir = parent_dir + # scen 4: rename dir -- as the last component of dst_rel_path is what we will rename move_me to, so dont create last bit (os.path.dirname), + # we will just change move_me into that last dir -> renaming the dir + dst_dir=CreateFSLocation( job, dst_storage_path, os.path.dirname(dst_rel_path) ) + + move_me.in_dir = dst_dir move_me.dir_details.rel_path = dst_rel_path move_me.name = os.path.basename(dst_rel_path) ResetAnySubdirPaths( move_me, dst_storage_path, dst_rel_path ) @@ -1070,25 +1058,8 @@ def MoveEntriesToOtherFolder(job, move_me, dst_storage_path, dst_rel_path): os.replace( orig_fs_pos, move_me.FullPathOnFS() ) return else: - # for a file, just get the top of the Path, and then we will use it to - # potentially make all the sub dirs for the new location - parent_dir=session.query(Dir).join(PathDirLink).join(Path).filter(Path.id==dst_storage_path.id).filter(Dir.rel_path=='').first() - - # - # TODO: this create Dirs in Path is everywhere, make it a function BUT - # also, the code above for the Dirs is very similar, could wrap them - # both with slightly different params and it would work... - # MAKE THIS A FUNCTION - # - part_rel_path="" - for dirname in dst_rel_path.split("/"): - part_rel_path += f"{dirname}" - parent_dir=AddDir( job, dirname, parent_dir, part_rel_path, dst_storage_path ) - part_rel_path += "/" - dst_dir=parent_dir - - # just make sure the Dir tree exists on the FS - os.makedirs( dst_storage_path.path_prefix + '/' + dst_rel_path, mode=0o777, exist_ok=True ) + # make (any needed) Dir for the new destination FS location + dst_dir=CreateFSLocation( job, dst_storage_path, dst_rel_path ) # check for duplicate name? (scen 2) e=session.query(Entry).join(EntryDirLink).join(Dir).filter(Entry.name==move_me.name,Dir.eid==dst_dir.eid).first() @@ -1110,6 +1081,20 @@ def MoveEntriesToOtherFolder(job, move_me, dst_storage_path, dst_rel_path): CleanUpDirInDB(job, old_dir) return +#################################################################################################################################### +# For a job, take a location in a path, traverse it making any new Dir's in the +# DB, and then make the dirs on the file system +#################################################################################################################################### +def CreateFSLocation( job, dst_path, dst_locn ): + part_rel_path="" + parent_dir=session.query(Dir).join(PathDirLink).join(Path).filter(Path.id==dst_path.id).filter(Dir.rel_path=='').first() + for dirname in dst_locn.split("/"): + part_rel_path += f"{dirname}" + parent_dir=AddDir( job, dirname, parent_dir, part_rel_path, dst_path ) + part_rel_path += "/" + os.makedirs( dst_path.path_prefix + '/' + dst_locn, mode=0o777, exist_ok=True ) + return parent_dir + #################################################################################################################################### # take a dir that is being moved, and reset its own and any sub dirs rel_paths,