ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix BUG-66: deleting files out from under and in general CleanUpDirInDB needs to use Entry not Dir

Browse Source
This commit is contained in:
Damien De Paoli
2021-09-25 12:11:13 +10:00
parent 0b22072f53
commit 5abd01bb4d
3 changed files with 25 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
BUGs
View File

@@ -5,11 +5,3 @@ BUG-61: in Fullscreen mode and next/prev occasionally dropped out of FS
this is just another consequence of going beyond Pagesize, when we get new entries from DB, it loses FS flag this is just another consequence of going beyond Pagesize, when we get new entries from DB, it loses FS flag
BUG-64: seems when we move files, they get a new? FILE entry -- the file, hash, thumb, faces, etc. should all still be connected to the FILE entry and just make it have a new home / new location... BUG-64: seems when we move files, they get a new? FILE entry -- the file, hash, thumb, faces, etc. should all still be connected to the FILE entry and just make it have a new home / new location...
- this is both with a move via GUI, and theoretically also when we find a moved file on the FS (could use hash check to validate if we can just keep connections) - this is both with a move via GUI, and theoretically also when we find a moved file on the FS (could use hash check to validate if we can just keep connections)
BUG-66: if I delete last image from a folder, OR, move the last image from a folder we don't try to clean it up? ALSO need to test when we do clean one up that theoretically has an empty parent after its deleted, etc.
-- deleted .xvpics in PROD, and its borked / cant import Camera Uploads at present
TEST:
Y - folder1/folder2/duplicate <- remove duplicate via rmdups
folder1/folder2/image <- remove via deleting image on underlying FS
folder1/folder2/image <- remove via deleting folder2/* on underlying FS
Do I want this functionality, it could remove the Camera-Uploads dir right? - folder1/folder2/image <- remove via move image to new folder
Do I want this functionality, it could remove the Camera-Uploads dir right? - folder1/folder2/image <- remove via deleting image in PA

2
TODO
View File

@@ -1,4 +1,6 @@
## GENERAL ## GENERAL
* close button on invalid password should look like danger/alert/close for jobs
* remember last import dir, so you can just go straight back to it * remember last import dir, so you can just go straight back to it
* maybe strip unnecessary / at end of directory name. i think i have left behind empty folders, e.g. check switzerland and austria * maybe strip unnecessary / at end of directory name. i think i have left behind empty folders, e.g. check switzerland and austria

44
pa_job_manager.py
View File

@@ -854,17 +854,17 @@ def ResetExistsOnFS(job, path):
#################################################################################################################################### ####################################################################################################################################
def RemoveEmtpyDirFromFS( job, del_me ): def RemoveEmtpyDirFromFS( job, del_me ):
try: try:
os.rmdir( del_me.PathOnFS() ) os.rmdir( del_me.FullPathOnFS() )
except Exception as e: except Exception as e:
print( f"ERROR: Failed to remove dir from filesystem - which={del_me.PathOnFS()}, err: {e}") print( f"ERROR: Failed to remove dir from filesystem - which={del_me.FullPathOnFS()}, err: {e}")
return return
def RemoveEmptyDirFromDB( job, del_me ): def RemoveEmptyDirFromDB( job, del_me ):
session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.eid).delete() session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.id).delete()
session.query(PathDirLink).filter(PathDirLink.dir_eid==del_me.eid).delete() session.query(PathDirLink).filter(PathDirLink.dir_eid==del_me.id).delete()
session.query(Dir).filter(Dir.eid==del_me.eid).delete() session.query(Dir).filter(Dir.eid==del_me.id).delete()
session.query(Entry).filter(Entry.id==del_me.eid).delete() session.query(Entry).filter(Entry.id==del_me.id).delete()
AddLogForJob( job, f"INFO: Removing {del_me.PathOnFS()} from system as removing duplicates has left it empty" ) AddLogForJob( job, f"INFO: Removing {del_me.FullPathOnFS()} from system as removing duplicates has left it empty" )
return return
#################################################################################################################################### ####################################################################################################################################
@@ -872,22 +872,22 @@ def RemoveEmptyDirFromDB( job, del_me ):
# this func will delete it and check that the parent dir is non-empty recursively, so this could trigger a cascading deletion of # this func will delete it and check that the parent dir is non-empty recursively, so this could trigger a cascading deletion of
# empty dirs in a hierachy, if the entry deletion leaves a dir with content, just finish # empty dirs in a hierachy, if the entry deletion leaves a dir with content, just finish
#################################################################################################################################### ####################################################################################################################################
def CleanUpDirInDB(job, d): def CleanUpDirInDB(job, e):
session.commit() session.commit()
print( f"CleanUpDirInDB(): checking dir: {d.PathOnFS()} ({d.eid})" ) print( f"CleanUpDirInDB(): checking dir: {e.FullPathOnFS()} ({e.id})" )
content = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==d.eid).first() content = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==e.id).first()
if not content: if not content:
print( f" Dir {d.PathOnFS()} - {d.eid} is empty - removing it" ) print( f" Dir {e.FullPathOnFS()} - {e.id} is empty - removing it" )
# get an Entry from DB (in_dir is a Dir)
parent_dir = session.query(Entry).get(d.in_dir.eid)
# okay remove this empty dir # okay remove this empty dir
RemoveEmtpyDirFromFS( job, d ) RemoveEmtpyDirFromFS( job, e )
RemoveEmptyDirFromDB( job, d) RemoveEmptyDirFromDB( job, e )
print( f" Dir {d.PathOnFS()} is in {parent_dir.PathOnFS()} ({parent_dir.eid}) -> check next" ) # get an Entry from DB (in_dir is a Dir/we need the ORM entry for code to work)
parent_dir = session.query(Entry).get(e.in_dir.eid)
print( f" Dir {e.FullPathOnFS()} is in {parent_dir.FullPathOnFS()} ({parent_dir.id}) -> check next" )
# check to see if removing the empty dir has left the parent dir empty # check to see if removing the empty dir has left the parent dir empty
CleanUpDirInDB(job, parent_dir) CleanUpDirInDB(job, parent_dir)
else: else:
print( f"There is content (first entry: {content.name}) in {d.PathOnFS()} - finished for this dir" ) print( f"There is content (first entry: {content.name}) in {e.FullPathOnFS()} - finished for this dir" )
return return
#################################################################################################################################### ####################################################################################################################################
@@ -895,7 +895,7 @@ def CleanUpDirInDB(job, d):
# used when scanning and a file has been removed out from under PA, or when we remove duplicates # used when scanning and a file has been removed out from under PA, or when we remove duplicates
#################################################################################################################################### ####################################################################################################################################
def RemoveFileFromDB(job, del_me): def RemoveFileFromDB(job, del_me):
parent_dir=del_me.in_dir parent_dir_e=session.query(Entry).get(del_me.in_dir.eid)
session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.id).delete() session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.id).delete()
connected_faces=session.query(FaceFileLink).filter(FaceFileLink.file_eid==del_me.id).all() connected_faces=session.query(FaceFileLink).filter(FaceFileLink.file_eid==del_me.id).all()
for ffl in connected_faces: for ffl in connected_faces:
@@ -904,7 +904,7 @@ def RemoveFileFromDB(job, del_me):
session.query(File).filter(File.eid==del_me.id).delete() session.query(File).filter(File.eid==del_me.id).delete()
session.query(Entry).filter(Entry.id==del_me.id).delete() session.query(Entry).filter(Entry.id==del_me.id).delete()
AddLogForJob( job, f"INFO: Removing {del_me.name} from system as it is no longer on the file system") AddLogForJob( job, f"INFO: Removing {del_me.name} from system as it is no longer on the file system")
CleanUpDirInDB(job, parent_dir) CleanUpDirInDB(job, parent_dir_e)
return return
#################################################################################################################################### ####################################################################################################################################
@@ -998,9 +998,11 @@ def MoveFileToRecycleBin(job,del_me):
parent_dir=new_dir parent_dir=new_dir
part_rel_path += "/" part_rel_path += "/"
# keep original parent_dir Entry before we 'move' it, as we need to see if its now empty, if so del it
parent_dir_e = session.query(Entry).get(del_me.in_dir.eid)
del_me.in_dir = new_dir del_me.in_dir = new_dir
AddLogForJob(job, f"Deleted file: {del_me.name} - (moved to {os.path.dirname(del_me.FullPathOnFS())})" ) AddLogForJob(job, f"Deleted file: {del_me.name} - (moved to {os.path.dirname(del_me.FullPathOnFS())})" )
CleanUpDirInDB(job, parent_dir) CleanUpDirInDB(job, parent_dir_e)
return return
#################################################################################################################################### ####################################################################################################################################
@@ -1356,7 +1358,7 @@ def JobGetFileDetails(job):
path=[jex.value for jex in job.extra if jex.name == "path"][0] path=[jex.value for jex in job.extra if jex.name == "path"][0]
path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0] path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0]
if DEBUG: if DEBUG:
print("DEBUG: JobGetFileDetails for path={path_prefix}" ) print( f"DEBUG: JobGetFileDetails for path={path_prefix}" )
p=session.query(Path).filter(Path.path_prefix==path_prefix).first() p=session.query(Path).filter(Path.path_prefix==path_prefix).first()
job.current_file_num = 0 job.current_file_num = 0
job.num_files = p.num_files job.num_files = p.num_files