fixed bugs 47, 48, 49 and reduced complexity of dup counting as well. Now removes dirs than become empty due to duplicate removals

pa_job_manager.py

@@ -815,25 +815,55 @@ def ResetExistsOnFS(job, path):
 # Convenience function to remove a file from the database - and its associated links
 # used when scanning and a file has been removed out from under PA, or when we remove duplicates
 ####################################################################################################################################
-def RemoveFileFromDB(id):
-    session.query(EntryDirLink).filter(EntryDirLink.entry_id==id).delete()
-    session.query(File).filter(File.eid==id).delete()
-    session.query(Entry).filter(Entry.id==id).delete()
+def RemoveEmtpyDirFromFS( job, del_me ):
+    try:
+        os.rmdir( del_me.FullPathOnFS() )
+    except Exception as e:
+        print( f"ERROR: Failed to remove file from filesystem - which={del_me.name}, err: {e}")
+    return
+
+def RemoveEmptyDirFromDB( job, del_me ):
+    session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.id).delete()
+    session.query(PathDirLink).filter(PathDirLink.dir_eid==del_me.id).delete()
+    session.query(Dir).filter(Dir.eid==del_me.id).delete()
+    session.query(Entry).filter(Entry.id==del_me.id).delete()
+    AddLogForJob( job, f"INFO: Removing {del_me.name} from system as removing duplicates has left it empty" )
     return
 
 ####################################################################################################################################
-# Actually moves the physical file from its current real directory to a subdir of the recycle bin path
+# Convenience function called after we delete an entry from a DB (always starts as a file deletion), but if we have an empty dir
+# this func will delete it and check that the parent dir is non-empty recursively, so this could trigger a cascading deletion of 
+# empty dirs in a hierachy, if the entry deletion leaves a dir with content, just finish
 ####################################################################################################################################
-def RemoveFileFromFS( del_me ):
-    try:
-        settings = session.query(Settings).first()
-        dst_dir=settings.recycle_bin_path + '/' + del_me.in_dir.in_path.path_prefix.replace('static/','') + '/' + del_me.in_dir.rel_path + '/'
-        os.makedirs( dst_dir,mode=0o777, exist_ok=True )
-        src=del_me.FullPathOnFS()
-        dst=dst_dir + '/' + del_me.name
-        os.replace( src, dst )
-    except Exception as e:
-        print( f"ERROR: Failed to remove file from filesystem - which={src}, err: {e}")
+def CleanUpDirInDB(job, d):
+    session.commit()
+    print( f"CleanUpDirInDB(): checking dir: {d.name} ({d.id})" )
+    content = session.query(Entry).join(EntryDirLink).filter(EntryDirLink.dir_eid==d.id).first()
+    if not content:
+        print( f"  Dir {d.name} - {d.id} is empty - removing it" )
+        # get an Entry from DB (in_dir is a Dir)
+        parent_dir = session.query(Entry).get(d.in_dir.eid)
+        # okay remove this empty dir
+        RemoveEmtpyDirFromFS( job, d )
+        RemoveEmptyDirFromDB( job, d)
+        print( f"  Dir {d.name} is in {parent_dir.name} ({parent_dir.id}) -> check next" )
+        # check to see if removing the empty dir has left the parent dir empty
+        CleanUpDirInDB(job, parent_dir)
+    else:
+        print( f"There is content (first entry: {content.name}) in {d.name} - finished for this dir" )
+    return
+
+####################################################################################################################################
+# Convenience function to remove a file from the database - and its associated links
+# used when scanning and a file has been removed out from under PA, or when we remove duplicates
+####################################################################################################################################
+def RemoveFileFromDB(job, del_me):
+    parent_dir=del_me.in_dir
+    session.query(EntryDirLink).filter(EntryDirLink.entry_id==del_me.id).delete()
+    session.query(File).filter(File.eid==del_me.id).delete()
+    session.query(Entry).filter(Entry.id==del_me.id).delete()
+    AddLogForJob( job, f"INFO: Removing {rm.name} from system as it is no longer on the file system")
+    CleanUpDirInDB(job, parent_dir)
     return
 
 ####################################################################################################################################
@@ -927,8 +957,10 @@ def MoveFileToRecycleBin(job,del_me):
         parent_dir=new_dir
         part_rel_path += "/"
     
+    parent_dir = session.query(Entry).get(del_me.in_dir.eid)
     del_me.in_dir = new_dir
     AddLogForJob(job, f"Deleted file: {del_me.name} - (moved to {os.path.dirname(del_me.FullPathOnFS())})" )
+    CleanUpDirInDB(job, parent_dir)
     return
 
 ####################################################################################################################################
@@ -986,14 +1018,12 @@ def HandleAnyFSDeletions(job):
     rms = session.query(Entry).filter(Entry.exists_on_fs==False,Entry.type_id!=dtype.id).all()
     rm_cnt=0
     for rm in rms:
-        RemoveFileFromDB(rm.id)
-        AddLogForJob( job, f"INFO: Removing {rm.name} from system as it is no longer on the file system")
+        RemoveFileFromDB(job, rm)
         rm_cnt+=1
 
     rmdirs = session.query(Entry).filter(Entry.exists_on_fs==False,Entry.type_id==1).order_by(Entry.id.desc()).all()
     for rmdir in rmdirs:
-        RemoveFileFromDB(rmdir.id)
-        AddLogForJob( job, f"INFO: Removing {rmdir.name} from system as it is no longer on the file system")
+        RemoveFileFromDB(job, rmdir)
         rm_cnt+=1
     return rm_cnt
 
@@ -1542,7 +1572,7 @@ def InitialValidationChecks():
             rbp_exists=1
             ptype = session.query(PathType).filter(PathType.name=='Bin').first().id
             symlink=CreateSymlink(job,ptype,path)
-            path, dirs, files = next(os.walk(path))
+            root, dirs, files = next(os.walk(path))
             if len(dirs) + len(files) > 0:
                 AddLogForJob(job, "INFO: the bin path contains content, cannot process to know where original deletes were form - skipping content!" )
                 AddLogForJob(job, "TODO: could be smart about what is known in the DB vs on the FS, and change below to an ERROR if it is one")