first cut of paths actually working with folder viewing - for now defaulting on for both Storage only

pa_job_manager.py

@@ -12,6 +12,10 @@
 
 # pylint: disable=no-member
 
+
+# global debug setting
+DEBUG=1
+
 ### SQLALCHEMY IMPORTS ###
 
 from sqlalchemy.ext.declarative import declarative_base
@@ -49,8 +53,6 @@ import re
 import sys
 
 
-# global debug setting
-DEBUG=1
 # this is required to handle the duplicate processing code
 sys.setrecursionlimit(50000)
 
@@ -300,8 +302,9 @@ def ProcessImportDirs(parent_job):
 def JobsForPaths( parent_job, paths, ptype ):
     now=datetime.now(pytz.utc)
     # make new set of Jobs per path... HandleJobs will make them run later
+    path_type = session.query(PathType).get(ptype)
     for path in paths:
-        p=session.query(Path).filter(Path.path_prefix==SymlinkName(path,path+'/')).first()
+        p=session.query(Path).filter(Path.path_prefix==SymlinkName(path_type.name,path,path+'/')).first()
         cfn=0
         if p:
             cfn=p.num_files
@@ -474,9 +477,12 @@ def JobForceScan(job):
     return
 
 # to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists
-def CreateSymlink(job,path):
-    symlink=SymlinkName(path, path)
+def CreateSymlink(job,ptype,path):
+    path_type = session.query(PathType).get(ptype)
+    symlink=SymlinkName(path_type.name, path, path)
     if not os.path.exists(symlink):
+        print( f"symlink does not exist - but is it missing path_type after static? s={symlink}" )
+        os.makedirs( os.path.dirname(symlink), mode=0o777, exist_ok=True )
         os.symlink(path, symlink)
     return symlink
 
@@ -584,21 +590,33 @@ def GetDateFromFile(file, stat):
     woy=c[1]
     return year, month, day, woy
 
+def AddJexToDependantJobs(job,name,value):
+    if DEBUG==1:
+        print( f"DEBUG: AddJexToDependantJobs({job}, {name}, {value}) ")
+    for j in session.query(Job).filter(Job.wait_for==job.id).all():
+        print( f"DEBUG: adding jex to this job.id == {j.id}" )
+        jex=JobExtra( name=name, value=value )
+        j.extra.append(jex)
+        AddJexToDependantJobs(j, name, value)
+    return
+
 def JobImportDir(job):
     JobProgressState( job, "In Progress" )
     settings = session.query(Settings).first()
     path=[jex.value for jex in job.extra if jex.name == "path"][0]
     path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0]
-    AddLogForJob(job, f"Checking 'path_type' Directory: {path}" )
+    AddLogForJob(job, f"Checking {path_type} Directory: {path}" )
     if DEBUG==1:
         print("DEBUG: Checking Directory: {}".format( path ) )
     if not os.path.exists( path ):
-        FinishJob( job, "Finished Importing: {} -- Path does not exist".format( path), "Failed" )
+        FinishJob( job, f"Finished Importing: {path} -- Path does not exist", "Failed" )
         return
-    symlink=CreateSymlink(job,path)
+    symlink=CreateSymlink(job,path_type,path)
 
     path_obj=Path( path_prefix=symlink, num_files=0, type_id=path_type )
     session.add(path_obj)
+    # find all jobs waiting on me and their children, etc. and add a path_prefix jex to symlink, so we can just reference it form here on in, rather than recreate that string
+    AddJexToDependantJobs(job,"path_prefix",symlink)
     ResetExistsOnFS(job, symlink)
 
     walk=os.walk(path, topdown=True)
@@ -623,7 +641,7 @@ def JobImportDir(job):
     for root, subdirs, files in ftree:
         # already create root above to work out num_files for whole os.walk
         if root != path:
-            pp=SymlinkName( path, root )+'/'+os.path.basename(root)
+            pp=SymlinkName( path_obj.type.name, path, root )+'/'+os.path.basename(root)
             print( F"pp={pp}, root={root}, symlink={symlink}" )
             rel_path=pp.replace(symlink+'/','')
             dir=AddDir(job, os.path.basename(root), parent_dir, rel_path, path_obj)
@@ -672,7 +690,8 @@ def RunFuncOnFilesInPath( job, path, file_func ):
 
 def JobProcessAI(job):
     path=[jex.value for jex in job.extra if jex.name == "path"][0]
-    path = SymlinkName(path, '/')
+    path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0]
+    path = SymlinkName(path_prefix, path, '/')
     p = session.query(Path).filter(Path.path_prefix==path).first()
     job.num_files=p.num_files
 
@@ -815,15 +834,18 @@ def ProcessFilesInDir(job, e, file_func):
 
 def JobGetFileDetails(job):
     JobProgressState( job, "In Progress" )
+    #### I think the fix here is to get JobImportDir (or whatever makes the PATH) to add a jex for path_prefix and just pull it here, and stop 're-creating' it via SymlinkName
     path=[jex.value for jex in job.extra if jex.name == "path"][0]
-    path=SymlinkName( path, path )
+    path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0]
+    print( f"JobGetFileDetails({job}) -- pp={path_prefix}" )
+#    path=SymlinkName( path_prefix, path, path )
     if DEBUG==1:
-        print("DEBUG: JobGetFileDetails for path={}".format( path ) )
-    p=session.query(Path).filter(Path.path_prefix==path).first()
+        print("DEBUG: JobGetFileDetails for path={}".format( path_prefix ) )
+    p=session.query(Path).filter(Path.path_prefix==path_prefix).first()
     job.current_file_num = 0
     job.num_files = p.num_files
     session.commit()
-    RunFuncOnFilesInPath( job, path, GenHashAndThumb )
+    RunFuncOnFilesInPath( job, path_prefix, GenHashAndThumb )
     FinishJob(job, "File Details job finished")
     session.commit()
     return