fixed up dup code to work with paths, added path_types throughout and updated TODO to be clear on what next

pa_job_manager.py

@@ -73,6 +73,15 @@ Base = declarative_base()
 # Class describing File in the database, and via sqlalchemy, connected to the DB as well
 # This has to match one-for-one the DB table
 ################################################################################
+class PathType(Base):
+    __tablename__ = "path_type"
+    id = Column(Integer, Sequence('path_type_id_seq'), primary_key=True )
+    name = Column(String, unique=True, nullable=False )
+
+    def __repr__(self):
+        return f"<id: {self.id}, name={self.name}>"
+
+
 class PathDirLink(Base):
     __tablename__ = "path_dir_link"
     path_id = Column(Integer, ForeignKey("path.id"), primary_key=True )
@@ -92,6 +101,8 @@ class EntryDirLink(Base):
 class Path(Base):
     __tablename__ = "path"
     id = Column(Integer, Sequence('path_id_seq'), primary_key=True )
+    type_id = Column(Integer, ForeignKey("path_type.id"))
+    type = relationship("PathType")
     path_prefix = Column(String, unique=True, nullable=False )
     num_files = Column(Integer)
 
@@ -121,7 +132,6 @@ class Entry(Base):
     in_dir = relationship ("Dir", secondary="entry_dir_link", uselist=False )
 
     def FullPathOnFS(self):
-        print( f"(FullPathOnFS: pp={self.in_dir.in_path.path_prefix}, rp={self.in_dir.rel_path}, n={self.name}" )
         s=self.in_dir.in_path.path_prefix + '/' 
         if len(self.in_dir.rel_path) > 0:
             s += self.in_dir.rel_path + '/'
@@ -274,7 +284,8 @@ def ProcessStorageDirs(parent_job):
     if settings == None:
         raise Exception("Cannot create file data with no settings / import path is missing")
     paths = settings.storage_path.split("#")
-    JobsForPaths( parent_job, paths )
+    ptype = session.query(PathType).filter(PathType.name=='Storage').first().id
+    JobsForPaths( parent_job, paths, ptype )
     return
 
 def ProcessImportDirs(parent_job):
@@ -282,10 +293,11 @@ def ProcessImportDirs(parent_job):
     if settings == None:
         raise Exception("Cannot create file data with no settings / import path is missing")
     paths = settings.import_path.split("#")
-    JobsForPaths( parent_job, paths )
+    ptype = session.query(PathType).filter(PathType.name=='Import').first().id
+    JobsForPaths( parent_job, paths, ptype )
     return
 
-def JobsForPaths( parent_job, paths ):
+def JobsForPaths( parent_job, paths, ptype ):
     now=datetime.now(pytz.utc)
     # make new set of Jobs per path... HandleJobs will make them run later
     for path in paths:
@@ -295,8 +307,10 @@ def JobsForPaths( parent_job, paths ):
             cfn=p.num_files
             
         jex=JobExtra( name="path", value=path )
+        jex2=JobExtra( name="path_type", value=ptype )
         job=Job(start_time=now, last_update=now, name="importdir", state="New", wait_for=None, pa_job_state="New", current_file_num=0, num_files=cfn )
         job.extra.append(jex)
+        job.extra.append(jex2)
         session.add(job)
         session.commit()
         if parent_job:
@@ -573,10 +587,9 @@ def GetDateFromFile(file, stat):
 def JobImportDir(job):
     JobProgressState( job, "In Progress" )
     settings = session.query(Settings).first()
-    if settings == None:
-        raise Exception("Cannot create file data with no settings / paths missing")
     path=[jex.value for jex in job.extra if jex.name == "path"][0]
-    AddLogForJob(job, "Checking Directory: {}".format( path ) )
+    path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0]
+    AddLogForJob(job, f"Checking 'path_type' Directory: {path}" )
     if DEBUG==1:
         print("DEBUG: Checking Directory: {}".format( path ) )
     if not os.path.exists( path ):
@@ -584,7 +597,7 @@ def JobImportDir(job):
         return
     symlink=CreateSymlink(job,path)
 
-    path_obj=Path( path_prefix=symlink, num_files=0 )
+    path_obj=Path( path_prefix=symlink, num_files=0, type_id=path_type )
     session.add(path_obj)
     ResetExistsOnFS(job, symlink)
 
@@ -922,7 +935,6 @@ def RemoveDups(job):
     cd_jobs=session.query(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all()
     for j in cd_jobs:
         FinishJob(j, "Just removed duplicates - so no need to do any other checkdups, we will force 1 last one after the remove step", "Withdrawn")
-        print("here-loop")
     session.commit()
 
     dup_cnt=0