added code to split out storage_path and import_path, still a bit clunky, but functional. SymlinkName also now in shared

files.py

@@ -23,6 +23,7 @@ from job import Job, JobExtra, Joblog, NewJob
 from person import Person, PersonRefimgLink
 from refimg import Refimg
 from settings import Settings
+from shared import SymlinkName
 
 ################################################################################
 # Class describing File in the database, and via sqlalchemy, connected to the DB as well
@@ -90,7 +91,7 @@ class FileType(db.Model):
 
 def ViewingOptions( request ):
     noo="Oldest"
-    grouping="Day"
+    grouping="None"
     how_many="50"
     offset=0
     size=128
@@ -113,24 +114,52 @@ def ViewingOptions( request ):
 ################################################################################
 # /file_list -> show detailed file list of files from import_path(s)
 ################################################################################
-@app.route("/file_list", methods=["GET"])
+@app.route("/file_list_ip", methods=["GET"])
-def file_list():
+def file_list_ip():
-    return render_template("file_list.html", page_title='View Files (details)', entry_data=Entry.query.order_by(Entry.name).all())
+    return render_template("file_list.html", page_title='View File Details (Import Path)', entry_data=Entry.query.order_by(Entry.name).all())
 
 ################################################################################
 # /files -> show thumbnail view of files from import_path(s)
 ################################################################################
-@app.route("/files", methods=["GET", "POST"])
+@app.route("/files_ip", methods=["GET", "POST"])
-def files():
+def files_ip():
 
     noo, grouping, how_many, offset, size = ViewingOptions( request )
     entries=[]
-    if noo == "Oldest":
-        entries=Entry.query.join(File).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all()
-    else:
-        entries=Entry.query.join(File).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all()
-    return render_template("files.html", page_title='View Files', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size )
 
+    # per import path, add entries to view 
+    settings=Settings.query.first()
+    paths = settings.import_path.split("#")
+    for path in paths:
+        prefix = SymlinkName(path,path+'/')
+
+        if noo == "Oldest":
+            entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all()
+        else:
+            entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all()
+
+    return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size )
+
+################################################################################
+# /files -> show thumbnail view of files from storage_path
+################################################################################
+@app.route("/files_sp", methods=["GET", "POST"])
+def files_sp():
+    noo, grouping, how_many, offset, size = ViewingOptions( request )
+    entries=[]
+
+    # per storage path, add entries to view 
+    settings=Settings.query.first()
+    paths = settings.storage_path.split("#")
+    for path in paths:
+        prefix = SymlinkName(path,path+'/')
+
+        if noo == "Oldest":
+            entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all()
+        else:
+            entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all()
+
+    return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size )
 
 ################################################################################
 # /search -> show thumbnail view of files from import_path(s)
@@ -167,6 +196,16 @@ def forcescan():
     st.SetMessage("force scan & rebuild data for files in:&nbsp;<a href=/job/{}>Job #{}</a>&nbsp;(Click the link to follow progress)".format( job.id, job.id) )
     return render_template("base.html")
 
+################################################################################
+# /files/scan_sp -> allows us to force a check for new files
+################################################################################
+@app.route("/files/scan_sp", methods=["GET"])
+def scan_sp():
+    job=NewJob("scan_sp" )
+    st.SetAlert("success")
+    st.SetMessage("scanning for new files in:&nbsp;<a href=/job/{}>Job #{}</a>&nbsp;(Click the link to follow progress)".format( job.id, job.id) )
+    return render_template("base.html")
+
 
 def TrimmedPath( prefix, path ):
     return path.replace(prefix, '' )

pa_job_manager.py

@@ -25,7 +25,7 @@ from sqlalchemy.orm import scoped_session
 
 ### LOCAL FILE IMPORTS ###
 
-from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE
+from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE, SymlinkName
 from datetime import datetime, timedelta, date
 import pytz
 import time
@@ -138,6 +138,7 @@ class Settings(Base):
     __tablename__ = "settings"
     id = Column(Integer, Sequence('settings_id_seq'), primary_key=True )
     import_path = Column(String)
+    storage_path = Column(String)
 
     def __repr__(self):
         return f"<id: {self.id}, import_path: {self.import_path}>"
@@ -235,11 +236,24 @@ def MessageToFE( job_id, alert, message ):
     session.commit()
     return msg.id
 
+def ProcessStorageDirs(parent_job):
+    settings = session.query(Settings).first()
+    if settings == None:
+        raise Exception("Cannot create file data with no settings / import path is missing")
+    paths = settings.storage_path.split("#")
+    JobsForPaths( parent_job, paths )
+    return
+
 def ProcessImportDirs(parent_job):
     settings = session.query(Settings).first()
     if settings == None:
         raise Exception("Cannot create file data with no settings / import path is missing")
     paths = settings.import_path.split("#")
+    JobsForPaths( parent_job, paths )
+    return
+
+def JobsForPaths( parent_job, paths ):
+
     now=datetime.now(pytz.utc)
     # make new set of Jobs per path... HandleJobs will make them run later
     for path in paths:
@@ -303,6 +317,8 @@ def RunJob(job):
         JobScanNow(job)
     elif job.name =="forcescan":
         JobForceScan(job)
+    elif job.name =="scan_sp":
+        JobScanStorageDir(job)
     elif job.name =="importdir":
         JobImportDir(job)
     elif job.name =="getfiledetails":
@@ -318,7 +334,7 @@ def RunJob(job):
     # okay, we finished a job, so check for any jobs that are dependant on this and run them...
 #    session.close()
     if job.pa_job_state != "Completed":
-        FinishJob(job, "PA Job Manager - This is a catchall to close of a Job, this sould never be seen and implies a job did not actually complete?", "Failed" )
+        FinishJob(job, "PA Job Manager - This is a catchall to close of a Job, this should never be seen and implies a job did not actually complete?", "Failed" )
     HandleJobs()
     return
 
@@ -386,6 +402,14 @@ def JobScanNow(job):
     session.commit()
     return
 
+def JobScanStorageDir(job):
+    JobProgressState( job, "In Progress" )
+    ProcessStorageDirs(job)
+    FinishJob( job, "Completed (scan for new files)" )
+    MessageToFE( job.id, "success", "Completed (scan for new files)" )
+    session.commit()
+    return
+
 def JobForceScan(job):
     JobProgressState( job, "In Progress" )
     session.query(FileRefimgLink).delete()
@@ -524,11 +548,11 @@ def JobImportDir(job):
     JobProgressState( job, "In Progress" )
     settings = session.query(Settings).first()
     if settings == None:
-        raise Exception("Cannot create file data with no settings / import path is missing")
+        raise Exception("Cannot create file data with no settings / paths missing")
     path=[jex.value for jex in job.extra if jex.name == "path"][0]
-    AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
+    AddLogForJob(job, "Checking Directory: {}".format( path ) )
     if DEBUG==1:
-        print("DEBUG: Checking Import Directory: {}".format( path ) )
+        print("DEBUG: Checking Directory: {}".format( path ) )
     if not os.path.exists( path ):
         FinishJob( job, "Finished Importing: {} -- Path does not exist".format( path), "Failed" )
         return
@@ -630,6 +654,7 @@ def GenHashAndThumb(job, e):
         return
 
     e.file_details[0].hash = md5( job, e.in_dir[0].path_prefix+'/'+ e.name )
+    print( e )
     if e.type.name == 'Image':
         e.file_details[0].thumbnail = GenImageThumbnail( job, e.in_dir[0].path_prefix+'/'+ e.name )
     elif e.type.name == 'Video':

templates/base.html

@@ -49,8 +49,9 @@
 				<div class="nav-item dropdown">
 					<a class="nav-item dropdown nav-link dropdown-toggle" href="#" id="FileMenu" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Files</a>
 					<div class="dropdown-menu" aria-labelledby="FileMenu">
-						<a class="dropdown-item" href="{{url_for('files')}}">View</a>
+						<a class="dropdown-item" href="{{url_for('files_ip')}}">View Photos to Import</a>
-						<a class="dropdown-item" href="{{url_for('file_list')}}">View Details</a>
+						<a class="dropdown-item" href="{{url_for('file_list_ip')}}">View Details of Photos to Import</a>
+						<a class="dropdown-item" href="{{url_for('files_sp')}}">View Stored Photos</a>
 					</div>
 				</div class="nav-item dropdown">
 				<div class="nav-item dropdown">
@@ -79,6 +80,7 @@
 						<a class="dropdown-item" href="{{url_for('settings')}}">Edit Settings</a>
 						<a class="dropdown-item" href="{{url_for('scannow')}}">Scan now (for new files)</a>
 						<a class="dropdown-item" href="{{url_for('forcescan')}}">Force Scan (delete data & rebuild)</a>
+						<a class="dropdown-item" href="{{url_for('scan_sp')}}">Scan Storage Path</a>
 						<a class="dropdown-item" href="{{url_for('wakeup')}}">Force wake the job manage</a>
 					</div class="dropdow-menu">
 				</div class="nav-item dropdown">