From 477bd6f099b20e0ed21cd163b8a0ce3afba04d6b Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 27 Feb 2021 18:30:54 +1100 Subject: [PATCH] added code to split out storage_path and import_path, still a bit clunky, but functional. SymlinkName also now in shared --- files.py | 61 +++++++++++++++++++++++++++++++++++++-------- pa_job_manager.py | 35 ++++++++++++++++++++++---- templates/base.html | 6 +++-- 3 files changed, 84 insertions(+), 18 deletions(-) diff --git a/files.py b/files.py index 886bd1b..19ad674 100644 --- a/files.py +++ b/files.py @@ -23,6 +23,7 @@ from job import Job, JobExtra, Joblog, NewJob from person import Person, PersonRefimgLink from refimg import Refimg from settings import Settings +from shared import SymlinkName ################################################################################ # Class describing File in the database, and via sqlalchemy, connected to the DB as well @@ -90,7 +91,7 @@ class FileType(db.Model): def ViewingOptions( request ): noo="Oldest" - grouping="Day" + grouping="None" how_many="50" offset=0 size=128 @@ -113,24 +114,52 @@ def ViewingOptions( request ): ################################################################################ # /file_list -> show detailed file list of files from import_path(s) ################################################################################ -@app.route("/file_list", methods=["GET"]) -def file_list(): - return render_template("file_list.html", page_title='View Files (details)', entry_data=Entry.query.order_by(Entry.name).all()) +@app.route("/file_list_ip", methods=["GET"]) +def file_list_ip(): + return render_template("file_list.html", page_title='View File Details (Import Path)', entry_data=Entry.query.order_by(Entry.name).all()) ################################################################################ # /files -> show thumbnail view of files from import_path(s) ################################################################################ -@app.route("/files", methods=["GET", "POST"]) -def files(): +@app.route("/files_ip", methods=["GET", "POST"]) +def files_ip(): noo, grouping, how_many, offset, size = ViewingOptions( request ) entries=[] - if noo == "Oldest": - entries=Entry.query.join(File).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all() - else: - entries=Entry.query.join(File).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - return render_template("files.html", page_title='View Files', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size ) + # per import path, add entries to view + settings=Settings.query.first() + paths = settings.import_path.split("#") + for path in paths: + prefix = SymlinkName(path,path+'/') + + if noo == "Oldest": + entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all() + else: + entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + + return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size ) + +################################################################################ +# /files -> show thumbnail view of files from storage_path +################################################################################ +@app.route("/files_sp", methods=["GET", "POST"]) +def files_sp(): + noo, grouping, how_many, offset, size = ViewingOptions( request ) + entries=[] + + # per storage path, add entries to view + settings=Settings.query.first() + paths = settings.storage_path.split("#") + for path in paths: + prefix = SymlinkName(path,path+'/') + + if noo == "Oldest": + entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all() + else: + entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).filter(Dir.path_prefix.like(prefix+'%')).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() + + return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size ) ################################################################################ # /search -> show thumbnail view of files from import_path(s) @@ -167,6 +196,16 @@ def forcescan(): st.SetMessage("force scan & rebuild data for files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) ) return render_template("base.html") +################################################################################ +# /files/scan_sp -> allows us to force a check for new files +################################################################################ +@app.route("/files/scan_sp", methods=["GET"]) +def scan_sp(): + job=NewJob("scan_sp" ) + st.SetAlert("success") + st.SetMessage("scanning for new files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) ) + return render_template("base.html") + def TrimmedPath( prefix, path ): return path.replace(prefix, '' ) diff --git a/pa_job_manager.py b/pa_job_manager.py index fa94f77..6d0d0de 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -25,7 +25,7 @@ from sqlalchemy.orm import scoped_session ### LOCAL FILE IMPORTS ### -from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE +from shared import DB_URL, PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT, THUMBSIZE, SymlinkName from datetime import datetime, timedelta, date import pytz import time @@ -138,6 +138,7 @@ class Settings(Base): __tablename__ = "settings" id = Column(Integer, Sequence('settings_id_seq'), primary_key=True ) import_path = Column(String) + storage_path = Column(String) def __repr__(self): return f"" @@ -235,11 +236,24 @@ def MessageToFE( job_id, alert, message ): session.commit() return msg.id +def ProcessStorageDirs(parent_job): + settings = session.query(Settings).first() + if settings == None: + raise Exception("Cannot create file data with no settings / import path is missing") + paths = settings.storage_path.split("#") + JobsForPaths( parent_job, paths ) + return + def ProcessImportDirs(parent_job): settings = session.query(Settings).first() if settings == None: raise Exception("Cannot create file data with no settings / import path is missing") paths = settings.import_path.split("#") + JobsForPaths( parent_job, paths ) + return + +def JobsForPaths( parent_job, paths ): + now=datetime.now(pytz.utc) # make new set of Jobs per path... HandleJobs will make them run later for path in paths: @@ -303,6 +317,8 @@ def RunJob(job): JobScanNow(job) elif job.name =="forcescan": JobForceScan(job) + elif job.name =="scan_sp": + JobScanStorageDir(job) elif job.name =="importdir": JobImportDir(job) elif job.name =="getfiledetails": @@ -318,7 +334,7 @@ def RunJob(job): # okay, we finished a job, so check for any jobs that are dependant on this and run them... # session.close() if job.pa_job_state != "Completed": - FinishJob(job, "PA Job Manager - This is a catchall to close of a Job, this sould never be seen and implies a job did not actually complete?", "Failed" ) + FinishJob(job, "PA Job Manager - This is a catchall to close of a Job, this should never be seen and implies a job did not actually complete?", "Failed" ) HandleJobs() return @@ -386,6 +402,14 @@ def JobScanNow(job): session.commit() return +def JobScanStorageDir(job): + JobProgressState( job, "In Progress" ) + ProcessStorageDirs(job) + FinishJob( job, "Completed (scan for new files)" ) + MessageToFE( job.id, "success", "Completed (scan for new files)" ) + session.commit() + return + def JobForceScan(job): JobProgressState( job, "In Progress" ) session.query(FileRefimgLink).delete() @@ -524,11 +548,11 @@ def JobImportDir(job): JobProgressState( job, "In Progress" ) settings = session.query(Settings).first() if settings == None: - raise Exception("Cannot create file data with no settings / import path is missing") + raise Exception("Cannot create file data with no settings / paths missing") path=[jex.value for jex in job.extra if jex.name == "path"][0] - AddLogForJob(job, "Checking Import Directory: {}".format( path ) ) + AddLogForJob(job, "Checking Directory: {}".format( path ) ) if DEBUG==1: - print("DEBUG: Checking Import Directory: {}".format( path ) ) + print("DEBUG: Checking Directory: {}".format( path ) ) if not os.path.exists( path ): FinishJob( job, "Finished Importing: {} -- Path does not exist".format( path), "Failed" ) return @@ -630,6 +654,7 @@ def GenHashAndThumb(job, e): return e.file_details[0].hash = md5( job, e.in_dir[0].path_prefix+'/'+ e.name ) + print( e ) if e.type.name == 'Image': e.file_details[0].thumbnail = GenImageThumbnail( job, e.in_dir[0].path_prefix+'/'+ e.name ) elif e.type.name == 'Video': diff --git a/templates/base.html b/templates/base.html index 0eaf72a..177d42e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -49,8 +49,9 @@
- View - View Details + View Photos to Import + View Details of Photos to Import + View Stored Photos
@@ -79,6 +80,7 @@ Edit Settings Scan now (for new files) Force Scan (delete data & rebuild) + Scan Storage Path Force wake the job manage