diff --git a/TODO b/TODO index 8fe3dda..a6bb7b3 100644 --- a/TODO +++ b/TODO @@ -1,13 +1,13 @@ ## GENERAL * issue where someone could call IP .../Imp/photos and SP .../Sto/photos and then static/photos is ambiguous: - -- TODO: make path prefix by static// so that files are in: static//in_path.pp/dir.rel_path/ - -- right now, I have hardcoded static/Storage in files.py so folder view will be broken until I do the above + -- DONE: make path prefix by static// so that files are in: static//in_path.pp/dir.rel_path/ -- then deleting below would just path_prefix from static/storage to .pa_bin/storage, etc. -- need to be able to view recycle bin (should be simple when we have path_types) &&& should able to consolidate the files_ip/files_sp/files_rb? route handling functions -- could also allow undelete per file / show content as another Files->View and more like storage (i.e. show folders) * storage_path viewing needs to be by folder / not a big grab bag of files (by default - DONE) -- BUG: issue with view by Day, etc. we print out day even if the Entry is not in the cwd -- TODO: Need to toggle the view if I want, and when viewing storage area, change single-click to be view file again, and right-click to be my context menu + * import_path can be folder view and works (need toggle as above) * need a way for search results to show we found something in import_path or storage_path: - now we can use the in_path, then have a series of icons, e.g. disk for storage, ? for import, and bin for recycling (before the blue path)--maybe even show different colours, e.g. info for import, primary for storage and danger for bin? * handle thumbs: diff --git a/files.py b/files.py index 171820e..9d4e606 100644 --- a/files.py +++ b/files.py @@ -139,6 +139,9 @@ def ViewingOptions( request ): else: folders=False cwd=None +# folders=True +# cwd='static/Import' + root=cwd if request.method=="POST": noo=request.form['noo'] @@ -160,7 +163,7 @@ def ViewingOptions( request ): if 'next' in request.form: offset += int(how_many) - return noo, grouping, how_many, offset, size, folders, cwd + return noo, grouping, how_many, offset, size, folders, cwd, root ################################################################################ # /file_list -> show detailed file list of files from import_path(s) @@ -175,28 +178,30 @@ def file_list_ip(): @app.route("/files_ip", methods=["GET", "POST"]) def files_ip(): - noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request ) + noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request ) entries=[] # per import path, add entries to view settings=Settings.query.first() paths = settings.import_path.split("#") for path in paths: - prefix = SymlinkName(path,path+'/') + prefix = SymlinkName("Import",path,path+'/') if noo == "Oldest": entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all() else: entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd ) + entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all() + + return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) ################################################################################ # /files -> show thumbnail view of files from storage_path ################################################################################ @app.route("/files_sp", methods=["GET", "POST"]) def files_sp(): - noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request ) + noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request ) entries=[] print( f"cwd={cwd}" ) @@ -205,16 +210,15 @@ def files_sp(): settings=Settings.query.first() paths = settings.storage_path.split("#") for path in paths: - prefix = SymlinkName(path,path+'/') + prefix = SymlinkName("Storage",path,path+'/') if noo == "Oldest": entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all() - entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all() else: entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all() - entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all() - return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd ) + entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all() + return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) ################################################################################ # /search -> show thumbnail view of files from import_path(s) @@ -222,7 +226,7 @@ def files_sp(): @app.route("/search", methods=["GET","POST"]) def search(): - noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request ) + noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request ) # always show flat results for search to start with folders=False @@ -233,7 +237,7 @@ def search(): all_entries = file_data + dir_data + ai_data - return render_template("files.html", page_title='View Files', search_term=request.form['term'], entry_data=all_entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd ) + return render_template("files.html", page_title='View Files', search_term=request.form['term'], entry_data=all_entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root ) ################################################################################ # /files/scannow -> allows us to force a check for new files @@ -336,6 +340,7 @@ def custom_static(filename): ################################################################################ @app.template_filter('TopLevelFolderOf') def _jinja2_filter_toplevelfolderof(path, cwd): + print(f"path={path}, cwd={cwd}, dirname=={os.path.dirname(path)}") if os.path.dirname(path) == cwd: return True else: diff --git a/pa_job_manager.py b/pa_job_manager.py index 0a8d52b..272702e 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -12,6 +12,10 @@ # pylint: disable=no-member + +# global debug setting +DEBUG=1 + ### SQLALCHEMY IMPORTS ### from sqlalchemy.ext.declarative import declarative_base @@ -49,8 +53,6 @@ import re import sys -# global debug setting -DEBUG=1 # this is required to handle the duplicate processing code sys.setrecursionlimit(50000) @@ -300,8 +302,9 @@ def ProcessImportDirs(parent_job): def JobsForPaths( parent_job, paths, ptype ): now=datetime.now(pytz.utc) # make new set of Jobs per path... HandleJobs will make them run later + path_type = session.query(PathType).get(ptype) for path in paths: - p=session.query(Path).filter(Path.path_prefix==SymlinkName(path,path+'/')).first() + p=session.query(Path).filter(Path.path_prefix==SymlinkName(path_type.name,path,path+'/')).first() cfn=0 if p: cfn=p.num_files @@ -474,9 +477,12 @@ def JobForceScan(job): return # to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists -def CreateSymlink(job,path): - symlink=SymlinkName(path, path) +def CreateSymlink(job,ptype,path): + path_type = session.query(PathType).get(ptype) + symlink=SymlinkName(path_type.name, path, path) if not os.path.exists(symlink): + print( f"symlink does not exist - but is it missing path_type after static? s={symlink}" ) + os.makedirs( os.path.dirname(symlink), mode=0o777, exist_ok=True ) os.symlink(path, symlink) return symlink @@ -584,21 +590,33 @@ def GetDateFromFile(file, stat): woy=c[1] return year, month, day, woy +def AddJexToDependantJobs(job,name,value): + if DEBUG==1: + print( f"DEBUG: AddJexToDependantJobs({job}, {name}, {value}) ") + for j in session.query(Job).filter(Job.wait_for==job.id).all(): + print( f"DEBUG: adding jex to this job.id == {j.id}" ) + jex=JobExtra( name=name, value=value ) + j.extra.append(jex) + AddJexToDependantJobs(j, name, value) + return + def JobImportDir(job): JobProgressState( job, "In Progress" ) settings = session.query(Settings).first() path=[jex.value for jex in job.extra if jex.name == "path"][0] path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0] - AddLogForJob(job, f"Checking 'path_type' Directory: {path}" ) + AddLogForJob(job, f"Checking {path_type} Directory: {path}" ) if DEBUG==1: print("DEBUG: Checking Directory: {}".format( path ) ) if not os.path.exists( path ): - FinishJob( job, "Finished Importing: {} -- Path does not exist".format( path), "Failed" ) + FinishJob( job, f"Finished Importing: {path} -- Path does not exist", "Failed" ) return - symlink=CreateSymlink(job,path) + symlink=CreateSymlink(job,path_type,path) path_obj=Path( path_prefix=symlink, num_files=0, type_id=path_type ) session.add(path_obj) + # find all jobs waiting on me and their children, etc. and add a path_prefix jex to symlink, so we can just reference it form here on in, rather than recreate that string + AddJexToDependantJobs(job,"path_prefix",symlink) ResetExistsOnFS(job, symlink) walk=os.walk(path, topdown=True) @@ -623,7 +641,7 @@ def JobImportDir(job): for root, subdirs, files in ftree: # already create root above to work out num_files for whole os.walk if root != path: - pp=SymlinkName( path, root )+'/'+os.path.basename(root) + pp=SymlinkName( path_obj.type.name, path, root )+'/'+os.path.basename(root) print( F"pp={pp}, root={root}, symlink={symlink}" ) rel_path=pp.replace(symlink+'/','') dir=AddDir(job, os.path.basename(root), parent_dir, rel_path, path_obj) @@ -672,7 +690,8 @@ def RunFuncOnFilesInPath( job, path, file_func ): def JobProcessAI(job): path=[jex.value for jex in job.extra if jex.name == "path"][0] - path = SymlinkName(path, '/') + path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0] + path = SymlinkName(path_prefix, path, '/') p = session.query(Path).filter(Path.path_prefix==path).first() job.num_files=p.num_files @@ -815,15 +834,18 @@ def ProcessFilesInDir(job, e, file_func): def JobGetFileDetails(job): JobProgressState( job, "In Progress" ) + #### I think the fix here is to get JobImportDir (or whatever makes the PATH) to add a jex for path_prefix and just pull it here, and stop 're-creating' it via SymlinkName path=[jex.value for jex in job.extra if jex.name == "path"][0] - path=SymlinkName( path, path ) + path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0] + print( f"JobGetFileDetails({job}) -- pp={path_prefix}" ) +# path=SymlinkName( path_prefix, path, path ) if DEBUG==1: - print("DEBUG: JobGetFileDetails for path={}".format( path ) ) - p=session.query(Path).filter(Path.path_prefix==path).first() + print("DEBUG: JobGetFileDetails for path={}".format( path_prefix ) ) + p=session.query(Path).filter(Path.path_prefix==path_prefix).first() job.current_file_num = 0 job.num_files = p.num_files session.commit() - RunFuncOnFilesInPath( job, path, GenHashAndThumb ) + RunFuncOnFilesInPath( job, path_prefix, GenHashAndThumb ) FinishJob(job, "File Details job finished") session.commit() return diff --git a/shared.py b/shared.py index c1522d2..dcbdcc3 100644 --- a/shared.py +++ b/shared.py @@ -29,7 +29,7 @@ def CreateSelect(name, selected, list, js=""): str += '' return str -def SymlinkName(path, file): +def SymlinkName(ptype, path, file): sig_bit=file.replace(path, "") last_dir=os.path.basename(path[0:-1]) @@ -37,7 +37,7 @@ def SymlinkName(path, file): last_bit = os.path.dirname(sig_bit)[0:-1] else: last_bit = os.path.dirname(sig_bit) - symlink = 'static/'+last_dir+'/'+last_bit + symlink = 'static/'+ptype+'/'+last_dir+'/'+last_bit if symlink[-1] == '/': symlink=symlink[0:-1] - return symlink \ No newline at end of file + return symlink diff --git a/templates/files.html b/templates/files.html index f8b3746..bb66c00 100644 --- a/templates/files.html +++ b/templates/files.html @@ -104,11 +104,6 @@ {% endif %} {% for obj in entry_data %} {% if loop.index==1 and folders %} - {% if obj.type.name != "Directory" %} - {% set root=obj.in_dir.in_path.path_prefix %} - {% else %} - {% set root=obj.dir_details.in_path.path_prefix %} - {% endif %} {% if cwd != root %}
@@ -165,9 +160,14 @@ {% endif %} {% else %} {% if folders %} - {# if not the top-level of the path and it is a top-level folder, then display a folder icon #} - {% if (cwd != obj.dir_details.in_path.path_prefix or obj.dir_details.rel_path|length > 0) and ((obj.dir_details.in_path.path_prefix+'/'+obj.dir_details.rel_path) | TopLevelFolderOf(cwd)) %} -
+ {% if obj.dir_details.rel_path | length %} + {% set dirname=obj.dir_details.in_path.path_prefix+'/'+obj.dir_details.rel_path %} + {% else %} + {% set dirname=obj.dir_details.in_path.path_prefix %} + {% endif %} + {# if this dir is the toplevel of the cwd, show the folder icon #} + {% if dirname| TopLevelFolderOf(cwd) %} +
{{obj.name}}