ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first cut of paths actually working with folder viewing - for now defaulting on for both Storage only

Browse Source
This commit is contained in:
Damien De Paoli
2021-05-01 18:16:45 +10:00
parent b6b98ae5ca
commit 5e03650ed5
5 changed files with 65 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TODO
View File

@@ -1,13 +1,13 @@
## GENERAL
* issue where someone could call IP .../Imp/photos and SP .../Sto/photos and then static/photos is ambiguous:
-- TODO: make path prefix by static/<ptype.name>/ so that files are in: static/<ptype.name>/in_path.pp/dir.rel_path/
-- right now, I have hardcoded static/Storage in files.py so folder view will be broken until I do the above
-- DONE: make path prefix by static/<ptype.name>/ so that files are in: static/<ptype.name>/in_path.pp/dir.rel_path/
-- then deleting below would just path_prefix from static/storage to .pa_bin/storage, etc.
-- need to be able to view recycle bin (should be simple when we have path_types) &&& should able to consolidate the files_ip/files_sp/files_rb? route handling functions
-- could also allow undelete per file / show content as another Files->View and more like storage (i.e. show folders)
* storage_path viewing needs to be by folder / not a big grab bag of files (by default - DONE)
-- BUG: issue with view by Day, etc. we print out day even if the Entry is not in the cwd
-- TODO: Need to toggle the view if I want, and when viewing storage area, change single-click to be view file again, and right-click to be my context menu
* import_path can be folder view and works (need toggle as above)
* need a way for search results to show we found something in import_path or storage_path:
- now we can use the in_path, then have a series of icons, e.g. disk for storage, ? for import, and bin for recycling (before the blue path)--maybe even show different colours, e.g. info for import, primary for storage and danger for bin?
* handle thumbs:

27
files.py
View File

@@ -139,6 +139,9 @@ def ViewingOptions( request ):
else:
folders=False
cwd=None
# folders=True
# cwd='static/Import'
root=cwd
if request.method=="POST":
noo=request.form['noo']
@@ -160,7 +163,7 @@ def ViewingOptions( request ):
if 'next' in request.form:
offset += int(how_many)
return noo, grouping, how_many, offset, size, folders, cwd
return noo, grouping, how_many, offset, size, folders, cwd, root
################################################################################
# /file_list -> show detailed file list of files from import_path(s)
@@ -175,28 +178,30 @@ def file_list_ip():
@app.route("/files_ip", methods=["GET", "POST"])
def files_ip():
noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request )
noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request )
entries=[]
# per import path, add entries to view
settings=Settings.query.first()
paths = settings.import_path.split("#")
for path in paths:
prefix = SymlinkName(path,path+'/')
prefix = SymlinkName("Import",path,path+'/')
if noo == "Oldest":
entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all()
else:
entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all()
return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd )
entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all()
return render_template("files.html", page_title='View Files (Import Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root )
################################################################################
# /files -> show thumbnail view of files from storage_path
################################################################################
@app.route("/files_sp", methods=["GET", "POST"])
def files_sp():
noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request )
noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request )
entries=[]
print( f"cwd={cwd}" )
@@ -205,16 +210,15 @@ def files_sp():
settings=Settings.query.first()
paths = settings.storage_path.split("#")
for path in paths:
prefix = SymlinkName(path,path+'/')
prefix = SymlinkName("Storage",path,path+'/')
if noo == "Oldest":
entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year,File.month,File.day,Entry.name).offset(offset).limit(how_many).all()
entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all()
else:
entries+=Entry.query.join(File).join(EntryDirLink).join(Dir).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(File.year.desc(),File.month.desc(),File.day.desc(),Entry.name).offset(offset).limit(how_many).all()
entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all()
return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd )
entries+=Entry.query.join(Dir).join(EntryDirLink).join(PathDirLink).join(Path).filter(Path.path_prefix==prefix).order_by(Entry.name).offset(offset).limit(how_many).all()
return render_template("files.html", page_title='View Files (Storage Path)', entry_data=entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root )
################################################################################
# /search -> show thumbnail view of files from import_path(s)
@@ -222,7 +226,7 @@ def files_sp():
@app.route("/search", methods=["GET","POST"])
def search():
noo, grouping, how_many, offset, size, folders, cwd = ViewingOptions( request )
noo, grouping, how_many, offset, size, folders, cwd, root = ViewingOptions( request )
# always show flat results for search to start with
folders=False
@@ -233,7 +237,7 @@ def search():
all_entries = file_data + dir_data + ai_data
return render_template("files.html", page_title='View Files', search_term=request.form['term'], entry_data=all_entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd )
return render_template("files.html", page_title='View Files', search_term=request.form['term'], entry_data=all_entries, noo=noo, grouping=grouping, how_many=how_many, offset=offset, size=size, folders=folders, cwd=cwd, root=root )
################################################################################
# /files/scannow -> allows us to force a check for new files
@@ -336,6 +340,7 @@ def custom_static(filename):
################################################################################
@app.template_filter('TopLevelFolderOf')
def _jinja2_filter_toplevelfolderof(path, cwd):
print(f"path={path}, cwd={cwd}, dirname=={os.path.dirname(path)}")
if os.path.dirname(path) == cwd:
return True
else:

50
pa_job_manager.py
View File

@@ -12,6 +12,10 @@
# pylint: disable=no-member
# global debug setting
DEBUG=1
### SQLALCHEMY IMPORTS ###
from sqlalchemy.ext.declarative import declarative_base
@@ -49,8 +53,6 @@ import re
import sys
# global debug setting
DEBUG=1
# this is required to handle the duplicate processing code
sys.setrecursionlimit(50000)
@@ -300,8 +302,9 @@ def ProcessImportDirs(parent_job):
def JobsForPaths( parent_job, paths, ptype ):
now=datetime.now(pytz.utc)
# make new set of Jobs per path... HandleJobs will make them run later
path_type = session.query(PathType).get(ptype)
for path in paths:
p=session.query(Path).filter(Path.path_prefix==SymlinkName(path,path+'/')).first()
p=session.query(Path).filter(Path.path_prefix==SymlinkName(path_type.name,path,path+'/')).first()
cfn=0
if p:
cfn=p.num_files
@@ -474,9 +477,12 @@ def JobForceScan(job):
return
# to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists
def CreateSymlink(job,path):
symlink=SymlinkName(path, path)
def CreateSymlink(job,ptype,path):
path_type = session.query(PathType).get(ptype)
symlink=SymlinkName(path_type.name, path, path)
if not os.path.exists(symlink):
print( f"symlink does not exist - but is it missing path_type after static? s={symlink}" )
os.makedirs( os.path.dirname(symlink), mode=0o777, exist_ok=True )
os.symlink(path, symlink)
return symlink
@@ -584,21 +590,33 @@ def GetDateFromFile(file, stat):
woy=c[1]
return year, month, day, woy
def AddJexToDependantJobs(job,name,value):
if DEBUG==1:
print( f"DEBUG: AddJexToDependantJobs({job}, {name}, {value}) ")
for j in session.query(Job).filter(Job.wait_for==job.id).all():
print( f"DEBUG: adding jex to this job.id == {j.id}" )
jex=JobExtra( name=name, value=value )
j.extra.append(jex)
AddJexToDependantJobs(j, name, value)
return
def JobImportDir(job):
JobProgressState( job, "In Progress" )
settings = session.query(Settings).first()
path=[jex.value for jex in job.extra if jex.name == "path"][0]
path_type=[jex.value for jex in job.extra if jex.name == "path_type"][0]
AddLogForJob(job, f"Checking 'path_type' Directory: {path}" )
AddLogForJob(job, f"Checking {path_type} Directory: {path}" )
if DEBUG==1:
print("DEBUG: Checking Directory: {}".format( path ) )
if not os.path.exists( path ):
FinishJob( job, "Finished Importing: {} -- Path does not exist".format( path), "Failed" )
FinishJob( job, f"Finished Importing: {path} -- Path does not exist", "Failed" )
return
symlink=CreateSymlink(job,path)
symlink=CreateSymlink(job,path_type,path)
path_obj=Path( path_prefix=symlink, num_files=0, type_id=path_type )
session.add(path_obj)
# find all jobs waiting on me and their children, etc. and add a path_prefix jex to symlink, so we can just reference it form here on in, rather than recreate that string
AddJexToDependantJobs(job,"path_prefix",symlink)
ResetExistsOnFS(job, symlink)
walk=os.walk(path, topdown=True)
@@ -623,7 +641,7 @@ def JobImportDir(job):
for root, subdirs, files in ftree:
# already create root above to work out num_files for whole os.walk
if root != path:
pp=SymlinkName( path, root )+'/'+os.path.basename(root)
pp=SymlinkName( path_obj.type.name, path, root )+'/'+os.path.basename(root)
print( F"pp={pp}, root={root}, symlink={symlink}" )
rel_path=pp.replace(symlink+'/','')
dir=AddDir(job, os.path.basename(root), parent_dir, rel_path, path_obj)
@@ -672,7 +690,8 @@ def RunFuncOnFilesInPath( job, path, file_func ):
def JobProcessAI(job):
path=[jex.value for jex in job.extra if jex.name == "path"][0]
path = SymlinkName(path, '/')
path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0]
path = SymlinkName(path_prefix, path, '/')
p = session.query(Path).filter(Path.path_prefix==path).first()
job.num_files=p.num_files
@@ -815,15 +834,18 @@ def ProcessFilesInDir(job, e, file_func):
def JobGetFileDetails(job):
JobProgressState( job, "In Progress" )
#### I think the fix here is to get JobImportDir (or whatever makes the PATH) to add a jex for path_prefix and just pull it here, and stop 're-creating' it via SymlinkName
path=[jex.value for jex in job.extra if jex.name == "path"][0]
path=SymlinkName( path, path )
path_prefix=[jex.value for jex in job.extra if jex.name == "path_prefix"][0]
print( f"JobGetFileDetails({job}) -- pp={path_prefix}" )
# path=SymlinkName( path_prefix, path, path )
if DEBUG==1:
print("DEBUG: JobGetFileDetails for path={}".format( path ) )
p=session.query(Path).filter(Path.path_prefix==path).first()
print("DEBUG: JobGetFileDetails for path={}".format( path_prefix ) )
p=session.query(Path).filter(Path.path_prefix==path_prefix).first()
job.current_file_num = 0
job.num_files = p.num_files
session.commit()
RunFuncOnFilesInPath( job, path, GenHashAndThumb )
RunFuncOnFilesInPath( job, path_prefix, GenHashAndThumb )
FinishJob(job, "File Details job finished")
session.commit()
return

4
shared.py
View File

@@ -29,7 +29,7 @@ def CreateSelect(name, selected, list, js=""):
str += '</select>'
return str
def SymlinkName(path, file):
def SymlinkName(ptype, path, file):
sig_bit=file.replace(path, "")
last_dir=os.path.basename(path[0:-1])
@@ -37,7 +37,7 @@ def SymlinkName(path, file):
last_bit = os.path.dirname(sig_bit)[0:-1]
else:
last_bit = os.path.dirname(sig_bit)
symlink = 'static/'+last_dir+'/'+last_bit
symlink = 'static/'+ptype+'/'+last_dir+'/'+last_bit
if symlink[-1] == '/':
symlink=symlink[0:-1]
return symlink

16
templates/files.html
View File

@@ -104,11 +104,6 @@
{% endif %}
{% for obj in entry_data %}
{% if loop.index==1 and folders %}
{% if obj.type.name != "Directory" %}
{% set root=obj.in_dir.in_path.path_prefix %}
{% else %}
{% set root=obj.dir_details.in_path.path_prefix %}
{% endif %}
{% if cwd != root %}
<figure class="px-1 dir" dir={{cwd|ParentPath}}>
<span style="font-size:{{(size|int-22)/2}}" class="fa-stack">
@@ -165,9 +160,14 @@
{% endif %}
{% else %}
{% if folders %}
{# if not the top-level of the path and it is a top-level folder, then display a folder icon #}
{% if (cwd != obj.dir_details.in_path.path_prefix or obj.dir_details.rel_path|length > 0) and ((obj.dir_details.in_path.path_prefix+'/'+obj.dir_details.rel_path) | TopLevelFolderOf(cwd)) %}
<figure class="px-1 dir" dir={{obj.dir_details.in_path.path_prefix+'/'+obj.dir_details.rel_path}}>
{% if obj.dir_details.rel_path | length %}
{% set dirname=obj.dir_details.in_path.path_prefix+'/'+obj.dir_details.rel_path %}
{% else %}
{% set dirname=obj.dir_details.in_path.path_prefix %}
{% endif %}
{# if this dir is the toplevel of the cwd, show the folder icon #}
{% if dirname| TopLevelFolderOf(cwd) %}
<figure class="px-1 dir" dir={{dirname}}>
<i style="font-size:{{size|int-22}};" class="fas fa-folder"></i>
<figcaption class="figure-caption text-center">{{obj.name}}</figcaption>
</figure class="figure">