ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finished moving GenerateFileData out of files.py into pa_job_manager.py

Browse Source
This commit is contained in:
Damien De Paoli
2021-01-17 11:31:06 +11:00
parent a89f7fdb2b
commit abff2d8bab
4 changed files with 33 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
files.py
View File

@@ -22,131 +22,6 @@ import time
from settings import Settings
from job import Job, Joblog, NewJob
class FileData():
def __init__(self):
self.view_list=[]
def getExif(self, file):
f = open(file, 'rb')
try:
tags = exifread.process_file(f)
except:
print('NO EXIF TAGS?!?!?!?')
f.close()
raise
f.close()
fthumbnail = base64.b64encode(tags['JPEGThumbnail'])
fthumbnail = str(fthumbnail)[2:-1]
return fthumbnail
def isVideo(self, file):
try:
fileInfo = MediaInfo.parse(file)
for track in fileInfo.tracks:
if track.track_type == "Video":
return True
return False
except Exception as e:
return False
# Converts linux paths into windows paths
# HACK: assumes c:, might be best to just look for [a-z]: ?
def FixPath(self, p):
if p.startswith('c:'):
p = p.replace('/', '\\')
return p
# Returns an md5 hash of the fnames' contents
def md5(self, fname):
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def isImage(self, file):
try:
img = Image.open(file)
return True
except:
return False
def generateVideoThumbnail(self, file):
#overall wrapper function for generating video thumbnails
vcap = cv2.VideoCapture(file)
res, im_ar = vcap.read()
while im_ar.mean() < 15 and res:
res, im_ar = vcap.read()
im_ar = cv2.resize(im_ar, (160, 90), 0, 0, cv2.INTER_LINEAR)
#save on a buffer for direct transmission
res, thumb_buf = cv2.imencode('.jpeg', im_ar)
# '.jpeg' etc are permitted
#get the bytes content
bt = thumb_buf.tostring()
fthumbnail = base64.b64encode(bt)
fthumbnail = str(fthumbnail)[2:-1]
return fthumbnail
##############################################################################
# HACK: At present this only handles one path (need to re-factor if we have #
# multiple valid paths in import_path) #
##############################################################################
def GenerateFileData(self):
settings = Settings.query.all()
if not settings:
return
last_import_date = settings[0].last_import_date
paths = settings[0].import_path.split("#")
for path in paths:
print( "GenerateFileData: Checking {}".format( path ) )
path = self.FixPath(path)
if os.path.exists( path ):
# to serve static content of the images, we create a symlink
# from inside the static subdir of each import_path that exists
symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1])))
if not os.path.exists(symlink):
os.symlink(path, symlink)
file_list=[]
file_list.append(glob.glob(path + '**', recursive=True))
for file in file_list[0]:
if file == path:
continue
stat = os.stat(file)
if last_import_date == 0 or stat.st_ctime > last_import_date:
print( "{} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
fthumbnail = None
if os.path.isdir(file):
ftype = 'Directory'
elif self.isImage(file):
ftype = 'Image'
fthumbnail = self.getExif(file)
elif self.isVideo(file):
ftype = 'Video'
fthumbnail = self.generateVideoThumbnail(file)
else:
ftype = 'File'
if ftype != "Directory":
fhash=self.md5(file)
else:
fhash=None
fsize = round(os.stat(file).st_size/(1024*1024))
fname=file.replace(path, "")
path_prefix=symlink.replace(path,"")
file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
db.session.add(file_obj)
else:
print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) )
settings[0].last_import_date = time.time()
db.session.commit()
self.view_list = File.query.all()
return self
################################################################################
# Class describing File in the database, and via sqlalchemy, connected to the DB as well
# This has to match one-for-one the DB table
@@ -165,48 +40,39 @@ class File(db.Model):
return "<id: {}, name: {}>".format(self.id, self.name )
### Initiatlise the file data set (GenerateFileData is clever enough to not
### re-process files when we run twice in quick succession, e.g. when running
### Flask in DEBUG mode
filedata = FileData()
filedata.GenerateFileData()
################################################################################
# /file_list -> show detailed file list of files from import_path(s)
################################################################################
@app.route("/file_list", methods=["GET"])
def file_list():
return render_template("file_list.html", page_title='View Files (details)', file_data=filedata)
return render_template("file_list.html", page_title='View Files (details)', file_data=File.query.all())
################################################################################
# /files -> show thumbnail view of files from import_path(s)
################################################################################
@app.route("/files", methods=["GET"])
def files():
return render_template("files.html", page_title='View Files', file_data=filedata)
return render_template("files.html", page_title='View Files', file_data=File.query.all())
################################################################################
# /files/scannow -> allows us to force a check for new files
################################################################################
@app.route("/files/scannow", methods=["GET"])
def scannow():
job=NewJob("scannow", 1 )
job=NewJob("scannow" )
st.SetAlert("success")
st.SetMessage("Created job to scan for new files")
return render_template("base.html", page_title='Forced look for new items', file_data=filedata)
return render_template("base.html")
################################################################################
# /files/forcescan -> deletes old data in DB, and does a brand new scan
################################################################################
@app.route("/files/forcescan", methods=["GET"])
def forcescan():
File.query.delete()
Settings.query.all()[0].last_import_date=0
db.session.commit()
filedata.GenerateFileData()
job=NewJob("forcescan" )
st.SetAlert("success")
st.SetMessage("Forced remove and recreation of all file data")
return render_template("base.html", page_title='Forced look for new items')
st.SetMessage("Created job to force scan & rebuild data for files")
return render_template("base.html")
################################################################################
# /static -> returns the contents of any file referenced inside /static.

24
pa_job_manager.py
View File

@@ -165,8 +165,10 @@ class FileData():
path_prefix=symlink.replace(path,"")
file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
session.add(file_obj)
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
AddLogForJob(job, "Found new file: {}".format(fname) )
else:
AddLogForJob(job, "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
settings.last_import_date = time.time()
session.commit()
return self
@@ -287,7 +289,7 @@ def RunJob(job):
if job.name =="scannow":
JobScanNow(job)
elif job.name =="forcescan":
print("force scan not being handled yet")
JobForceScan(job)
else:
print("Requested to process unknown job type: {}".format(job.name))
except Exception as e:
@@ -298,6 +300,8 @@ def HandleJobs():
print("PA job manager is scanning for jobs")
pa_eng.state = 'Scanning Jobs'
jobs=GetJobs()
pa_eng.num_active_jobs=0
pa_eng.num_completed_jobs=0
for job in jobs:
if job.pa_job_state != 'Completed':
RunJob(job)
@@ -317,6 +321,22 @@ def JobScanNow(job):
session.commit()
return
def JobForceScan(job):
session.query(File).delete()
settings = session.query(Settings).first()
if settings == None:
raise Exception("Cannot create file data with no settings / import path is missing")
settings.last_import_date = 0
session.commit()
filedata.GenerateFileData(job)
job.state="Completed"
job.pa_job_state="Completed"
job.last_update=datetime.now(pytz.utc)
MessageToFE( job.id, "success", "Completed (forced remove and recreation of all file data)" )
session.commit()
return
if __name__ == "__main__":
print("PA job manager starting")
try:

4
templates/file_list.html
View File

@@ -1,10 +1,10 @@
{% extends "base.html" %} {% block main_content %}
<div class="container">
<h3 class="offset-lg-2">{{page_title}} -- {{file_data.view_path}}</h3>
<h3 class="offset-lg-2">{{page_title}}</h3>
<div class="row">
<table class="table table-striped table-sm col-xl-12">
<thead><tr class="thead-light"><th>Name</th><th>Size (MB)</th><th>Path Prefix</th><th>Hash</th></tr></thead><tbody>
{% for obj in file_data.view_list %}
{% for obj in file_data %}
<tr><td>
{% if obj.type == "Directory" %}
<i style="font-size:48;" class="fas fa-folder"></i><br><span class="figure-caption">{{obj.name}}</span>

4
templates/files.html
View File

@@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block main_content %}
<div class="container-fluid">
<h3 class="offset-lg-2">{{page_title}} -- {{file_data.view_path}}</h3>
<h3 class="offset-lg-2">{{page_title}}</h3>
<div class="form-row input-group">
<div class="input-group-prepend">
<button style="width:98%" class="btn btn-outline-info disabled" disabled>Size:</button>
@@ -24,7 +24,7 @@
</div>
<br>
<div class="row">
{% for obj in file_data.view_list %}
{% for obj in file_data %}
{% if obj.type != "Directory" %}
<center>
<figure class="figure px-2" font-size: 24px;>