diff --git a/files.py b/files.py index a20cc4d..9cafe7d 100644 --- a/files.py +++ b/files.py @@ -22,131 +22,6 @@ import time from settings import Settings from job import Job, Joblog, NewJob -class FileData(): - def __init__(self): - self.view_list=[] - - def getExif(self, file): - f = open(file, 'rb') - try: - tags = exifread.process_file(f) - except: - print('NO EXIF TAGS?!?!?!?') - f.close() - raise - f.close() - - fthumbnail = base64.b64encode(tags['JPEGThumbnail']) - fthumbnail = str(fthumbnail)[2:-1] - return fthumbnail - - def isVideo(self, file): - try: - fileInfo = MediaInfo.parse(file) - for track in fileInfo.tracks: - if track.track_type == "Video": - return True - return False - except Exception as e: - return False - - # Converts linux paths into windows paths - # HACK: assumes c:, might be best to just look for [a-z]: ? - def FixPath(self, p): - if p.startswith('c:'): - p = p.replace('/', '\\') - return p - - # Returns an md5 hash of the fnames' contents - def md5(self, fname): - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - - def isImage(self, file): - try: - img = Image.open(file) - return True - except: - return False - - def generateVideoThumbnail(self, file): - #overall wrapper function for generating video thumbnails - vcap = cv2.VideoCapture(file) - res, im_ar = vcap.read() - while im_ar.mean() < 15 and res: - res, im_ar = vcap.read() - im_ar = cv2.resize(im_ar, (160, 90), 0, 0, cv2.INTER_LINEAR) - #save on a buffer for direct transmission - res, thumb_buf = cv2.imencode('.jpeg', im_ar) - # '.jpeg' etc are permitted - #get the bytes content - bt = thumb_buf.tostring() - fthumbnail = base64.b64encode(bt) - fthumbnail = str(fthumbnail)[2:-1] - return fthumbnail - - - ############################################################################## - # HACK: At present this only handles one path (need to re-factor if we have # - # multiple valid paths in import_path) # - ############################################################################## - def GenerateFileData(self): - settings = Settings.query.all() - if not settings: - return - last_import_date = settings[0].last_import_date - paths = settings[0].import_path.split("#") - - for path in paths: - print( "GenerateFileData: Checking {}".format( path ) ) - path = self.FixPath(path) - if os.path.exists( path ): - # to serve static content of the images, we create a symlink - # from inside the static subdir of each import_path that exists - symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1]))) - if not os.path.exists(symlink): - os.symlink(path, symlink) - - file_list=[] - file_list.append(glob.glob(path + '**', recursive=True)) - for file in file_list[0]: - if file == path: - continue - stat = os.stat(file) - if last_import_date == 0 or stat.st_ctime > last_import_date: - print( "{} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) ) - fthumbnail = None - if os.path.isdir(file): - ftype = 'Directory' - elif self.isImage(file): - ftype = 'Image' - fthumbnail = self.getExif(file) - elif self.isVideo(file): - ftype = 'Video' - fthumbnail = self.generateVideoThumbnail(file) - else: - ftype = 'File' - - if ftype != "Directory": - fhash=self.md5(file) - else: - fhash=None - - fsize = round(os.stat(file).st_size/(1024*1024)) - fname=file.replace(path, "") - path_prefix=symlink.replace(path,"") - file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail ) - db.session.add(file_obj) - else: - print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) ) - settings[0].last_import_date = time.time() - db.session.commit() - self.view_list = File.query.all() - return self - ################################################################################ # Class describing File in the database, and via sqlalchemy, connected to the DB as well # This has to match one-for-one the DB table @@ -165,48 +40,39 @@ class File(db.Model): return "".format(self.id, self.name ) -### Initiatlise the file data set (GenerateFileData is clever enough to not -### re-process files when we run twice in quick succession, e.g. when running -### Flask in DEBUG mode -filedata = FileData() -filedata.GenerateFileData() - ################################################################################ # /file_list -> show detailed file list of files from import_path(s) ################################################################################ @app.route("/file_list", methods=["GET"]) def file_list(): - return render_template("file_list.html", page_title='View Files (details)', file_data=filedata) + return render_template("file_list.html", page_title='View Files (details)', file_data=File.query.all()) ################################################################################ # /files -> show thumbnail view of files from import_path(s) ################################################################################ @app.route("/files", methods=["GET"]) def files(): - return render_template("files.html", page_title='View Files', file_data=filedata) + return render_template("files.html", page_title='View Files', file_data=File.query.all()) ################################################################################ # /files/scannow -> allows us to force a check for new files ################################################################################ @app.route("/files/scannow", methods=["GET"]) def scannow(): - job=NewJob("scannow", 1 ) + job=NewJob("scannow" ) st.SetAlert("success") st.SetMessage("Created job to scan for new files") - return render_template("base.html", page_title='Forced look for new items', file_data=filedata) + return render_template("base.html") ################################################################################ # /files/forcescan -> deletes old data in DB, and does a brand new scan ################################################################################ @app.route("/files/forcescan", methods=["GET"]) def forcescan(): - File.query.delete() - Settings.query.all()[0].last_import_date=0 - db.session.commit() - filedata.GenerateFileData() + job=NewJob("forcescan" ) st.SetAlert("success") - st.SetMessage("Forced remove and recreation of all file data") - return render_template("base.html", page_title='Forced look for new items') + st.SetMessage("Created job to force scan & rebuild data for files") + return render_template("base.html") ################################################################################ # /static -> returns the contents of any file referenced inside /static. diff --git a/pa_job_manager.py b/pa_job_manager.py index f620e2f..923f4a5 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -165,8 +165,10 @@ class FileData(): path_prefix=symlink.replace(path,"") file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail ) session.add(file_obj) + AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) + AddLogForJob(job, "Found new file: {}".format(fname) ) else: - AddLogForJob(job, "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) + AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) settings.last_import_date = time.time() session.commit() return self @@ -287,7 +289,7 @@ def RunJob(job): if job.name =="scannow": JobScanNow(job) elif job.name =="forcescan": - print("force scan not being handled yet") + JobForceScan(job) else: print("Requested to process unknown job type: {}".format(job.name)) except Exception as e: @@ -298,6 +300,8 @@ def HandleJobs(): print("PA job manager is scanning for jobs") pa_eng.state = 'Scanning Jobs' jobs=GetJobs() + pa_eng.num_active_jobs=0 + pa_eng.num_completed_jobs=0 for job in jobs: if job.pa_job_state != 'Completed': RunJob(job) @@ -317,6 +321,22 @@ def JobScanNow(job): session.commit() return +def JobForceScan(job): + session.query(File).delete() + settings = session.query(Settings).first() + if settings == None: + raise Exception("Cannot create file data with no settings / import path is missing") + settings.last_import_date = 0 + session.commit() + filedata.GenerateFileData(job) + job.state="Completed" + job.pa_job_state="Completed" + job.last_update=datetime.now(pytz.utc) + MessageToFE( job.id, "success", "Completed (forced remove and recreation of all file data)" ) + session.commit() + return + + if __name__ == "__main__": print("PA job manager starting") try: diff --git a/templates/file_list.html b/templates/file_list.html index 7534897..0f197bf 100644 --- a/templates/file_list.html +++ b/templates/file_list.html @@ -1,10 +1,10 @@ {% extends "base.html" %} {% block main_content %}
-

{{page_title}} -- {{file_data.view_path}}

+

{{page_title}}

- {% for obj in file_data.view_list %} + {% for obj in file_data %}
NameSize (MB)Path PrefixHash
{% if obj.type == "Directory" %}
{{obj.name}} diff --git a/templates/files.html b/templates/files.html index 430f6f1..7ce8411 100644 --- a/templates/files.html +++ b/templates/files.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% block main_content %}
-

{{page_title}} -- {{file_data.view_path}}

+

{{page_title}}

@@ -24,7 +24,7 @@

- {% for obj in file_data.view_list %} + {% for obj in file_data %} {% if obj.type != "Directory" %}