finished moving GenerateFileData out of files.py into pa_job_manager.py

files.py

@@ -22,131 +22,6 @@ import time
 from settings import Settings
 from job import Job, Joblog, NewJob
 
-class FileData():
-    def __init__(self):
-        self.view_list=[]
-
-    def getExif(self, file):
-        f = open(file, 'rb')
-        try:
-            tags = exifread.process_file(f)
-        except:
-            print('NO EXIF TAGS?!?!?!?')
-            f.close()
-            raise
-        f.close()
-
-        fthumbnail = base64.b64encode(tags['JPEGThumbnail'])
-        fthumbnail = str(fthumbnail)[2:-1]
-        return fthumbnail
-    
-    def isVideo(self, file):
-        try:
-            fileInfo = MediaInfo.parse(file)
-            for track in fileInfo.tracks:
-                if track.track_type == "Video":
-                    return True
-            return False
-        except Exception as e:
-            return False
-
-    # Converts linux paths into windows paths 
-    # HACK: assumes c:, might be best to just look for [a-z]: ?
-    def FixPath(self, p):
-        if p.startswith('c:'):
-            p = p.replace('/', '\\')
-        return p
-
-    # Returns an md5 hash of the fnames' contents 
-    def md5(self, fname):
-        hash_md5 = hashlib.md5()
-        with open(fname, "rb") as f:
-            for chunk in iter(lambda: f.read(4096), b""):
-                hash_md5.update(chunk)
-        return hash_md5.hexdigest()
-
-    def isImage(self, file):
-        try:
-            img = Image.open(file)
-            return True
-        except:
-            return False
-
-    def generateVideoThumbnail(self, file):
-        #overall wrapper function for generating video thumbnails
-        vcap = cv2.VideoCapture(file)
-        res, im_ar = vcap.read()
-        while im_ar.mean() < 15 and res:
-            res, im_ar = vcap.read()
-        im_ar = cv2.resize(im_ar, (160, 90), 0, 0, cv2.INTER_LINEAR)
-        #save on a buffer for direct transmission
-        res, thumb_buf = cv2.imencode('.jpeg', im_ar)
-        # '.jpeg' etc are permitted
-        #get the bytes content
-        bt = thumb_buf.tostring()
-        fthumbnail = base64.b64encode(bt)
-        fthumbnail = str(fthumbnail)[2:-1]
-        return fthumbnail
-
-
-    ##############################################################################
-    # HACK: At present this only handles one path (need to re-factor if we have  #
-    # multiple valid paths in import_path)                                       #
-    ##############################################################################
-    def GenerateFileData(self):
-        settings = Settings.query.all()
-        if not settings:
-            return
-        last_import_date = settings[0].last_import_date
-        paths = settings[0].import_path.split("#")
-
-        for path in paths:
-            print( "GenerateFileData: Checking {}".format( path ) )
-            path = self.FixPath(path)
-            if os.path.exists( path ):
-                # to serve static content of the images, we create a symlink
-                # from inside the static subdir of each import_path that exists
-                symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1])))
-                if not os.path.exists(symlink):
-                    os.symlink(path, symlink)
-
-                file_list=[]
-                file_list.append(glob.glob(path + '**', recursive=True))
-                for file in file_list[0]:
-                    if file == path:
-                        continue
-                    stat = os.stat(file)
-                    if last_import_date == 0 or stat.st_ctime > last_import_date:
-                        print( "{} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
-                        fthumbnail = None
-                        if os.path.isdir(file):
-                            ftype = 'Directory'
-                        elif self.isImage(file):
-                            ftype = 'Image'
-                            fthumbnail = self.getExif(file)
-                        elif self.isVideo(file):
-                            ftype = 'Video'
-                            fthumbnail = self.generateVideoThumbnail(file)
-                        else:
-                            ftype = 'File'
-
-                        if ftype != "Directory":
-                            fhash=self.md5(file)
-                        else:
-                            fhash=None
-
-                        fsize = round(os.stat(file).st_size/(1024*1024))
-                        fname=file.replace(path, "")
-                        path_prefix=symlink.replace(path,"")
-                        file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
-                        db.session.add(file_obj)
-                    else:
-                        print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) )
-            settings[0].last_import_date = time.time()
-            db.session.commit()
-        self.view_list = File.query.all()
-        return self
-
 ################################################################################
 # Class describing File in the database, and via sqlalchemy, connected to the DB as well
 # This has to match one-for-one the DB table
@@ -165,48 +40,39 @@ class File(db.Model):
         return "<id: {}, name: {}>".format(self.id, self.name )
 
 
-### Initiatlise the file data set (GenerateFileData is clever enough to not
-### re-process files when we run twice in quick succession, e.g. when running
-### Flask in DEBUG mode
-filedata = FileData()
-filedata.GenerateFileData()
-
 ################################################################################
 # /file_list -> show detailed file list of files from import_path(s)
 ################################################################################
 @app.route("/file_list", methods=["GET"])
 def file_list():
-    return render_template("file_list.html", page_title='View Files (details)', file_data=filedata)
+    return render_template("file_list.html", page_title='View Files (details)', file_data=File.query.all())
 
 ################################################################################
 # /files -> show thumbnail view of files from import_path(s)
 ################################################################################
 @app.route("/files", methods=["GET"])
 def files():
-    return render_template("files.html", page_title='View Files', file_data=filedata)
+    return render_template("files.html", page_title='View Files', file_data=File.query.all())
 
 ################################################################################
 # /files/scannow -> allows us to force a check for new files
 ################################################################################
 @app.route("/files/scannow", methods=["GET"])
 def scannow():
-    job=NewJob("scannow", 1 )
+    job=NewJob("scannow" )
     st.SetAlert("success")
     st.SetMessage("Created job to scan for new files")
-    return render_template("base.html", page_title='Forced look for new items', file_data=filedata)
+    return render_template("base.html")
 
 ################################################################################
 # /files/forcescan -> deletes old data in DB, and does a brand new scan
 ################################################################################
 @app.route("/files/forcescan", methods=["GET"])
 def forcescan():
-    File.query.delete()
+    job=NewJob("forcescan" )
-    Settings.query.all()[0].last_import_date=0
-    db.session.commit()
-    filedata.GenerateFileData()
     st.SetAlert("success")
-    st.SetMessage("Forced remove and recreation of all file data")
+    st.SetMessage("Created job to force scan & rebuild data for files")
-    return render_template("base.html", page_title='Forced look for new items')
+    return render_template("base.html")
 
 ################################################################################
 # /static -> returns the contents of any file referenced inside /static.

pa_job_manager.py

@@ -165,8 +165,10 @@ class FileData():
                         path_prefix=symlink.replace(path,"")
                         file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
                         session.add(file_obj)
+                        AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
+                        AddLogForJob(job, "Found new file: {}".format(fname) )
                     else:
-                        AddLogForJob(job, "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
+                        AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
             settings.last_import_date = time.time()
             session.commit()
         return self
@@ -287,7 +289,7 @@ def RunJob(job):
         if job.name =="scannow":
             JobScanNow(job)
         elif job.name =="forcescan":
-            print("force scan not being handled yet")
+            JobForceScan(job)
         else:
             print("Requested to process unknown job type: {}".format(job.name))
     except Exception as e:
@@ -298,6 +300,8 @@ def HandleJobs():
     print("PA job manager is scanning for jobs")
     pa_eng.state = 'Scanning Jobs'
     jobs=GetJobs()
+    pa_eng.num_active_jobs=0
+    pa_eng.num_completed_jobs=0
     for job in jobs:
         if job.pa_job_state != 'Completed':
             RunJob(job)
@@ -317,6 +321,22 @@ def JobScanNow(job):
     session.commit()
     return
 
+def JobForceScan(job):
+    session.query(File).delete()
+    settings = session.query(Settings).first()
+    if settings == None:
+        raise Exception("Cannot create file data with no settings / import path is missing")
+    settings.last_import_date = 0
+    session.commit()
+    filedata.GenerateFileData(job)
+    job.state="Completed"
+    job.pa_job_state="Completed"
+    job.last_update=datetime.now(pytz.utc)
+    MessageToFE( job.id, "success", "Completed (forced remove and recreation of all file data)" )
+    session.commit()
+    return
+
+
 if __name__ == "__main__":
     print("PA job manager starting")
     try:

templates/file_list.html

@@ -1,10 +1,10 @@
 {% extends "base.html" %} {% block main_content %}
 		<div class="container">
-			<h3 class="offset-lg-2">{{page_title}} -- {{file_data.view_path}}</h3>
+			<h3 class="offset-lg-2">{{page_title}}</h3>
 			<div class="row">
 				<table class="table table-striped table-sm col-xl-12">
 				<thead><tr class="thead-light"><th>Name</th><th>Size (MB)</th><th>Path Prefix</th><th>Hash</th></tr></thead><tbody>
-					{% for obj in file_data.view_list %}
+					{% for obj in file_data %}
 						<tr><td>
 							{% if obj.type == "Directory" %}
 								<i style="font-size:48;" class="fas fa-folder"></i><br><span class="figure-caption">{{obj.name}}</span>

templates/files.html

@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 {% block main_content %}
 	<div class="container-fluid">
-		<h3 class="offset-lg-2">{{page_title}} -- {{file_data.view_path}}</h3>
+		<h3 class="offset-lg-2">{{page_title}}</h3>
 		<div class="form-row input-group">
 			<div class="input-group-prepend">
 				<button style="width:98%" class="btn btn-outline-info disabled" disabled>Size:</button>
@@ -24,7 +24,7 @@
 		</div>
 		<br>
 		<div class="row">
-			{% for obj in file_data.view_list %}
+			{% for obj in file_data %}
 				{% if obj.type != "Directory" %}
 					<center>
 						<figure class="figure px-2" font-size: 24px;>