cleaned up code, before rewrite for viewing

TODO

@@ -1,3 +1,11 @@
+DDP:
+    files.py -> entry.py?
+    but basically I need to put the new classes into files.py, and when I
+    render, I use: Entry.query.all()
+
+
+
+
 ### DB	
 
 ### BACKEND

pa_job_manager.py

@@ -134,63 +134,6 @@ class FileData():
             session.add(job)
         return
         
-    ##############################################################################
-    def GenerateFileData(self, job):
-        settings = session.query(Settings).first()
-        if settings == None:
-            return
-        last_import_date = settings.last_import_date
-        paths = settings.import_path.split("#")
-
-        for path in paths:
-            AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
-            path = self.FixPath(path)
-            if os.path.exists( path ):
-                # to serve static content of the images, we create a symlink
-                # from inside the static subdir of each import_path that exists
-                symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1])))
-                if not os.path.exists(symlink):
-                    os.symlink(path, symlink)
-
-                file_list=[]
-                file_list.append(glob.glob(path + '**', recursive=True))
-                for file in file_list[0]:
-                    if file == path:
-                        continue
-                    fname=file.replace(path, "")
-                    stat = os.stat(file)
-                    if last_import_date == 0 or stat.st_ctime > last_import_date:
-                        AddLogForJob(job, "DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
-                        fthumbnail = None
-                        if os.path.isdir(file):
-                            ftype = 'Directory'
-                        elif self.isImage(file):
-                            ftype = 'Image'
-                            fthumbnail = self.getExif(file)
-                        elif self.isVideo(file):
-                            ftype = 'Video'
-                            fthumbnail = self.generateVideoThumbnail(file)
-                        else:
-                            ftype = 'File'
-
-                        if ftype != "Directory":
-                            fhash=self.md5(file)
-                        else:
-                            fhash=None
-
-                        fsize = round(os.stat(file).st_size/(1024*1024))
-                        path_prefix=symlink.replace(path,"")
-                        file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
-                        session.add(file_obj)
-                        AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
-                        AddLogForJob(job, "Found new file: {}".format(fname) )
-                    else:
-                        AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
-                    # include this for making the job be a bit slower, and # allowing testing of active jobs/job refresh page, etc.
-                    #time.sleep(0.4)
-            settings.last_import_date = time.time()
-            session.commit()
-        return self
 
 ################################################################################
 # Class describing File in the database, and via sqlalchemy, connected to the DB as well
@@ -359,7 +302,6 @@ def AddLogForJob(job, message, current_file=''):
     return
 
 def RunJob(job):
-#    try:
     if job.name =="scannow":
         JobScanNow(job)
     elif job.name =="forcescan":
@@ -368,17 +310,7 @@ def RunJob(job):
         JobImportDir(job)
     else:
         print("Requested to process unknown job type: {}".format(job.name))
-#    except Exception as e:
-        if DEBUG==0:
-            try:
-                MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
-            except Exception as e:
-                print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
-#        else:
-#            print("back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
-#            exit(1)
     return
-#    return
 
 def HandleJobs():
     global pa_eng
@@ -390,16 +322,17 @@ def HandleJobs():
     pa_eng.num_completed_jobs=0
     for job in jobs:
         if job.pa_job_state != 'Completed':
-            # use this to remove threads for easier debuggin
+            # use this to remove threads for easier debugging, and errors will stacktrace to the console
             if DEBUG==1:
                 RunJob(job)
             else:
-                print ("WTF")
+                try:
                     threading.Thread(target=RunJob, args=(job,)).start()
-            print ("HandleJobs setting num_active jobs to +1")
+                except Exception as e:
-            pa_eng.num_active_jobs = pa_eng.num_active_jobs + 1
+                    try:
-        else:
+                        MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
-            pa_eng.num_completed_jobs = pa_eng.num_completed_jobs +1
+                    except Exception as e2:
+                        print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- orig exep was: {}, this exception was: {})".format( job.id, job.name, e, e2) )
     print("PA job manager is waiting jobs")
     pa_eng.state = 'Waiting for new Jobs'
     return
@@ -428,32 +361,48 @@ def JobForceScan(job):
     session.commit()
     return
 
+# to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists
+def MakeSymlink(job,path):
+    symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
+    if not os.path.exists(symlink):
+        os.symlink(path, symlink)
+    return symlink
+
+def AddDir(job, dirname, path_prefix, in_dir):
+    dir=Dir( path_prefix=path_prefix )
+    dtype = FileType(name='Directory')
+    e=Entry( name=dirname, type=dtype.id )
+    e.dir_details.append(dir)
+    # this occurs when we Add the actual Dir for the import_path
+    if in_dir:
+        e.in_dir.append(in_dir)
+    AddLogForJob(job, "DEBUG: AddDir: {} in (dir_id={})".format(dirname, in_dir) )
+    session.add(e)
+    return dir
+
+def AddFile(job, fname, ftype, fsize, in_dir ):
+    e=Entry( name=fname, type=ftype )
+    f=New_File( size_mb=fsize )
+    e.file_details.append(f)
+    e.in_dir.append(in_dir)
+    AddLogForJob(job, "Found new file: {}".format(fname) )
+    session.add(e)
+    return e
+
 def JobImportDir(job):
-    print("Working on this - import dir: {}".format(job.id))
+    print("DEBUG: Importing dir: {}".format(job.id))
     settings = session.query(Settings).first()
     if settings == None:
         raise Exception("Cannot create file data with no settings / import path is missing")
     last_import_date = settings.last_import_date
     for jex in job.extra:
         if jex.name =="path":
-            print("Should be importing: {}".format(jex.value))
+            path = FixPath( jex.value)
-            path=jex.value
             AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
-            path = FixPath(path)
             if os.path.exists( path ):
-                # to serve static content of the images, we create a symlink
+                symlink=MakeSymlink(job,path)
-                # from inside the static subdir of each import_path that exists
+                dir=AddDir(job, os.path.basename(path[0:-1]), symlink, None )
-                symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
+                for file in glob.glob(path + '**', recursive=True):
-                if not os.path.exists(symlink):
-                    os.symlink(path, symlink)
-                file_list=[]
-                file_list.append(glob.glob(path + '**', recursive=True))
-                dir=Dir( path_prefix=symlink )
-                dtype = FileType(name='Directory')
-                e=Entry( name=os.path.basename(path[0:-1]), type=dtype.id )
-                e.dir_details.append(dir)
-                session.add(e)
-                for file in file_list[0]:
                     if file == path:
                         continue
                     fname=file.replace(path, "")
@@ -463,12 +412,7 @@ def JobImportDir(job):
                         print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
                         if os.path.isdir(file):
                             path_prefix=os.path.join(symlink,fname)
-                            e=Entry( name=fname, type=dtype.id )
+                            dir=AddDir( job, fname, path_prefix, dir )
-                            dir=Dir( path_prefix=path_prefix )
-                            e.dir_details.append(dir)
-                            print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
-                            print("DEBUG: DIR- path={}, pp={}, sl={}".format( path, path_prefix, symlink ) )
-# DEBUG: DIR- path=/home/ddp/src/photoassistant/images_to_process/, pp=static/images_to_process, sl=static/images_to_process
                         else:
                             if isImage(file):
                                 ftype = FileType(name='Image')
@@ -477,22 +421,13 @@ def JobImportDir(job):
                             else:
                                 ftype = FileType('File')
                             fsize = round(os.stat(file).st_size/(1024*1024))
-                            e=Entry( name=os.path.basename(fname), type=ftype.id )
+                            e=AddFile( job, os.path.basename(fname), ftype.id, fsize, dir )
-                            f=New_File( size_mb=fsize )
-                            e.file_details.append(f)
-                        e.in_dir.append(dir)
-                        session.add(e)
-                        print( session.new )
-
-                        AddLogForJob(job, "Found new file: {}".format(fname) )
-                        print("Found new file: {}".format(fname) )
                     else:
                         AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
                         print("DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
             #settings.last_import_date = time.time()
             session.commit()
-            print( "Ending, list session new objects" )
+    print ("DEBUG: finished Job import dir")
-    print ("fake finished import dir")
     return
  
 def isVideo(file):