From 47085b7a43a2fe9d56e125c0a4788956e3e28d9c Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 19 Jan 2021 11:10:25 +1100 Subject: [PATCH] cleaned up code, before rewrite for viewing --- TODO | 8 +++ pa_job_manager.py | 171 ++++++++++++++-------------------------------- 2 files changed, 61 insertions(+), 118 deletions(-) diff --git a/TODO b/TODO index ce79fc3..31d42a9 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,11 @@ +DDP: + files.py -> entry.py? + but basically I need to put the new classes into files.py, and when I + render, I use: Entry.query.all() + + + + ### DB ### BACKEND diff --git a/pa_job_manager.py b/pa_job_manager.py index a693dd2..7485acf 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -134,63 +134,6 @@ class FileData(): session.add(job) return - ############################################################################## - def GenerateFileData(self, job): - settings = session.query(Settings).first() - if settings == None: - return - last_import_date = settings.last_import_date - paths = settings.import_path.split("#") - - for path in paths: - AddLogForJob(job, "Checking Import Directory: {}".format( path ) ) - path = self.FixPath(path) - if os.path.exists( path ): - # to serve static content of the images, we create a symlink - # from inside the static subdir of each import_path that exists - symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1]))) - if not os.path.exists(symlink): - os.symlink(path, symlink) - - file_list=[] - file_list.append(glob.glob(path + '**', recursive=True)) - for file in file_list[0]: - if file == path: - continue - fname=file.replace(path, "") - stat = os.stat(file) - if last_import_date == 0 or stat.st_ctime > last_import_date: - AddLogForJob(job, "DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) ) - fthumbnail = None - if os.path.isdir(file): - ftype = 'Directory' - elif self.isImage(file): - ftype = 'Image' - fthumbnail = self.getExif(file) - elif self.isVideo(file): - ftype = 'Video' - fthumbnail = self.generateVideoThumbnail(file) - else: - ftype = 'File' - - if ftype != "Directory": - fhash=self.md5(file) - else: - fhash=None - - fsize = round(os.stat(file).st_size/(1024*1024)) - path_prefix=symlink.replace(path,"") - file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail ) - session.add(file_obj) - AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) - AddLogForJob(job, "Found new file: {}".format(fname) ) - else: - AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) - # include this for making the job be a bit slower, and # allowing testing of active jobs/job refresh page, etc. - #time.sleep(0.4) - settings.last_import_date = time.time() - session.commit() - return self ################################################################################ # Class describing File in the database, and via sqlalchemy, connected to the DB as well @@ -359,26 +302,15 @@ def AddLogForJob(job, message, current_file=''): return def RunJob(job): -# try: - if job.name =="scannow": - JobScanNow(job) - elif job.name =="forcescan": - JobForceScan(job) - elif job.name =="importdir": - JobImportDir(job) - else: - print("Requested to process unknown job type: {}".format(job.name)) -# except Exception as e: - if DEBUG==0: - try: - MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) ) - except Exception as e: - print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) ) -# else: -# print("back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) ) -# exit(1) - return -# return + if job.name =="scannow": + JobScanNow(job) + elif job.name =="forcescan": + JobForceScan(job) + elif job.name =="importdir": + JobImportDir(job) + else: + print("Requested to process unknown job type: {}".format(job.name)) + return def HandleJobs(): global pa_eng @@ -390,16 +322,17 @@ def HandleJobs(): pa_eng.num_completed_jobs=0 for job in jobs: if job.pa_job_state != 'Completed': - # use this to remove threads for easier debuggin + # use this to remove threads for easier debugging, and errors will stacktrace to the console if DEBUG==1: RunJob(job) else: - print ("WTF") - threading.Thread(target=RunJob, args=(job,)).start() - print ("HandleJobs setting num_active jobs to +1") - pa_eng.num_active_jobs = pa_eng.num_active_jobs + 1 - else: - pa_eng.num_completed_jobs = pa_eng.num_completed_jobs +1 + try: + threading.Thread(target=RunJob, args=(job,)).start() + except Exception as e: + try: + MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) ) + except Exception as e2: + print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- orig exep was: {}, this exception was: {})".format( job.id, job.name, e, e2) ) print("PA job manager is waiting jobs") pa_eng.state = 'Waiting for new Jobs' return @@ -428,32 +361,48 @@ def JobForceScan(job): session.commit() return +# to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists +def MakeSymlink(job,path): + symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1]))) + if not os.path.exists(symlink): + os.symlink(path, symlink) + return symlink + +def AddDir(job, dirname, path_prefix, in_dir): + dir=Dir( path_prefix=path_prefix ) + dtype = FileType(name='Directory') + e=Entry( name=dirname, type=dtype.id ) + e.dir_details.append(dir) + # this occurs when we Add the actual Dir for the import_path + if in_dir: + e.in_dir.append(in_dir) + AddLogForJob(job, "DEBUG: AddDir: {} in (dir_id={})".format(dirname, in_dir) ) + session.add(e) + return dir + +def AddFile(job, fname, ftype, fsize, in_dir ): + e=Entry( name=fname, type=ftype ) + f=New_File( size_mb=fsize ) + e.file_details.append(f) + e.in_dir.append(in_dir) + AddLogForJob(job, "Found new file: {}".format(fname) ) + session.add(e) + return e + def JobImportDir(job): - print("Working on this - import dir: {}".format(job.id)) + print("DEBUG: Importing dir: {}".format(job.id)) settings = session.query(Settings).first() if settings == None: raise Exception("Cannot create file data with no settings / import path is missing") last_import_date = settings.last_import_date for jex in job.extra: if jex.name =="path": - print("Should be importing: {}".format(jex.value)) - path=jex.value + path = FixPath( jex.value) AddLogForJob(job, "Checking Import Directory: {}".format( path ) ) - path = FixPath(path) if os.path.exists( path ): - # to serve static content of the images, we create a symlink - # from inside the static subdir of each import_path that exists - symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1]))) - if not os.path.exists(symlink): - os.symlink(path, symlink) - file_list=[] - file_list.append(glob.glob(path + '**', recursive=True)) - dir=Dir( path_prefix=symlink ) - dtype = FileType(name='Directory') - e=Entry( name=os.path.basename(path[0:-1]), type=dtype.id ) - e.dir_details.append(dir) - session.add(e) - for file in file_list[0]: + symlink=MakeSymlink(job,path) + dir=AddDir(job, os.path.basename(path[0:-1]), symlink, None ) + for file in glob.glob(path + '**', recursive=True): if file == path: continue fname=file.replace(path, "") @@ -463,12 +412,7 @@ def JobImportDir(job): print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) ) if os.path.isdir(file): path_prefix=os.path.join(symlink,fname) - e=Entry( name=fname, type=dtype.id ) - dir=Dir( path_prefix=path_prefix ) - e.dir_details.append(dir) - print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) ) - print("DEBUG: DIR- path={}, pp={}, sl={}".format( path, path_prefix, symlink ) ) -# DEBUG: DIR- path=/home/ddp/src/photoassistant/images_to_process/, pp=static/images_to_process, sl=static/images_to_process + dir=AddDir( job, fname, path_prefix, dir ) else: if isImage(file): ftype = FileType(name='Image') @@ -477,22 +421,13 @@ def JobImportDir(job): else: ftype = FileType('File') fsize = round(os.stat(file).st_size/(1024*1024)) - e=Entry( name=os.path.basename(fname), type=ftype.id ) - f=New_File( size_mb=fsize ) - e.file_details.append(f) - e.in_dir.append(dir) - session.add(e) - print( session.new ) - - AddLogForJob(job, "Found new file: {}".format(fname) ) - print("Found new file: {}".format(fname) ) + e=AddFile( job, os.path.basename(fname), ftype.id, fsize, dir ) else: AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) print("DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file ) #settings.last_import_date = time.time() session.commit() - print( "Ending, list session new objects" ) - print ("fake finished import dir") + print ("DEBUG: finished Job import dir") return def isVideo(file):