ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned up code, before rewrite for viewing

Browse Source
This commit is contained in:
Damien De Paoli
2021-01-19 11:10:25 +11:00
parent 5285bf66ab
commit 47085b7a43
2 changed files with 61 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
TODO
View File

@@ -1,3 +1,11 @@
DDP:
files.py -> entry.py?
but basically I need to put the new classes into files.py, and when I
render, I use: Entry.query.all()
### DB
### BACKEND

171
pa_job_manager.py
View File

@@ -134,63 +134,6 @@ class FileData():
session.add(job)
return
##############################################################################
def GenerateFileData(self, job):
settings = session.query(Settings).first()
if settings == None:
return
last_import_date = settings.last_import_date
paths = settings.import_path.split("#")
for path in paths:
AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
path = self.FixPath(path)
if os.path.exists( path ):
# to serve static content of the images, we create a symlink
# from inside the static subdir of each import_path that exists
symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1])))
if not os.path.exists(symlink):
os.symlink(path, symlink)
file_list=[]
file_list.append(glob.glob(path + '**', recursive=True))
for file in file_list[0]:
if file == path:
continue
fname=file.replace(path, "")
stat = os.stat(file)
if last_import_date == 0 or stat.st_ctime > last_import_date:
AddLogForJob(job, "DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
fthumbnail = None
if os.path.isdir(file):
ftype = 'Directory'
elif self.isImage(file):
ftype = 'Image'
fthumbnail = self.getExif(file)
elif self.isVideo(file):
ftype = 'Video'
fthumbnail = self.generateVideoThumbnail(file)
else:
ftype = 'File'
if ftype != "Directory":
fhash=self.md5(file)
else:
fhash=None
fsize = round(os.stat(file).st_size/(1024*1024))
path_prefix=symlink.replace(path,"")
file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
session.add(file_obj)
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
AddLogForJob(job, "Found new file: {}".format(fname) )
else:
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
# include this for making the job be a bit slower, and # allowing testing of active jobs/job refresh page, etc.
#time.sleep(0.4)
settings.last_import_date = time.time()
session.commit()
return self
################################################################################
# Class describing File in the database, and via sqlalchemy, connected to the DB as well
@@ -359,26 +302,15 @@ def AddLogForJob(job, message, current_file=''):
return
def RunJob(job):
# try:
if job.name =="scannow":
JobScanNow(job)
elif job.name =="forcescan":
JobForceScan(job)
elif job.name =="importdir":
JobImportDir(job)
else:
print("Requested to process unknown job type: {}".format(job.name))
# except Exception as e:
if DEBUG==0:
try:
MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
except Exception as e:
print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
# else:
# print("back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
# exit(1)
return
# return
if job.name =="scannow":
JobScanNow(job)
elif job.name =="forcescan":
JobForceScan(job)
elif job.name =="importdir":
JobImportDir(job)
else:
print("Requested to process unknown job type: {}".format(job.name))
return
def HandleJobs():
global pa_eng
@@ -390,16 +322,17 @@ def HandleJobs():
pa_eng.num_completed_jobs=0
for job in jobs:
if job.pa_job_state != 'Completed':
# use this to remove threads for easier debuggin
# use this to remove threads for easier debugging, and errors will stacktrace to the console
if DEBUG==1:
RunJob(job)
else:
print ("WTF")
threading.Thread(target=RunJob, args=(job,)).start()
print ("HandleJobs setting num_active jobs to +1")
pa_eng.num_active_jobs = pa_eng.num_active_jobs + 1
else:
pa_eng.num_completed_jobs = pa_eng.num_completed_jobs +1
try:
threading.Thread(target=RunJob, args=(job,)).start()
except Exception as e:
try:
MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
except Exception as e2:
print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- orig exep was: {}, this exception was: {})".format( job.id, job.name, e, e2) )
print("PA job manager is waiting jobs")
pa_eng.state = 'Waiting for new Jobs'
return
@@ -428,32 +361,48 @@ def JobForceScan(job):
session.commit()
return
# to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists
def MakeSymlink(job,path):
symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
if not os.path.exists(symlink):
os.symlink(path, symlink)
return symlink
def AddDir(job, dirname, path_prefix, in_dir):
dir=Dir( path_prefix=path_prefix )
dtype = FileType(name='Directory')
e=Entry( name=dirname, type=dtype.id )
e.dir_details.append(dir)
# this occurs when we Add the actual Dir for the import_path
if in_dir:
e.in_dir.append(in_dir)
AddLogForJob(job, "DEBUG: AddDir: {} in (dir_id={})".format(dirname, in_dir) )
session.add(e)
return dir
def AddFile(job, fname, ftype, fsize, in_dir ):
e=Entry( name=fname, type=ftype )
f=New_File( size_mb=fsize )
e.file_details.append(f)
e.in_dir.append(in_dir)
AddLogForJob(job, "Found new file: {}".format(fname) )
session.add(e)
return e
def JobImportDir(job):
print("Working on this - import dir: {}".format(job.id))
print("DEBUG: Importing dir: {}".format(job.id))
settings = session.query(Settings).first()
if settings == None:
raise Exception("Cannot create file data with no settings / import path is missing")
last_import_date = settings.last_import_date
for jex in job.extra:
if jex.name =="path":
print("Should be importing: {}".format(jex.value))
path=jex.value
path = FixPath( jex.value)
AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
path = FixPath(path)
if os.path.exists( path ):
# to serve static content of the images, we create a symlink
# from inside the static subdir of each import_path that exists
symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
if not os.path.exists(symlink):
os.symlink(path, symlink)
file_list=[]
file_list.append(glob.glob(path + '**', recursive=True))
dir=Dir( path_prefix=symlink )
dtype = FileType(name='Directory')
e=Entry( name=os.path.basename(path[0:-1]), type=dtype.id )
e.dir_details.append(dir)
session.add(e)
for file in file_list[0]:
symlink=MakeSymlink(job,path)
dir=AddDir(job, os.path.basename(path[0:-1]), symlink, None )
for file in glob.glob(path + '**', recursive=True):
if file == path:
continue
fname=file.replace(path, "")
@@ -463,12 +412,7 @@ def JobImportDir(job):
print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
if os.path.isdir(file):
path_prefix=os.path.join(symlink,fname)
e=Entry( name=fname, type=dtype.id )
dir=Dir( path_prefix=path_prefix )
e.dir_details.append(dir)
print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
print("DEBUG: DIR- path={}, pp={}, sl={}".format( path, path_prefix, symlink ) )
# DEBUG: DIR- path=/home/ddp/src/photoassistant/images_to_process/, pp=static/images_to_process, sl=static/images_to_process
dir=AddDir( job, fname, path_prefix, dir )
else:
if isImage(file):
ftype = FileType(name='Image')
@@ -477,22 +421,13 @@ def JobImportDir(job):
else:
ftype = FileType('File')
fsize = round(os.stat(file).st_size/(1024*1024))
e=Entry( name=os.path.basename(fname), type=ftype.id )
f=New_File( size_mb=fsize )
e.file_details.append(f)
e.in_dir.append(dir)
session.add(e)
print( session.new )
AddLogForJob(job, "Found new file: {}".format(fname) )
print("Found new file: {}".format(fname) )
e=AddFile( job, os.path.basename(fname), ftype.id, fsize, dir )
else:
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
print("DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
#settings.last_import_date = time.time()
session.commit()
print( "Ending, list session new objects" )
print ("fake finished import dir")
print ("DEBUG: finished Job import dir")
return
def isVideo(file):