cleaned up code, before rewrite for viewing
This commit is contained in:
@@ -134,63 +134,6 @@ class FileData():
|
||||
session.add(job)
|
||||
return
|
||||
|
||||
##############################################################################
|
||||
def GenerateFileData(self, job):
|
||||
settings = session.query(Settings).first()
|
||||
if settings == None:
|
||||
return
|
||||
last_import_date = settings.last_import_date
|
||||
paths = settings.import_path.split("#")
|
||||
|
||||
for path in paths:
|
||||
AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
|
||||
path = self.FixPath(path)
|
||||
if os.path.exists( path ):
|
||||
# to serve static content of the images, we create a symlink
|
||||
# from inside the static subdir of each import_path that exists
|
||||
symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1])))
|
||||
if not os.path.exists(symlink):
|
||||
os.symlink(path, symlink)
|
||||
|
||||
file_list=[]
|
||||
file_list.append(glob.glob(path + '**', recursive=True))
|
||||
for file in file_list[0]:
|
||||
if file == path:
|
||||
continue
|
||||
fname=file.replace(path, "")
|
||||
stat = os.stat(file)
|
||||
if last_import_date == 0 or stat.st_ctime > last_import_date:
|
||||
AddLogForJob(job, "DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
|
||||
fthumbnail = None
|
||||
if os.path.isdir(file):
|
||||
ftype = 'Directory'
|
||||
elif self.isImage(file):
|
||||
ftype = 'Image'
|
||||
fthumbnail = self.getExif(file)
|
||||
elif self.isVideo(file):
|
||||
ftype = 'Video'
|
||||
fthumbnail = self.generateVideoThumbnail(file)
|
||||
else:
|
||||
ftype = 'File'
|
||||
|
||||
if ftype != "Directory":
|
||||
fhash=self.md5(file)
|
||||
else:
|
||||
fhash=None
|
||||
|
||||
fsize = round(os.stat(file).st_size/(1024*1024))
|
||||
path_prefix=symlink.replace(path,"")
|
||||
file_obj = File( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail )
|
||||
session.add(file_obj)
|
||||
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
|
||||
AddLogForJob(job, "Found new file: {}".format(fname) )
|
||||
else:
|
||||
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
|
||||
# include this for making the job be a bit slower, and # allowing testing of active jobs/job refresh page, etc.
|
||||
#time.sleep(0.4)
|
||||
settings.last_import_date = time.time()
|
||||
session.commit()
|
||||
return self
|
||||
|
||||
################################################################################
|
||||
# Class describing File in the database, and via sqlalchemy, connected to the DB as well
|
||||
@@ -359,26 +302,15 @@ def AddLogForJob(job, message, current_file=''):
|
||||
return
|
||||
|
||||
def RunJob(job):
|
||||
# try:
|
||||
if job.name =="scannow":
|
||||
JobScanNow(job)
|
||||
elif job.name =="forcescan":
|
||||
JobForceScan(job)
|
||||
elif job.name =="importdir":
|
||||
JobImportDir(job)
|
||||
else:
|
||||
print("Requested to process unknown job type: {}".format(job.name))
|
||||
# except Exception as e:
|
||||
if DEBUG==0:
|
||||
try:
|
||||
MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
|
||||
except Exception as e:
|
||||
print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
|
||||
# else:
|
||||
# print("back-end Failed to run job (id: {}, name: {} -- exception was: {})".format( job.id, job.name, e) )
|
||||
# exit(1)
|
||||
return
|
||||
# return
|
||||
if job.name =="scannow":
|
||||
JobScanNow(job)
|
||||
elif job.name =="forcescan":
|
||||
JobForceScan(job)
|
||||
elif job.name =="importdir":
|
||||
JobImportDir(job)
|
||||
else:
|
||||
print("Requested to process unknown job type: {}".format(job.name))
|
||||
return
|
||||
|
||||
def HandleJobs():
|
||||
global pa_eng
|
||||
@@ -390,16 +322,17 @@ def HandleJobs():
|
||||
pa_eng.num_completed_jobs=0
|
||||
for job in jobs:
|
||||
if job.pa_job_state != 'Completed':
|
||||
# use this to remove threads for easier debuggin
|
||||
# use this to remove threads for easier debugging, and errors will stacktrace to the console
|
||||
if DEBUG==1:
|
||||
RunJob(job)
|
||||
else:
|
||||
print ("WTF")
|
||||
threading.Thread(target=RunJob, args=(job,)).start()
|
||||
print ("HandleJobs setting num_active jobs to +1")
|
||||
pa_eng.num_active_jobs = pa_eng.num_active_jobs + 1
|
||||
else:
|
||||
pa_eng.num_completed_jobs = pa_eng.num_completed_jobs +1
|
||||
try:
|
||||
threading.Thread(target=RunJob, args=(job,)).start()
|
||||
except Exception as e:
|
||||
try:
|
||||
MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e) )
|
||||
except Exception as e2:
|
||||
print("Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- orig exep was: {}, this exception was: {})".format( job.id, job.name, e, e2) )
|
||||
print("PA job manager is waiting jobs")
|
||||
pa_eng.state = 'Waiting for new Jobs'
|
||||
return
|
||||
@@ -428,32 +361,48 @@ def JobForceScan(job):
|
||||
session.commit()
|
||||
return
|
||||
|
||||
# to serve static content of the images, we create a symlink from inside the static subdir of each import_path that exists
|
||||
def MakeSymlink(job,path):
|
||||
symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
|
||||
if not os.path.exists(symlink):
|
||||
os.symlink(path, symlink)
|
||||
return symlink
|
||||
|
||||
def AddDir(job, dirname, path_prefix, in_dir):
|
||||
dir=Dir( path_prefix=path_prefix )
|
||||
dtype = FileType(name='Directory')
|
||||
e=Entry( name=dirname, type=dtype.id )
|
||||
e.dir_details.append(dir)
|
||||
# this occurs when we Add the actual Dir for the import_path
|
||||
if in_dir:
|
||||
e.in_dir.append(in_dir)
|
||||
AddLogForJob(job, "DEBUG: AddDir: {} in (dir_id={})".format(dirname, in_dir) )
|
||||
session.add(e)
|
||||
return dir
|
||||
|
||||
def AddFile(job, fname, ftype, fsize, in_dir ):
|
||||
e=Entry( name=fname, type=ftype )
|
||||
f=New_File( size_mb=fsize )
|
||||
e.file_details.append(f)
|
||||
e.in_dir.append(in_dir)
|
||||
AddLogForJob(job, "Found new file: {}".format(fname) )
|
||||
session.add(e)
|
||||
return e
|
||||
|
||||
def JobImportDir(job):
|
||||
print("Working on this - import dir: {}".format(job.id))
|
||||
print("DEBUG: Importing dir: {}".format(job.id))
|
||||
settings = session.query(Settings).first()
|
||||
if settings == None:
|
||||
raise Exception("Cannot create file data with no settings / import path is missing")
|
||||
last_import_date = settings.last_import_date
|
||||
for jex in job.extra:
|
||||
if jex.name =="path":
|
||||
print("Should be importing: {}".format(jex.value))
|
||||
path=jex.value
|
||||
path = FixPath( jex.value)
|
||||
AddLogForJob(job, "Checking Import Directory: {}".format( path ) )
|
||||
path = FixPath(path)
|
||||
if os.path.exists( path ):
|
||||
# to serve static content of the images, we create a symlink
|
||||
# from inside the static subdir of each import_path that exists
|
||||
symlink = FixPath('static/{}'.format( os.path.basename(path[0:-1])))
|
||||
if not os.path.exists(symlink):
|
||||
os.symlink(path, symlink)
|
||||
file_list=[]
|
||||
file_list.append(glob.glob(path + '**', recursive=True))
|
||||
dir=Dir( path_prefix=symlink )
|
||||
dtype = FileType(name='Directory')
|
||||
e=Entry( name=os.path.basename(path[0:-1]), type=dtype.id )
|
||||
e.dir_details.append(dir)
|
||||
session.add(e)
|
||||
for file in file_list[0]:
|
||||
symlink=MakeSymlink(job,path)
|
||||
dir=AddDir(job, os.path.basename(path[0:-1]), symlink, None )
|
||||
for file in glob.glob(path + '**', recursive=True):
|
||||
if file == path:
|
||||
continue
|
||||
fname=file.replace(path, "")
|
||||
@@ -463,12 +412,7 @@ def JobImportDir(job):
|
||||
print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
|
||||
if os.path.isdir(file):
|
||||
path_prefix=os.path.join(symlink,fname)
|
||||
e=Entry( name=fname, type=dtype.id )
|
||||
dir=Dir( path_prefix=path_prefix )
|
||||
e.dir_details.append(dir)
|
||||
print("DEBUG: {} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) )
|
||||
print("DEBUG: DIR- path={}, pp={}, sl={}".format( path, path_prefix, symlink ) )
|
||||
# DEBUG: DIR- path=/home/ddp/src/photoassistant/images_to_process/, pp=static/images_to_process, sl=static/images_to_process
|
||||
dir=AddDir( job, fname, path_prefix, dir )
|
||||
else:
|
||||
if isImage(file):
|
||||
ftype = FileType(name='Image')
|
||||
@@ -477,22 +421,13 @@ def JobImportDir(job):
|
||||
else:
|
||||
ftype = FileType('File')
|
||||
fsize = round(os.stat(file).st_size/(1024*1024))
|
||||
e=Entry( name=os.path.basename(fname), type=ftype.id )
|
||||
f=New_File( size_mb=fsize )
|
||||
e.file_details.append(f)
|
||||
e.in_dir.append(dir)
|
||||
session.add(e)
|
||||
print( session.new )
|
||||
|
||||
AddLogForJob(job, "Found new file: {}".format(fname) )
|
||||
print("Found new file: {}".format(fname) )
|
||||
e=AddFile( job, os.path.basename(fname), ftype.id, fsize, dir )
|
||||
else:
|
||||
AddLogForJob(job, "DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
|
||||
print("DEBUG: {} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ), file )
|
||||
#settings.last_import_date = time.time()
|
||||
session.commit()
|
||||
print( "Ending, list session new objects" )
|
||||
print ("fake finished import dir")
|
||||
print ("DEBUG: finished Job import dir")
|
||||
return
|
||||
|
||||
def isVideo(file):
|
||||
|
||||
Reference in New Issue
Block a user