broad (small) changes to make style of all routes and jobs to be consistent, e.g. use underscords between words
This commit is contained in:
8
TODO
8
TODO
@@ -1,9 +1,9 @@
|
|||||||
### GENERAL
|
### GENERAL
|
||||||
* all routes should be consistent naming conventions (with or without _ )
|
|
||||||
|
|
||||||
* get build process to create a random string for secret for PROD, otherwise use builtin for dev
|
|
||||||
|
|
||||||
* think about security - in job_mgr anywhere I can os.replace/remove NEED to protect, etc
|
* think about security - in job_mgr anywhere I can os.replace/remove NEED to protect, etc
|
||||||
|
- just need to use this I think:
|
||||||
|
from werkzeug.utils import secure_filename
|
||||||
|
|
||||||
|
secure_filename(xxxx)
|
||||||
|
|
||||||
* change the rotation code to use that jpeg util to reduce/remove compression loss?
|
* change the rotation code to use that jpeg util to reduce/remove compression loss?
|
||||||
|
|
||||||
|
|||||||
8
ai.py
8
ai.py
@@ -18,11 +18,11 @@ from face import Face, FaceFileLink, FaceRefimgLink, FaceNoMatchOverride, FaceFo
|
|||||||
# pylint: disable=no-member
|
# pylint: disable=no-member
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# /aistats -> placholder for some sort of stats
|
# /ai_stats -> placholder for some sort of stats
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/aistats", methods=["GET"])
|
@app.route("/ai_stats", methods=["GET"])
|
||||||
@login_required
|
@login_required
|
||||||
def aistats():
|
def ai_stats():
|
||||||
stats = db.session.execute( "select p.tag, count(f.id) from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag order by 2 desc" )
|
stats = db.session.execute( "select p.tag, count(f.id) from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag order by 2 desc" )
|
||||||
cnt_res = db.session.execute( "select count(1) from ( select p.tag from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag ) as foo" )
|
cnt_res = db.session.execute( "select count(1) from ( select p.tag from person p, face f, face_file_link ffl, face_refimg_link frl, person_refimg_link prl where p.id = prl.person_id and prl.refimg_id = frl.refimg_id and frl.face_id = ffl.face_id and ffl.face_id = f.id group by p.tag ) as foo" )
|
||||||
num_stats=cnt_res.first()[0]
|
num_stats=cnt_res.first()[0]
|
||||||
@@ -38,7 +38,7 @@ def aistats():
|
|||||||
fstats['all_matched_faces'] = db.session.execute( "select count(distinct face_id) as count from face_refimg_link" ).first()[0]
|
fstats['all_matched_faces'] = db.session.execute( "select count(distinct face_id) as count from face_refimg_link" ).first()[0]
|
||||||
fstats['all_unmatched_faces'] = db.session.execute( "select count(f.id) from face f left join face_refimg_link frl on f.id = frl.face_id where frl.refimg_id is null" ).first()[0]
|
fstats['all_unmatched_faces'] = db.session.execute( "select count(f.id) from face f left join face_refimg_link frl on f.id = frl.face_id where frl.refimg_id is null" ).first()[0]
|
||||||
|
|
||||||
return render_template("aistats.html", page_title='AI Statistics', stats=stats, num_stats=num_stats, fstats=fstats )
|
return render_template("ai_stats.html", page_title='AI Statistics', stats=stats, num_stats=num_stats, fstats=fstats )
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|||||||
34
files.py
34
files.py
@@ -320,7 +320,7 @@ def SetOrderStrings( OPT ):
|
|||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# /GetEntries -> helper function that Gets Entries for required files to show
|
# /GetEntries -> helper function that Gets Entries for required files to show
|
||||||
# for several routes (files_ip, files_sp, files_rbp, search, viewlist)
|
# for several routes (files_ip, files_sp, files_rbp, search, view_list)
|
||||||
################################################################################
|
################################################################################
|
||||||
def GetEntries( OPT ):
|
def GetEntries( OPT ):
|
||||||
entries=[]
|
entries=[]
|
||||||
@@ -367,9 +367,9 @@ def GetEntries( OPT ):
|
|||||||
UpdatePref( pref, OPT )
|
UpdatePref( pref, OPT )
|
||||||
return entries
|
return entries
|
||||||
|
|
||||||
@app.route("/ChangeFileOpts", methods=["POST"])
|
@app.route("/change_file_opts", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def ChangeFileOpts():
|
def change_file_opts():
|
||||||
# reset options based on form post, then redirect back to orig page (with a GET to allow back button to work)
|
# reset options based on form post, then redirect back to orig page (with a GET to allow back button to work)
|
||||||
OPT=States( request )
|
OPT=States( request )
|
||||||
return redirect( request.referrer )
|
return redirect( request.referrer )
|
||||||
@@ -462,12 +462,12 @@ def scan_ip():
|
|||||||
return redirect("/jobs")
|
return redirect("/jobs")
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# /files/forcescan -> deletes old data in DB, and does a brand new scan
|
# /files/force_scan -> deletes old data in DB, and does a brand new scan
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/files/forcescan", methods=["GET"])
|
@app.route("/files/force_scan", methods=["GET"])
|
||||||
@login_required
|
@login_required
|
||||||
def forcescan():
|
def force_scan():
|
||||||
job=NewJob( name="forcescan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" )
|
job=NewJob( name="force_scan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" )
|
||||||
return redirect("/jobs")
|
return redirect("/jobs")
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
@@ -497,7 +497,7 @@ def fix_dups():
|
|||||||
if 'pagesize' not in request.form:
|
if 'pagesize' not in request.form:
|
||||||
# default to 10, see if we have a larger value as someone reset it in the gui, rather than first time invoked
|
# default to 10, see if we have a larger value as someone reset it in the gui, rather than first time invoked
|
||||||
pagesize = 10
|
pagesize = 10
|
||||||
jexes = JobExtra.query.join(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all()
|
jexes = JobExtra.query.join(Job).filter(Job.name=='check_dups').filter(Job.pa_job_state=='New').all()
|
||||||
jexes.append( JobExtra( name="pagesize", value=pagesize ) )
|
jexes.append( JobExtra( name="pagesize", value=pagesize ) )
|
||||||
else:
|
else:
|
||||||
pagesize=int(request.form['pagesize'])
|
pagesize=int(request.form['pagesize'])
|
||||||
@@ -533,7 +533,7 @@ def rm_dups():
|
|||||||
|
|
||||||
jex.append( JobExtra( name="pagesize", value=10 ) )
|
jex.append( JobExtra( name="pagesize", value=10 ) )
|
||||||
|
|
||||||
job=NewJob( name="rmdups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" )
|
job=NewJob( name="rm_dups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" )
|
||||||
|
|
||||||
return redirect("/jobs")
|
return redirect("/jobs")
|
||||||
|
|
||||||
@@ -578,8 +578,8 @@ def move_files():
|
|||||||
return make_response( jsonify( job_id=job.id ) )
|
return make_response( jsonify( job_id=job.id ) )
|
||||||
|
|
||||||
@login_required
|
@login_required
|
||||||
@app.route("/viewlist", methods=["POST"])
|
@app.route("/view_list", methods=["POST"])
|
||||||
def viewlist():
|
def view_list():
|
||||||
OPT=States( request )
|
OPT=States( request )
|
||||||
# Get next/prev set of data - e.g. if next set, then it will use orig_url
|
# Get next/prev set of data - e.g. if next set, then it will use orig_url
|
||||||
# to go forward how_many from offset and then use viewer.html to show that
|
# to go forward how_many from offset and then use viewer.html to show that
|
||||||
@@ -705,7 +705,7 @@ def view_img_post(id):
|
|||||||
|
|
||||||
# route called from front/end - if multiple images are being transformed, each transorm == a separate call
|
# route called from front/end - if multiple images are being transformed, each transorm == a separate call
|
||||||
# to this route (and therefore a separate transorm job. Each reponse allows the f/e to check the
|
# to this route (and therefore a separate transorm job. Each reponse allows the f/e to check the
|
||||||
# specific transorm job is finished (/checktransformjob) which will be called (say) every 1 sec. from f/e
|
# specific transorm job is finished (/check_transform_job) which will be called (say) every 1 sec. from f/e
|
||||||
# with a spinning wheel, then when pa_job_mgr has finished it will return the transformed thumb
|
# with a spinning wheel, then when pa_job_mgr has finished it will return the transformed thumb
|
||||||
@app.route("/transform", methods=["POST"])
|
@app.route("/transform", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
@@ -721,14 +721,14 @@ def transform():
|
|||||||
return make_response( jsonify( job_id=job.id ) )
|
return make_response( jsonify( job_id=job.id ) )
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# /checktransformjob -> URL that is called repeatedly by front-end waiting for the
|
# /check_transform_job -> URL that is called repeatedly by front-end waiting for the
|
||||||
# b/e to finish the transform job. Once done, the new / now
|
# b/e to finish the transform job. Once done, the new / now
|
||||||
# transformed image's thumbnail is returned so the f/e can
|
# transformed image's thumbnail is returned so the f/e can
|
||||||
# update with it
|
# update with it
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/checktransformjob", methods=["POST"])
|
@app.route("/check_transform_job", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def checktransformjob():
|
def check_transform_job():
|
||||||
job_id = request.form['job_id']
|
job_id = request.form['job_id']
|
||||||
job = Job.query.get(job_id)
|
job = Job.query.get(job_id)
|
||||||
j=jsonify( finished=False )
|
j=jsonify( finished=False )
|
||||||
@@ -779,9 +779,9 @@ def _jinja2_filter_parentpath(path):
|
|||||||
# json list of existing dir names that could be near it in time. Starting
|
# json list of existing dir names that could be near it in time. Starting
|
||||||
# simple, by using YYYYMM-1, YYYYMM, YYYYMM+1 dirs
|
# simple, by using YYYYMM-1, YYYYMM, YYYYMM+1 dirs
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@app.route("/getexistingpaths/<dt>", methods=["POST"])
|
@app.route("/get_existing_paths/<dt>", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def GetExistingPathsAsDiv(dt):
|
def get_existing_paths(dt):
|
||||||
dir_ft=FileType.query.filter(FileType.name=='Directory').first()
|
dir_ft=FileType.query.filter(FileType.name=='Directory').first()
|
||||||
dirs_arr=[]
|
dirs_arr=[]
|
||||||
for delta in range(-7, 8):
|
for delta in range(-7, 8):
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ function change_rp_sel()
|
|||||||
function GetExistingDirsAsDiv( dt, divname, ptype )
|
function GetExistingDirsAsDiv( dt, divname, ptype )
|
||||||
{
|
{
|
||||||
$.ajax({
|
$.ajax({
|
||||||
type: 'POST', data: null, url: '/getexistingpaths/'+dt,
|
type: 'POST', data: null, url: '/get_existing_paths/'+dt,
|
||||||
success: function(data) {
|
success: function(data) {
|
||||||
$('#'+divname).html(data)
|
$('#'+divname).html(data)
|
||||||
dirs = JSON.parse(data)
|
dirs = JSON.parse(data)
|
||||||
|
|||||||
@@ -4,9 +4,10 @@
|
|||||||
// newly created one that was sent back in the response to the POST
|
// newly created one that was sent back in the response to the POST
|
||||||
function CheckTransformJob(id,job_id)
|
function CheckTransformJob(id,job_id)
|
||||||
{
|
{
|
||||||
|
CheckForJobs()
|
||||||
$.ajax(
|
$.ajax(
|
||||||
{
|
{
|
||||||
type: 'POST', data: '&job_id='+job_id, url: '/checktransformjob', success: function(data) {
|
type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', success: function(data) {
|
||||||
if( data.finished )
|
if( data.finished )
|
||||||
{
|
{
|
||||||
$('#s'+id).hide()
|
$('#s'+id).hide()
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ function StatusMsg(st)
|
|||||||
// clear message only when toast is hidden (either timeout OR user clicks close btn)
|
// clear message only when toast is hidden (either timeout OR user clicks close btn)
|
||||||
$('#' + el).on( 'hidden.bs.toast',
|
$('#' + el).on( 'hidden.bs.toast',
|
||||||
function() {
|
function() {
|
||||||
$.ajax( { type: 'POST', url: '/clearmsg/'+st.id, success: function(data) {} } )
|
$.ajax( { type: 'POST', url: '/clear_msg/'+st.id, success: function(data) {} } )
|
||||||
} )
|
} )
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ function CheckForJobs()
|
|||||||
{
|
{
|
||||||
$.ajax(
|
$.ajax(
|
||||||
{
|
{
|
||||||
type: 'POST', url: '/checkforjobs',
|
type: 'POST', url: '/check_for_jobs',
|
||||||
success: function(data) {
|
success: function(data) {
|
||||||
data.sts.forEach(
|
data.sts.forEach(
|
||||||
function(el)
|
function(el)
|
||||||
|
|||||||
@@ -4,9 +4,10 @@
|
|||||||
// newly created one that was sent back in the response to the POST
|
// newly created one that was sent back in the response to the POST
|
||||||
function CheckTransformJob(id,job_id)
|
function CheckTransformJob(id,job_id)
|
||||||
{
|
{
|
||||||
|
CheckForJobs()
|
||||||
$.ajax(
|
$.ajax(
|
||||||
{
|
{
|
||||||
type: 'POST', data: '&job_id='+job_id, url: '/checktransformjob', success: function(data) {
|
type: 'POST', data: '&job_id='+job_id, url: '/check_transform_job', success: function(data) {
|
||||||
if( data.finished )
|
if( data.finished )
|
||||||
{
|
{
|
||||||
// stop throbber, remove grayscale & then force reload with timestamped version of im.src
|
// stop throbber, remove grayscale & then force reload with timestamped version of im.src
|
||||||
|
|||||||
22
job.py
22
job.py
@@ -236,12 +236,12 @@ def joblog(id):
|
|||||||
return render_template("joblog.html", job=joblog, logs=logs, duration=duration, display_more=display_more, order=order, estimate=estimate, refresh=refresh)
|
return render_template("joblog.html", job=joblog, logs=logs, duration=duration, display_more=display_more, order=order, estimate=estimate, refresh=refresh)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# /wakeup -> GET -> forces the job manager to wake up, and check the queue
|
# /wake_up -> GET -> forces the job manager to wake up, and check the queue
|
||||||
# should not be needed, but in DEV can be helpful
|
# should not be needed, but in DEV can be helpful
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/wakeup", methods=["GET"])
|
@app.route("/wake_up", methods=["GET"])
|
||||||
@login_required
|
@login_required
|
||||||
def wakeup():
|
def wake_up():
|
||||||
WakePAJobManager(job_id=None)
|
WakePAJobManager(job_id=None)
|
||||||
return redirect("/")
|
return redirect("/")
|
||||||
|
|
||||||
@@ -317,12 +317,12 @@ def joblog_search():
|
|||||||
|
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# / -> POST -> looks for pa_job_manager status to F/E jobs and sends json of
|
# /check_for_jobs -> POST -> looks for pa_job_manager status to F/E jobs and sends json of
|
||||||
# them back to F/E (called form internal/js/jobs.js:CheckForJobs()
|
# them back to F/E called form internal/js/jobs.js:check_for_jobs()
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/checkforjobs", methods=["POST"])
|
@app.route("/check_for_jobs", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def CheckForJobs():
|
def check_for_jobs():
|
||||||
num=GetNumActiveJobs()
|
num=GetNumActiveJobs()
|
||||||
sts=[]
|
sts=[]
|
||||||
for msg in PA_JobManager_Message.query.all():
|
for msg in PA_JobManager_Message.query.all():
|
||||||
@@ -333,12 +333,12 @@ def CheckForJobs():
|
|||||||
return make_response( jsonify( num_active_jobs=num, sts=sts ) )
|
return make_response( jsonify( num_active_jobs=num, sts=sts ) )
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# / -> POST -> looks for pa_job_manager status to F/E jobs and sends json of
|
# /clear_msg -> POST -> clears out a F/E message based on passed in <id>
|
||||||
# them back to F/E (called form internal/js/jobs.js:CheckForJobs()
|
# called form internal/js/jobs.js:CheckForJobs()
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/clearmsg/<id>", methods=["POST"])
|
@app.route("/clear_msg/<id>", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def ClearMessage(id):
|
def clear_message(id):
|
||||||
PA_JobManager_Message.query.filter(PA_JobManager_Message.id==id).delete()
|
PA_JobManager_Message.query.filter(PA_JobManager_Message.id==id).delete()
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
# no real need for this response, as if it succeeded/failed the F/E ignores it
|
# no real need for this response, as if it succeeded/failed the F/E ignores it
|
||||||
|
|||||||
@@ -687,22 +687,22 @@ def JobsForPath( parent_job, path, ptype ):
|
|||||||
jex=[]
|
jex=[]
|
||||||
jex.append( JobExtra( name="path", value=path ) )
|
jex.append( JobExtra( name="path", value=path ) )
|
||||||
jex.append( JobExtra( name="path_type", value=ptype.id ) )
|
jex.append( JobExtra( name="path_type", value=ptype.id ) )
|
||||||
job1=NewJob( name="importdir", num_files=cfn, wait_for=None, jex=jex, parent_job=parent_job, desc=f"scan for files from {ptype.name} path" )
|
job1=NewJob( name="import_dir", num_files=cfn, wait_for=None, jex=jex, parent_job=parent_job, desc=f"scan for files from {ptype.name} path" )
|
||||||
|
|
||||||
# then get file details (hash/thumbs)
|
# then get file details (hash/thumbs)
|
||||||
jex=[]
|
jex=[]
|
||||||
jex.append( JobExtra( name="path", value=path ) )
|
jex.append( JobExtra( name="path", value=path ) )
|
||||||
job2=NewJob( name="getfiledetails", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"get details of files from {ptype.name} path" )
|
job2=NewJob( name="get_file_details", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"get details of files from {ptype.name} path" )
|
||||||
|
|
||||||
# can start straight after importdir - job1, does not need details (job2)
|
# can start straight after import_dir - job1, does not need details (job2)
|
||||||
jex=[]
|
jex=[]
|
||||||
jex.append( JobExtra( name="person", value="all" ) )
|
jex.append( JobExtra( name="person", value="all" ) )
|
||||||
jex.append( JobExtra( name="path_type", value=ptype.id ) )
|
jex.append( JobExtra( name="path_type", value=ptype.id ) )
|
||||||
job3=NewJob( name="run_ai_on_path", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"match faces on files from {ptype.name} path" )
|
job3=NewJob( name="run_ai_on_path", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"match faces on files from {ptype.name} path" )
|
||||||
|
|
||||||
# careful here, wait for getfiledetails (job2), the ai job cannot cause a dup
|
# careful here, wait for get_file_details (job2), the ai job cannot cause a dup
|
||||||
# but it can fail - in which case the checkdup will be withdrawn
|
# but it can fail - in which case the check_dup will be withdrawn
|
||||||
job4=NewJob( name="checkdups", num_files=0, wait_for=job2.id, jex=None, parent_job=parent_job, desc="check for duplicate files" )
|
job4=NewJob( name="check_dups", num_files=0, wait_for=job2.id, jex=None, parent_job=parent_job, desc="check for duplicate files" )
|
||||||
|
|
||||||
# okay, now process all the new jobs
|
# okay, now process all the new jobs
|
||||||
HandleJobs(False)
|
HandleJobs(False)
|
||||||
@@ -835,17 +835,17 @@ def RunJob(job):
|
|||||||
job.start_time=datetime.now(pytz.utc)
|
job.start_time=datetime.now(pytz.utc)
|
||||||
if job.name =="scan_ip":
|
if job.name =="scan_ip":
|
||||||
JobScanImportDir(job)
|
JobScanImportDir(job)
|
||||||
elif job.name =="forcescan":
|
elif job.name =="force_scan":
|
||||||
JobForceScan(job)
|
JobForceScan(job)
|
||||||
elif job.name =="scan_sp":
|
elif job.name =="scan_sp":
|
||||||
JobScanStorageDir(job)
|
JobScanStorageDir(job)
|
||||||
elif job.name =="importdir":
|
elif job.name =="import_dir":
|
||||||
JobImportDir(job)
|
JobImportDir(job)
|
||||||
elif job.name =="getfiledetails":
|
elif job.name =="get_file_details":
|
||||||
JobGetFileDetails(job)
|
JobGetFileDetails(job)
|
||||||
elif job.name == "checkdups":
|
elif job.name == "check_dups":
|
||||||
JobCheckForDups(job)
|
JobCheckForDups(job)
|
||||||
elif job.name == "rmdups":
|
elif job.name == "rm_dups":
|
||||||
JobRemoveDups(job)
|
JobRemoveDups(job)
|
||||||
elif job.name == "delete_files":
|
elif job.name == "delete_files":
|
||||||
JobDeleteFiles(job)
|
JobDeleteFiles(job)
|
||||||
@@ -1570,7 +1570,7 @@ def WithdrawDependantJobs( job, id, reason ):
|
|||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# next 3 funcs used to optimise whether to do dependant jobs (i.e. no new files, dont keep doing file details, ai scans
|
# next 3 funcs used to optimise whether to do dependant jobs (i.e. no new files, dont keep doing file details, ai scans
|
||||||
# find last successful importdir job for this path
|
# find last successful import_dir job for this path
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
def find_last_time_new_files_found(job):
|
def find_last_time_new_files_found(job):
|
||||||
path=[jex.value for jex in job.extra if jex.name == "path"][0]
|
path=[jex.value for jex in job.extra if jex.name == "path"][0]
|
||||||
@@ -1581,11 +1581,11 @@ def find_last_time_new_files_found(job):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# find time of last getfiledetails job for this path
|
# find time of last get_file_details job for this path
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
def find_last_successful_gfd_job(job):
|
def find_last_successful_gfd_job(job):
|
||||||
path=[jex.value for jex in job.extra if jex.name == "path"][0]
|
path=[jex.value for jex in job.extra if jex.name == "path"][0]
|
||||||
jobs=session.query(Job).join(JobExtra).filter(Job.name=="getfiledetails").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
|
jobs=session.query(Job).join(JobExtra).filter(Job.name=="get_file_details").filter(JobExtra.value==path).filter(Job.state=='Completed').order_by(Job.id.desc()).limit(1).all()
|
||||||
for j in jobs:
|
for j in jobs:
|
||||||
return j.last_update.timestamp()
|
return j.last_update.timestamp()
|
||||||
return 0
|
return 0
|
||||||
@@ -1731,7 +1731,7 @@ def JobImportDir(job):
|
|||||||
last_ai_scan=find_last_successful_ai_scan(job)
|
last_ai_scan=find_last_successful_ai_scan(job)
|
||||||
|
|
||||||
for j in session.query(Job).filter(Job.wait_for==job.id).all():
|
for j in session.query(Job).filter(Job.wait_for==job.id).all():
|
||||||
if j.name == "getfiledetails" and last_file_details > last_scan:
|
if j.name == "get_file_details" and last_file_details > last_scan:
|
||||||
FinishJob(j, f"Job #{j.id} has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
|
FinishJob(j, f"Job #{j.id} has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
|
||||||
# scan found no new files and last ai scan was after the last file scan
|
# scan found no new files and last ai scan was after the last file scan
|
||||||
if j.name == "run_ai_on_path" and last_ai_scan > last_scan:
|
if j.name == "run_ai_on_path" and last_ai_scan > last_scan:
|
||||||
@@ -2032,10 +2032,10 @@ def GenVideoThumbnail( job, fname):
|
|||||||
# /removedups, but some other job has since created another dup message...
|
# /removedups, but some other job has since created another dup message...
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
def ClearOtherDupMessagesAndJobs():
|
def ClearOtherDupMessagesAndJobs():
|
||||||
msgs=session.query(PA_JobManager_FE_Message).join(Job).filter(Job.name=='checkdups')
|
msgs=session.query(PA_JobManager_FE_Message).join(Job).filter(Job.name=='check_dups')
|
||||||
for msg in msgs:
|
for msg in msgs:
|
||||||
session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==msg.id).delete()
|
session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==msg.id).delete()
|
||||||
cd_jobs=session.query(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all()
|
cd_jobs=session.query(Job).filter(Job.name=='check_dups').filter(Job.pa_job_state=='New').all()
|
||||||
for j in cd_jobs:
|
for j in cd_jobs:
|
||||||
FinishJob(j, "New CheckForDups job/removal supercedes this job, withdrawing it", "Withdrawn")
|
FinishJob(j, "New CheckForDups job/removal supercedes this job, withdrawing it", "Withdrawn")
|
||||||
session.commit()
|
session.commit()
|
||||||
@@ -2065,7 +2065,7 @@ def JobCheckForDups(job):
|
|||||||
def JobRemoveDups(job):
|
def JobRemoveDups(job):
|
||||||
JobProgressState( job, "In Progress" )
|
JobProgressState( job, "In Progress" )
|
||||||
AddLogForJob(job, f"INFO: Starting Remove Duplicates job...")
|
AddLogForJob(job, f"INFO: Starting Remove Duplicates job...")
|
||||||
# as checkdups covers all dups, delete all future dups messages, and Withdraw future checkdups jobs
|
# as check_dups covers all dups, delete all future dups messages, and Withdraw future check_dups jobs
|
||||||
ClearOtherDupMessagesAndJobs()
|
ClearOtherDupMessagesAndJobs()
|
||||||
|
|
||||||
dup_cnt=0
|
dup_cnt=0
|
||||||
@@ -2120,8 +2120,8 @@ def JobRemoveDups(job):
|
|||||||
dup_cnt += 1
|
dup_cnt += 1
|
||||||
|
|
||||||
|
|
||||||
# Need to put another checkdups job in now to force / validate we have no dups
|
# Need to put another check_dups job in now to force / validate we have no dups
|
||||||
next_job=NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
next_job=NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
||||||
AddLogForJob(job, f"adding <a href='/job/{next_job.id}'>job id={next_job.id} {next_job.name}</a> to confirm there are no more duplicates" )
|
AddLogForJob(job, f"adding <a href='/job/{next_job.id}'>job id={next_job.id} {next_job.name}</a> to confirm there are no more duplicates" )
|
||||||
FinishJob(job, f"Finished removing {dup_cnt} duplicate files" )
|
FinishJob(job, f"Finished removing {dup_cnt} duplicate files" )
|
||||||
return
|
return
|
||||||
@@ -2152,7 +2152,7 @@ def JobMoveFiles(job):
|
|||||||
if 'eid-' in jex.name:
|
if 'eid-' in jex.name:
|
||||||
move_me=session.query(Entry).get(jex.value)
|
move_me=session.query(Entry).get(jex.value)
|
||||||
MoveEntriesToOtherFolder( job, move_me, dst_storage_path, f"{prefix}{suffix}" )
|
MoveEntriesToOtherFolder( job, move_me, dst_storage_path, f"{prefix}{suffix}" )
|
||||||
NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
||||||
FinishJob(job, f"Finished move selected file(s)")
|
FinishJob(job, f"Finished move selected file(s)")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -2166,7 +2166,7 @@ def JobDeleteFiles(job):
|
|||||||
if 'eid-' in jex.name:
|
if 'eid-' in jex.name:
|
||||||
del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
|
del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
|
||||||
MoveFileToRecycleBin(job,del_me)
|
MoveFileToRecycleBin(job,del_me)
|
||||||
NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
||||||
FinishJob(job, f"Finished deleting selected file(s)")
|
FinishJob(job, f"Finished deleting selected file(s)")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -2180,7 +2180,7 @@ def JobRestoreFiles(job):
|
|||||||
if 'eid-' in jex.name:
|
if 'eid-' in jex.name:
|
||||||
restore_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
|
restore_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
|
||||||
RestoreFile(job,restore_me)
|
RestoreFile(job,restore_me)
|
||||||
NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
NewJob( name="check_dups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
|
||||||
FinishJob(job, f"Finished restoring selected file(s)")
|
FinishJob(job, f"Finished restoring selected file(s)")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
16
states.py
16
states.py
@@ -60,14 +60,14 @@ class States(PA):
|
|||||||
|
|
||||||
# this is any next/prev or noo, grouping, etc. change (so use referrer to work out what to do with this)
|
# this is any next/prev or noo, grouping, etc. change (so use referrer to work out what to do with this)
|
||||||
# because this can happen on a view, or files_up, etc. change this FIRST
|
# because this can happen on a view, or files_up, etc. change this FIRST
|
||||||
if 'ChangeFileOpts' in request.path:
|
if 'change_file_opts' in request.path:
|
||||||
ref=request.referrer
|
ref=request.referrer
|
||||||
base=request.base_url
|
base=request.base_url
|
||||||
base=base.replace("ChangeFileOpts", "")
|
base=base.replace("change_file_opts", "")
|
||||||
self.url = "/"+ref.replace(base, "" )
|
self.url = "/"+ref.replace(base, "" )
|
||||||
|
|
||||||
# if viewlist, then we really are a view, and view_eid should be in the form
|
# if view_list, then we really are a view, and view_eid should be in the form
|
||||||
if 'viewlist' in request.path:
|
if 'view_list' in request.path:
|
||||||
self.path_type = 'View'
|
self.path_type = 'View'
|
||||||
self.view_eid = request.form['view_eid']
|
self.view_eid = request.form['view_eid']
|
||||||
self.url = request.form['orig_url']
|
self.url = request.form['orig_url']
|
||||||
@@ -122,7 +122,7 @@ class States(PA):
|
|||||||
self.view_eid = self.url[6:]
|
self.view_eid = self.url[6:]
|
||||||
self.path_type="View"
|
self.path_type="View"
|
||||||
self.orig_url=self.url
|
self.orig_url=self.url
|
||||||
elif 'ChangeFileOpts' not in self.url:
|
elif 'change_file_opts' not in self.url:
|
||||||
print( f"ERROR: DDP messed up, failed to match URL {self.url} for settings this will fail, redirecting to home" )
|
print( f"ERROR: DDP messed up, failed to match URL {self.url} for settings this will fail, redirecting to home" )
|
||||||
print( f"referrer={request.referrer}" )
|
print( f"referrer={request.referrer}" )
|
||||||
return
|
return
|
||||||
@@ -205,7 +205,7 @@ class States(PA):
|
|||||||
if request.method=="POST":
|
if request.method=="POST":
|
||||||
if self.path_type != "View" and 'noo' in request.form:
|
if self.path_type != "View" and 'noo' in request.form:
|
||||||
# we are changing values based on a POST to the form, if we changed the noo option, we need to reset things
|
# we are changing values based on a POST to the form, if we changed the noo option, we need to reset things
|
||||||
if 'ChangeFileOpts' in request.path and self.noo != request.form['noo']:
|
if 'change_file_opts' in request.path and self.noo != request.form['noo']:
|
||||||
self.noo=request.form['noo']
|
self.noo=request.form['noo']
|
||||||
self.first_eid=0
|
self.first_eid=0
|
||||||
self.last_eid=0
|
self.last_eid=0
|
||||||
@@ -222,7 +222,7 @@ class States(PA):
|
|||||||
# seems html cant do boolean, but uses strings so convert
|
# seems html cant do boolean, but uses strings so convert
|
||||||
if self.path_type != "View" and 'folders' in request.form:
|
if self.path_type != "View" and 'folders' in request.form:
|
||||||
# we are changing values based on a POST to the form, if we are in folder view and we changed the folders option, we need to reset things
|
# we are changing values based on a POST to the form, if we are in folder view and we changed the folders option, we need to reset things
|
||||||
if 'ChangeFileOpts' in request.path:
|
if 'change_file_opts' in request.path:
|
||||||
if self.folders and self.folders != request.form['folders']:
|
if self.folders and self.folders != request.form['folders']:
|
||||||
self.num_entries=0
|
self.num_entries=0
|
||||||
self.first_eid=0
|
self.first_eid=0
|
||||||
@@ -283,7 +283,7 @@ class States(PA):
|
|||||||
pref.first_eid = self.first_eid
|
pref.first_eid = self.first_eid
|
||||||
pref.last_eid = self.last_eid
|
pref.last_eid = self.last_eid
|
||||||
pref.num_entries = self.num_entries
|
pref.num_entries = self.num_entries
|
||||||
# only passed in (at the moment) in viewlist
|
# only passed in (at the moment) in view_list
|
||||||
pref.current = self.current
|
pref.current = self.current
|
||||||
|
|
||||||
db.session.add(pref)
|
db.session.add(pref)
|
||||||
|
|||||||
@@ -84,7 +84,7 @@
|
|||||||
<div class="dropdown-menu" aria-labelledby="PersonMenu">
|
<div class="dropdown-menu" aria-labelledby="PersonMenu">
|
||||||
<a class="dropdown-item" href="{{url_for('new_person')}}">Create Person</a>
|
<a class="dropdown-item" href="{{url_for('new_person')}}">Create Person</a>
|
||||||
<a class="dropdown-item" href="{{url_for('persons')}}">Show People</a>
|
<a class="dropdown-item" href="{{url_for('persons')}}">Show People</a>
|
||||||
<a class="dropdown-item" href="{{url_for('aistats')}}">View stats of matches</a>
|
<a class="dropdown-item" href="{{url_for('ai_stats')}}">View stats of matches</a>
|
||||||
<a class="dropdown-item" href="{{url_for('run_ai_on_import')}}">Match in import path(s) </a>
|
<a class="dropdown-item" href="{{url_for('run_ai_on_import')}}">Match in import path(s) </a>
|
||||||
<a class="dropdown-item" href="{{url_for('run_ai_on_storage')}}">Match in storage path(s) </a>
|
<a class="dropdown-item" href="{{url_for('run_ai_on_storage')}}">Match in storage path(s) </a>
|
||||||
{% if config.ENV != "production" %}
|
{% if config.ENV != "production" %}
|
||||||
@@ -99,9 +99,9 @@
|
|||||||
<a class="dropdown-item" href="{{url_for('scan_ip')}}">Scan now (for new files)</a>
|
<a class="dropdown-item" href="{{url_for('scan_ip')}}">Scan now (for new files)</a>
|
||||||
<a class="dropdown-item" href="{{url_for('scan_sp')}}">Scan Storage Path</a>
|
<a class="dropdown-item" href="{{url_for('scan_sp')}}">Scan Storage Path</a>
|
||||||
{% if config.ENV != "production" %}
|
{% if config.ENV != "production" %}
|
||||||
<a class="dropdown-item" href="{{url_for('forcescan')}}">Force Scan (delete data & rebuild)</a>
|
<a class="dropdown-item" href="{{url_for('force_scan')}}">Force Scan (delete data & rebuild)</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a class="dropdown-item" href="{{url_for('wakeup')}}">Force wake the job manager</a>
|
<a class="dropdown-item" href="{{url_for('wake_up')}}">Force wake the job manager</a>
|
||||||
</div class="dropdow-menu">
|
</div class="dropdow-menu">
|
||||||
</div class="nav-item dropdown">
|
</div class="nav-item dropdown">
|
||||||
<div class="d-flex nav-link flex-grow-1 justify-content-center p-0">
|
<div class="d-flex nav-link flex-grow-1 justify-content-center p-0">
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<div class="container-fluid">
|
<div class="container-fluid">
|
||||||
<form id="main_form" method="POST" action="/ChangeFileOpts">
|
<form id="main_form" method="POST" action="/change_file_opts">
|
||||||
<input type="hidden" name="cwd" id="cwd" value="{{OPT.cwd}}">
|
<input type="hidden" name="cwd" id="cwd" value="{{OPT.cwd}}">
|
||||||
{% if search_term is defined %}
|
{% if search_term is defined %}
|
||||||
<input type="hidden" name="search_term" id="view_term" value="{{search_term}}">
|
<input type="hidden" name="search_term" id="view_term" value="{{search_term}}">
|
||||||
@@ -253,7 +253,7 @@
|
|||||||
|
|
||||||
</div>
|
</div>
|
||||||
<div class="container-fluid">
|
<div class="container-fluid">
|
||||||
<form id="nav_form" method="POST" action="/ChangeFileOpts">
|
<form id="nav_form" method="POST" action="/change_file_opts">
|
||||||
<input type="hidden" name="cwd" id="cwd" value="{{OPT.cwd}}">
|
<input type="hidden" name="cwd" id="cwd" value="{{OPT.cwd}}">
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col my-auto d-flex justify-content-center">
|
<div class="col my-auto d-flex justify-content-center">
|
||||||
|
|||||||
@@ -27,9 +27,9 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
row+='<a href="{{url_for('joblog', id=job.id)}}">Job #{{job.id}} - {{job.name}}</a>'
|
row+='<a href="{{url_for('joblog', id=job.id)}}">Job #{{job.id}} - {{job.name}}</a>'
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if job.name != "rmdups" %}
|
{% if job.name != "rm_dups" %}
|
||||||
{% for ex in job.extra %}
|
{% for ex in job.extra %}
|
||||||
{% if 'path' == ex.name or ('path_prefix'==ex.name and job.name == 'checkdups') %}
|
{% if 'path' == ex.name or ('path_prefix'==ex.name and job.name == 'check_dups') %}
|
||||||
row+=' ({{ex.name}} == {{ ex.value }})'
|
row+=' ({{ex.name}} == {{ ex.value }})'
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ function ChangeDefault()
|
|||||||
data+="&default_grouping="+$('#grouping').val()
|
data+="&default_grouping="+$('#grouping').val()
|
||||||
data+="&default_import_folders="+$('#import_folders').val()
|
data+="&default_import_folders="+$('#import_folders').val()
|
||||||
data+="&default_storage_folders="+$('#storage_folders').val()
|
data+="&default_storage_folders="+$('#storage_folders').val()
|
||||||
$.ajax({ type: 'POST', data: data, url: '/changedefaults', success: function(data){ window.location='/states'; return false; } })
|
$.ajax({ type: 'POST', data: data, url: '/change_defaults', success: function(data){ window.location='/states'; return false; } })
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
{% endblock script_content %}
|
{% endblock script_content %}
|
||||||
|
|||||||
@@ -111,7 +111,7 @@
|
|||||||
{% if search_term is defined %}
|
{% if search_term is defined %}
|
||||||
data+="&search_term={{search_term}}"
|
data+="&search_term={{search_term}}"
|
||||||
{% endif %}
|
{% endif %}
|
||||||
$.ajax({ type: 'POST', data: data, url: '/viewlist', success: function(res){
|
$.ajax({ type: 'POST', data: data, url: '/view_list', success: function(res){
|
||||||
current=res.current
|
current=res.current
|
||||||
eids=res.eids
|
eids=res.eids
|
||||||
objs=res.objs
|
objs=res.objs
|
||||||
|
|||||||
10
user.py
10
user.py
@@ -1,6 +1,6 @@
|
|||||||
from main import db
|
from main import db
|
||||||
from sqlalchemy import Sequence
|
from sqlalchemy import Sequence
|
||||||
from flask import request, redirect
|
from flask import request, redirect, make_response, jsonify
|
||||||
from flask_login import UserMixin, login_required
|
from flask_login import UserMixin, login_required
|
||||||
from main import db, app, ma
|
from main import db, app, ma
|
||||||
from sqlalchemy.exc import SQLAlchemyError
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
@@ -39,12 +39,12 @@ class PAUser(UserMixin,db.Model):
|
|||||||
return self.dn
|
return self.dn
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# /changedefaults -> POST only -> changes defaults in DB, states.html reloads
|
# /change_defaults -> POST only -> changes defaults in DB, states.html reloads
|
||||||
# /states once its successful
|
# /states once its successful
|
||||||
################################################################################
|
################################################################################
|
||||||
@app.route("/changedefaults", methods=["POST"])
|
@app.route("/change_defaults", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
def changedefaults():
|
def change_defaults():
|
||||||
user=PAUser.query.filter(PAUser.dn==request.form['dn']).one()
|
user=PAUser.query.filter(PAUser.dn==request.form['dn']).one()
|
||||||
user.default_size = request.form['default_size']
|
user.default_size = request.form['default_size']
|
||||||
user.default_import_noo = request.form['default_import_noo']
|
user.default_import_noo = request.form['default_import_noo']
|
||||||
@@ -62,4 +62,4 @@ def changedefaults():
|
|||||||
user.default_storage_folders = False
|
user.default_storage_folders = False
|
||||||
db.session.add(user)
|
db.session.add(user)
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
return make_response( status="success" )
|
return make_response( jsonify( status="success" ) )
|
||||||
|
|||||||
Reference in New Issue
Block a user