diff --git a/BUGs b/BUGs
index 08a5a4e..80a7dba 100644
--- a/BUGs
+++ b/BUGs
@@ -1,4 +1,4 @@
-### Next: 113
+### Next: 114
BUG-100: I managed to get 2 photos matching mich in the NOT_WORKING photo (probably dif refimgs but same p.tag?)
= /photos/2012/20120414-damien/IMG_8467.JPG
BUG-106: cant add trudy /pat? as refimgs via FaceDBox
@@ -6,4 +6,3 @@ BUG-106: cant add trudy /pat? as refimgs via FaceDBox
(it came from a face bbox, BUT, I have grown the face seln by 10%?)
BUG-109: add mich force override, removed it, then re-added it, then rebuild DB form scratch and metadata has a duplicate
- redo disco metadata with md5 not UUID of face dataS
-BUG-113: with a stale job, we keep checking each 1 second, and so we keep creating new status messages
diff --git a/TODO b/TODO
index d78f430..cc6f236 100644
--- a/TODO
+++ b/TODO
@@ -1,19 +1,14 @@
### GENERAL
- * get all status messages to use toasts AND get func to also increase/descrease the job counter as appropriate)
- - [DONE] all (success/creation) status messages use toasts
- -- [TODO] ensure all pa_job_mgr jobs are sending messages back to the FE -- SHOULD??? I just hook FinishJob???
-
* should be using jsonify to return real json to my API calls, e.g:
use make_response( jsonify (... ) )
-- [TODO] all POSTs should follow new status behaviour (unless its a form POST, as that immediately renders the form in flask and will show the Status)
all GETs stay as is for now (as they expect a html reply, not data, and then html is base.html+ and it handles the status)
- -- ONLY time this will fail is multiple status messages (which can occur I believe if a Wake of job mgr and then a job is created in DB, the success will only be shown)
- -- [TODO] simple fix will be to get 'hidden' jobs (wake job_mgr and maybe? metadata) to post failure events to the JobMgr_FE DB queue
- -- [TODO] -- change class Status to class UILog
- -- [TODO] -- change alert to level
+ -- [TODO] find persons needs an array returned - test this, also viewlist
* delete files should behave like /move_files (stay on same page) as well as the status messages above
+ * all routes should be consistent naming conventions (with or without _ )
+
* change the rotation code to use that jpeg util to reduce/remove compression loss?
* ignore face should ignore ALL matching faces (re: Declan)
diff --git a/ai.py b/ai.py
index 46c0070..416f625 100644
--- a/ai.py
+++ b/ai.py
@@ -4,7 +4,6 @@ from flask import request, render_template, redirect
from main import db, app, ma
from sqlalchemy import Sequence
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
from path import Path, PathType
from files import Entry, Dir, File, PathDirLink
from person import Refimg, Person, PersonRefimgLink
@@ -52,8 +51,7 @@ def run_ai_on():
jex=[]
for el in request.form:
jex.append( JobExtra( name=f"{el}", value=request.form[el] ) )
- job=NewJob( "run_ai_on", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to Look for face(s) in selected file(s)")
+ job=NewJob( "run_ai_on", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in selected file(s)" )
return redirect("/jobs")
@app.route("/run_ai_on_import", methods=["GET"])
@@ -64,8 +62,7 @@ def run_ai_on_import():
ptype=PathType.query.filter(PathType.name=='Import').first()
jex.append( JobExtra( name=f"person", value="all" ) )
jex.append( JobExtra( name=f"path_type", value=ptype.id ) )
- job=NewJob( "run_ai_on_path", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to Look for face(s) in import path(s)")
+ job=NewJob( "run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in import path(s)")
return redirect("/jobs")
@app.route("/run_ai_on_storage", methods=["GET"])
@@ -75,8 +72,7 @@ def run_ai_on_storage():
ptype=PathType.query.filter(PathType.name=='Storage').first()
jex.append( JobExtra( name=f"person", value="all" ) )
jex.append( JobExtra( name=f"path_type", value=ptype.id ) )
- job=NewJob( "run_ai_on_path", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to Look for face(s) in storage path(s)")
+ job=NewJob( "run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in storage path(s)")
return redirect("/jobs")
@app.route("/unmatched_faces", methods=["GET"])
diff --git a/dups.py b/dups.py
index a3769e6..3106933 100644
--- a/dups.py
+++ b/dups.py
@@ -4,7 +4,6 @@ from flask import request, render_template, send_from_directory
from main import db, app, ma
from sqlalchemy import Sequence
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
import os
import glob
from PIL import Image
@@ -20,7 +19,6 @@ import re
################################################################################
# Local Class imports
################################################################################
-from job import Job, JobExtra, Joblog, NewJob
from settings import Settings
from shared import SymlinkName, PA
from path import PathType
diff --git a/files.py b/files.py
index dbbf540..385aaa0 100644
--- a/files.py
+++ b/files.py
@@ -4,7 +4,6 @@ from flask import request, render_template, redirect, send_from_directory, url_f
from main import db, app, ma
from sqlalchemy import Sequence
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
import os
import glob
from PIL import Image
@@ -24,7 +23,7 @@ from states import States, PA_UserState
################################################################################
# Local Class imports
################################################################################
-from job import Job, JobExtra, Joblog, NewJob
+from job import Job, JobExtra, Joblog, NewJob, SetFELog
from path import PathType, Path, MovePathDetails
from person import Refimg, Person, PersonRefimgLink
from settings import Settings, SettingsIPath, SettingsSPath, SettingsRBPath
@@ -463,13 +462,12 @@ def search(search_term):
return render_template("files.html", page_title='View Files', search_term=search_term, entry_data=entries, OPT=OPT, move_paths=move_paths )
################################################################################
-# /files/scannow -> allows us to force a check for new files
+# /files/scan_ip -> allows us to force a check for new files
################################################################################
-@app.route("/files/scannow", methods=["GET"])
+@app.route("/files/scan_ip", methods=["GET"])
@login_required
-def scannow():
- job=NewJob("scannow" )
- st.SetMessage("scanning for new files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) )
+def scan_ip():
+ job=NewJob( name="scan_ip", num_files=0, wait_for=None, jex=None, desc="scan for new files in import path" )
return redirect("/jobs")
################################################################################
@@ -478,8 +476,7 @@ def scannow():
@app.route("/files/forcescan", methods=["GET"])
@login_required
def forcescan():
- job=NewJob("forcescan" )
- st.SetMessage("force scan & rebuild data for files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) )
+ job=NewJob( name="forcescan", num_files=0, wait_for=None, jex=None, desc="remove data and rescan import & storage paths" )
return redirect("/jobs")
################################################################################
@@ -488,8 +485,7 @@ def forcescan():
@app.route("/files/scan_sp", methods=["GET"])
@login_required
def scan_sp():
- job=NewJob("scan_sp" )
- st.SetMessage("scanning for new files in: Job #{} (Click the link to follow progress)".format( job.id, job.id) )
+ job=NewJob( name="scan_sp", num_files=0, wait_for=None, jex=None, desc="scan for new files in storage path" )
return redirect("/jobs")
@@ -504,7 +500,7 @@ def fix_dups():
rows = db.engine.execute( "select e1.id as id1, f1.hash, d1.rel_path as rel_path1, d1.eid as did1, e1.name as fname1, p1.id as path1, p1.type_id as path_type1, e2.id as id2, d2.rel_path as rel_path2, d2.eid as did2, e2.name as fname2, p2.id as path2, p2.type_id as path_type2 from entry e1, file f1, dir d1, entry_dir_link edl1, path_dir_link pdl1, path p1, entry e2, file f2, dir d2, entry_dir_link edl2, path_dir_link pdl2, path p2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and p1.type_id != (select id from path_type where name = 'Bin') and p1.id = pdl1.path_id and pdl1.dir_eid = d1.eid and p2.type_id != (select id from path_type where name = 'Bin') and p2.id = pdl2.path_id and pdl2.dir_eid = d2.eid and f1.hash = f2.hash and e1.id != e2.id and f1.size_mb = f2.size_mb order by path1, rel_path1, fname1");
if rows.returns_rows == False:
- st.SetMessage(f"Err, no dups - should now clear the FE 'danger' message?")
+ SetFELog(f"Err, no dups - should now clear the FE 'danger' message?", "warning")
return redirect("/")
if 'pagesize' not in request.form:
@@ -546,8 +542,7 @@ def rm_dups():
jex.append( JobExtra( name="pagesize", value=10 ) )
- job=NewJob( "rmdups", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to delete duplicate files")
+ job=NewJob( name="rmdups", num_files=0, wait_for=None, jex=jex, desc="to delete duplicate files" )
return redirect("/jobs")
@@ -561,8 +556,7 @@ def restore_files():
for el in request.form:
jex.append( JobExtra( name=f"{el}", value=request.form[el] ) )
- job=NewJob( "restore_files", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to restore selected file(s)")
+ job=NewJob( name="restore_files", num_files=0, wait_for=None, jex=jex, desc="to restore selected file(s)" )
return redirect("/jobs")
################################################################################
@@ -575,8 +569,7 @@ def delete_files():
for el in request.form:
jex.append( JobExtra( name=f"{el}", value=request.form[el] ) )
- job=NewJob( "delete_files", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to delete selected file(s)")
+ job=NewJob( name="delete_files", num_files=0, wait_for=None, jex=jex, desc="to delete selected file(s)" )
return redirect("/jobs")
################################################################################
@@ -589,11 +582,9 @@ def move_files():
jex=[]
for el in request.form:
jex.append( JobExtra( name=f"{el}", value=request.form[el] ) )
- job=NewJob( "move_files", 0, None, jex )
- return make_response( jsonify(
- job_id=job.id,
- message=f"Created Job #{job.id} to move selected file(s)",
- level="success", alert="success", persistent=False, cant_close=False ) )
+ job=NewJob( name="move_files", num_files=0, wait_for=None, jex=jex, desc="to move selected file(s)" )
+ # data is not used, but send response to trigger CheckForJobs()
+ return make_response( jsonify( job_id=job.id ) )
@login_required
@app.route("/viewlist", methods=["POST"])
@@ -700,7 +691,7 @@ def view(id):
msg += "Clearing out all states. This means browser back buttons will not work, please start a new tab and try again"
PA_UserState.query.delete()
db.session.commit()
- st.SetMessage( msg, "warning" )
+ SetFELog( msg, "warning" )
return redirect("/")
else:
NMO_data = FaceOverrideType.query.all()
@@ -735,12 +726,8 @@ def transform():
for el in request.form:
jex.append( JobExtra( name=f"{el}", value=request.form[el] ) )
- job=NewJob( "transform_image", 0, None, jex )
-
- resp={}
- resp['job_id']=job.id
-
- return resp
+ job=NewJob( name="transform_image", num_files=0, wait_for=None, jex=jex, desc="to transform selected file(s)" )
+ return make_response( jsonify( job_id=job.id ) )
################################################################################
# /checktransformjob -> URL that is called repeatedly by front-end waiting for the
@@ -753,14 +740,12 @@ def transform():
def checktransformjob():
job_id = request.form['job_id']
job = Job.query.get(job_id)
- resp={}
- resp['finished']=False
+ j=jsonify( finished=False )
if job.pa_job_state == 'Completed':
id=[jex.value for jex in job.extra if jex.name == "id"][0]
e=Entry.query.join(File).filter(Entry.id==id).first()
- resp['thumbnail']=e.file_details.thumbnail
- resp['finished']=True
- return resp
+ j=jsonify( finished=True, thumbnail=e.file_details.thumbnail )
+ return make_response( j )
################################################################################
# /include -> return contents on /include and does not need a login, so we
diff --git a/internal/js/files_support.js b/internal/js/files_support.js
index 29d2f74..445875a 100644
--- a/internal/js/files_support.js
+++ b/internal/js/files_support.js
@@ -64,7 +64,7 @@ function MoveSubmit()
// reorder the images via ecnt again, so highlighting can work
document.mf_id=0; $('.figure').each( function() { $(this).attr('ecnt', document.mf_id ); document.mf_id++ } )
$('#dbox').modal('hide')
- $.ajax({ type: 'POST', data: $('#mv_fm').serialize(), url: '/move_files', success: function(data){ console.log(data); StatusMsg(data); CheckForJobs(); return false; } })
+ $.ajax({ type: 'POST', data: $('#mv_fm').serialize(), url: '/move_files', success: function(data){ CheckForJobs(); return false; } })
}
// show the DBox for a move file, includes all thumbnails of selected files to move
diff --git a/internal/js/jobs.js b/internal/js/jobs.js
index f26bab2..c97f6ca 100644
--- a/internal/js/jobs.js
+++ b/internal/js/jobs.js
@@ -1,30 +1,31 @@
-// global
-var next_toast_id=1
-
function NewToast(data)
{
- console.log(data)
- // make new div, include data.alert as background colour, and data.message as toast body
- d_id='st' + String(next_toast_id)
+ // toast "id" is based on msg_id, for any persistent/long-lived msg's we will try to reshow them
+ // dont bother, if it already exists it is visible, just move on
+ d_id='st' + String(data.id)
+ if( $('#'+d_id).length !== 0 )
+ return
+ // make new div, include data.level as background colour, and data.message as toast body
div='
+ div += ' bg-' + data.level + '" '
+ if( data.level == "success" )
+ div += 'data-bs-delay="2000" '
+ div += `role="alert" aria-live="assertive" aria-atomic="true">
+
`
div += data.message
- div += '
'
+ div += '
'
if( data.cant_close !== true )
{
div += ' '
}
@@ -35,9 +36,6 @@ function NewToast(data)
// insert this as the first element in the status_container
$('#status_container').prepend(div)
- // make sure we have a new id for next toast
- next_toast_id++
-
return d_id
}
@@ -45,20 +43,13 @@ function NewToast(data)
// can reuse any that are hidden, OR, create a new one by appending as needed (so we can have 2+ toasts on screen)
function StatusMsg(st)
{
- console.log('StatusMsg' + st )
el=NewToast(st)
$('#' + el ).toast("show")
- // if there is a job_id, then clear the message for it or it will be picked up again on reload
- // BUT, we dont want to do this immediately, should hook on close, but for
- // now, we will do this to get a first pass working
- if( st.job_id !== undefined )
- {
- console.log( 'set hidden.bs.toast handler for: ' + st.job_id )
- $('#' + el).on( 'hidden.bs.toast',
- function() {
- $.ajax( { type: 'POST', url: '/clearmsgforjob/'+st.job_id, success: function(data) { console.log('cleared job id' )} } )
- } )
- }
+ // clear message only when toast is hidden (either timeout OR user clicks close btn)
+ $('#' + el).on( 'hidden.bs.toast',
+ function() {
+ $.ajax( { type: 'POST', url: '/clearmsg/'+st.id, success: function(data) {} } )
+ } )
}
// this will make the active jobs badge red with a > 0 value, or navbar colours
diff --git a/internal/js/view_support.js b/internal/js/view_support.js
index 9403352..fd602eb 100644
--- a/internal/js/view_support.js
+++ b/internal/js/view_support.js
@@ -305,6 +305,7 @@ function CreatePersonAndRefimg( key )
$('#dbox').modal('hide')
$('#faces').prop('checked',true)
DrawImg()
+ CheckForJobs()
}
})
}
diff --git a/job.py b/job.py
index b746815..d03cb26 100644
--- a/job.py
+++ b/job.py
@@ -5,7 +5,6 @@ from settings import Settings
from main import db, app, ma
from sqlalchemy import Sequence, func
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
from datetime import datetime, timedelta
import pytz
import socket
@@ -68,24 +67,22 @@ class Job(db.Model):
# DB. has to be about a specific job_id and is success/danger, etc. (alert)
# and a message
################################################################################
-class PA_JobManager_Message(db.Model):
+class PA_JobManager_Message(PA,db.Model):
__tablename__ = "pa_job_manager_fe_message"
id = db.Column(db.Integer, db.Sequence('pa_job_manager_fe_message_id_seq'), primary_key=True )
job_id = db.Column(db.Integer, db.ForeignKey('job.id') )
- alert = db.Column(db.String)
+ level = db.Column(db.String)
message = db.Column(db.String)
persistent = db.Column(db.Boolean)
cant_close = db.Column(db.Boolean)
job = db.relationship ("Job" )
- def __repr__(self):
- return f"Job #{job.id} to {desc}', level="success" )
- WakePAJobManager()
+ WakePAJobManager(job.id)
return job
################################################################################
@@ -161,6 +159,20 @@ def FinishJob(job, last_log, state="Completed", pa_job_state="Completed"):
if job.state=="Failed":
WithdrawDependantJobs( job, job.id, "dependant job failed" )
db.session.commit()
+ if state=="Completed" :
+ level="success"
+ elif state=="Withdrawn" :
+ level="warning"
+ SetFELog( message=last_log, level=level )
+ return
+
+################################################################################
+# This allows a log to be picked up in jscript on the FE
+################################################################################
+def SetFELog(message, level="success", job_id=None, persistent=False, cant_close=False):
+ m=PA_JobManager_Message( message=message, level=level, job_id=job_id, persistent=persistent, cant_close=cant_close)
+ db.session.add(m)
+ db.session.commit()
return
################################################################################
@@ -230,7 +242,7 @@ def joblog(id):
@app.route("/wakeup", methods=["GET"])
@login_required
def wakeup():
- WakePAJobManager()
+ WakePAJobManager(job_id=None)
return redirect("/")
################################################################################
@@ -259,7 +271,7 @@ def stale_job(id):
db.engine.execute( f"delete from pa_job_manager_fe_message where job_id = {id}" )
db.session.commit()
- WakePAJobManager()
+ WakePAJobManager(job.id)
return redirect("/jobs")
################################################################################
@@ -311,11 +323,12 @@ def joblog_search():
@login_required
def CheckForJobs():
num=GetNumActiveJobs()
- print( f"called: /checkforjobs -- num={num}" )
sts=[]
for msg in PA_JobManager_Message.query.all():
- u='Job #' + str(msg.job_id) + ': '
- sts.append( { 'message': u+msg.message, 'alert': msg.alert, 'job_id': msg.job_id, 'persistent': msg.persistent, 'cant_close': msg.cant_close } )
+ u=''
+ if 'Job #' not in msg.message and msg.job_id:
+ u='Job #' + str(msg.job_id) + ': '
+ sts.append( { 'id': msg.id, 'message': u+msg.message, 'level': msg.level, 'job_id': msg.job_id, 'persistent': msg.persistent, 'cant_close': msg.cant_close } )
return make_response( jsonify( num_active_jobs=num, sts=sts ) )
###############################################################################
@@ -330,6 +343,18 @@ def ClearMessageForJob(id):
# no real need for this response, as if it succeeded/failed the F/E ignores it
return make_response( jsonify( status="success" ) )
+###############################################################################
+# / -> POST -> looks for pa_job_manager status to F/E jobs and sends json of
+# them back to F/E (called form internal/js/jobs.js:CheckForJobs()
+################################################################################
+@app.route("/clearmsg/", methods=["POST"])
+@login_required
+def ClearMessage(id):
+ PA_JobManager_Message.query.filter(PA_JobManager_Message.id==id).delete()
+ db.session.commit()
+ # no real need for this response, as if it succeeded/failed the F/E ignores it
+ return make_response( jsonify( id=id, status="success" ) )
+
###############################################################################
# This func creates a new filter in jinja2 to format the time from the db in a
# way that is more readable (converted to local tz too)
diff --git a/main.py b/main.py
index 5cceddf..0e3a2ca 100644
--- a/main.py
+++ b/main.py
@@ -11,7 +11,6 @@ from datetime import datetime
import os
import re
import socket
-from status import st, Status
from shared import CreateSelect, CreateFoldersSelect, LocationIcon, DB_URL, PROD_HOST, OLDEST_LOG_LIMIT
# for ldap auth
@@ -59,19 +58,12 @@ Compress(app)
from ai import aistats
from files import Entry
from person import Person
-from job import Job, GetNumActiveJobs, GetJM_Message, ClearJM_Message
from settings import Settings
from user import PAUser
####################################### GLOBALS #######################################
# allow jinja2 to call these python functions directly
app.jinja_env.add_extension('jinja2.ext.loopcontrols')
-app.jinja_env.globals['ClearStatus'] = st.ClearStatus
-app.jinja_env.globals['GetAlert'] = st.GetAlert
-app.jinja_env.globals['GetMessage'] = st.GetMessage
-app.jinja_env.globals['GetNumActiveJobs'] = GetNumActiveJobs
-app.jinja_env.globals['GetJM_Message'] = GetJM_Message
-app.jinja_env.globals['ClearJM_Message'] = ClearJM_Message
app.jinja_env.globals['CreateSelect'] = CreateSelect
app.jinja_env.globals['CreateFoldersSelect'] = CreateFoldersSelect
app.jinja_env.globals['LocationIcon'] = LocationIcon
diff --git a/pa_job_manager.py b/pa_job_manager.py
index deb52aa..5fff9f9 100644
--- a/pa_job_manager.py
+++ b/pa_job_manager.py
@@ -494,19 +494,19 @@ class Job(Base):
##############################################################################
# Class describing PA_JobManager_FE_Message and in the DB (via sqlalchemy)
# the job manager (this code) can send a message back to the front end via the
-# DB. has to be about a specific job_id and is success/danger, etc. (alert)
+# DB. has to be about a specific job_id and is success/danger, etc. (level)
# and a message
################################################################################
class PA_JobManager_FE_Message(Base):
__tablename__ = "pa_job_manager_fe_message"
id = Column(Integer, Sequence('pa_job_manager_fe_message_id_seq'), primary_key=True )
job_id = Column(Integer, ForeignKey('job.id') )
- alert = Column(String)
+ level = Column(String)
message = Column(String)
persistent = Column(Boolean)
cant_close = Column(Boolean)
def __repr__(self):
- return "Job #{job.id} to {desc}',
+ level="success", persistent=False, cant_close=False )
if parent_job:
str=f"adding job id={job.id} {job.name}"
if job.wait_for:
@@ -560,11 +562,11 @@ def NewJob(name, num_files=0, wait_for=None, jex=None, parent_job=None ):
return job
##############################################################################
-# MessageToFE(): sends a specific alert/messasge for a given job via the DB to
+# MessageToFE(): sends a specific level/messasge for a given job via the DB to
# the front end
##############################################################################
-def MessageToFE( job_id, alert, message, persistent, cant_close ):
- msg = PA_JobManager_FE_Message( job_id=job_id, alert=alert, message=message, persistent=persistent, cant_close=cant_close)
+def MessageToFE( job_id, message, level, persistent, cant_close ):
+ msg = PA_JobManager_FE_Message( job_id=job_id, message=message, level=level, persistent=persistent, cant_close=cant_close)
session.add(msg)
session.commit()
return msg.id
@@ -686,22 +688,22 @@ def JobsForPath( parent_job, path, ptype ):
jex=[]
jex.append( JobExtra( name="path", value=path ) )
jex.append( JobExtra( name="path_type", value=ptype.id ) )
- job1=NewJob( "importdir", cfn, None, jex, parent_job )
+ job1=NewJob( name="importdir", num_files=cfn, wait_for=None, jex=jex, parent_job=parent_job, desc=f"scan for files from {ptype.name} path" )
# then get file details (hash/thumbs)
jex=[]
jex.append( JobExtra( name="path", value=path ) )
- job2=NewJob("getfiledetails", 0, job1.id, jex, parent_job )
+ job2=NewJob( name="getfiledetails", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"get details of files from {ptype.name} path" )
# can start straight after importdir - job1, does not need details (job2)
jex=[]
jex.append( JobExtra( name="person", value="all" ) )
jex.append( JobExtra( name="path_type", value=ptype.id ) )
- job3=NewJob("run_ai_on_path", 0, job1.id, jex, parent_job )
+ job3=NewJob( name="run_ai_on_path", num_files=0, wait_for=job1.id, jex=jex, parent_job=parent_job, desc=f"match faces on files from {ptype.name} path" )
# careful here, wait for getfiledetails (job2), the ai job cannot cause a dup
# but it can fail - in which case the checkdup will be withdrawn
- job4=NewJob( "checkdups", 0, job2.id, None, parent_job )
+ job4=NewJob( name="checkdups", num_files=0, wait_for=job2.id, jex=None, parent_job=parent_job, desc="check for duplicate files" )
# okay, now process all the new jobs
HandleJobs(False)
@@ -832,8 +834,8 @@ def RunJob(job):
# only update start_time if we have never set it - stops restarts resetting start_time
if not job.start_time:
job.start_time=datetime.now(pytz.utc)
- if job.name =="scannow":
- JobScanNow(job)
+ if job.name =="scan_ip":
+ JobScanImportDir(job)
elif job.name =="forcescan":
JobForceScan(job)
elif job.name =="scan_sp":
@@ -874,7 +876,7 @@ def RunJob(job):
##############################################################################
# FinishJob(): finish this job off (if no overrides), its just marked completed
##############################################################################
-def FinishJob(job, last_log, state="Completed", pa_job_state="Completed"):
+def FinishJob(job, last_log, state="Completed", pa_job_state="Completed", level="success", persistent=False, cant_close=False):
job.state=state
job.pa_job_state=pa_job_state
if not job.start_time:
@@ -884,6 +886,7 @@ def FinishJob(job, last_log, state="Completed", pa_job_state="Completed"):
if job.state=="Failed":
WithdrawDependantJobs( job, job.id, "failed" )
session.commit()
+ MessageToFE( job_id=job.id, message=last_log, level=level, persistent=persistent, cant_close=cant_close )
if DEBUG:
print( f"DEBUG: {last_log}" )
return
@@ -906,7 +909,7 @@ def HandleJobs(first_run=False):
job.pa_job_state = 'Stale'
session.add(job)
AddLogForJob( job, "ERROR: Job has been marked stale as it did not complete" )
- MessageToFE( job.id, "danger", f'Stale job, click here to restart or cancel', True, False )
+ MessageToFE( job_id=job.id, message=f'Stale job, click here to restart or cancel', level="danger", persistent=True, cant_close=False )
session.commit()
continue
if job.pa_job_state == 'New':
@@ -932,7 +935,7 @@ def HandleJobs(first_run=False):
# threading.Thread(target=RunJob, args=(job,)).start()
except Exception as e:
try:
- MessageToFE( job.id, "danger", "Failed with: {} (try job log for details)".format(e), True, False )
+ MessageToFE( job_id=job.id, level="danger", message="Failed with: {} (try job log for details)".format(e), persistent=True, cant_close=False )
except Exception as e2:
print("ERROR: Failed to let front-end know, but back-end Failed to run job (id: {}, name: {} -- orig exep was: {}, this exception was: {})".format( job.id, job.name, e, e2) )
print("INFO: PA job manager is waiting for a job")
@@ -950,13 +953,12 @@ def JobProgressState( job, state ):
return
##############################################################################
-# JobScanNow(): start and process the job to start scanning now (import paths)
+# JobScanImportDir(): start and process the job to start scanning now (import paths)
##############################################################################
-def JobScanNow(job):
+def JobScanImportDir(job):
JobProgressState( job, "In Progress" )
ProcessImportDirs(job)
FinishJob( job, "Completed (scan for new files)" )
- MessageToFE( job.id, "success", "Completed (scan for new files)", False, False )
return
##############################################################################
@@ -966,7 +968,6 @@ def JobScanStorageDir(job):
JobProgressState( job, "In Progress" )
ProcessStorageDirs(job)
FinishJob( job, "Completed (scan for new files)" )
- MessageToFE( job.id, "success", "Completed (scan for new files)", False, False )
return
@@ -1080,7 +1081,6 @@ def JobForceScan(job):
ProcessImportDirs(job)
ProcessStorageDirs(job)
FinishJob(job, "Completed (forced remove and recreation of all file data)")
- MessageToFE( job.id, "success", "Completed (forced remove and recreation of all file data)", False, False )
return
##############################################################################
@@ -1553,7 +1553,7 @@ def AddJexToDependantJobs(job,name,value):
####################################################################################################################################
def WithdrawDependantJobs( job, id, reason ):
for j in session.query(Job).filter(Job.wait_for==id).all():
- FinishJob(j, f"Job (#{j.id}) has been withdrawn -- #{job.id} {reason}", "Withdrawn" )
+ FinishJob(j, f"Job #{j.id} has been withdrawn -- #{job.id} {reason}", "Withdrawn" )
WithdrawDependantJobs(j, j.id, reason)
return
@@ -1722,17 +1722,18 @@ def JobImportDir(job):
for j in session.query(Job).filter(Job.wait_for==job.id).all():
if j.name == "getfiledetails" and last_file_details > last_scan:
- FinishJob(j, f"Job (#{j.id}) has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
+ FinishJob(j, f"Job #{j.id} has been withdrawn -- #{job.id} (scan job) did not find new files", "Withdrawn" )
# scan found no new files and last ai scan was after the last file scan
if j.name == "run_ai_on_path" and last_ai_scan > last_scan:
newest_refimg = session.query(Refimg).order_by(Refimg.created_on.desc()).limit(1).all()
# IF we also have no new refimgs since last scan, then no need to run any AI again
if newest_refimg and newest_refimg[0].created_on < last_scan:
- FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
+ FinishJob(j, f"Job #{j.id} has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
# IF we also have no new refimgs since last AI scan, then no need to run any AI again
elif newest_refimg and newest_refimg[0].created_on < last_ai_scan:
- FinishJob(j, f"Job (#{j.id}) has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
- FinishJob(job, f"Finished Importing: {path} - Processed {overall_file_cnt} files, Found {found_new_files} new files, Removed {rm_cnt} file(s)")
+ FinishJob(j, f"Job #{j.id} has been withdrawn -- scan did not find new files, and no new reference images since last scan", "Withdrawn" )
+ AddLogForJob(job, f"Finished importing: {path} - Processed {overall_file_cnt} files, Found {found_new_files} new files, Removed {rm_cnt} file(s)")
+ FinishJob(job, f"Finished import of {path_obj.type.name} path")
return
####################################################################################################################################
@@ -2042,10 +2043,9 @@ def JobCheckForDups(job):
for row in res:
if row.count > 0:
AddLogForJob(job, f"Found duplicates, Creating Status message in front-end for attention")
- MessageToFE( job.id, "danger", f'Found duplicate(s), click here to finalise import by removing duplicates', True, True )
+ FinishJob( job=job, last_log=f'Found duplicate(s), click here to finalise import by removing duplicates', state="Completed", pa_job_state="Completed", level="danger", persistent=True, cant_close=True )
else:
FinishJob(job, f"No duplicates found")
- FinishJob(job, f"Finished looking for duplicates")
return
####################################################################################################################################
@@ -2109,12 +2109,11 @@ def JobRemoveDups(job):
MoveFileToRecycleBin(job,del_me)
dup_cnt += 1
- FinishJob(job, f"Finished removing {dup_cnt} duplicate files" )
# Need to put another checkdups job in now to force / validate we have no dups
- next_job=NewJob( "checkdups" )
+ next_job=NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
AddLogForJob(job, f"adding job id={next_job.id} {next_job.name} to confirm there are no more duplicates" )
- MessageToFE( job.id, "success", f"Finished Job#{job.id} removing duplicate files", False, False )
+ FinishJob(job, f"Finished removing {dup_cnt} duplicate files" )
return
####################################################################################################################################
@@ -2143,8 +2142,7 @@ def JobMoveFiles(job):
if 'eid-' in jex.name:
move_me=session.query(Entry).get(jex.value)
MoveEntriesToOtherFolder( job, move_me, dst_storage_path, f"{prefix}{suffix}" )
- next_job=NewJob( "checkdups" )
- MessageToFE( job.id, "success", "Completed (move of selected files)", False, False )
+ NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
FinishJob(job, f"Finished move selected file(s)")
return
@@ -2158,8 +2156,7 @@ def JobDeleteFiles(job):
if 'eid-' in jex.name:
del_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
MoveFileToRecycleBin(job,del_me)
- next_job=NewJob( "checkdups" )
- MessageToFE( job.id, "success", "Completed (delete of selected files)", False, False )
+ NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
FinishJob(job, f"Finished deleting selected file(s)")
return
@@ -2173,8 +2170,7 @@ def JobRestoreFiles(job):
if 'eid-' in jex.name:
restore_me=session.query(Entry).join(File).filter(Entry.id==jex.value).first()
RestoreFile(job,restore_me)
- next_job=NewJob( "checkdups" )
- MessageToFE( job.id, "success", "Completed (restore of selected files)", False, False )
+ NewJob( name="checkdups", num_files=0, wait_for=None, jex=None, parent_job=None, desc="check for duplicate files" )
FinishJob(job, f"Finished restoring selected file(s)")
return
@@ -2339,7 +2335,7 @@ def ReloadMetadata(job):
####################################################################################################################################
def InitialValidationChecks():
now=datetime.now(pytz.utc)
- job=NewJob( "init" )
+ job=NewJob( name="init", num_files=0, wait_for=None, jex=None, parent_job=None, desc="initialise photo assistant" )
job.start_time=datetime.now(pytz.utc)
JobProgressState( job, "In Progress" )
AddLogForJob(job, f"INFO: Starting Initial Validation checks...")
@@ -2640,7 +2636,7 @@ def CheckAndRunBinClean():
now=datetime.now(pytz.utc)
if not j or (now-j.last_update).days >= settings.scheduled_bin_cleanup:
print( f"INFO: Should force clean up bin path, del files older than {settings.bin_cleanup_file_age} days old" )
- job=NewJob( "clean_bin" )
+ NewJob( name="clean_bin", num_files=0, wait_for=None, jex=None, parent_job=None, desc="periodic clean up on Bin path" )
created_jobs=True
return created_jobs
@@ -2660,11 +2656,11 @@ def ScheduledJobs():
now=datetime.now(pytz.utc)
if ndays_since_last_im_scan >= settings.scheduled_import_scan:
print( f"INFO: Time to force an import scan, last scan was {ndays_since_last_im_scan} days ago" )
- job=NewJob( "scannow" )
+ NewJob( name="scan_ip", num_files=0, wait_for=None, jex=None, parent_job=None, desc="periodic clean scan for new files in Import path" )
created_jobs=True
if ndays_since_last_st_scan >= settings.scheduled_storage_scan:
print( f"INFO: Time to force a storage scan, last scan was {ndays_since_last_st_scan}" )
- job=NewJob( "scan_sp" )
+ NewJob( name="scan_sp", num_files=0, wait_for=None, jex=None, parent_job=None, desc="periodic clean scan for new files in Storage path" )
created_jobs=True
if CheckAndRunBinClean():
created_jobs=True
diff --git a/person.py b/person.py
index 99bd75b..e6b0212 100644
--- a/person.py
+++ b/person.py
@@ -1,17 +1,16 @@
from wtforms import SubmitField, StringField, HiddenField, validators, Form
from flask_wtf import FlaskForm
-from flask import request, render_template, redirect, url_for
+from flask import request, render_template, redirect, url_for, make_response, jsonify
from main import db, app, ma
from settings import Settings, AIModel
from sqlalchemy import Sequence
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
from flask_login import login_required, current_user
from werkzeug.utils import secure_filename
from shared import GenFace, GenThumb, PA
from face import Face, FaceRefimgLink, FaceOverrideType, FaceNoMatchOverride, FaceForceMatchOverride
from path import Path, PathType
-from job import JobExtra, NewJob
+from job import JobExtra, NewJob, SetFELog
import os
import time
@@ -98,7 +97,7 @@ def AddRefimgToPerson( filename, person ):
print( f"Failed to delete tmp file for refimg addition: {e}" )
if not face_locn:
- st.SetMessage( f"Failed to find face in Refimg:" )
+ SetFELog( f"Failed to find face in Refimg:", "danger" )
raise Exception("Could not find face in uploaded reference image" )
return
refimg.face_top = face_locn[0]
@@ -111,11 +110,11 @@ def AddRefimgToPerson( filename, person ):
db.session.add(person)
db.session.add(refimg)
db.session.commit()
- st.SetMessage( f"Associated new Refimg ({refimg.fname}) with person: {person.tag}" )
+ SetFELog( f"Associated new Refimg ({refimg.fname}) with person: {person.tag}" )
except SQLAlchemyError as e:
- st.SetMessage( f"Failed to add Refimg: {e.orig}", "danger" )
+ SetFELog( f"Failed to add Refimg: {e.orig}", "danger" )
except Exception as e:
- st.SetMessage( f"Failed to modify Refimg: {e}", "danger" )
+ SetFELog( f"Failed to modify Refimg: {e}", "danger" )
return
################################################################################
@@ -171,10 +170,10 @@ def new_person():
try:
db.session.add(person)
db.session.commit()
- st.SetMessage( "Created new Person ({})".format(person.tag) )
+ SetFELog( f"Created new Person ({person.tag})" )
return redirect( url_for( 'person', id=person.id) )
except SQLAlchemyError as e:
- st.SetMessage( f"Failed to add Person: {e.orig}", "danger" )
+ SetFELog( f"Failed to add Person: {e.orig}", "danger" )
return redirect( url_for( '/persons') )
else:
return render_template("person.html", person=None, form=form, page_title=page_title )
@@ -186,10 +185,8 @@ def match_with_create_person():
# add this fname (of temp refimg) to person
fname=TempRefimgFile( request.form['refimg_data'], p.tag )
AddRefimgToPerson( fname, p )
- resp={}
- resp['who']=p.tag
- resp['distance']='0.0'
- return resp
+ SetFELog( f"Created person: {p.tag}" )
+ return make_response( jsonify( who=p.tag, distance='0.0' ) )
################################################################################
# /person/ -> GET/POST(save or delete) -> shows/edits/delets a single person
@@ -204,7 +201,7 @@ def person(id):
try:
person = Person.query.get(id)
if 'delete' in request.form:
- st.SetMessage("Successfully deleted Person: ({})".format( person.tag ) )
+ SetFELog( f"Successfully deleted Person: ({person.tag})" )
# do linkages by hand, or one day replace with delete cascade in the DB defintions
db.session.execute( f"delete from face_refimg_link frl where refimg_id in ( select refimg_id from person_refimg_link where person_id = {id} )" )
db.session.execute( f"delete from person_refimg_link where person_id = {id}" )
@@ -225,23 +222,23 @@ def person(id):
# delete the "match" between a face found in a file and this ref img
FaceRefimgLink.query.filter(FaceRefimgLink.refimg_id==deld[0].id).delete()
Refimg.query.filter(Refimg.id==deld[0].id).delete()
- st.SetMessage( f"Successfully Updated Person: removed reference image {deld[0].fname}" )
+ SetFELog( f"Successfully Updated Person: removed reference image {deld[0].fname}" )
else:
- st.SetMessage("Successfully Updated Person: (From: {}, {}, {})".format(person.tag, person.firstname, person.surname) )
+ s=f"Successfully Updated Person: (From: {person.tag}, {person.firstname}, {person.surname})"
person.tag = request.form['tag']
person.surname = request.form['surname']
person.firstname = request.form['firstname']
- st.AppendMessage(" To: ({}, {}, {})".format(person.tag, person.firstname, person.surname) )
+ SetFELog( f"{s} To: ({person.tag}, {person.firstname}, {person.surname})" )
db.session.add(person)
db.session.commit()
return redirect( url_for( 'person', id=person.id) )
except SQLAlchemyError as e:
- st.SetMessage( f"Failed to modify Person: {e}", "danger" )
+ SetFELog( f"Failed to modify Person: {e}", "danger" )
return redirect( url_for( 'persons' ) )
else:
person = Person.query.get(id)
if not person:
- st.SetMessage( f"No such person with id: {id}", "danger" )
+ SetFELog( f"No such person with id: {id}", "danger" )
return redirect("/")
form = PersonForm(request.values, obj=person)
return render_template("person.html", person=person, form=form, page_title = page_title)
@@ -266,7 +263,7 @@ def add_refimg():
fname = f"/tmp/{fname}"
f.save( fname )
except Exception as e:
- st.SetMessage( f"Failed to load reference image: {e}", "danger" )
+ SetFELog( f"Failed to load reference image: {e}", "danger" )
AddRefimgToPerson( fname, person )
return redirect( url_for( 'person', id=person.id) )
@@ -297,7 +294,6 @@ def find_persons(who):
@app.route("/add_refimg_to_person", methods=["POST"])
@login_required
def add_refimg_to_person():
- resp={}
f = Face.query.get( request.form['face_id'] )
p = Person.query.get( request.form['person_id'] )
@@ -310,19 +306,15 @@ def add_refimg_to_person():
ptype=PathType.query.filter(PathType.name=='Import').first()
jex.append( JobExtra( name=f"person", value="all" ) )
jex.append( JobExtra( name=f"path_type", value=ptype.id ) )
- job=NewJob( "run_ai_on_path", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to Look for face(s) in import path(s)")
+ job=NewJob( name="run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in import path(s)" )
jex=[]
ptype=PathType.query.filter(PathType.name=='Storage').first()
jex.append( JobExtra( name=f"person", value="all" ) )
jex.append( JobExtra( name=f"path_type", value=ptype.id ) )
- job=NewJob( "run_ai_on_path", 0, None, jex )
- st.SetMessage( f"Created Job #{job.id} to Look for face(s) in storage path(s)")
+ job=NewJob( name="run_ai_on_path", num_files=0, wait_for=None, jex=jex, desc="Look for face(s) in storage path(s)" )
- resp['who']=p.tag
- resp['distance']='0.0'
- return resp
+ return make_response( jsonify( who=p.tag, distance='0.0' ) )
################################################################################
# /add_force_match_override -> POST
@@ -349,13 +341,11 @@ def add_force_match_override():
jex.append( JobExtra( name="face_id", value=f.id ) )
jex.append( JobExtra( name="person_id", value=p.id ) )
# dont do status update here, the F/E is in the middle of a dbox, just send metadata through to the B/E
- NewJob( "metadata", 0, None, jex )
+ NewJob( "metadata", num_files=0, wait_for=None, jex=jex, desc="create metadata for adding forced match" )
print( f"Placing an override match with face_id {face_id}, for person: {p.tag}" )
# this will reply to the Ajax / POST, and cause the page to re-draw with new face override to person_tag
- resp={}
- resp['person_tag']=p.tag
- return resp
+ return make_response( jsonify( person_tag=p.tag ) )
################################################################################
# /remove_force_match_override -> POST
@@ -379,11 +369,10 @@ def remove_force_match_override():
jex.append( JobExtra( name="face_id", value=face_id ) )
jex.append( JobExtra( name="person_id", value=p.id ) )
# dont do status update here, the F/E is in the middle of a dbox, just send metadata through to the B/E
- NewJob( "metadata", 0, None, jex )
+ NewJob( "metadata", num_files=0, wait_for=None, jex=jex, desc="create metadata for removing forced match" )
- # this will reply to the Ajax / POST, and cause the page to re-draw with new face override
- resp={}
- return resp
+ # this will reply to the Ajax / POST, and cause the page to re-draw with new face override (data is not used)
+ return make_response( jsonify( person_tag=p.tag ) )
################################################################################
# /remove_no_match_override -> POST
@@ -403,11 +392,10 @@ def remove_no_match_override():
jex.append( JobExtra( name="face_id", value=face_id ) )
jex.append( JobExtra( name="type_id", value=type_id ) )
# dont do status update here, the F/E is in the middle of a dbox, just send metadata through to the B/E
- NewJob( "metadata", 0, None, jex )
+ NewJob( "metadata", num_files=0, wait_for=None, jex=jex, desc="create metadata for removing forced non-match" )
- # this will reply to the Ajax / POST, and cause the page to re-draw with new face override
- resp={}
- return resp
+ # this will reply to the Ajax / POST, and cause the page to re-draw with new face override (data is not used)
+ return make_response( jsonify( face_id=face_id ) )
################################################################################
@@ -435,10 +423,8 @@ def add_no_match_override():
jex.append( JobExtra( name="face_id", value=f.id ) )
jex.append( JobExtra( name="type_id", value=t.id ) )
# dont do status update here, the F/E is in the middle of a dbox, just send metadata through to the B/E
- NewJob( "metadata", 0, None, jex )
+ NewJob( "metadata", num_files=0, wait_for=None, jex=jex, desc="create metadata for adding forced non-match" )
print( f"Placing an override of NO Match for face_id {face_id}" )
# this will reply to the Ajax / POST, and cause the page to re-draw with new face override to person_tag
- resp={}
- resp['type']=t.name
- return resp
+ return make_response( jsonify( type=t.name ) )
diff --git a/settings.py b/settings.py
index 9fd4684..ce78de4 100644
--- a/settings.py
+++ b/settings.py
@@ -3,7 +3,6 @@ from flask_wtf import FlaskForm
from flask import request, render_template, redirect, url_for
from sqlalchemy import Sequence
from sqlalchemy.exc import SQLAlchemyError
-from status import st, Status
from flask_login import login_required, current_user
from main import db, app, ma
@@ -98,10 +97,10 @@ def settings():
if request.method == 'POST' and form.validate():
try:
-
+ from job import SetFELog
s = Settings.query.one()
if 'submit' in request.form:
- st.SetMessage("Successfully Updated Settings" )
+ SetFELog("Successfully Updated Settings" )
s.import_path = request.form['import_path']
s.storage_path = request.form['storage_path']
s.recycle_bin_path = request.form['recycle_bin_path']
@@ -122,7 +121,7 @@ def settings():
db.session.commit()
return redirect( url_for( 'settings' ) )
except SQLAlchemyError as e:
- st.SetMessage( f"Failed to modify Setting: {e.orig}", "danger" )
+ SetFELog( f"Failed to modify Setting: {e.orig}", "danger" )
return render_template("settings.html", form=form, page_title=page_title, HELP=HELP)
else:
form = SettingsForm( obj=Settings.query.first() )
diff --git a/templates/base.html b/templates/base.html
index 058bae1..1afab3f 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -96,7 +96,7 @@
Admin