From a67c20d72b4c78585c5c6b3b68b12208ec683702 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 11 Jan 2022 13:18:21 +1100 Subject: [PATCH] new BUG (very minor), reordered TODOs and now have basic stale job handling - they are detected, and can be cancelled or restarted from GUI --- BUGs | 3 ++- TODO | 12 +++--------- job.py | 39 ++++++++++++++++++++++++++++++++++++++- pa_job_manager.py | 35 ++++++++++++++++++++++++----------- templates/base.html | 2 +- templates/jobs.html | 24 ++++++++++++++++++++++-- 6 files changed, 90 insertions(+), 25 deletions(-) diff --git a/BUGs b/BUGs index 9df42af..0fdc739 100644 --- a/BUGs +++ b/BUGs @@ -1,4 +1,5 @@ -### Next: 77 +### Next: 78 BUG-56: when making a viewing list of AI:mich, (any search?) and going past the page_size, it gets the wrong data from the DB for the 'next' entry BUG-60: entries per page (in folders view) ignores pagesize, and this also contributes to BUG-56 I think BUG-74: search/others? remembers start/offset, and if you reset view (e.g. another search) it doesnt show first page of results +BUG-77: when moving folders out from a parent folder (storage/2020 off-camera-to-oct), it did not delete the empty 2020 off-camera-to-oct folder diff --git a/TODO b/TODO index f3b329d..39d4cc6 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,4 @@ ## GENERAL - * dont allow me to stupidly move a folder to itself - - * move all unsorted photos/* -> import/ - * per file you could select an unknown face and add it as a ref img to an existing person, or make a new person and attach? * when search, have a way to hide deleted files @@ -19,6 +15,8 @@ * put a delete option on viewer page + * delete folder + * in Fullscreen mode and next/prev dropped out of FS when calling /viewlist route -- only way to fix this, is for when we POST to viewlist, it returns json, and we never leave /view/X -- then we can stay on-page, and stay in FS and then just call ViewImageOrVide() @@ -26,8 +24,7 @@ * metadata at folder level with file level to add more richness - store in DB? or store in hidden file (or both)... IF it is outside the DB, then I can 'rebuild' the DB at anytime from scratch - * why .xcf is seen as a video??? - - actually only 1 is... I think if type == 'Unknown' then do file display and use ? as the image again + * dont allow me to stupidly move a folder to itself * get build process to create a random string for secret for PROD, otherwise use builtin for dev @@ -59,9 +56,6 @@ * put weekly? job to scan storage dir * put weekly? job to scan storage dir - need a manual button to restart a job in the GUI, - (based on file-level optims, just run the job as new and it will optim over already done parts and continue) - Admin -> do I want to have admin roles/users? -> purge deleted files (and associated DB data) needs a dbox or privs diff --git a/job.py b/job.py index b4030a1..2a895e2 100644 --- a/job.py +++ b/job.py @@ -143,7 +143,8 @@ def joblog(id): return render_template("joblog.html", job=joblog, logs=logs, log_cnt=log_cnt, duration=duration, page_title=page_title, first_logs_only=first_logs_only, estimate=estimate) ############################################################################### -# /job/ -> GET -> shows status/history of jobs +# /wakeup -> GET -> forces the job manager to wake up, and check the queue +# should not be needed, but in DEV can be helpful ################################################################################ @app.route("/wakeup", methods=["GET"]) @login_required @@ -151,6 +152,42 @@ def wakeup(): WakePAJobManager() return render_template("base.html") +################################################################################ +@app.route("/stale_job/", methods=["GET", "POST"]) +@login_required +def stale_job(id): + print( f"Handle Stale Job#{id} -> {request.form['action']} it") + job=Job.query.get(id) + now=datetime.now(pytz.utc) + db.session.add(job) + if request.form['action'] == "restart": + log=Joblog( job_id=id, log="(Stale) Job restarted manually by user", log_date=now ) + job.pa_job_state='New' + job.state='New' + elif request.form['action'] == "cancel": + log=Joblog( job_id=id, log="(Stale) Job withdrawn manually by user", log_date=now ) + job.pa_job_state='Completed' + job.state='Withdrawn' + + job.last_update=now + db.session.add(log) + + # clear out message for this job being stale (and do this via raw sql to + # avoid circulr import) + db.engine.execute( f"delete from pa_job_manager_fe_message where job_id = {id}" ) + + db.session.commit() + WakePAJobManager() + return render_template("base.html") + +################################################################################ +@app.route("/stale_jobs", methods=["GET", "POST"]) +@login_required +def stale_jobs(): + page_title='Stale job list' + jobs = Job.query.filter(Job.pa_job_state=='Stale').order_by(Job.id.desc()).all() + return render_template("jobs.html", jobs=jobs, page_title=page_title) + ############################################################################### # This func creates a new filter in jinja2 to format the time from the db in a # way that is more readable (converted to local tz too) diff --git a/pa_job_manager.py b/pa_job_manager.py index 6e4d09b..2dc1f31 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -558,7 +558,7 @@ def JobsForPaths( parent_job, paths, ptype ): session.commit() if parent_job: AddLogForJob(parent_job, "adding job id={} {} (wait for: {})".format( job4.id, job4.id, job4.name, job4.wait_for ) ) - HandleJobs() + HandleJobs(False) return ############################################################################## @@ -636,7 +636,7 @@ def RunJob(job): # session.close() if job.pa_job_state != "Completed": FinishJob(job, "PA Job Manager - This is a catchall to close of a Job, this should never be seen and implies a job did not actually complete?", "Failed" ) - HandleJobs() + HandleJobs(False) return ############################################################################## @@ -665,13 +665,26 @@ def FinishJob(job, last_log, state="Completed", pa_job_state="Completed"): return ############################################################################## -# HandleJobs(): go through each job, if it New, then tackle it -- -# TODO: why not only retrieve New jobs from DB? +# HandleJobs(first_run): go through each job, if it New, then tackle it -- +# if first_run is True, then we are restarting the job manager and any job +# that was "In Progress" is stale, and should be handled -- mark it as Stale +# and that allows user in F/E to cancel or restart it ############################################################################## -def HandleJobs(): - if DEBUG: - print("INFO: PA job manager is scanning for new jobs to process") - for job in session.query(Job).all(): +def HandleJobs(first_run=False): + if first_run: + print("INFO: PA job manager is starting up - check for stale jobs" ) + else: + if DEBUG: + print("INFO: PA job manager is scanning for new jobs to process") + for job in session.query(Job).filter(Job.pa_job_state != 'Complete').all(): + if first_run and job.pa_job_state == 'In Progress': + print( f"INFO: Found stale job#{job.id} - {job.name}" ) + job.pa_job_state = 'Stale' + session.add(job) + AddLogForJob( job, "ERROR: Job has been marked stale as it did not complete" ) + MessageToFE( job.id, "danger", f'Stale job, click  here to restart or cancel' ) + session.commit() + continue if job.pa_job_state == 'New': if job.wait_for != None: j2 = session.query(Job).get(job.wait_for) @@ -1657,7 +1670,7 @@ def JobMoveFiles(job): # Sanity check, if prefix starts with /, reject it -> no /etc/shadow potentials # Sanity check, if .. in prefix or suffix, reject it -> no ../../etc/shadow potentials # Sanity check, if // in prefix or suffix, reject it -> not sure code wouldnt try to make empty dirs, and I dont want to chase /////// cases, any 2 in a row is enough to reject - if '..' in prefix or '..' in suffix or prefix[0] == '/' or '//' in prefix or '//' in suffix: + if '..' in prefix or '..' in suffix or (prefix and prefix[0] == '/') or '//' in prefix or '//' in suffix: FinishJob( job, f"ERROR: Not processing move as the paths contain illegal chars", "Failed" ) return # also remove unecessary slashes, jic @@ -1917,10 +1930,10 @@ if __name__ == "__main__": InitialValidationChecks() - HandleJobs() + HandleJobs(True) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind((PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT)) s.listen() while True: conn, addr = s.accept() - HandleJobs() + HandleJobs(False) diff --git a/templates/base.html b/templates/base.html index cffd212..e2f39fb 100644 --- a/templates/base.html +++ b/templates/base.html @@ -127,7 +127,7 @@ {% endif %} {% if GetJM_Message() != None %} {% set msg=GetJM_Message() %} - {% if request.endpoint != "fix_dups" and request.endpoint != "rm_dups" %} + {% if request.endpoint != "fix_dups" and request.endpoint != "rm_dups" and request.endpoint != "stale_jobs" %}
{% if msg.alert != "success" and msg.job.name != "checkdups" %}
diff --git a/templates/jobs.html b/templates/jobs.html index 63b2507..630db7b 100644 --- a/templates/jobs.html +++ b/templates/jobs.html @@ -6,13 +6,26 @@ var active_rows=Array() var completed_rows=Array() + function HandleStaleJob(id, action) + { + s='' + s+='' + s+='
' + $(s).appendTo('body').submit(); + } + {% for job in jobs %} {% if job.state == "Failed" %} row='Job #{{job.id}} - {{job.name}}' {% elif job.state == "Withdrawn" %} row='Job #{{job.id}} - {{job.name}}' {% else %} - row='Job #{{job.id}} - {{job.name}}' + row="" + {% if job.pa_job_state == 'Stale' %} + row+='' + row+='  ' + {% endif %} + row+='Job #{{job.id}} - {{job.name}}' {% endif %} {% if job.name != "rmdups" %} {% for ex in job.extra %} @@ -27,7 +40,10 @@ {% else %} row+= '{{job.start_time|vicdate}}' {% endif %} - {% if job.pa_job_state != "Completed" %} + {% if job.pa_job_state == "Stale" %} + row += 'In Progress' + active_rows.push(row) + {% elif job.pa_job_state != "Completed" %} {% if job.num_files and job.num_files > 0 %} {% set prog=(job.current_file_num/job.num_files*100)|round|int %} row +=` @@ -52,9 +68,11 @@

{{page_title}}

+ {% if 'Stale' not in page_title %} + {% endif %}
Active JobsJob StartedProgress
@@ -63,8 +81,10 @@ {% endblock main_content %}