new BUG (very minor), reordered TODOs and now have basic stale job handling - they are detected, and can be cancelled or restarted from GUI

job.py

@@ -143,7 +143,8 @@ def joblog(id):
     return render_template("joblog.html", job=joblog, logs=logs, log_cnt=log_cnt, duration=duration, page_title=page_title, first_logs_only=first_logs_only, estimate=estimate)
 
 ###############################################################################
-# /job/<id> -> GET -> shows status/history of jobs
+# /wakeup -> GET -> forces the job manager to wake up, and check the queue
+# should not be needed, but in DEV can be helpful
 ################################################################################
 @app.route("/wakeup", methods=["GET"])
 @login_required
@@ -151,6 +152,42 @@ def wakeup():
     WakePAJobManager()
     return render_template("base.html")
 
+################################################################################
+@app.route("/stale_job/<id>", methods=["GET", "POST"])
+@login_required
+def stale_job(id):
+    print( f"Handle Stale Job#{id} -> {request.form['action']} it")
+    job=Job.query.get(id)
+    now=datetime.now(pytz.utc)
+    db.session.add(job)
+    if request.form['action'] == "restart":
+        log=Joblog( job_id=id, log="(Stale) Job restarted manually by user", log_date=now )
+        job.pa_job_state='New'
+        job.state='New'
+    elif request.form['action'] == "cancel":
+        log=Joblog( job_id=id, log="(Stale) Job withdrawn manually by user", log_date=now )
+        job.pa_job_state='Completed'
+        job.state='Withdrawn'
+
+    job.last_update=now
+    db.session.add(log)
+
+    # clear out message for this job being stale (and do this via raw sql to
+    # avoid circulr import)
+    db.engine.execute( f"delete from pa_job_manager_fe_message where job_id = {id}" )
+
+    db.session.commit()
+    WakePAJobManager()
+    return render_template("base.html")
+
+################################################################################
+@app.route("/stale_jobs", methods=["GET", "POST"])
+@login_required
+def stale_jobs():
+    page_title='Stale job list'
+    jobs = Job.query.filter(Job.pa_job_state=='Stale').order_by(Job.id.desc()).all() 
+    return render_template("jobs.html", jobs=jobs, page_title=page_title)
+
 ###############################################################################
 # This func creates a new filter in jinja2 to format the time from the db in a
 # way that is more readable (converted to local tz too)