actually create a job for scan now, and let the job manager do the work and send back output via new PA_JobManager_FE_Message table

files.py

@@ -191,10 +191,8 @@ def files():
 @app.route("/files/scannow", methods=["GET"])
 def scannow():
     job=NewJob("scannow", 1 )
-    print("beginning of using a job to scan for new files, rather than do it in code here: {}".format(job))
-    filedata.GenerateFileData()
     st.SetAlert("success")
-    st.SetMessage("Scanned for new files")
+    st.SetMessage("Created job to scan for new files")
     return render_template("base.html", page_title='Forced look for new items', file_data=filedata)
 
 ################################################################################

job.py

@@ -75,6 +75,9 @@ def joblog(id):
     page_title='Show Job Details'
     joblog = Job.query.get(id)
     logs=Joblog.query.filter(Joblog.job_id==id).all()
+    if joblog.pa_job_state == "Completed":
+        duration=(joblog.last_update-joblog.start_time)
+    else:
         duration=(datetime.now(pytz.utc)-joblog.start_time)
     duration= duration-timedelta(microseconds=duration.microseconds)
     return render_template("joblog.html", job=joblog, logs=logs, duration=duration, page_title=page_title)

main.py

@@ -36,12 +36,35 @@ from ai import *
 
 ####################################### CLASSES / DB model #######################################
 
+class PA_JobManager_Message(db.Model):
+    __tablename__ = "pa_job_manager_fe_message"
+    id = db.Column(db.Integer, db.Sequence('pa_job_manager_fe_message_id_seq'), primary_key=True )
+    job_id = db.Column(db.Integer, db.ForeignKey('job.id') )
+    alert = db.Column(db.String)
+    message = db.Column(db.String)
+    def __repr__(self):
+        return "<id: {}, job_id: {}, alert: {}, message: {}".format(self.id, self.job_id, self.alert, self.message)
+
+def GetJM_Message():
+    msg=PA_JobManager_Message.query.first()
+    return msg
+
+def ClearJM_Message(id):
+    msg=PA_JobManager_Message.query.get(id)
+    print("ClearJM_Message: {} ".format(msg))
+    PA_JobManager_Message.query.filter(PA_JobManager_Message.id==id).delete()
+    db.session.commit()
+    return
+
+
 ####################################### GLOBALS #######################################
 # allow jinja2 to call these python functions directly
 app.jinja_env.globals['ClearStatus'] = st.ClearStatus
 app.jinja_env.globals['GetAlert'] = st.GetAlert
 app.jinja_env.globals['GetMessage'] = st.GetMessage
 app.jinja_env.globals['GetNumActiveJobs'] = GetNumActiveJobs
+app.jinja_env.globals['GetJM_Message'] = GetJM_Message
+app.jinja_env.globals['ClearJM_Message'] = ClearJM_Message
 
 # default page, just the navbar
 @app.route("/", methods=["GET"])

pa_job_manager.py

@@ -202,7 +202,12 @@ class Settings(Base):
 ### Initiatlise the file data set
 filedata = FileData()
 
-#### DDP: work out the class creation line for just sqlalchemy
+################################################################################
+# classes for the job manager:
+#    PA_JobManager overall status tracking), 
+#    Job (and Joblog) for each JOb, and 
+#    PA_Jobmanager_fe_message (to pass messages to the front-end web)
+################################################################################
 class PA_JobManager(Base):
     __tablename__ = "pa_job_manager"
     id = Column(Integer, Sequence('pa_job_manager_id_seq'), primary_key=True)
@@ -213,20 +218,16 @@ class PA_JobManager(Base):
     def __repr__(self):
         return "<id={}, state={}, num_active_jobs={}, num_completed_jobs={}>".format( self.id, self.state, self.num_active_jobs, self.num_completed_jobs )
 
-
 class Joblog(Base):
     __tablename__ = "joblog"
     id = Column(Integer, Sequence('ill_id_seq'), primary_key=True )
-    job_id = Column(Integer, ForeignKey('job.id'), primary_key=True )
+    job_id = Column(Integer, ForeignKey('job.id') )
     log_date = Column(DateTime(timezone=True))
     log = Column(String)
 
     def __repr__(self):
         return "<id: {}, job_id: {}, log: {}".format(self.id, self.job_id, self.log )
 
-################################################################################
-# Class describing Action in the database, and via sqlalchemy, connected to the DB as well
-################################################################################
 class Job(Base):
     __tablename__ = "job"
     id = Column(Integer, Sequence('joblog_id_seq'), primary_key=True )
@@ -245,6 +246,19 @@ class Job(Base):
     def __repr__(self):
         return "<id: {}, start_time: {}, last_update: {}, name: {}, state: {}, num_passes: {}, current_passes: {}, num_files: {}, current_file_num: {}, current_file: {}>".format(self.id, self.start_time, self.last_update, self.name, self.state, self.num_passes, self.current_pass, self.num_files, self.num_files, self.current_file_num, self.current_file)
 
+class PA_JobManager_FE_Message(Base):
+    __tablename__ = "pa_job_manager_fe_message"
+    id = Column(Integer, Sequence('pa_job_manager_fe_message_id_seq'), primary_key=True )
+    job_id = Column(Integer, ForeignKey('job.id'), primary_key=True )
+    alert = Column(String)
+    message = Column(String)
+    def __repr__(self):
+        return "<id: {}, job_id: {}, alert: {}, message: {}".format(self.id, self.job_id, self.alert, self.message)
+
+def MessageToFE( job_id, alert, message ):
+    msg = PA_JobManager_FE_Message( job_id=job_id, alert=alert, message=message)
+    session.add(msg)
+    session.commit()
 
 def GetJobs():
     return session.query(Job).all()
@@ -261,16 +275,19 @@ def InitialiseManager():
 
 def RunJob(job):
     print("Run job: {}, pa_eng state: {}, internal job state: {}".format( job.name, job.pa_job_state, job.state) )
+    try:
         if job.name =="scannow":
-        print("scannow not being handled yet")
             JobScanNow(job)
         elif job.name =="forcescan":
             print("force scan not being handled yet")
         else:
             print("Requested to process unknown job type: {}".format(job.name))
+    except:
+        MessageToFE( job.id, "danger", "Failed (see log for details)" )
     return
 
 def HandleJobs():
+    print("PA job manager is scanning for jobs")
     pa_eng.state = 'Scanning Jobs'
     jobs=GetJobs()
     for job in jobs:
@@ -279,23 +296,19 @@ def HandleJobs():
             pa_eng.num_active_jobs = pa_eng.num_active_jobs + 1
         else:
             pa_eng.num_completed_jobs = pa_eng.num_completed_jobs +1
-    print("PA job manager is up")
+    print("PA job manager is waiting jobs")
     pa_eng.state = 'Waiting for new Jobs'
     return
 
 def JobScanNow(job):
-    print("About to call GenerateFileData()")
     filedata.GenerateFileData()
-    print( "need to update job");
     job.state="Completed"
     job.pa_job_state="Completed"
     job.last_update=datetime.now(pytz.utc)
+    MessageToFE( job.id, "success", "Completed (scan for new files)" )
     session.commit()
     return
 
-def UpdateJob():
-    return
-
 if __name__ == "__main__":
     print("PA job manager starting")
     try:
@@ -304,10 +317,5 @@ if __name__ == "__main__":
     except Exception as e:
         print( "Failed to initialise PA Job Manager: {}".format(e) )
         session.rollback()
-    print("Exiting for now: {}".format( pa_eng ))
-    cnt=0
-    while cnt < 3:
     HandleJobs()
-        time.sleep(60)
-        cnt=cnt+1
-
+    print("Exiting for now: {}".format( pa_eng ))

templates/base.html

@@ -101,6 +101,13 @@
 		    {{ ClearStatus() }}
 		</div>
 	{% endif %}
+	{% if GetJM_Message() != None %}
+		{% set msg=GetJM_Message() %}
+		<div class="row alert alert-{{msg.alert}}">
+            <a href="{{url_for('joblog', id=msg.job_id)}}">Job #{{msg.job_id}}</a>: {{msg.message|safe}}
+        </div>
+		{% set dont_print=ClearJM_Message(msg.id) %}
+	{% endif %}
 {% endif %}
 
 	{% block main_content %}