diff --git a/ai.py b/ai.py index bf306b5..3f1a5c5 100644 --- a/ai.py +++ b/ai.py @@ -11,4 +11,4 @@ from status import st, Status ################################################################################ @app.route("/aistats", methods=["GET", "POST"]) def aistats(): - return render_template("aistats.html", page_title='Placeholder', alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("aistats.html", page_title='Placeholder') diff --git a/files.py b/files.py index e0338c5..50a9f5a 100644 --- a/files.py +++ b/files.py @@ -20,7 +20,7 @@ import time # Local Class imports ################################################################################ from settings import Settings -from job import Job, Joblog +from job import Job, Joblog, NewJob class FileData(): def __init__(self): @@ -176,22 +176,26 @@ filedata.GenerateFileData() ################################################################################ @app.route("/file_list", methods=["GET"]) def file_list(): - return render_template("file_list.html", page_title='View Files (details)', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("file_list.html", page_title='View Files (details)', file_data=filedata) ################################################################################ # /files -> show thumbnail view of files from import_path(s) ################################################################################ @app.route("/files", methods=["GET"]) def files(): - return render_template("files.html", page_title='View Files', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("files.html", page_title='View Files', file_data=filedata) ################################################################################ # /files/scannow -> allows us to force a check for new files ################################################################################ @app.route("/files/scannow", methods=["GET"]) def scannow(): + job=NewJob("scannow", 1 ) + print("beginning of using a job to scan for new files, rather than do it in code here: {}".format(job)) filedata.GenerateFileData() - return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Scanned for new files" ) + st.SetAlert("success") + st.SetMessage("Scanned for new files") + return render_template("base.html", page_title='Forced look for new items', file_data=filedata) ################################################################################ # /files/forcescan -> deletes old data in DB, and does a brand new scan @@ -202,7 +206,9 @@ def forcescan(): Settings.query.all()[0].last_import_date=0 db.session.commit() filedata.GenerateFileData() - return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Forced remove and recreation of all file data" ) + st.SetAlert("success") + st.SetMessage("Forced remove and recreation of all file data") + return render_template("base.html", page_title='Forced look for new items') ################################################################################ # /static -> returns the contents of any file referenced inside /static. diff --git a/job.py b/job.py index 8535fc7..aeebd79 100644 --- a/job.py +++ b/job.py @@ -24,17 +24,39 @@ class Job(db.Model): id = db.Column(db.Integer, db.Sequence('joblog_id_seq'), primary_key=True ) start_time = db.Column(db.DateTime(timezone=True)) last_update = db.Column(db.DateTime(timezone=True)) + name = db.Column(db.String) state = db.Column(db.String) num_passes = db.Column(db.Integer) current_pass = db.Column(db.Integer) num_files = db.Column(db.Integer) current_file_num = db.Column(db.Integer) current_file = db.Column(db.String) + wait_for = db.Column(db.Integer) + pa_job_state = db.Column(db.String) def __repr__(self): - return "".format(self.id, self.start_time, self.last_update, self.state, self.num_passes, self.current_pass, self.num_files, self.num_files, self.current_file_num, self.current_file) + return "".format(self.id, self.start_time, self.last_update, self.name, self.state, self.num_passes, self.current_pass, self.num_files, self.num_files, self.current_file_num, self.current_file) +################################################################################ +# Utility classes for Jobs +################################################################################ +def GetNumActiveJobs(): + ret = db.engine.execute("select num_active_jobs from pa_job_manager").first(); + if( ret != None ): + return ret.num_active_jobs + +############################################################################### +# NewJob takes a name (which will be matched in pa_job_manager.py to run +# the appropriate job - which will update the Job() until complete +############################################################################### +def NewJob(name, num_passes="1", num_files="0", wait_for=None ): + job=Job(start_time='now()', last_update='now()', name=name, state="New", num_passes=num_passes, num_files=num_files, + current_pass=0, current_file_num=0, current_file='', + wait_for=wait_for, pa_job_state="New" ) + db.session.add(job) + db.session.commit() + ################################################################################ # /jobs -> show current settings ################################################################################ @@ -42,7 +64,7 @@ class Job(db.Model): def jobs(): page_title='Job actions' jobs = Job.query.all() - return render_template("jobs.html", jobs=jobs, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("jobs.html", jobs=jobs, page_title=page_title) ############################################################################### @@ -55,7 +77,7 @@ def joblog(id): logs=Joblog.query.filter(Joblog.job_id==id).all() duration=(datetime.now(pytz.utc)-joblog.start_time) duration= duration-timedelta(microseconds=duration.microseconds) - return render_template("joblog.html", imp=joblog, logs=logs, duration=duration, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("joblog.html", job=joblog, logs=logs, duration=duration, page_title=page_title) ############################################################################### # This func creates a new filter in jinja2 to format the time from the db in a diff --git a/main.py b/main.py index c8e0873..bdb5f73 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,7 @@ from settings import Settings from files import File from person import Person from refimg import Refimg -from job import Job +from job import Job, GetNumActiveJobs from ai import * ####################################### CLASSES / DB model ####################################### @@ -39,11 +39,14 @@ from ai import * ####################################### GLOBALS ####################################### # allow jinja2 to call these python functions directly app.jinja_env.globals['ClearStatus'] = st.ClearStatus +app.jinja_env.globals['GetAlert'] = st.GetAlert +app.jinja_env.globals['GetMessage'] = st.GetMessage +app.jinja_env.globals['GetNumActiveJobs'] = GetNumActiveJobs # default page, just the navbar @app.route("/", methods=["GET"]) def main_page(): - return render_template("base.html", alert=st.GetAlert(), message=st.GetMessage()) + return render_template("base.html") if __name__ == "__main__": if hostname == PROD_HOST: diff --git a/pa_job_engine.py b/pa_job_engine.py deleted file mode 100644 index 110bc67..0000000 --- a/pa_job_engine.py +++ /dev/null @@ -1,19 +0,0 @@ -### -# -# This file controls the 'external' job control engine, that (periodically # -# looks / somehow is pushed an event?) picks up new jobs, and processes them. -# -# It then stores the progress/status, etc. in job and joblog tables as needed -# via wrapper functions. -# -# The whole pa_job_engine is multi-threaded, and uses the database tables for -# state management and communication back to the pa web site -# -### - -class PA_JobEngine(db.Model): - __tablename__ = "pa_job_engine" - id = db.Column(db.Integer, db.Sequence('pa_job_engine_id_seq'), primary_key=True) - state db.Column(db.String) - num_jobs_active = db.Column(db.Integer) - num_jobs_complete = db.Column(db.Integer) diff --git a/person.py b/person.py index e49aff8..0f828cd 100644 --- a/person.py +++ b/person.py @@ -58,7 +58,7 @@ class PersonForm(FlaskForm): @app.route("/persons", methods=["GET"]) def persons(): persons = Person.query.all() - return render_template("persons.html", persons=persons, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("persons.html", persons=persons) ################################################################################ @@ -83,7 +83,7 @@ def new_person(): except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to add Person: {}".format(e.orig) ) - return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title) ################################################################################ # /person/ -> GET/POST(save or delete) -> shows/edits/delets a single @@ -118,9 +118,9 @@ def person(id): except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Person: {}".format(e) ) - return render_template("person.html", form=form, reference_imgs="test", page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("person.html", form=form, reference_imgs="test", page_title=page_title) else: person = Person.query.get(id) print(person) form = PersonForm(request.values, obj=person) - return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title) diff --git a/refimg.py b/refimg.py index 596c25c..6841436 100644 --- a/refimg.py +++ b/refimg.py @@ -51,7 +51,7 @@ class RefimgForm(FlaskForm): @app.route("/refimgs", methods=["GET"]) def refimgs(): refimgs = Refimg.query.all() - return render_template("refimgs.html", refimgs=refimgs, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("refimgs.html", refimgs=refimgs) ################################################################################ # /refimg -> GET/POST -> creates a new refimg type and when created, takes you back to /refimgs @@ -79,7 +79,7 @@ def new_refimg(): except Exception as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Refimg: {}".format(e) ) - return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("refimg.html", form=form, page_title=page_title) ################################################################################ # /refimg/ -> GET/POST(save or delete) -> shows/edits/delets a single @@ -111,8 +111,8 @@ def refimg(id): except Exception as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Refimg: {}".format(e) ) - return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("refimg.html", form=form, page_title=page_title) else: refimg = Refimg.query.get(id) form = RefimgForm(request.values, obj=refimg) - return render_template("refimg.html", object=refimg, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("refimg.html", object=refimg, form=form, page_title = page_title) diff --git a/settings.py b/settings.py index a16b23e..f9b3e15 100644 --- a/settings.py +++ b/settings.py @@ -60,9 +60,9 @@ def settings(): except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Setting: {}".format(e.orig) ) - return render_template("settings.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("settings.html", form=form, page_title=page_title) else: tmp_sets = Settings.query.all() sets = settings_schema.dump( tmp_sets ) form = SettingsForm(obj=tmp_sets[0]) - return render_template("settings.html", form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("settings.html", form=form, page_title = page_title) diff --git a/tables.sql b/tables.sql index fde4cd3..0c57c41 100644 --- a/tables.sql +++ b/tables.sql @@ -20,22 +20,44 @@ create table person_refimg_link ( person_id integer, refimg_id integer, constraint u_prl_refimg_id unique(refimg_id) ); create table job ( - id integer, start_time timestamptz, last_update timestamptz, state varchar(128), num_passes integer, current_pass integer, - num_files integer, current_file_num integer, current_file integer, wait_for integer, pa_job_state varchar(48), + id integer, start_time timestamptz, last_update timestamptz, name varchar(64), state varchar(128), num_passes integer, current_pass integer, + num_files integer, current_file_num integer, current_file varchar(256), wait_for integer, pa_job_state varchar(48), constraint pk_job_id primary key(id) ); create table joblog ( id integer, job_id integer, log_date timestamptz, log varchar, constraint pk_jl_id primary key(id), constraint fk_jl_job_id foreign key(job_id) references job(id) ); -create table pa_job_engine ( id integer, state varchar(128), num_jobs_active integer, num_jobs_complete integer, constraint pa_job_engine_id primary key(id) ); +create table pa_job_manager ( id integer, state varchar(128), num_active_jobs integer, num_completed_jobs integer, constraint pa_job_manager_id primary key(id) ); create sequence file_id_seq; -create sequence ill_id_seq; -create sequence importlog_id_seq; +create sequence joblog_id_seq; +create sequence job_id_seq; create sequence person_id_seq; create sequence refimg_id_seq; create sequence settings_id_seq; -create sequence pa_job_engine_id_seq; +create sequence pa_job_manager_id_seq; --- this should have sensible paths, or really an empty path on real initialisation and handling of this situation too +-- fake data only for making testing easier +insert into person values ( (select nextval('person_id_seq')), 'dad', 'Damien', 'De Paoli' ); +insert into person values ( (select nextval('person_id_seq')), 'mum', 'Mandy', 'De Paoli' ); +insert into person values ( (select nextval('person_id_seq')), 'cam', 'Cameron', 'De Paoli' ); +insert into person values ( (select nextval('person_id_seq')), 'mich', 'Michelle', 'De Paoli' ); +insert into refimg values ( (select nextval('refimg_id_seq')), 'dad.jpg'); +insert into refimg values ( (select nextval('refimg_id_seq')), 'mum.jpg'); +insert into refimg values ( (select nextval('refimg_id_seq')), 'cam.jpg'); +insert into refimg values ( (select nextval('refimg_id_seq')), 'mich.jpg'); +insert into person_refimg_link values ( 1, 1 ); +insert into person_refimg_link values ( 2, 2 ); +insert into person_refimg_link values ( 3, 3 ); +insert into person_refimg_link values ( 4, 4 ); insert into settings values ( (select nextval('settings_id_seq')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', 0 ); +insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Full Import', 'Completed', 4, 4, 157, 157, 'last_fake_data.img', null, 'Completed' ); +insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Full Import', 'Processing AI', 4, 3, 157, 45, 'fake_data.img', null, 'Running' ); +insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Scan Files', 'New', 3, 0, 157, 0, '', null, 'New' ); +insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Gen Hashes', 'New', 3, 0, 157, 0, '', 3, 'New' ); +insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Process AI', 'New', 3, 0, 157, 0, '', 4, 'New' ); +insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Scanning Files' ); +insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Finished Scanning Files' ); +insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Generating Hashes and thumbnails' ); +insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Finished Generating Hashes and thumbnails' ); +insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Processing AI for "Cam"' ); diff --git a/templates/base.html b/templates/base.html index ce1994e..48e5d7e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -77,7 +77,17 @@ Force Scan (delete data & rebuild) - +
+ Active Jobs: + {% if GetNumActiveJobs() != None %} + 4 + {% else %} + 0 + {% endif %} + +
+ +
@@ -85,9 +95,9 @@ - {% if message is defined and message|length %} -
- {{message|safe}} + {% if GetMessage()|length %} +
+ {{ GetMessage()|safe}} {{ ClearStatus() }}
{% endif %} diff --git a/templates/joblog.html b/templates/joblog.html index 761839f..3197f8e 100644 --- a/templates/joblog.html +++ b/templates/joblog.html @@ -3,9 +3,13 @@ {% block main_content %}

{{page_title}}

+
+ + +
- +
@@ -13,36 +17,46 @@
- +
- +
+ {% if job.wait_for != None %} +
+ + +
+ {% endif %}
- +
- {% set prog=(imp.current_pass/imp.num_passes*100)|round|int %} + {% set prog=(job.current_pass/job.num_passes*100)|round|int %}
-
{{imp.current_pass}} of {{imp.num_passes}} - {{prog}}%
+
{{job.current_pass}} of {{job.num_passes}} - {{prog}}%
- {% set prog=(imp.current_file_num/imp.num_files*100)|round|int %}
+ {% if job.num_files > 0 %} + {% set prog=(job.current_file_num/job.num_files*100)|round|int %}
-
{{imp.current_file_num}} of {{imp.num_files}} - {{prog}}%
+
{{job.current_file_num}} of {{job.num_files}} - {{prog}}%
+ {% else %} + N/A + {% endif %}
- +
{% for log in logs %} diff --git a/templates/jobs.html b/templates/jobs.html index 3c0be4e..ee543a9 100644 --- a/templates/jobs.html +++ b/templates/jobs.html @@ -1,22 +1,48 @@ {% extends "base.html" %} {% block main_content %} + + +

{{page_title}}

-
WhenDetails
+
- + - {% for imp in imports %} - - - - {% endfor %} +
Import IdImport StartedPasses
Active JobsJob StartedPasses
{{imp.id}}{{imp.start_time}} -
- {% set prog=(imp.current_pass/imp.num_passes*100)|round|int %} -
{{imp.current_pass}} of {{imp.num_passes}}
-
-
{% endblock main_content %}