ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

git commit, converted over to base.html pulling Status*, rather than every render_template calling it. Tightened up naming for job manager, fixed a few bugs in there with items like div by zero, created the active jobs link/badge in navbar, have it invoked each time by base.html template and it gets active jobs from DB, pa_job_manager now initiliases from an empty DB and can work out where it is at, no loop/thread/or actual job code yet. jobs.py has basics of a NewJob(), so next step is to force that job to be executed in pa_job_manager, but its tea time

Browse Source
This commit is contained in:
Damien De Paoli
2021-01-16 17:51:16 +11:00
parent dc2ea1265f
commit e138ab22aa
12 changed files with 156 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ai.py
View File

@@ -11,4 +11,4 @@ from status import st, Status
################################################################################
@app.route("/aistats", methods=["GET", "POST"])
def aistats():
return render_template("aistats.html", page_title='Placeholder', alert=st.GetAlert(), message=st.GetMessage() )
return render_template("aistats.html", page_title='Placeholder')

16
files.py
View File

@@ -20,7 +20,7 @@ import time
# Local Class imports
################################################################################
from settings import Settings
from job import Job, Joblog
from job import Job, Joblog, NewJob
class FileData():
def __init__(self):
@@ -176,22 +176,26 @@ filedata.GenerateFileData()
################################################################################
@app.route("/file_list", methods=["GET"])
def file_list():
return render_template("file_list.html", page_title='View Files (details)', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("file_list.html", page_title='View Files (details)', file_data=filedata)
################################################################################
# /files -> show thumbnail view of files from import_path(s)
################################################################################
@app.route("/files", methods=["GET"])
def files():
return render_template("files.html", page_title='View Files', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("files.html", page_title='View Files', file_data=filedata)
################################################################################
# /files/scannow -> allows us to force a check for new files
################################################################################
@app.route("/files/scannow", methods=["GET"])
def scannow():
job=NewJob("scannow", 1 )
print("beginning of using a job to scan for new files, rather than do it in code here: {}".format(job))
filedata.GenerateFileData()
return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Scanned for new files" )
st.SetAlert("success")
st.SetMessage("Scanned for new files")
return render_template("base.html", page_title='Forced look for new items', file_data=filedata)
################################################################################
# /files/forcescan -> deletes old data in DB, and does a brand new scan
@@ -202,7 +206,9 @@ def forcescan():
Settings.query.all()[0].last_import_date=0
db.session.commit()
filedata.GenerateFileData()
return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Forced remove and recreation of all file data" )
st.SetAlert("success")
st.SetMessage("Forced remove and recreation of all file data")
return render_template("base.html", page_title='Forced look for new items')
################################################################################
# /static -> returns the contents of any file referenced inside /static.

28
job.py
View File

@@ -24,17 +24,39 @@ class Job(db.Model):
id = db.Column(db.Integer, db.Sequence('joblog_id_seq'), primary_key=True )
start_time = db.Column(db.DateTime(timezone=True))
last_update = db.Column(db.DateTime(timezone=True))
name = db.Column(db.String)
state = db.Column(db.String)
num_passes = db.Column(db.Integer)
current_pass = db.Column(db.Integer)
num_files = db.Column(db.Integer)
current_file_num = db.Column(db.Integer)
current_file = db.Column(db.String)
wait_for = db.Column(db.Integer)
pa_job_state = db.Column(db.String)
def __repr__(self):
return "<id: {}, start_time: {}, last_update: {}, state: {}, num_passes: {}, current_passes: {}, num_files: {}, current_file_num: {}, current_file: {}>".format(self.id, self.start_time, self.last_update, self.state, self.num_passes, self.current_pass, self.num_files, self.num_files, self.current_file_num, self.current_file)
return "<id: {}, start_time: {}, last_update: {}, name: {}, state: {}, num_passes: {}, current_passes: {}, num_files: {}, current_file_num: {}, current_file: {}>".format(self.id, self.start_time, self.last_update, self.name, self.state, self.num_passes, self.current_pass, self.num_files, self.num_files, self.current_file_num, self.current_file)
################################################################################
# Utility classes for Jobs
################################################################################
def GetNumActiveJobs():
ret = db.engine.execute("select num_active_jobs from pa_job_manager").first();
if( ret != None ):
return ret.num_active_jobs
###############################################################################
# NewJob takes a name (which will be matched in pa_job_manager.py to run
# the appropriate job - which will update the Job() until complete
###############################################################################
def NewJob(name, num_passes="1", num_files="0", wait_for=None ):
job=Job(start_time='now()', last_update='now()', name=name, state="New", num_passes=num_passes, num_files=num_files,
current_pass=0, current_file_num=0, current_file='',
wait_for=wait_for, pa_job_state="New" )
db.session.add(job)
db.session.commit()
################################################################################
# /jobs -> show current settings
################################################################################
@@ -42,7 +64,7 @@ class Job(db.Model):
def jobs():
page_title='Job actions'
jobs = Job.query.all()
return render_template("jobs.html", jobs=jobs, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("jobs.html", jobs=jobs, page_title=page_title)
###############################################################################
@@ -55,7 +77,7 @@ def joblog(id):
logs=Joblog.query.filter(Joblog.job_id==id).all()
duration=(datetime.now(pytz.utc)-joblog.start_time)
duration= duration-timedelta(microseconds=duration.microseconds)
return render_template("joblog.html", imp=joblog, logs=logs, duration=duration, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("joblog.html", job=joblog, logs=logs, duration=duration, page_title=page_title)
###############################################################################
# This func creates a new filter in jinja2 to format the time from the db in a

7
main.py
View File

@@ -31,7 +31,7 @@ from settings import Settings
from files import File
from person import Person
from refimg import Refimg
from job import Job
from job import Job, GetNumActiveJobs
from ai import *
####################################### CLASSES / DB model #######################################
@@ -39,11 +39,14 @@ from ai import *
####################################### GLOBALS #######################################
# allow jinja2 to call these python functions directly
app.jinja_env.globals['ClearStatus'] = st.ClearStatus
app.jinja_env.globals['GetAlert'] = st.GetAlert
app.jinja_env.globals['GetMessage'] = st.GetMessage
app.jinja_env.globals['GetNumActiveJobs'] = GetNumActiveJobs
# default page, just the navbar
@app.route("/", methods=["GET"])
def main_page():
return render_template("base.html", alert=st.GetAlert(), message=st.GetMessage())
return render_template("base.html")
if __name__ == "__main__":
if hostname == PROD_HOST:

19
pa_job_engine.py
View File

@@ -1,19 +0,0 @@
###
#
# This file controls the 'external' job control engine, that (periodically #
# looks / somehow is pushed an event?) picks up new jobs, and processes them.
#
# It then stores the progress/status, etc. in job and joblog tables as needed
# via wrapper functions.
#
# The whole pa_job_engine is multi-threaded, and uses the database tables for
# state management and communication back to the pa web site
#
###
class PA_JobEngine(db.Model):
__tablename__ = "pa_job_engine"
id = db.Column(db.Integer, db.Sequence('pa_job_engine_id_seq'), primary_key=True)
state db.Column(db.String)
num_jobs_active = db.Column(db.Integer)
num_jobs_complete = db.Column(db.Integer)

8
person.py
View File

@@ -58,7 +58,7 @@ class PersonForm(FlaskForm):
@app.route("/persons", methods=["GET"])
def persons():
persons = Person.query.all()
return render_template("persons.html", persons=persons, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("persons.html", persons=persons)
################################################################################
@@ -83,7 +83,7 @@ def new_person():
except SQLAlchemyError as e:
st.SetAlert( "danger" )
st.SetMessage( "<b>Failed to add Person:</b>&nbsp;{}".format(e.orig) )
return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title)
################################################################################
# /person/<id> -> GET/POST(save or delete) -> shows/edits/delets a single
@@ -118,9 +118,9 @@ def person(id):
except SQLAlchemyError as e:
st.SetAlert( "danger" )
st.SetMessage( "<b>Failed to modify Person:</b>&nbsp;{}".format(e) )
return render_template("person.html", form=form, reference_imgs="test", page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("person.html", form=form, reference_imgs="test", page_title=page_title)
else:
person = Person.query.get(id)
print(person)
form = PersonForm(request.values, obj=person)
return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("person.html", object=person, form=form, reference_imgs=reference_imgs, page_title = page_title)

8
refimg.py
View File

@@ -51,7 +51,7 @@ class RefimgForm(FlaskForm):
@app.route("/refimgs", methods=["GET"])
def refimgs():
refimgs = Refimg.query.all()
return render_template("refimgs.html", refimgs=refimgs, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("refimgs.html", refimgs=refimgs)
################################################################################
# /refimg -> GET/POST -> creates a new refimg type and when created, takes you back to /refimgs
@@ -79,7 +79,7 @@ def new_refimg():
except Exception as e:
st.SetAlert( "danger" )
st.SetMessage( "<b>Failed to modify Refimg:</b>&nbsp;{}".format(e) )
return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("refimg.html", form=form, page_title=page_title)
################################################################################
# /refimg/<id> -> GET/POST(save or delete) -> shows/edits/delets a single
@@ -111,8 +111,8 @@ def refimg(id):
except Exception as e:
st.SetAlert( "danger" )
st.SetMessage( "<b>Failed to modify Refimg:</b>&nbsp;{}".format(e) )
return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("refimg.html", form=form, page_title=page_title)
else:
refimg = Refimg.query.get(id)
form = RefimgForm(request.values, obj=refimg)
return render_template("refimg.html", object=refimg, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("refimg.html", object=refimg, form=form, page_title = page_title)

4
settings.py
View File

@@ -60,9 +60,9 @@ def settings():
except SQLAlchemyError as e:
st.SetAlert( "danger" )
st.SetMessage( "<b>Failed to modify Setting:</b>&nbsp;{}".format(e.orig) )
return render_template("settings.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("settings.html", form=form, page_title=page_title)
else:
tmp_sets = Settings.query.all()
sets = settings_schema.dump( tmp_sets )
form = SettingsForm(obj=tmp_sets[0])
return render_template("settings.html", form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() )
return render_template("settings.html", form=form, page_title = page_title)

36
tables.sql
View File

@@ -20,22 +20,44 @@ create table person_refimg_link ( person_id integer, refimg_id integer,
constraint u_prl_refimg_id unique(refimg_id) );
create table job (
id integer, start_time timestamptz, last_update timestamptz, state varchar(128), num_passes integer, current_pass integer,
num_files integer, current_file_num integer, current_file integer, wait_for integer, pa_job_state varchar(48),
id integer, start_time timestamptz, last_update timestamptz, name varchar(64), state varchar(128), num_passes integer, current_pass integer,
num_files integer, current_file_num integer, current_file varchar(256), wait_for integer, pa_job_state varchar(48),
constraint pk_job_id primary key(id) );
create table joblog ( id integer, job_id integer, log_date timestamptz, log varchar,
constraint pk_jl_id primary key(id), constraint fk_jl_job_id foreign key(job_id) references job(id) );
create table pa_job_engine ( id integer, state varchar(128), num_jobs_active integer, num_jobs_complete integer, constraint pa_job_engine_id primary key(id) );
create table pa_job_manager ( id integer, state varchar(128), num_active_jobs integer, num_completed_jobs integer, constraint pa_job_manager_id primary key(id) );
create sequence file_id_seq;
create sequence ill_id_seq;
create sequence importlog_id_seq;
create sequence joblog_id_seq;
create sequence job_id_seq;
create sequence person_id_seq;
create sequence refimg_id_seq;
create sequence settings_id_seq;
create sequence pa_job_engine_id_seq;
create sequence pa_job_manager_id_seq;
-- this should have sensible paths, or really an empty path on real initialisation and handling of this situation too
-- fake data only for making testing easier
insert into person values ( (select nextval('person_id_seq')), 'dad', 'Damien', 'De Paoli' );
insert into person values ( (select nextval('person_id_seq')), 'mum', 'Mandy', 'De Paoli' );
insert into person values ( (select nextval('person_id_seq')), 'cam', 'Cameron', 'De Paoli' );
insert into person values ( (select nextval('person_id_seq')), 'mich', 'Michelle', 'De Paoli' );
insert into refimg values ( (select nextval('refimg_id_seq')), 'dad.jpg');
insert into refimg values ( (select nextval('refimg_id_seq')), 'mum.jpg');
insert into refimg values ( (select nextval('refimg_id_seq')), 'cam.jpg');
insert into refimg values ( (select nextval('refimg_id_seq')), 'mich.jpg');
insert into person_refimg_link values ( 1, 1 );
insert into person_refimg_link values ( 2, 2 );
insert into person_refimg_link values ( 3, 3 );
insert into person_refimg_link values ( 4, 4 );
insert into settings values ( (select nextval('settings_id_seq')), '/home/ddp/src/photoassistant/images_to_process/#c:/Users/cam/Desktop/code/python/photoassistant/photos/#/home/ddp/src/photoassistant/new_img_dir/', 0 );
insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Full Import', 'Completed', 4, 4, 157, 157, 'last_fake_data.img', null, 'Completed' );
insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Full Import', 'Processing AI', 4, 3, 157, 45, 'fake_data.img', null, 'Running' );
insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Scan Files', 'New', 3, 0, 157, 0, '', null, 'New' );
insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Gen Hashes', 'New', 3, 0, 157, 0, '', 3, 'New' );
insert into job values ( (select nextval('job_id_seq')), now(), now(), 'Process AI', 'New', 3, 0, 157, 0, '', 4, 'New' );
insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Scanning Files' );
insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Finished Scanning Files' );
insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Generating Hashes and thumbnails' );
insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Finished Generating Hashes and thumbnails' );
insert into joblog values ( (select nextval('joblog_id_seq')), 1, now(), 'Started Processing AI for "Cam"' );

18
templates/base.html
View File

@@ -77,7 +77,17 @@
<a class="dropdown-item" href="{{url_for('forcescan')}}">Force Scan (delete data & rebuild)</a>
</div class="dropdow-menu">
</div class="nav-item dropdown">
</div clas="navbar-nav">
<div class="nav-item ml-5">
<a href="{{url_for('jobs')}}"}}<span class="navbar-text">Active Jobs:
{% if GetNumActiveJobs() != None %}
<span class="badge badge-danger text-white"}}>4</span>
{% else %}
<span class="badge">0</span>
{% endif %}
</a>
</div class="nav-item">
</div class="navbar-nav">
<form class="form-inline my-2 my-lg-0" method="POST" action="/search">
<input class="form-control mr-sm-2" type="search" placeholder="by file, date (YYYMMDD) or tag" aria-label="Search" name="term">
<button class="btn btn-outline-success my-2 my-sm-0" type="submit">Search</button>
@@ -85,9 +95,9 @@
</div class="collapse navbar-collapse">
</nav>
{% if message is defined and message|length %}
<div class="row alert alert-{{alert}}">
{{message|safe}}
{% if GetMessage()|length %}
<div class="row alert alert-{{GetAlert()}}">
{{ GetMessage()|safe}}
{{ ClearStatus() }}
</div>
{% endif %}

32
templates/joblog.html
View File

@@ -3,9 +3,13 @@
{% block main_content %}
<div class="container">
<h3>{{page_title}}</h3>
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Name:</label>
<label class="form-control-plaintext col-lg-10">{{job.name}}</label>
</div>
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Start Time:</label>
<label class="form-control-plaintext col-lg-10">{{imp.start_time|vicdate}}</label>
<label class="form-control-plaintext col-lg-10">{{job.start_time|vicdate}}</label>
</div>
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Duration:</label>
@@ -13,36 +17,46 @@
</div>
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Last Update:</label>
<label class="form-control-plaintext col-lg-10">{{imp.last_update|vicdate}}</label>
<label class="form-control-plaintext col-lg-10">{{job.last_update|vicdate}}</label>
</div>
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Current state:</label>
<label class="form-control-plaintext col-lg-10">{{imp.state}}</label>
<label class="form-control-plaintext col-lg-10">{{job.state}}</label>
</div>
{% if job.wait_for != None %}
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Waiting on Job:</label>
<label class="form-control-plaintext col-lg-10"><a href="{{url_for('joblog', id=job.wait_for)}}">Job (id={{job.wait_for}})</a></label>
</div>
{% endif %}
<div class="row col-lg-12">
<label class="form-control-plaintext col-lg-2">Current File:</label>
<label class="form-control-plaintext col-lg-10">{{imp.current_file}}</label>
<label class="form-control-plaintext col-lg-10">{{job.current_file}}</label>
</div>
<div class="row col-lg-12">
{% set prog=(imp.current_pass/imp.num_passes*100)|round|int %}
{% set prog=(job.current_pass/job.num_passes*100)|round|int %}
<label class="form-control-plaintext col-lg-2">Passes:</label>
<div class="col-lg-10 px-0">
<div class="progress" style="height:80%">
<div class="progress-bar" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{imp.current_pass}} of {{imp.num_passes}} - {{prog}}%</div>
<div class="progress-bar" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{job.current_pass}} of {{job.num_passes}} - {{prog}}%</div>
</div>
</div>
</div>
<div class="row col-lg-12">
{% set prog=(imp.current_file_num/imp.num_files*100)|round|int %}
<label class="form-control-plaintext col-lg-2">Files in pass:</label>
<div class="col-lg-10 px-0">
{% if job.num_files > 0 %}
{% set prog=(job.current_file_num/job.num_files*100)|round|int %}
<div class="progress" style="height:80%">
<div class="progress-bar bg-info" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{imp.current_file_num}} of {{imp.num_files}} - {{prog}}%</div>
<div class="progress-bar bg-info" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{job.current_file_num}} of {{job.num_files}} - {{prog}}%</div>
</div>
{% else %}
N/A
{% endif %}
</div>
</div>
<div class="row col-lg-12">
<table id="import_tbl" class="table table-striped table-sm" data-toolbar="#toolbar" data-search="true">
<table id="jobort_tbl" class="table table-striped table-sm" data-toolbar="#toolbar" data-search="true">
<thead><tr class="thead-light"><th>When</th><th>Details</th></tr></thead>
<tbody>
{% for log in logs %}

50
templates/jobs.html
View File

@@ -1,22 +1,48 @@
{% extends "base.html" %}
{% block main_content %}
<script>
var active_rows=Array()
var completed_rows=Array()
{% for job in jobs %}
row=`
<tr><td>
<a href="{{url_for('joblog', id=job.id )}}">{{job.name}}</td><td>{{job.start_time}}</td>
<td>
`
{% if job.pa_job_state != "Completed" %}
row +=`
<div class="progress">
{% set prog=(job.current_pass/job.num_passes*100)|round|int %}
<div class="progress-bar" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{job.current_pass}} of {{job.num_passes}}</div>
</div>
</td></tr>
`
active_rows.push(row)
{% else %}
row +=`
{{job.last_update}}</td></td></tr>
`
completed_rows.push(row)
{% endif %}
{% endfor %}
</script>
<h3>{{page_title}}</h3>
<table id="import_tbl" class="table table-striped table-sm" data-toolbar="#toolbar" data-search="true">
<table id="job_tbl" class="table table-striped table-sm" data-toolbar="#toolbar" data-search="true">
<thead>
<tr class="thead-light"><th>Import Id</th><th>Import Started</th><th>Passes</th></tr>
<tr class="thead-light"><th>Active Jobs</th><th>Job Started</th><th>Passes</th></tr>
</thead>
<tbody>
{% for imp in imports %}
<tr><td><a href="{{url_for('importlog', id=imp.id )}}">{{imp.id}}</td><td>{{imp.start_time}}</td>
<td>
<div class="progress">
{% set prog=(imp.current_pass/imp.num_passes*100)|round|int %}
<div class="progress-bar" role="progressbar" style="width: {{prog}}%;" aria-valuenow="{{prog}}" aria-valuemin="0" aria-valuemax="100">{{imp.current_pass}} of {{imp.num_passes}}</div>
</div>
</td>
</tr>
{% endfor %}
<script>
for(el in active_rows)
document.write(active_rows[el])
document.write( '<tr class="thead-light"><th>Completed Jobs</th><th>Job Started</th><th>Job Completed</th></tr>' )
for(el in completed_rows)
document.write(completed_rows[el])
</script>
</tbody>
</table>
{% endblock main_content %}