From 4c662b3a0e1c0418445434fd7801ce33389f9893 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 13:49:05 +1100 Subject: [PATCH 01/17] removed debug print of video thumbnail, and made file importer look to see if file is newer than last import, if so, process it and store it in the database. If its older skip it. Then at the end of the function return the database as the list we process --- files.py | 57 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/files.py b/files.py index 0f3c467..36e4804 100644 --- a/files.py +++ b/files.py @@ -14,6 +14,7 @@ import exifread import base64 import numpy import cv2 +import time ################################################################################ # Local Class imports @@ -91,7 +92,6 @@ class FileData(): bt = thumb_buf.tostring() fthumbnail = base64.b64encode(bt) fthumbnail = str(fthumbnail)[2:-1] - print(fthumbnail) return fthumbnail @@ -100,8 +100,10 @@ class FileData(): # multiple valid paths in import_path) # ############################################################################## def GenerateFileData(self): - sets = Settings.query.filter(Settings.name=="import_path").all() - paths= sets[0].value.split("#") + import_path = Settings.query.filter(Settings.name=="import_path").all()[0].value + last_import_date_obj = Settings.query.filter(Settings.name=="last_import_date").all() + last_import_date = float(last_import_date_obj[0].value) + paths= import_path.split("#") for path in paths: path = self.FixPath(path) @@ -115,28 +117,39 @@ class FileData(): self.file_list.append(glob.glob(self.view_path + '**', recursive=True)) for file in self.file_list[0]: - fthumbnail = None if file == path: continue - if os.path.isdir(file): - ftype = 'Directory' - elif self.isImage(file): - ftype = 'Image' - fthumbnail = self.getExif(file) - elif self.isVideo(file): - ftype = 'Video' - fthumbnail = self.generateVideoThumbnail(file) - else: - ftype = 'File' + stat = os.stat(file) + if last_import_date == 0 or stat.st_ctime > last_import_date: + print( "{} - {} is newer than {}".format( file, stat.st_ctime, last_import_date ) ) + fthumbnail = None + if os.path.isdir(file): + ftype = 'Directory' + elif self.isImage(file): + ftype = 'Image' + fthumbnail = self.getExif(file) + elif self.isVideo(file): + ftype = 'Video' + fthumbnail = self.generateVideoThumbnail(file) + else: + ftype = 'File' - if ftype != "Directory": - fhash=self.md5(file) - else: - fhash=None + if ftype != "Directory": + fhash=self.md5(file) + else: + fhash=None - fsize = round(os.stat(file).st_size/(1024*1024)) - fname=file.replace(path, "") - self.view_list.append( Files( name=fname, type=ftype, size_mb=fsize, hash=fhash, thumbnail=fthumbnail )) + fsize = round(os.stat(file).st_size/(1024*1024)) + fname=file.replace(path, "") + file_obj = Files( name=fname, type=ftype, size_mb=fsize, hash=fhash, thumbnail=fthumbnail ) + print( file_obj ) + db.session.add(file_obj) + else: + print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) ) + last_import_date_obj[0].value = str(time.time()) + db.session.commit() + file_obj = Files.query.filter().all() + self.view_list = file_obj return self ################################################################################ @@ -150,7 +163,7 @@ class Files(db.Model): size_mb = db.Column(db.Integer, unique=False, nullable=False) # hash might not be unique, this could be the source of dupe problems hash = db.Column(db.Integer, unique=True, nullable=True) - thumbnail = db.Column(db.LargeBinary, unique=False, nullable=True) + thumbnail = db.Column(db.String, unique=False, nullable=True) def __repr__(self): return "".format(self.id, self.name ) From e35247ea7009c33596c7a8c0723a34c60a769af6 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 14:44:11 +1100 Subject: [PATCH 02/17] video files now have a icon to indicate its a video as well --- templates/files.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/files.html b/templates/files.html index df3fcb7..9240e66 100644 --- a/templates/files.html +++ b/templates/files.html @@ -22,6 +22,7 @@ +
{% for obj in file_data.view_list %} {% if obj.type != "Directory" %} @@ -30,7 +31,12 @@ {% if obj.type=="Image" %} {% elif obj.type == "Video" %} - +
+ +
+ +
+
{% endif %}
{{obj.name}}
From e82adde1f06ad5b43be7a40d55a2e78011bbcb90 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 14:59:48 +1100 Subject: [PATCH 03/17] minor code cleanup, removed debug --- files.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/files.py b/files.py index 36e4804..49eceff 100644 --- a/files.py +++ b/files.py @@ -142,14 +142,12 @@ class FileData(): fsize = round(os.stat(file).st_size/(1024*1024)) fname=file.replace(path, "") file_obj = Files( name=fname, type=ftype, size_mb=fsize, hash=fhash, thumbnail=fthumbnail ) - print( file_obj ) db.session.add(file_obj) else: print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) ) last_import_date_obj[0].value = str(time.time()) db.session.commit() - file_obj = Files.query.filter().all() - self.view_list = file_obj + self.view_list = Files.query.all() return self ################################################################################ From 48748c23bed8c660387cf92eb739e4a96390fbf8 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 15:17:46 +1100 Subject: [PATCH 04/17] used this to remove unique ids on buttons --- templates/files.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/templates/files.html b/templates/files.html index 9240e66..047f4a7 100644 --- a/templates/files.html +++ b/templates/files.html @@ -7,19 +7,19 @@
- +
- +
- +
- +
- +

@@ -34,7 +34,7 @@
- +
{% endif %} @@ -48,10 +48,11 @@ {% endblock main_content %} {% block script_content %} From d8572337233c567c0cabe9e4de8ff8dec7983b04 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 16:58:41 +1100 Subject: [PATCH 05/17] files now processes multiple paths, and view shows images for any images in appropriate path --- files.py | 20 ++++++++++---------- templates/files.html | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/files.py b/files.py index 49eceff..2e110d1 100644 --- a/files.py +++ b/files.py @@ -23,10 +23,7 @@ from settings import Settings class FileData(): def __init__(self): - self.view_path='' self.view_list=[] - self.symlink='' - self.file_list=[] ################################################################################ # Utility Functions for Files @@ -106,17 +103,18 @@ class FileData(): paths= import_path.split("#") for path in paths: + print( "GenerateFileData: Checking {}".format( path ) ) path = self.FixPath(path) if os.path.exists( path ): - self.view_path = path # to serve static content of the images, we create a symlink # from inside the static subdir of each import_path that exists - self.symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1]))) - if not os.path.exists(self.symlink): - os.symlink(path, self.symlink) + symlink = self.FixPath('static/{}'.format( os.path.basename(path[0:-1]))) + if not os.path.exists(symlink): + os.symlink(path, symlink) - self.file_list.append(glob.glob(self.view_path + '**', recursive=True)) - for file in self.file_list[0]: + file_list=[] + file_list.append(glob.glob(path + '**', recursive=True)) + for file in file_list[0]: if file == path: continue stat = os.stat(file) @@ -141,7 +139,8 @@ class FileData(): fsize = round(os.stat(file).st_size/(1024*1024)) fname=file.replace(path, "") - file_obj = Files( name=fname, type=ftype, size_mb=fsize, hash=fhash, thumbnail=fthumbnail ) + path_prefix=symlink.replace(path,"") + file_obj = Files( name=fname, type=ftype, size_mb=fsize, hash=fhash, path_prefix=path_prefix, thumbnail=fthumbnail ) db.session.add(file_obj) else: print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) ) @@ -158,6 +157,7 @@ class Files(db.Model): id = db.Column(db.Integer, db.Sequence('files_id_seq'), primary_key=True ) name = db.Column(db.String, unique=True, nullable=False ) type = db.Column(db.String, unique=False, nullable=False) + path_prefix = db.Column(db.String, unique=False, nullable=False) size_mb = db.Column(db.Integer, unique=False, nullable=False) # hash might not be unique, this could be the source of dupe problems hash = db.Column(db.Integer, unique=True, nullable=True) diff --git a/templates/files.html b/templates/files.html index 047f4a7..1c43db3 100644 --- a/templates/files.html +++ b/templates/files.html @@ -29,7 +29,7 @@
{% if obj.type=="Image" %} - + {% elif obj.type == "Video" %}
From 21b163acc79fc5eb6d893faff8d0bd56df2855b7 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 16:59:15 +1100 Subject: [PATCH 06/17] ignore /home/ddp/src/photoassistant/new_img_dir/ as well, so we can test 2 active paths --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index caaa717..1e265ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ photos/ images_to_process/ +new_img_dir/ static/* From 09f588e7e43aaf4bd74618a3ea9dc91f20f29077 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 17:13:12 +1100 Subject: [PATCH 07/17] only call GenerateFileData once now --- files.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/files.py b/files.py index 2e110d1..4446faf 100644 --- a/files.py +++ b/files.py @@ -24,7 +24,7 @@ from settings import Settings class FileData(): def __init__(self): self.view_list=[] - + ################################################################################ # Utility Functions for Files ################################################################################ @@ -167,24 +167,25 @@ class Files(db.Model): return "".format(self.id, self.name ) +### Initiatlise the file data set (GenerateFileData is clever enough to not +### re-process files when we run twice in quick succession, e.g. when running +### Flask in DEBUG mode +filedata = FileData() +filedata.GenerateFileData() ################################################################################ # /file_list -> show detailed file list of files from import_path(s) ################################################################################ @app.route("/file_list", methods=["GET"]) def file_list(): - filedata = FileData() - file_data=filedata.GenerateFileData() - return render_template("file_list.html", page_title='View Files (details)', file_data=file_data, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("file_list.html", page_title='View Files (details)', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() ) ################################################################################ # /files -> show thumbnail view of files from import_path(s) ################################################################################ @app.route("/files", methods=["GET"]) def files(): - filedata = FileData() - file_data=filedata.GenerateFileData() - return render_template("files.html", page_title='View Files', file_data=file_data, alert=st.GetAlert(), message=st.GetMessage() ) + return render_template("files.html", page_title='View Files', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() ) @app.route("/static/") From 594a4803cdeb2fac3f92699b1fa4e7047198345a Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 17:34:31 +1100 Subject: [PATCH 08/17] made file_list.html better as well --- templates/file_list.html | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/templates/file_list.html b/templates/file_list.html index ddf5c1e..0165d09 100644 --- a/templates/file_list.html +++ b/templates/file_list.html @@ -6,19 +6,32 @@ NameSize (MB)Hash {% for obj in file_data.view_list %} - {% if obj.type=="Directory" %} - - {% elif obj.type=="Image" %} - - {% elif obj.type=="Video" %} - - {% else %} - - {% endif %} - {% if obj.type=="Image" %} - - {% endif %} - {{obj.name}} + {% if obj.type == "Directory" %} +
{{obj.name}} + {% else %} +
+ +
{{obj.name}}
+
+ {% endif %} {{obj.size_mb}}{{obj.hash}} {% endfor %} From 86c7fbd0bef7595a8e9927aad6ad5d14308e6d14 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 17:35:42 +1100 Subject: [PATCH 09/17] made file_list.html better as well --- templates/file_list.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/file_list.html b/templates/file_list.html index 0165d09..7534897 100644 --- a/templates/file_list.html +++ b/templates/file_list.html @@ -2,8 +2,8 @@

{{page_title}} -- {{file_data.view_path}}

- - +
NameSize (MB)Hash
+ {% for obj in file_data.view_list %} + {% endfor %}
NameSize (MB)Path PrefixHash
{% if obj.type == "Directory" %} @@ -32,7 +32,7 @@
{{obj.name}}
{% endif %} - 		{{obj.size_mb}}{{obj.hash}}
{{obj.size_mb}}{{obj.path_prefix}}{{obj.hash}}
From e83a40ee5291fea42ad0ef0e36f9fa7fbd6551c3 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 18:32:47 +1100 Subject: [PATCH 10/17] converted over to a flat settings table, no more key-value pairs of settings --- files.py | 9 ++++---- settings.py | 50 +++++++++++++++++++---------------------- templates/setting.html | 25 --------------------- templates/settings.html | 31 ++++++++++++++++--------- 4 files changed, 47 insertions(+), 68 deletions(-) delete mode 100644 templates/setting.html diff --git a/files.py b/files.py index 4446faf..6247be7 100644 --- a/files.py +++ b/files.py @@ -97,10 +97,9 @@ class FileData(): # multiple valid paths in import_path) # ############################################################################## def GenerateFileData(self): - import_path = Settings.query.filter(Settings.name=="import_path").all()[0].value - last_import_date_obj = Settings.query.filter(Settings.name=="last_import_date").all() - last_import_date = float(last_import_date_obj[0].value) - paths= import_path.split("#") + settings = Settings.query.all() + last_import_date = settings[0].last_import_date + paths = settings[0].import_path.split("#") for path in paths: print( "GenerateFileData: Checking {}".format( path ) ) @@ -144,7 +143,7 @@ class FileData(): db.session.add(file_obj) else: print( "{} - {} is OLDER than {}".format( file, stat.st_ctime, last_import_date ) ) - last_import_date_obj[0].value = str(time.time()) + settings[0].last_import_date = time.time() db.session.commit() self.view_list = Files.query.all() return self diff --git a/settings.py b/settings.py index 8f1a4a1..f2a1acc 100644 --- a/settings.py +++ b/settings.py @@ -1,4 +1,4 @@ -from wtforms import SubmitField, StringField, HiddenField, validators, Form +from wtforms import SubmitField, StringField, FloatField, HiddenField, validators, Form from flask_wtf import FlaskForm from flask import request, render_template, redirect from main import db, app, ma @@ -11,13 +11,13 @@ from status import st, Status ################################################################################ class Settings(db.Model): id = db.Column(db.Integer, db.Sequence('settings_id_seq'), primary_key=True ) - name = db.Column(db.String, unique=True, nullable=True ) - value = db.Column(db.String, unique=True, nullable=True ) + import_path = db.Column(db.String) + last_import_date = db.Column(db.Float) def __repr__(self): - return "".format(self.id, self.name, self.value) + return "".format(self.id, self.import_path, self.last_import_date) -############################################################################### +################################################################################ # Helper class that inherits a .dump() method to turn class Author into json / useful in jinja2 ################################################################################ class SettingsSchema(ma.SQLAlchemyAutoSchema): @@ -25,7 +25,7 @@ class SettingsSchema(ma.SQLAlchemyAutoSchema): model = Settings ordered = True -settings_schema=SettingsSchema() +settings_schema = SettingsSchema(many=True) ################################################################################ # Helper class that defines a form for Settings, used to make html
@@ -33,41 +33,37 @@ settings_schema=SettingsSchema() ################################################################################ class SettingsForm(FlaskForm): id = HiddenField() - name = StringField('Name:', [validators.DataRequired()]) - value = StringField('value:', [validators.DataRequired()]) + import_path = StringField('Path to import from:', [validators.DataRequired()]) + last_import_date = FloatField('Date of last import:', [validators.DataRequired()]) submit = SubmitField('Save' ) ################################################################################ # /settings -> show current settings ################################################################################ -@app.route("/settings", methods=["GET"]) +@app.route("/settings", methods=["GET", "POST"]) def settings(): - sets = Settings.query.all() - form = SettingsForm() - page_title='Show Settings' - return render_template("settings.html", objects=sets, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) - -################################################################################ -# /setting/ -> show current settings -################################################################################ -@app.route("/setting/", methods=["GET", "POST"]) -def setting(id): form = SettingsForm(request.form) if request.method == 'POST' and form.validate(): try: + # HACK, I don't really need an id here, but sqlalchemy get weird + # without one, so just grab the id of the only row there, it will + # do... + id = Settings.query.all()[0].id s = Settings.query.get(id) if 'submit' in request.form: - st.SetMessage("Successfully Updated Setting (name={})".format(s.name) ) - s.name = request.form['name'] - s.value = request.form['value'] + st.SetMessage("Successfully Updated Settings" ) + s.import_path = request.form['import_path'] + s.last_import_date = request.form['last_import_date'] db.session.commit() return redirect( '/settings' ) except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Setting: {}".format(e.orig) ) - return render_template("edit_setting.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + page_title='Show Settings' + return render_template("settings.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) else: - s = Settings.query.get(id) - page_title='Edit Setting: {}'.format(s.name) - form = SettingsForm(request.values, obj=s) - return render_template("setting.html", objects=s, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + tmp_sets = Settings.query.all() + sets = settings_schema.dump( tmp_sets ) + form = SettingsForm(obj=tmp_sets[0]) + page_title='Show Settings' + return render_template("settings.html", form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) diff --git a/templates/setting.html b/templates/setting.html deleted file mode 100644 index 843da7c..0000000 --- a/templates/setting.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "base.html" %} {% block main_content %} -
-

{{page_title}}

-
- - {% for field in form %} - {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} - {{field}} - {% elif field.type != 'SubmitField' %} -
- {{ field.label( class="col-lg-2" ) }} - {{ field( class="form-control col-lg-4" ) }} -
- {% endif %} - {% endfor %} -
-
-
-
- {{ form.submit( class="btn btn-primary offset-lg-2 col-lg-2" )}} -
- -
-
-{% endblock main_content %} diff --git a/templates/settings.html b/templates/settings.html index 2ac004a..70dd336 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -1,13 +1,22 @@ {% extends "base.html" %} {% block main_content %} -
-

{{page_title}}

-
- - - {% for obj in objects %} - - {% endfor %} -
NameValue
{{obj.name}}{{obj.value}}
-
-
+
+

{{page_title}}

+
+
+ {% for field in form %} + {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} + {{field}}
+ {% elif field.type != 'SubmitField' %} +
+ {{ field.label( class="col-lg-2" ) }} + {{ field( class="form-control col-lg-10" ) }} +
+ {% endif %} + {% endfor %} +
+ {{form.submit(class="btn btn-primary offset-lg-2 col-lg-2" )}} +
+
+
+
{% endblock main_content %} From 684d057e5910d290db0dcc7bf9223716f5dcf7a7 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 18:45:13 +1100 Subject: [PATCH 11/17] added admin routes/code to allow a scan now and a forced scan --- files.py | 22 ++++++++++++++++++++++ settings.py | 3 +-- templates/base.html | 4 +++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/files.py b/files.py index 6247be7..1ba02a0 100644 --- a/files.py +++ b/files.py @@ -186,7 +186,29 @@ def file_list(): def files(): return render_template("files.html", page_title='View Files', file_data=filedata, alert=st.GetAlert(), message=st.GetMessage() ) +################################################################################ +# /files/scannow -> allows us to force a check for new files +################################################################################ +@app.route("/files/scannow", methods=["GET"]) +def scannow(): + filedata.GenerateFileData() + return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Scanned for new files" ) +################################################################################ +# /files/forcescan -> deletes old data in DB, and does a brand new scan +################################################################################ +@app.route("/files/forcescan", methods=["GET"]) +def forcescan(): + Files.query.delete() + Settings.query.all()[0].last_import_date=0 + db.session.commit() + filedata.GenerateFileData() + return render_template("base.html", page_title='Forced look for new items', file_data=filedata, alert="success", message="Forced remove and recreation of all file data" ) + +################################################################################ +# /static -> returns the contents of any file referenced inside /static. +# we create/use symlinks in static/ to reference the images to show +################################################################################ @app.route("/static/") def custom_static(filename): return send_from_directory("static/", filename) diff --git a/settings.py b/settings.py index f2a1acc..ea89d31 100644 --- a/settings.py +++ b/settings.py @@ -43,6 +43,7 @@ class SettingsForm(FlaskForm): @app.route("/settings", methods=["GET", "POST"]) def settings(): form = SettingsForm(request.form) + page_title='Settings' if request.method == 'POST' and form.validate(): try: # HACK, I don't really need an id here, but sqlalchemy get weird @@ -59,11 +60,9 @@ def settings(): except SQLAlchemyError as e: st.SetAlert( "danger" ) st.SetMessage( "Failed to modify Setting: {}".format(e.orig) ) - page_title='Show Settings' return render_template("settings.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) else: tmp_sets = Settings.query.all() sets = settings_schema.dump( tmp_sets ) form = SettingsForm(obj=tmp_sets[0]) - page_title='Show Settings' return render_template("settings.html", form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) diff --git a/templates/base.html b/templates/base.html index e62e29f..fb62788 100644 --- a/templates/base.html +++ b/templates/base.html @@ -52,7 +52,9 @@
- Settings + Edit Settings + Scan now (for new files) + Force Scan (delete data & rebuild)
From 2c170e2a9cfb83e7f1ed385ed6d861aa6bf60012 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 21:12:02 +1100 Subject: [PATCH 12/17] not used anymore --- templates/edit_setting.html | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 templates/edit_setting.html diff --git a/templates/edit_setting.html b/templates/edit_setting.html deleted file mode 100644 index 35ac3b1..0000000 --- a/templates/edit_setting.html +++ /dev/null @@ -1,28 +0,0 @@ -{% extends "base.html" %} {% block main_content %} -
-

{{page_title}}

-
- - {% for field in form %} - {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} - {{field}}
- {% elif field.type != 'SubmitField' %} -
- {{ field.label( class="col-lg-2" ) }} - {{ field( class="form-control col-lg-4" ) }} -
- {% endif %} - {% endfor %} -
-
-
-
- {{ form.submit( class="btn btn-primary offset-lg-2 col-lg-2" )}} - {% if 'Edit' in page_title %} - {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} - {% endif %} -
- -
-
-{% endblock main_content %} From a3a95f636e1f1139b03e946b61a2047d708d14a5 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 21:51:33 +1100 Subject: [PATCH 13/17] now have a Person class & associated CReate/Update/Delete functions, and menu in base.html --- main.py | 1 + person.py | 108 +++++++++++++++++++++++++++++++++++++++++ templates/base.html | 16 +++++- templates/person.html | 28 +++++++++++ templates/persons.html | 16 ++++++ 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 person.py create mode 100644 templates/person.html create mode 100644 templates/persons.html diff --git a/main.py b/main.py index 748240f..df21d11 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,7 @@ Bootstrap(app) ################################# Now, import non-book classes ################################### from settings import Settings from files import Files +from person import Person ####################################### GLOBALS ####################################### # allow jinja2 to call these python functions directly diff --git a/person.py b/person.py new file mode 100644 index 0000000..79b9a46 --- /dev/null +++ b/person.py @@ -0,0 +1,108 @@ +from wtforms import SubmitField, StringField, HiddenField, validators, Form +from flask_wtf import FlaskForm +from flask import request, render_template, redirect +from main import db, app, ma +from sqlalchemy import Sequence +from sqlalchemy.exc import SQLAlchemyError +from status import st, Status + +################################################################################ +# Class describing Person in the database, and via sqlalchemy, connected to the DB as well +################################################################################ +class Person(db.Model): + id = db.Column(db.Integer, db.Sequence('person_id_seq'), primary_key=True ) + tag = db.Column(db.String(48), unique=False, nullable=False) + surname = db.Column(db.String(48), unique=False, nullable=False) + firstname = db.Column(db.String(48), unique=False, nullable=False) + + def __repr__(self): + return "".format(self.tag,self.firstname, self.surname) + +################################################################################ +# Helper class that inherits a .dump() method to turn class Person into json / useful in jinja2 +################################################################################ +class PersonSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = Person + ordered = True + +################################################################################ +# Helper class that defines a form for person, used to make html
, with field validation (via wtforms) +################################################################################ +class PersonForm(FlaskForm): + id = HiddenField() + tag = StringField('Tag (searchable name):', [validators.DataRequired()]) + firstname = StringField('FirstName(s):', [validators.DataRequired()]) + surname = StringField('Surname:', [validators.DataRequired()]) + submit = SubmitField('Save' ) + delete = SubmitField('Delete' ) + +################################################################################ +# Routes for person data +# +# /persons -> GET only -> prints out list of all persons +################################################################################ +@app.route("/persons", methods=["GET"]) +def persons(): + persons = Person.query.all() + return render_template("persons.html", persons=persons, alert=st.GetAlert(), message=st.GetMessage() ) + + +################################################################################ +# /person -> GET/POST -> creates a new person type and when created, takes you back to /persons +################################################################################ +@app.route("/person", methods=["GET", "POST"]) +def new_person(): + form = PersonForm(request.form) + page_title='Create new Person' + if 'surname' not in request.form: + return render_template("person.html", form=form, page_title=page_title ) + else: + person = Person( tag=request.form["tag"], surname=request.form["surname"], firstname=request.form["firstname"] ) + try: + db.session.add(person) + db.session.commit() + print(person) + st.SetMessage( "Created new Person ({})".format(person.tag) ) + return redirect( '/persons' ) + except SQLAlchemyError as e: + st.SetAlert( "danger" ) + st.SetMessage( "Failed to add Person: {}".format(e.orig) ) + return render_template("person.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + +################################################################################ +# /person/ -> GET/POST(save or delete) -> shows/edits/delets a single +# person +################################################################################ +@app.route("/person/", methods=["GET", "POST"]) +def person(id): + form = PersonForm(request.form) + page_title='Edit Person' + if request.method == 'POST': + try: + person = Person.query.get(id) + if 'delete' in request.form: + st.SetMessage("Successfully deleted Person: ({})".format( person.tag ) ) + person = Person.query.filter(Person.id==id).delete() + if 'submit' in request.form and form.validate(): + st.SetMessage("Successfully Updated Person: (From: {}, {}, {})".format(person.tag, person.firstname, person.surname) ) + person.tag = request.form['tag'] + person.surname = request.form['surname'] + person.firstname = request.form['firstname'] + st.AppendMessage(" To: ({}, {}, {})".format(person.tag, person.firstname, person.surname) ) + db.session.commit() + return redirect( '/persons' ) + except SQLAlchemyError as e: + st.SetAlert( "danger" ) + st.SetMessage( "Failed to modify Person: {}".format(e.orig) ) + return render_template("person.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + else: + person = Person.query.get(id) + form = PersonForm(request.values, obj=person) + return render_template("person.html", object=person, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) + +################################################################################ +# helper fund to GetPersons -> person_list -> jinja2 for person drop-down in book.html +################################################################################ +def GetPersons(): + return Person.query.order_by('surname','firstname').all() diff --git a/templates/base.html b/templates/base.html index fb62788..97aaf3a 100644 --- a/templates/base.html +++ b/templates/base.html @@ -49,6 +49,20 @@ View Details +
+ +
+ Reference Images + View Stats +
+
+
+ +
+ Create Person + Show People +
+
@@ -59,7 +73,7 @@
- + diff --git a/templates/person.html b/templates/person.html new file mode 100644 index 0000000..8ace7b2 --- /dev/null +++ b/templates/person.html @@ -0,0 +1,28 @@ +{% extends "base.html" %} {% block main_content %} +
+

{{page_title}}

+
+
+ {% for field in form %} + {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} + {{field}}
+ {% elif field.type != 'SubmitField' %} +
+ {{ field.label( class="col-lg-4" ) }} + {{ field( class="form-control col-lg-4" ) }} +
+ {% endif %} + {% endfor %} +
+
+
+
+ {{ form.submit( class="btn btn-primary offset-lg-4 col-lg-2" )}} + {% if 'Edit' in page_title %} + {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} + {% endif %} +
+
+
+
+{% endblock main_content %} diff --git a/templates/persons.html b/templates/persons.html new file mode 100644 index 0000000..e68328e --- /dev/null +++ b/templates/persons.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block main_content %} +

Show All People

+ + + + + + {% for person in persons %} + + + {% endfor %} + +
TagFirstname(s)Surname
{{person.tag}}{{person.firstname}}{{person.surname}}
+{% endblock main_content %} From ed3a85b8f00680b4b52660590c5a7fb71346dbf0 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 23:58:12 +1100 Subject: [PATCH 14/17] added reference image class, also tweaked DB classes to have foreign keys, so you cant delete a person connected to a file (tested). Also made refimg class do some quirky bootstrap/jquery to get file dialogs not looking like crap --- main.py | 11 +++++++ person.py | 16 ++++++---- templates/refimg.html | 69 ++++++++++++++++++++++++++++++++++++++++++ templates/refimgs.html | 15 +++++++++ 4 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 templates/refimg.html create mode 100644 templates/refimgs.html diff --git a/main.py b/main.py index df21d11..a874f44 100644 --- a/main.py +++ b/main.py @@ -30,6 +30,17 @@ Bootstrap(app) from settings import Settings from files import Files from person import Person +from refimg import Refimg +from ai import * + +####################################### CLASSES / DB model ####################################### +class Person_Refimg_Link(db.Model): + __tablename__ = "person_refimg_link" + person_id = db.Column(db.Integer, db.ForeignKey('person.id'), unique=True, nullable=False, primary_key=True) + refimg_id = db.Column(db.Integer, db.ForeignKey('refimg.id'), unique=True, nullable=False, primary_key=True) + + def __repr__(self): + return "".format(self.person_id, self.refimg_id) ####################################### GLOBALS ####################################### # allow jinja2 to call these python functions directly diff --git a/person.py b/person.py index 79b9a46..e56c105 100644 --- a/person.py +++ b/person.py @@ -6,6 +6,8 @@ from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError from status import st, Status +from files import Files + ################################################################################ # Class describing Person in the database, and via sqlalchemy, connected to the DB as well ################################################################################ @@ -18,6 +20,14 @@ class Person(db.Model): def __repr__(self): return "".format(self.tag,self.firstname, self.surname) +class File_Person_Link(db.Model): + __tablename__ = "file_person_link" + file_id = db.Column(db.Integer, db.ForeignKey('files.id'), unique=True, nullable=False, primary_key=True) + person_id = db.Column(db.Integer, db.ForeignKey('person.id'), unique=True, nullable=False, primary_key=True) + + def __repr__(self): + return "".format(self.file_id, self.person_id) + ################################################################################ # Helper class that inherits a .dump() method to turn class Person into json / useful in jinja2 ################################################################################ @@ -100,9 +110,3 @@ def person(id): person = Person.query.get(id) form = PersonForm(request.values, obj=person) return render_template("person.html", object=person, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) - -################################################################################ -# helper fund to GetPersons -> person_list -> jinja2 for person drop-down in book.html -################################################################################ -def GetPersons(): - return Person.query.order_by('surname','firstname').all() diff --git a/templates/refimg.html b/templates/refimg.html new file mode 100644 index 0000000..d33bd5e --- /dev/null +++ b/templates/refimg.html @@ -0,0 +1,69 @@ +{% extends "base.html" %} +{% block main_content %} +
+

{{page_title}}

+
+
+ {% for field in form %} + {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} + {{field}}
+ {% elif field.type != 'SubmitField' %} +
+ {% if 'Edit' in page_title %} + {{ field.label( class="col-lg-2" ) }} +
+
+ +
+
+ {{field.data}} + +
+ {% else %} + {{ field.label( class="col-lg-2" ) }} + +
+
+ {{field.data}} +
+
+ +
+
+ {% endif %} +
+ {% endif %} + {% endfor %} +
+
+
+
+ {{ form.submit( class="btn btn-primary offset-lg-2 col-lg-2" )}} + {% if 'Edit' in page_title %} + {{ form.delete( class="btn btn-outline-danger col-lg-2" )}} + {% endif %} +
+ +
+
+{% endblock main_content %} + +{% block script_content %} + +{% endblock script_content %} diff --git a/templates/refimgs.html b/templates/refimgs.html new file mode 100644 index 0000000..1af2343 --- /dev/null +++ b/templates/refimgs.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} + +{% block main_content %} +

Show All Reference Images

+ + + + + + {% for refimg in refimgs %} + + {% endfor %} + +
Filename
{{refimg.fname}}
+{% endblock main_content %} From 5bf41ce6a8f447b3921b5b221a8124691ce4a837 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 23:58:27 +1100 Subject: [PATCH 15/17] token placholder for AI menu with html --- ai.py | 15 +++++++++++++++ templates/aistats.html | 5 +++++ templates/base.html | 20 +++++++++++++------- 3 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 ai.py create mode 100644 templates/aistats.html diff --git a/ai.py b/ai.py new file mode 100644 index 0000000..53982f0 --- /dev/null +++ b/ai.py @@ -0,0 +1,15 @@ +from wtforms import SubmitField, StringField, HiddenField, validators, Form +from flask_wtf import FlaskForm +from flask import request, render_template, redirect +from main import db, app, ma +from sqlalchemy import Sequence +from sqlalchemy.exc import SQLAlchemyError +from status import st, Status + +################################################################################ +# /refimg/ -> GET/POST(save or delete) -> shows/edits/delets a single +# refimg +################################################################################ +@app.route("/aistats", methods=["GET", "POST"]) +def aistats(): + return render_template("aistats.html", page_title='Placeholder', alert=st.GetAlert(), message=st.GetMessage() ) diff --git a/templates/aistats.html b/templates/aistats.html new file mode 100644 index 0000000..985119a --- /dev/null +++ b/templates/aistats.html @@ -0,0 +1,5 @@ +{% extends "base.html" %} + +{% block main_content %} +

Placeholder

+{% endblock main_content %} diff --git a/templates/base.html b/templates/base.html index 97aaf3a..ce1994e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -49,13 +49,6 @@ View Details
-
- -
- Reference Images - View Stats -
-
@@ -63,6 +56,19 @@ Show People
+
+ +
+ Create Reference Image + View Reference Images +
+
+
+ +
+ View Stats +
+
From 991b654d4c52da2f1fa57ba5f7cff9b3cdcc2f7b Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Tue, 12 Jan 2021 23:58:39 +1100 Subject: [PATCH 16/17] added reference image class, also tweaked DB classes to have foreign keys, so you cant delete a person connected to a file (tested). Also made refimg class do some quirky bootstrap/jquery to get file dialogs not looking like crap --- refimg.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 refimg.py diff --git a/refimg.py b/refimg.py new file mode 100644 index 0000000..ff5e7d6 --- /dev/null +++ b/refimg.py @@ -0,0 +1,97 @@ +from wtforms import SubmitField, StringField, HiddenField, FileField, validators, Form +from flask_wtf import FlaskForm +from flask import request, render_template, redirect +from main import db, app, ma +from sqlalchemy import Sequence +from sqlalchemy.exc import SQLAlchemyError +from status import st, Status + +################################################################################ +# Class describing Refimg in the database, and via sqlalchemy, connected to the DB as well +################################################################################ +class Refimg(db.Model): + id = db.Column(db.Integer, db.Sequence('refimg_id_seq'), primary_key=True ) + fname = db.Column(db.String(256), unique=True, nullable=False) + + def __repr__(self): + return "".format(self.id, self.fname ) + +################################################################################ +# Helper class that inherits a .dump() method to turn class Refimg into json / useful in jinja2 +################################################################################ +class RefimgSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = Refimg + ordered = True + +################################################################################ +# Helper class that defines a form for refimg, used to make html
, with field validation (via wtforms) +################################################################################ +class RefimgForm(FlaskForm): + id = HiddenField() +# fname = StringField('File name:', [validators.DataRequired()]) + fname = FileField('File name:') + submit = SubmitField('Save' ) + delete = SubmitField('Delete' ) + +################################################################################ +# Routes for refimg data +# +# /refimgs -> GET only -> prints out list of all refimgs +################################################################################ +@app.route("/refimgs", methods=["GET"]) +def refimgs(): + refimgs = Refimg.query.all() + return render_template("refimgs.html", refimgs=refimgs, alert=st.GetAlert(), message=st.GetMessage() ) + + +################################################################################ +# /refimg -> GET/POST -> creates a new refimg type and when created, takes you back to /refimgs +################################################################################ +@app.route("/refimg", methods=["GET", "POST"]) +def new_refimg(): + form = RefimgForm(request.form) + page_title='Create new Reference Image' + if 'fname' not in request.form: + return render_template("refimg.html", form=form, page_title=page_title ) + else: + refimg = Refimg( fname=request.form["fname"] ) + try: + db.session.add(refimg) + db.session.commit() + print(refimg) + st.SetMessage( "Created new Refimg ({})".format(refimg.fname) ) + return redirect( '/refimgs' ) + except SQLAlchemyError as e: + st.SetAlert( "danger" ) + st.SetMessage( "Failed to add Refimg: {}".format(e.orig) ) + return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + +################################################################################ +# /refimg/ -> GET/POST(save or delete) -> shows/edits/delets a single +# refimg +################################################################################ +@app.route("/refimg/", methods=["GET", "POST"]) +def refimg(id): + form = RefimgForm(request.form) + page_title='Edit Reference Image' + if request.method == 'POST': + try: + refimg = Refimg.query.get(id) + if 'delete' in request.form: + st.SetMessage("Successfully deleted Refimg: ({})".format( refimg.fname ) ) + refimg = Refimg.query.filter(Refimg.id==id).delete() + if 'submit' in request.form and form.validate(): + st.SetMessage("Successfully Updated Refimg: (From: {})".format(refimg.fname)) + refimg.fname = request.form['fname'] + st.AppendMessage(" To: ({})".format(refimg.fname) ) + db.session.commit() + return redirect( '/refimgs' ) + except SQLAlchemyError as e: + st.SetAlert( "danger" ) + st.SetMessage( "Failed to modify Refimg: {}".format(e.orig) ) + return render_template("refimg.html", form=form, page_title=page_title, alert=st.GetAlert(), message=st.GetMessage() ) + else: + refimg = Refimg.query.get(id) + form = RefimgForm(request.values, obj=refimg) + return render_template("refimg.html", object=refimg, form=form, page_title = page_title, alert=st.GetAlert(), message=st.GetMessage() ) From 2a7151dc669064a02d9cfe8d6518a035171f8b33 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 13 Jan 2021 13:52:56 +1100 Subject: [PATCH 17/17] tried to add a reference_images directory in git, that contains actual uploaded images and the modify/remove also deals with filesystem issues --- .gitignore | 1 + refimg.py | 18 +++++++++++---- templates/refimg.html | 53 +++++++++++-------------------------------- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 1e265ed..d5066e8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ photos/ images_to_process/ new_img_dir/ static/* +reference_images/* diff --git a/refimg.py b/refimg.py index ff5e7d6..f9bb3ba 100644 --- a/refimg.py +++ b/refimg.py @@ -5,6 +5,7 @@ from main import db, app, ma from sqlalchemy import Sequence from sqlalchemy.exc import SQLAlchemyError from status import st, Status +import os ################################################################################ # Class describing Refimg in the database, and via sqlalchemy, connected to the DB as well @@ -29,8 +30,8 @@ class RefimgSchema(ma.SQLAlchemyAutoSchema): ################################################################################ class RefimgForm(FlaskForm): id = HiddenField() -# fname = StringField('File name:', [validators.DataRequired()]) - fname = FileField('File name:') + fname = StringField('File name:', [validators.DataRequired()]) + refimg_file = FileField('File name:') submit = SubmitField('Save' ) delete = SubmitField('Delete' ) @@ -44,7 +45,6 @@ def refimgs(): refimgs = Refimg.query.all() return render_template("refimgs.html", refimgs=refimgs, alert=st.GetAlert(), message=st.GetMessage() ) - ################################################################################ # /refimg -> GET/POST -> creates a new refimg type and when created, takes you back to /refimgs ################################################################################ @@ -52,14 +52,18 @@ def refimgs(): def new_refimg(): form = RefimgForm(request.form) page_title='Create new Reference Image' - if 'fname' not in request.form: + if request.method == 'GET': return render_template("refimg.html", form=form, page_title=page_title ) else: + # save the actual uploaded image to reference_images/ + f=request.files['refimg_file'] + f.save(os.path.join("reference_images/", request.form["fname"])) + + # now save into the DB refimg = Refimg( fname=request.form["fname"] ) try: db.session.add(refimg) db.session.commit() - print(refimg) st.SetMessage( "Created new Refimg ({})".format(refimg.fname) ) return redirect( '/refimgs' ) except SQLAlchemyError as e: @@ -78,6 +82,7 @@ def refimg(id): if request.method == 'POST': try: refimg = Refimg.query.get(id) + os.remove("reference_images/{}".format(refimg.fname) ) if 'delete' in request.form: st.SetMessage("Successfully deleted Refimg: ({})".format( refimg.fname ) ) refimg = Refimg.query.filter(Refimg.id==id).delete() @@ -85,6 +90,9 @@ def refimg(id): st.SetMessage("Successfully Updated Refimg: (From: {})".format(refimg.fname)) refimg.fname = request.form['fname'] st.AppendMessage(" To: ({})".format(refimg.fname) ) + # save the actual uploaded image to reference_images/ + f=request.files['refimg_file'] + f.save(os.path.join("reference_images/", request.form["fname"])) db.session.commit() return redirect( '/refimgs' ) except SQLAlchemyError as e: diff --git a/templates/refimg.html b/templates/refimg.html index d33bd5e..162a47a 100644 --- a/templates/refimg.html +++ b/templates/refimg.html @@ -3,43 +3,18 @@

{{page_title}}

- - {% for field in form %} - {% if field.type == 'HiddenField' or field.type == 'CSRFTokenField' %} - {{field}}
- {% elif field.type != 'SubmitField' %} -
- {% if 'Edit' in page_title %} - {{ field.label( class="col-lg-2" ) }} -
-
- -
-
- {{field.data}} - -
- {% else %} - {{ field.label( class="col-lg-2" ) }} - -
-
- {{field.data}} -
-
- -
-
- {% endif %} -
- {% endif %} - {% endfor %} + + {{form.id}} + {{form.csrf_token}} +
+ {{ form.fname.label( class="col-lg-2" ) }} + {{ form.fname( id="fname", class="form-control col-lg-4" ) }} +   + +

@@ -61,9 +36,7 @@ console.log(str) str=str.replace('C:\\fakepath\\', '' ) console.log(str) - $("#fname_span").html(str) - $("#fname_id").val(str) - $("#new_file_chooser").val('') + $("#fname").val(str) } {% endblock script_content %}