From b952fe82f6eedf17a7c5db43d99a358f68d6bb06 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 31 Dec 2022 15:42:03 +1100 Subject: [PATCH] large refactor of code to use venv (in .python) for DEV, to use consistent python versions with PROD, sepcifically flask 2.2.2 to address the deprecation of FLASK_ENV -> replaced with ENV. Fixed up git and dockerignores to be more stringent --- .dockerignore | 16 +++++++++++++++- .gitignore | 8 ++++---- Dockerfile | 2 +- README | 19 ++++++++++++++++--- TODO | 5 ++--- files.py | 2 +- main.py | 2 +- pa_job_manager.py | 2 +- requirements.txt | 1 + shared.py | 8 ++++---- wrapper.sh | 5 ++--- 11 files changed, 48 insertions(+), 22 deletions(-) diff --git a/.dockerignore b/.dockerignore index 55963de..00767eb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,21 @@ -DB_BACKUP images_to_process new_img_dir +photos +reference_images +storage static/Bin/* static/Import/* static/Storage/* static/Metadata/* +.pa_metadata +.pa_bin +.python +__pycache__ +BUGS +README +TODO +DB_BACKUP +db-container +.gitignore +.dockerignore +Dockerfile diff --git a/.gitignore b/.gitignore index ff82ebb..c302072 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,8 @@ photos/ storage/ images_to_process/ DB_BACKUP/ -new_img_dir/ -static/ -internal/upstream internal/git-log.txt -.pa_metadata +internal/upstream +static/ +.pa_metadata/ +.python/ diff --git a/Dockerfile b/Dockerfile index 6658aa4..df3604e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN truncate -s0 /tmp/preseed.cfg && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ apt-get install -y tzdata -RUN apt-get update && apt-get -y dist-upgrade && apt-get -y install python3-pip python3-psycopg2 libpq-dev gunicorn mediainfo cmake libgl1-mesa-glx libglib2.0-0 python3-ldap libjpeg-turbo-progs ffmpeg git +RUN apt-get update && apt-get -y dist-upgrade && apt-get -y install python3-pip libpq-dev gunicorn mediainfo cmake libgl1-mesa-glx libglib2.0-0 python3-ldap libjpeg-turbo-progs ffmpeg git COPY requirements.txt requirements.txt RUN pip3 install -r requirements.txt RUN pip3 install --upgrade pillow --user diff --git a/README b/README index 000c8b9..7b2dbe2 100644 --- a/README +++ b/README @@ -9,6 +9,14 @@ CAM: fill this in pls ubuntu packages: sudo apt-get install -y mediainfo cmake python3-flask + +REDO THIS SOON (good to test on a clean install): + # use a venv (to get consistent python versions with Prod container) + python3 -m venv .python + source ./.python/bin/activate + pip3 install -r requirements.txt + + pip packages: * pymediainfo * PIL (should be there by default) @@ -50,14 +58,19 @@ upstream packages... sudo apt-get install libcairo2-dev libjpeg-dev libgif-dev libgirepository1.0-dev libcups2-dev pip3 list | tail -n +3 | grep -v mysqlclient | grep -v duplicity | grep -v gpg | awk ' { print $1 } ' | xargs pip3 install --upgrade -To run debug version of web server: - FLASK_APP=main FLASK_ENV=development flask run --host=192.168.0.2 +To run debug version of web server (gunicorn deprecates FLASK_ENV, so co-exist for now): + + # older flask: + FLASK_APP=main ENV=development FLASK_ENV=development flask run --host=192.168.0.2 + + # flask 2.2.2+ (in venv .python) + FLASK_APP=main ENV=development ./.python/bin/flask --debug run --host=192.168.0.2 to run prod version of web server: gunicorn --bind="192.168.0.2:5000" --threads=2 --workers=2 main:app Also have to run the job manager for jobs to work: - FLASK_ENV="development" python3 pa_job_manager.py + ENV="development" python3 pa_job_manager.py To rebuild DB from scratch/empty data: diff --git a/TODO b/TODO index e303f1c..617fe7e 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,4 @@ ## GENERAL - * get rid of deprecation with FLASK_ENV - * after move, instead of back to home page, go to an ammended view of the thumbs to keep moving * ignore face should ignore ALL matching faces (re: Declan) @@ -124,4 +122,5 @@ ### FUTURE: * can emby use nfo for images (for AI/tags?) -NO sadly - + + * work out why gitignore does not ignore say .pa_bin but has to ignore .pa_metada diff --git a/files.py b/files.py index 33c44ae..bfe653f 100644 --- a/files.py +++ b/files.py @@ -726,7 +726,7 @@ def view(id): if id not in eids: print( f"ERROR: viewing an id, but its not in eids OPT={OPT}, id={id}, eids={eids}") msg="Sorry, viewing data is confused, cannot view this image now" - if os.environ['FLASK_ENV'] == "production": + if os.environ['ENV'] == "production": msg += "Clearing out all states. This means browser back buttons will not work, please start a new tab and try again" PA_UserState.query.delete() db.session.commit() diff --git a/main.py b/main.py index b39abda..1e04d91 100644 --- a/main.py +++ b/main.py @@ -28,7 +28,7 @@ app = Flask(__name__) app.config['SQLALCHEMY_DATABASE_URI'] = DB_URL app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -app.config['ENV'] = os.environ['FLASK_ENV'] +app.config['ENV'] = os.environ['ENV'] app.config['SECRET_KEY'] = b'my_insecure_PA_token_with_random_2134876adsfjhlkasdf87' app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 31536000 diff --git a/pa_job_manager.py b/pa_job_manager.py index 4a503c0..49dc19d 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -50,7 +50,7 @@ import uuid # global debug setting -if 'FLASK_ENV' not in os.environ or os.environ['FLASK_ENV'] != "production": +if 'ENV' not in os.environ or os.environ['ENV'] != "production": DEBUG=True else: DEBUG=False diff --git a/requirements.txt b/requirements.txt index b89d373..ae228ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ face_recognition Werkzeug flask-compress ffmpeg-python +psycopg2 diff --git a/shared.py b/shared.py index cd7fd40..d76196f 100644 --- a/shared.py +++ b/shared.py @@ -40,14 +40,14 @@ OLDEST_LOG_LIMIT = 5 if hostname == "lappy": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@localhost:5432/pa' -# if we dont set the env or we are explicitly DEV, run web server on localhost & db on mara (port 65432) PA_EXIF_ROTATER = './utils/pa_exifautotran' -elif 'FLASK_ENV' not in os.environ or os.environ['FLASK_ENV'] == "development": +# if we dont set the env or we are explicitly DEV, run web server on localhost & db on mara (port 65432) +elif 'ENV' not in os.environ or os.environ['ENV'] == "development": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@mara.ddp.net:65432/pa' -# if we explicitly are on PROD, run web server on localhost (pa_web container) & db on mara (port 5432 on padb container)- only accessed via internal docker ports) PA_EXIF_ROTATER = './utils/pa_exifautotran' -elif os.environ['FLASK_ENV'] == "production": +# if we explicitly are on PROD, run web server on localhost (pa_web container) & db on mara (port 5432 on padb container)- only accessed via internal docker ports) +elif os.environ['ENV'] == "production": PA_JOB_MANAGER_HOST="localhost" DB_URL = 'postgresql+psycopg2://pa:for_now_pa@padb/pa' PA_EXIF_ROTATER = '/code/utils/pa_exifautotran' diff --git a/wrapper.sh b/wrapper.sh index 7ab3c3b..43bfaff 100755 --- a/wrapper.sh +++ b/wrapper.sh @@ -1,5 +1,4 @@ #!/bin/bash -#su mythtv -g mythtv -c 'FLASK_ENV="production" python3 -u /code/pa_job_manager.py' &> /var/log/pa_job_manager.out & -gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --env FLASK_ENV="production" --error-logfile gunicorn.error.log --access-logfile gunicorn.log --capture-output & -su mythtv -g mythtv -c 'FLASK_ENV="production" python3 -u /code/pa_job_manager.py' &> /var/log/pa_job_manager.out +su mythtv -g mythtv -c 'ENV="production" python3 -u /code/pa_job_manager.py' &> /var/log/pa_job_manager.out & +gunicorn --bind=0.0.0.0:443 --workers=4 --threads=16 --certfile /etc/letsencrypt/live/pa.depaoli.id.au/fullchain.pem --keyfile /etc/letsencrypt/live/pa.depaoli.id.au/privkey.pem main:app --env ENV="production" --error-logfile gunicorn.error.log --access-logfile gunicorn.log --capture-output