From 71ec54807a0f222f560493b0454e9f6adb707187 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Sat, 17 Sep 2022 22:40:50 +1000 Subject: [PATCH] remove last remnants of Paths, now only 1 directory per path AND created a new db-container subdir that takes the latest postgres image and adds a crontab / uses cron to run a bkup_users.sh script that dumps the person, refimg and linkage table so that if we ever blow the DB away, we dont lose the users --- TODO | 6 +----- db-container/Dockerfile | 1 + db-container/bkup_users.sh | 7 +++++++ db-container/crontab | 2 +- files.py | 29 ++++++++++------------------- pa_job_manager.py | 5 ++++- settings.py | 26 ++++++++++++-------------- 7 files changed, 36 insertions(+), 40 deletions(-) create mode 100755 db-container/bkup_users.sh diff --git a/TODO b/TODO index 0bdc403..7a407d0 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,5 @@ ## GENERAL - * remove multiple Paths from SettingsIPath, etc. - - * put a 'job' in to dump those users out to the filesystem so when we rebuild the padb docker container, we dont have to re-add the users/refimgs, etc. - - * should I change the rotation code to use that jpeg util to reduce/remove compression loss? + * change the rotation code to use that jpeg util to reduce/remove compression loss? * should allow context menu from View thumbs (particularly useful on search) to show other files around this one by date (maybe that folder or something?) diff --git a/db-container/Dockerfile b/db-container/Dockerfile index 67c8512..ef58fa6 100644 --- a/db-container/Dockerfile +++ b/db-container/Dockerfile @@ -13,5 +13,6 @@ RUN truncate -s0 /tmp/preseed.cfg && \ RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN apt-get update && apt-get -y install cron COPY crontab /etc/crontab +COPY bkup_users.sh /tmp/ EXPOSE 5432 CMD ["sh", "-c", "cron ; /usr/local/bin/docker-entrypoint.sh postgres"] diff --git a/db-container/bkup_users.sh b/db-container/bkup_users.sh new file mode 100755 index 0000000..f7fc757 --- /dev/null +++ b/db-container/bkup_users.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +pg_dump -a --user=pa --table=person --table=refimg --table=person_refimg_link > /tmp/new_users.sql +tmp=`diff -q /docker-entrypoint-initdb.d/users.sql /tmp/new_users.sql` +if (( $? !=0 && $(stat -c%s "/tmp/new_users.sql") > 200000 )) ; then + cp -f /tmp/new_users.sql /docker-entrypoint-initdb.d/users.sql +fi diff --git a/db-container/crontab b/db-container/crontab index 9699ea6..0be8096 100644 --- a/db-container/crontab +++ b/db-container/crontab @@ -1 +1 @@ -* * * * * root pg_dump -a --user=pa --table=person --table=refimg --table=person_refimg_link > /tmp/new_users.sql +* * * * * root /tmp/bkup_users.sh diff --git a/files.py b/files.py index 634b5ca..4aa914b 100644 --- a/files.py +++ b/files.py @@ -371,13 +371,11 @@ def GetEntries( OPT ): paths = [] if OPT.path_type == 'Storage': - paths = SettingsSPath() + path = SettingsSPath() elif OPT.path_type == 'Import': - paths = SettingsIPath() + path = SettingsIPath() elif OPT.path_type == 'Bin': - paths.append(SettingsRBPath()) - - num_paths = len(paths) + path = SettingsRBPath() num_entries=0 path_cnt=1 @@ -388,20 +386,13 @@ def GetEntries( OPT ): update_last_eid = True else: update_last_eid = False - for path in paths: - if not os.path.exists(path): - continue - prefix = SymlinkName(OPT.path_type,path,path+'/') - if OPT.folders: - tmp_ents, tmp_num_ents = GetEntriesInFolderView( OPT, prefix ) - else: - tmp_ents, tmp_num_ents = GetEntriesInFlatView( OPT, prefix ) - entries += tmp_ents - num_entries += tmp_num_ents - # if we have another path, keep adding num_etnries, and last_eid is the last path, not this one, so reset to 0 - if update_last_eid and path_cnt < num_paths: - OPT.last_eid=0 - path_cnt += 1 + prefix = SymlinkName(OPT.path_type,path,path+'/') + if OPT.folders: + tmp_ents, tmp_num_ents = GetEntriesInFolderView( OPT, prefix ) + else: + tmp_ents, tmp_num_ents = GetEntriesInFlatView( OPT, prefix ) + entries += tmp_ents + num_entries += tmp_num_ents if update_last_eid: # find pref... via path_type if we are here diff --git a/pa_job_manager.py b/pa_job_manager.py index c77ff32..c12ff73 100644 --- a/pa_job_manager.py +++ b/pa_job_manager.py @@ -2245,7 +2245,10 @@ def ReloadMetadata(job): match=re.search( '(\d+)_([^_]+)', fname ) face_id=match.group(1) person_tag=match.group(2) - p = session.query(Person).filter(Person.tag==person_tag).one() + p = session.query(Person).filter(Person.tag==person_tag).first() + if not p: + print( f"There is a metadata override on the file system for person: {person_tag} - but they are no longer in the DB - skip" ) + continue face_data=GetFaceInMetadata(fname) if DEBUG: print( f"Found metadata showing Override match for person: {person_tag}" ) diff --git a/settings.py b/settings.py index 09a9862..1aaf7af 100644 --- a/settings.py +++ b/settings.py @@ -146,9 +146,9 @@ def SettingsRBPath(): return # path setting is an absolute path, just use it, otherwise prepend base_path first if settings.recycle_bin_path[0] == '/': - path = settings.recycle_bin_path + path=settings.recycle_bin_path else: - path = settings.base_path+settings.recycle_bin_path + path=settings.base_path+settings.recycle_bin_path return path ############################################################################## @@ -161,12 +161,11 @@ def SettingsSPath(): if settings == None: print("Cannot create file data with no settings / storage path is missing") return - for p in settings.storage_path.split("#"): - if p[0] == '/': - paths.append(p) - else: - paths.append(settings.base_path+p) - return paths + if settings.storage_path[0] == '/': + path=settings.storage_path + else: + path=settings.base_path+settings.storage_path + return path ############################################################################## # SettingsIPath(): return modified array of paths (take each path in @@ -178,12 +177,11 @@ def SettingsIPath(): if settings == None: print ("Cannot create file data with no settings / import path is missing") return - for p in settings.import_path.split("#"): - if p[0] == '/': - paths.append(p) - else: - paths.append(settings.base_path+p) - return paths + if settings.import_path[0] == '/': + path=settings.import_path + else: + path=settings.base_path+settings.import_path + return path ############################################################################## # SettingsMPath(): return path to actual metadata path from settings