ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved to storing DB ids rather than names of files/dirs for deletions

Browse Source
This commit is contained in:
Damien De Paoli
2021-02-14 16:20:48 +11:00
parent 7fa0d3e02b
commit 9e84236f49
1 changed files with 44 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
files.py
View File

@@ -19,7 +19,7 @@ import time
################################################################################ ################################################################################
# Local Class imports # Local Class imports
################################################################################ ################################################################################
from job import Job, Joblog, NewJob from job import Job, JobExtra, Joblog, NewJob
from person import Person, PersonRefimgLink from person import Person, PersonRefimgLink
from refimg import Refimg from refimg import Refimg
from settings import Settings from settings import Settings
@@ -166,8 +166,8 @@ def TrimmedPath( prefix, path ):
def AddDup( prefix, row, dups ): def AddDup( prefix, row, dups ):
if row.hash not in dups: if row.hash not in dups:
dups[row.hash]=[] dups[row.hash]=[]
dups[row.hash].append( { 'f': row.fname1, 'd':TrimmedPath(prefix, row.path1) } ) dups[row.hash].append( { 'f': row.fname1, 'd':TrimmedPath(prefix, row.path1), 'did': row.did1, 'h':row.hash, 'id':row.id1 } )
dups[row.hash].append( { 'f': row.fname2, 'd':TrimmedPath(prefix, row.path2) } ) dups[row.hash].append( { 'f': row.fname2, 'd':TrimmedPath(prefix, row.path2), 'did': row.did2, 'h':row.hash, 'id':row.id2 } )
else: else:
# process path1 / fname1 -- if that combo is not in the dups[hash], add it # process path1 / fname1 -- if that combo is not in the dups[hash], add it
found=0 found=0
@@ -176,7 +176,7 @@ def AddDup( prefix, row, dups ):
found=1 found=1
continue continue
if not found: if not found:
dups[row.hash].append( { 'f': row.fname1, 'd':TrimmedPath(prefix, row.path1) } ) dups[row.hash].append( { 'f': row.fname1, 'd':TrimmedPath(prefix, row.path1), 'did': row.did1, 'h':row.hash, 'id':row.id1 } )
# process path2 / fname2 -- if that combo is not in the dups[hash], add it # process path2 / fname2 -- if that combo is not in the dups[hash], add it
found=0 found=0
@@ -185,12 +185,12 @@ def AddDup( prefix, row, dups ):
found=1 found=1
continue continue
if not found: if not found:
dups[row.hash].append( { 'f': row.fname2, 'd':TrimmedPath(prefix, row.path2) } ) dups[row.hash].append( { 'f': row.fname2, 'd':TrimmedPath(prefix, row.path2), 'did': row.did2, 'h':row.hash, 'id':row.id2 } )
return return
@app.route("/fix_dups", methods=["GET"]) @app.route("/fix_dups", methods=["POST"])
def fix_dups(): def fix_dups():
rows = db.engine.execute( "select f1.hash, d1.path_prefix as path1, e1.name as fname1, d2.path_prefix as path2, e2.name as fname2 from entry e1, file f1, dir d1, entry_dir_link edl1, entry e2, file f2, dir d2, entry_dir_link edl2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and f1.hash = f2.hash and e1.id != e2.id order by path1, fname1;" ) rows = db.engine.execute( "select e1.id as id1, f1.hash, d1.path_prefix as path1, d1.eid as did1, e1.name as fname1, e2.id as id2, d2.path_prefix as path2, d2.eid as did2, e2.name as fname2 from entry e1, file f1, dir d1, entry_dir_link edl1, entry e2, file f2, dir d2, entry_dir_link edl2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and f1.hash = f2.hash and e1.id != e2.id order by path1, fname1;" )
if rows.returns_rows == False: if rows.returns_rows == False:
st.SetAlert("success") st.SetAlert("success")
@@ -200,7 +200,6 @@ def fix_dups():
# use import_path setting to remove the dup path prefix of static/basename(<import_path>) # use import_path setting to remove the dup path prefix of static/basename(<import_path>)
# -- static isn't really seen, and the import path basename is always going to be the same # -- static isn't really seen, and the import path basename is always going to be the same
s=Settings.query.first() s=Settings.query.first()
print (s.import_path)
if s.import_path[-1] == '/': if s.import_path[-1] == '/':
prefix = os.path.basename(s.import_path[0:-1]) prefix = os.path.basename(s.import_path[0:-1])
else: else:
@@ -212,10 +211,13 @@ def fix_dups():
d1="" d1=""
d2="" d2=""
did1=""
did2=""
str="" str=""
dup_cnt=1 dup_cnt=1
per_file_dups=[] per_file_dups=[]
per_path_dups=[] per_path_dups=[]
hashes=""
for hash in dups: for hash in dups:
if len(dups[hash]) > 2: if len(dups[hash]) > 2:
per_file_dups.append(dups[hash]) per_file_dups.append(dups[hash])
@@ -228,15 +230,46 @@ def fix_dups():
elif d1 != dups[hash][0]['d']: elif d1 != dups[hash][0]['d']:
if d1 != '': if d1 != '':
dup_cnt=1 dup_cnt=1
per_path_dups.append({'count': dup_cnt, 'd1': d1, 'd2': d2 }) per_path_dups.append({'count': dup_cnt, 'd1': d1, 'd2': d2, 'did1': did1, 'did2': did2, 'hashes' : hashes })
d1 = dups[hash][0]['d'] d1 = dups[hash][0]['d']
d2 = dups[hash][1]['d'] d2 = dups[hash][1]['d']
did1 = dups[hash][0]['did']
did2 = dups[hash][1]['did']
str=f"duplicates found in {d1} and {d2}" str=f"duplicates found in {d1} and {d2}"
hashes = f"{hash},"
else: else:
dup_cnt += 1 dup_cnt += 1
hashes += f"{hash},"
per_path_dups.append({'count': dup_cnt, 'd1': d1, 'd2': d2 }) per_path_dups.append({'count': dup_cnt, 'd1': d1, 'd2': d2, 'did1': did1, 'did2': did2, 'hashes' : hashes })
return render_template("dups.html", per_file_dups=per_file_dups, per_path_dups=per_path_dups) print( f"msg={request.form['fe_msg_id']}" )
return render_template("dups.html", per_file_dups=per_file_dups, per_path_dups=per_path_dups, fe_msg_id=request.form['fe_msg_id'] )
@app.route("/rm_dups", methods=["POST"])
def rm_dups():
jex=[]
for el in request.form:
if 'kfhash-' in el:
# get which row/number kf it is...
pfx, which = el.split('-')
jex.append( JobExtra( name=f"kfid-{which}", value=request.form['kfname-'+which] ) )
jex.append( JobExtra( name=f"kfhash-{which}", value=request.form[el] ) )
if 'kdhash-' in el:
# get which row/number kd it is...
pfx, which = el.split('-')
jex.append( JobExtra( name=f"kdid-{which}", value=request.form['kdid-'+which] ) )
jex.append( JobExtra( name=f"kdhash-{which}", value=request.form[el] ) )
fe_msg_id=request.form['fe_msg_id']
# allow backend to delete FE message once delete is being processed
jex.append( JobExtra( name="fe_msg_id", value=fe_msg_id ) )
job=NewJob( "rmdups", 0, None, jex )
st.SetAlert("success")
st.SetMessage("created job to delete duplicate files")
return render_template("base.html")
@app.route("/move_files", methods=["POST"]) @app.route("/move_files", methods=["POST"])
def move_files(): def move_files():