ddp/photoassistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

made just one checkdups/rmdups front end message once we dups

Browse Source
This commit is contained in:
Damien De Paoli
2021-04-02 16:19:40 +11:00
parent e3f2870197
commit e6fd9a9b13
2 changed files with 33 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
files.py
View File

@@ -247,7 +247,7 @@ def fix_dups():
if 'pagesize' not in request.form: if 'pagesize' not in request.form:
# default to 10, see if we have a larger value as someone reset it in the gui, rather than first time invoked # default to 10, see if we have a larger value as someone reset it in the gui, rather than first time invoked
pagesize = 10 pagesize = 10
jexes = JobExtra.query.join(Job).join(PA_JobManager_Message).filter(PA_JobManager_Message.id==request.form['fe_msg_id']).all() jexes = JobExtra.query.join(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all()
jexes.append( JobExtra( name="pagesize", value=pagesize ) ) jexes.append( JobExtra( name="pagesize", value=pagesize ) )
else: else:
pagesize=int(request.form['pagesize']) pagesize=int(request.form['pagesize'])
@@ -258,7 +258,7 @@ def fix_dups():
DD.SecondPass() DD.SecondPass()
# DD.Dump() # DD.Dump()
return render_template("dups.html", DD=DD, fe_msg_id=request.form['fe_msg_id'], pagesize=pagesize ) return render_template("dups.html", DD=DD, pagesize=pagesize )
@app.route("/rm_dups", methods=["POST"]) @app.route("/rm_dups", methods=["POST"])
def rm_dups(): def rm_dups():
@@ -276,9 +276,6 @@ def rm_dups():
jex.append( JobExtra( name=f"kdid-{which}", value=request.form['kdid-'+which] ) ) jex.append( JobExtra( name=f"kdid-{which}", value=request.form['kdid-'+which] ) )
jex.append( JobExtra( name=f"kdhash-{which}", value=request.form[el] ) ) jex.append( JobExtra( name=f"kdhash-{which}", value=request.form[el] ) )
fe_msg_id=request.form['fe_msg_id']
# allow backend to delete FE message once delete is being processed
jex.append( JobExtra( name="fe_msg_id", value=fe_msg_id ) )
jex.append( JobExtra( name="pagesize", value=10 ) ) jex.append( JobExtra( name="pagesize", value=10 ) )
job=NewJob( "rmdups", 0, None, jex ) job=NewJob( "rmdups", 0, None, jex )

53
pa_job_manager.py
View File

@@ -282,22 +282,20 @@ def JobsForPaths( parent_job, paths ):
if parent_job: if parent_job:
AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job2.id, job2.id, job2.name, job2.wait_for ) ) AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job2.id, job2.id, job2.name, job2.wait_for ) )
"""
jex3=JobExtra( name="path", value=path ) jex3=JobExtra( name="path", value=path )
job3=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=job2.id, pa_job_state="New", current_file_num=0 ) job3=Job(start_time=now, last_update=now, name="processai", state="New", wait_for=job2.id, pa_job_state="New", current_file_num=0 )
job3.extra.append(jex3) job3.extra.append(jex4)
session.add(job3) session.add(job3)
session.commit() session.commit()
if parent_job: if parent_job:
AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job3.id, job3.id, job3.name, job3.wait_for ) ) AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job3.id, job3.id, job3.name, job3.wait_for ) )
""" """
jex4=JobExtra( name="path", value=path ) job4=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=job2.id, pa_job_state="New", current_file_num=0 )
job4=Job(start_time=now, last_update=now, name="processai", state="New", wait_for=job2.id, pa_job_state="New", current_file_num=0 ) session.add(job4)
job4.extra.append(jex4) session.commit()
session.add(job4) if parent_job:
session.commit() AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job4.id, job4.id, job4.name, job4.wait_for ) )
if parent_job:
AddLogForJob(parent_job, "adding <a href='/job/{}'>job id={} {}</a> (wait for: {})".format( job3.id, job3.id, job3.name, job3.wait_for ) )
"""
HandleJobs() HandleJobs()
return return
@@ -857,17 +855,16 @@ def GenVideoThumbnail(job, file):
return thumbnail return thumbnail
def CheckForDups(job): def CheckForDups(job):
path=[jex.value for jex in job.extra if jex.name == "path"][0] AddLogForJob( job, f"Check for duplicates" )
path=SymlinkName( path, path )
AddLogForJob( job, f"Check for duplicates in import path: {path}" ) res = session.execute( f"select count(e1.name) as count from entry e1, file f1, dir d1, entry_dir_link edl1, entry e2, file f2, dir d2, entry_dir_link edl2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and f1.hash = f2.hash and e1.id != e2.id" )
res = session.execute( f"select count(e1.name) as count from entry e1, file f1, dir d1, entry_dir_link edl1, entry e2, file f2, dir d2, entry_dir_link edl2 where e1.id = f1.eid and e2.id = f2.eid and d1.eid = edl1.dir_eid and edl1.entry_id = e1.id and edl2.dir_eid = d2.eid and edl2.entry_id = e2.id and d1.path_prefix like '%{path}%' and f1.hash = f2.hash and e1.id != e2.id" )
for row in res: for row in res:
if row.count > 0: if row.count > 0:
AddLogForJob(job, f"Found duplicates, Creating Status message in front-end for attention") AddLogForJob(job, f"Found duplicates, Creating Status message in front-end for attention")
msg_id=MessageToFE( job.id, "danger", 'replaceme' ) MessageToFE( job.id, "danger", f'Found duplicate(s), click&nbsp; <a href="javascript:document.body.innerHTML+=\'<form id=_fm method=POST action=/fix_dups></form>\'; document.getElementById(\'_fm\').submit();">here</a>&nbsp;to finalise import by removing duplicates' )
session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==msg_id).update( { 'message' : f'Found duplicate(s), click&nbsp; <a href="javascript:document.body.innerHTML+=\'<form id=_fm method=POST action=/fix_dups><input type=hidden name=fe_msg_id value={msg_id}></form>\'; document.getElementById(\'_fm\').submit();">here</a>&nbsp;to finalise import by removing duplicates' } ) else:
FinishJob(job, f"Finished Looking for Duplicates") FinishJob(job, f"No duplicates found")
FinishJob(job, f"Finished looking for duplicates")
return return
def RemoveFileFromFS( del_me ): def RemoveFileFromFS( del_me ):
@@ -884,12 +881,17 @@ def RemoveFileFromFS( del_me ):
return return
def RemoveDups(job): def RemoveDups(job):
# clear FE message we are deleting dups for this now... AddLogForJob(job, f"INFO: Starting Remove Duplicates job...")
fe_msg_id =[jex.value for jex in job.extra if jex.name == "fe_msg_id"][0] # as checkdups covers all dups, delete all future dups messages, and Withdraw future checkdups jobs
session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==fe_msg_id).delete() msgs=session.query(PA_JobManager_FE_Message).join(Job).filter(Job.name=='checkdups')
for msg in msgs:
session.query(PA_JobManager_FE_Message).filter(PA_JobManager_FE_Message.id==msg.id).delete()
cd_jobs=session.query(Job).filter(Job.name=='checkdups').filter(Job.pa_job_state=='New').all()
for j in cd_jobs:
FinishJob(j, "Just removed duplicates - so no need to do any other checkdups, we will force 1 last one after the remove step", "Withdrawn")
print("here-loop")
session.commit() session.commit()
AddLogForJob(job, f"INFO: Starting Remove Duplicates job...")
dup_cnt=0 dup_cnt=0
for jex in job.extra: for jex in job.extra:
if 'kfid-' in jex.name: if 'kfid-' in jex.name:
@@ -925,6 +927,7 @@ def RemoveDups(job):
for hash in hashes.split(","): for hash in hashes.split(","):
files=session.query(Entry).join(File).filter(File.hash==hash).all() files=session.query(Entry).join(File).filter(File.hash==hash).all()
found=None found=None
del_me=None
for f in files: for f in files:
if os.path.isfile(f.in_dir[0].path_prefix+'/'+f.name) == False: if os.path.isfile(f.in_dir[0].path_prefix+'/'+f.name) == False:
AddLogForJob( job, f"ERROR: (per path del) file (DB id: {f.eid} - {f.in_dir[0].path_prefix}/{f.name}) does not exist? ignorning file") AddLogForJob( job, f"ERROR: (per path del) file (DB id: {f.eid} - {f.in_dir[0].path_prefix}/{f.name}) does not exist? ignorning file")
@@ -943,8 +946,14 @@ def RemoveDups(job):
dup_cnt += 1 dup_cnt += 1
FinishJob(job, f"Finished removing {dup_cnt} duplicate files" ) FinishJob(job, f"Finished removing {dup_cnt} duplicate files" )
return
# Need to put another checkdups job in now to force / validate we have no dups
now=datetime.now(pytz.utc)
next_job=Job(start_time=now, last_update=now, name="checkdups", state="New", wait_for=None, pa_job_state="New", current_file_num=0 )
session.add(next_job)
session.commit()
AddLogForJob(job, "adding <a href='/job/{}'>job id={} {}</a> to confirm there are no more duplicates".format( next_job.id, next_job.id, next_job.name ) )
return
if __name__ == "__main__": if __name__ == "__main__":
print("INFO: PA job manager starting - listening on {}:{}".format( PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT) ) print("INFO: PA job manager starting - listening on {}:{}".format( PA_JOB_MANAGER_HOST, PA_JOB_MANAGER_PORT) )