268 lines
13 KiB
Python
268 lines
13 KiB
Python
from wtforms import SubmitField, StringField, HiddenField, validators, Form
|
|
from flask_wtf import FlaskForm
|
|
from flask import request, render_template, send_from_directory
|
|
from main import db, app, ma
|
|
from sqlalchemy import Sequence
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
from status import st, Status
|
|
import os
|
|
import glob
|
|
from PIL import Image
|
|
from pymediainfo import MediaInfo
|
|
import hashlib
|
|
import exifread
|
|
import base64
|
|
import numpy
|
|
import cv2
|
|
import time
|
|
import re
|
|
|
|
################################################################################
|
|
# Local Class imports
|
|
################################################################################
|
|
from job import Job, JobExtra, Joblog, NewJob
|
|
from settings import Settings
|
|
from shared import SymlinkName, PA
|
|
from path import PathType
|
|
|
|
################################################################################
|
|
# DupRow class is a simple 'struct' to keep data per duplicate file / just to
|
|
# avoid using python list/dicts intermixed, and be able to consistently use
|
|
# dot-notation of fields
|
|
class DupRow(PA):
|
|
def __init__(self, hash, file, dir, did, fid):
|
|
### DupRow Attributes -- note, simple class, no methods ###
|
|
self.h=hash
|
|
self.f=file
|
|
self.d=dir
|
|
self.did=did
|
|
self.id=fid
|
|
return
|
|
|
|
################################################################################
|
|
# DupPathRow class is a simple 'struct' to keep data per files in duplicate paths
|
|
# just to avoid using python list/dicts intermixed, and be able to consistently use
|
|
# dot-notation of fields
|
|
class DupPathRow(PA):
|
|
def __init__(self, count, d1, d2, did1, did2, hashes ):
|
|
self.count=count
|
|
self.d1=d1
|
|
self.d2=d2
|
|
self.did1=did1
|
|
self.did2=did2
|
|
self.hashes=hashes
|
|
return
|
|
|
|
################################################################################
|
|
# Duplicates class is used with one instance/object to process all the
|
|
# 'duplicate' data from the Database, and parse it into more usable data
|
|
# structures. This is needed also, as the database content shows duplicates
|
|
# more than once, e.g.
|
|
# file1 and file2 are a duplicate, then later file2 and file 1 are 'another' duplicate
|
|
# The class passes over the data in 2 passes. The first pass in AddDup() finds
|
|
# any files in the import and storage path and marks the storage ones to keep,
|
|
# the import ones to delete. Anything else is either a set of files duplicated
|
|
# inside the import path or set of files duplicated in the storage path
|
|
# The first pass, simply concatenates these into a data structure
|
|
# (im_same_dups) that contains all the duplicates with a key of the md5 hash
|
|
#
|
|
# The second pass (), processes these duplicates to see if there are any in the
|
|
# storage path that follow the pattern 'YYYY/YYYYMMDD' -> if so mark these to
|
|
# keep and the rest to be deleted.
|
|
#
|
|
# After the 2 passes, we have data structures that allow the web to break up
|
|
# the duplicates into batches to process:
|
|
# 1) auto delete any in the import path that are also in the storage path
|
|
# - careful here, if we have any in the import path and 2+ in the storage path, leave it for manual intervention
|
|
# 2) auto delete any in the storage path that are in a set where 1 of them match the 'YYYY/YYYYMMDD' format, the rest are deleted
|
|
# 3) a set of directories where there are only 2 duplicate files (with the same file name), just in a different dir - allow user to choose the dir to keep
|
|
# 4) a set of individual files where I want the user to make a decision (3 or more copies, those with different filenames, or in the same dir) - allow user to choose file to keep
|
|
class Duplicates(PA):
|
|
def __init__(self):
|
|
### Duplicates Attributes ###
|
|
self.ip_to_sp_dups_keep={}
|
|
self.ip_to_sp_dups_del={}
|
|
self.dups_to_process={}
|
|
self.per_file_dups=[]
|
|
self.per_path_dups=[]
|
|
self.preferred_file={}
|
|
self.preferred_path={}
|
|
self.hashes_processed={}
|
|
self.eids_processed={}
|
|
self.uniq_dups=0
|
|
self.total_dups=0
|
|
|
|
self.import_ptype_id = PathType.query.filter(PathType.name=='Import').first().id
|
|
self.storage_ptype_id = PathType.query.filter(PathType.name=='Storage').first().id
|
|
|
|
# is this file in the import path?
|
|
def InImportPath( self, path_type ):
|
|
if path_type == self.import_ptype_id:
|
|
return True
|
|
return False
|
|
|
|
# is this file in the storage path?
|
|
def InStoragePath( self, path_type ):
|
|
if path_type == self.storage_ptype_id:
|
|
return True
|
|
return False
|
|
|
|
# this stores this object into the keep from same path list (sometimes there can be more than 1 SP, e.g SP to SP to IP)
|
|
# for now, by not dealing with the extra SP, we will just delete the IP, and force a check_dups after deleting, it will then
|
|
# pick up and process the SP to SP - if still needed -- if there is only SP1 to SP2, then the per_path_dup will pick it up and
|
|
# I believe this will all work, but doesn't hurt to do an extra check_dups again
|
|
def KeepInIPSPDups( self, obj ):
|
|
if obj.h not in self.ip_to_sp_dups_keep:
|
|
self.ip_to_sp_dups_keep[obj.h]= obj
|
|
return
|
|
|
|
# this stores this object into the Delete from same path list (if it is not
|
|
# already there)
|
|
def DelInIPSPDups( self, obj ):
|
|
if obj.h not in self.ip_to_sp_dups_del:
|
|
self.ip_to_sp_dups_del[obj.h]=[]
|
|
self.ip_to_sp_dups_del[obj.h].append( obj )
|
|
else:
|
|
for el in self.ip_to_sp_dups_del[obj.h]:
|
|
if el.id == obj.id:
|
|
return
|
|
self.ip_to_sp_dups_del[obj.h].append( obj )
|
|
return
|
|
|
|
# this function takes a duplicate file (in the import path and the storage path)
|
|
# and then puts the storage path file in the keep list (self.ip_to_sp_dups_keep) via self.KeepInIPSPDups()
|
|
# and then puts the import path file in the delete list (self.ip_to_sp_dups_keep) via self.DelInIPSPDups()
|
|
def DupInImportAndStoragePath( self, row, dr1, dr2 ):
|
|
if self.InStoragePath(row.path_type1) and self.InImportPath(row.path_type2):
|
|
self.KeepInIPSPDups( dr1 )
|
|
self.DelInIPSPDups( dr2 )
|
|
return True
|
|
if self.InStoragePath(row.path_type2) and self.InImportPath(row.path_type1):
|
|
self.KeepInIPSPDups( dr2 )
|
|
self.DelInIPSPDups( dr1 )
|
|
return True
|
|
return False
|
|
|
|
# AddDup: takes a row from the database effectively file1 & file2
|
|
# we process these into appropriate data structures on this first pass
|
|
def AddDup( self, row ):
|
|
self.hashes_processed[row.hash]=1
|
|
self.eids_processed[row.id1]=1
|
|
self.eids_processed[row.id2]=1
|
|
dr1=DupRow( row.hash, row.fname1, row.rel_path1, row.did1, row.id1 )
|
|
dr2=DupRow( row.hash, row.fname2, row.rel_path2, row.did2, row.id2 )
|
|
# if in both import and storage path, just keep the storage path file,
|
|
# and del import path file. This function checks and keeps/dels as needed
|
|
if self.DupInImportAndStoragePath( row, dr1, dr2 ):
|
|
return
|
|
|
|
# if the hash is not in dups_to_process, created / append
|
|
if row.hash not in self.dups_to_process:
|
|
self.dups_to_process[row.hash]=[]
|
|
self.dups_to_process[row.hash].append( dr1 )
|
|
self.dups_to_process[row.hash].append( dr2 )
|
|
else:
|
|
# process path1 / fname1 -- if that combo is not in the dups_to_process[hash], add it
|
|
found=0
|
|
for dup in self.dups_to_process[row.hash]:
|
|
if dup.id == row.id1:
|
|
found=1
|
|
continue
|
|
if not found:
|
|
self.dups_to_process[row.hash].append( dr1 )
|
|
|
|
# process path2 / fname2 -- if that combo is not in the dups_to_process[hash], add it
|
|
for dup in self.dups_to_process[row.hash]:
|
|
if dup.id == row.id2:
|
|
found=1
|
|
continue
|
|
if not found:
|
|
self.dups_to_process[row.hash].append( dr2 )
|
|
return
|
|
|
|
# AddDupPath: takes a row from the database effectively with a dup pair in dir1 & dir2
|
|
# we process these into appropriate data structures on this second pass
|
|
# working through if a dir is in th estorage path and is
|
|
def AddDupPath(self, hash):
|
|
# this gets complex, if this hash is also in a shared imp / sp - then dont deal with it now, let the imp files be deleted and
|
|
# the repeat check_dups validation step catch it as a cleander (potential) for still more duplicates just in sp
|
|
dpr=DupPathRow( 2, self.dups_to_process[hash][0].d, self.dups_to_process[hash][1].d, self.dups_to_process[hash][0].did, self.dups_to_process[hash][1].did, hash )
|
|
if hash in self.ip_to_sp_dups_keep:
|
|
return False
|
|
new=1
|
|
for el in self.per_path_dups:
|
|
# if this new hash / dup in dpr has same dirs as existing per_path_dups row, then just another file in same dup dir...
|
|
if el.d1 == dpr.d1 and el.d2 == dpr.d2:
|
|
el.count += 2
|
|
el.hashes = f"{el.hashes},{hash}"
|
|
new=0
|
|
# okay, we have a new pair of duplicate dirs... Add them, and if either has matching regex its preferred
|
|
# FIXME: what if both do? what if one is in SP and the other not, etc...
|
|
if new:
|
|
self.per_path_dups.append( dpr )
|
|
if re.search( r'\d{4}/\d{8}', dpr.d1):
|
|
self.preferred_path[dpr.did1]=1
|
|
if re.search( r'\d{4}/\d{8}', dpr.d2):
|
|
self.preferred_path[dpr.did2]=1
|
|
return True
|
|
|
|
# The second pass processes row by row of dups_to_process, looking for per_file_dups and per_path_dups
|
|
# AND works out counts to display overall tallies of types of keeping and deletion of files, and choices to make
|
|
def SecondPass(self):
|
|
# okay, go for each duplicate that should be processed (they are stored
|
|
# by hash, and have at least 2 entries, but can have more, and be in
|
|
# the IP or SP and any combo, cater for all below
|
|
for hash in self.dups_to_process:
|
|
# more than 2 files (just ask per file) OR (implied) only 2 copies, and files are in same dir (so must be diff name, so just ask) OR (implied) on 2 copies in same dir & filename different (ask per file)
|
|
# will force ask per file
|
|
if (len(self.dups_to_process[hash]) > 2) or (self.dups_to_process[hash][0].d == self.dups_to_process[hash][1].d) or (self.dups_to_process[hash][0].f != self.dups_to_process[hash][1].f):
|
|
self.per_file_dups.append(self.dups_to_process[hash])
|
|
for el in self.dups_to_process[hash]:
|
|
if re.search( r'\d{4}/\d{8}', el.d):
|
|
self.preferred_file[hash] = el.id
|
|
else:
|
|
# will force ask per path
|
|
self.AddDupPath( hash )
|
|
|
|
# provide convenience counts
|
|
self.uniq_dups = len(self.hashes_processed)
|
|
self.total_dups = len(self.eids_processed)
|
|
return
|
|
|
|
# quick debugger to see the data in the data structure (not used by default)
|
|
def Dump(self):
|
|
if len(self.ip_to_sp_dups_keep) > 0:
|
|
print( "############ Files that are in both Import and Storage Paths ###########")
|
|
for h in self.ip_to_sp_dups_keep:
|
|
print( f"hash={h} keep 1 of {len(self.ip_to_sp_dups_del[h])+1}, keep: {self.ip_to_sp_dups_keep[h]} | ", end='' )
|
|
for d in self.ip_to_sp_dups_del[h]:
|
|
print( f"Del: {d}", end='' )
|
|
print( "" )
|
|
print( f"{len(self.ip_to_sp_dups_keep)} sets of duplicate files to delete at least 1, anything with 2 or more dups is printed above explicitly" )
|
|
|
|
if len(self.dups_to_process) > 0:
|
|
print( "############ Duplicate Files that are needing to be futher processed ###########")
|
|
for h in self.dups_to_process:
|
|
print( f"hash={h} keep 1 of {len(self.dups_to_process[h])} from: {self.dups_to_process[h]}" )
|
|
print( f"which is a total of {len(self.dups_to_process)} set(s) of duplicate files to keep only 1 of" )
|
|
|
|
if len(self.preferred_file) > 0:
|
|
print( " We have preferred (regexp matched) ###########")
|
|
for h in self.preferred_file:
|
|
print( f"hash={h}, keep this one: {self.preferred_file[h]} from ", end='' )
|
|
for d in self.dups_to_process[h]:
|
|
print( f"{d.id}, ", end='' )
|
|
print ("")
|
|
print( f"which is a total of {len(self.preferred_file)} duplicate files we will keep as they match the regexp" )
|
|
|
|
if len(self.per_path_dups) > 0:
|
|
print( "############ Duplicate Files in Paths that are needing to be futher processed ###########")
|
|
for pair in self.per_path_dups:
|
|
print( f"{pair.count} dups in dir1: {pair.did1} dir2: {pair.did2}" )
|
|
if pair.did1 in self.preferred_path:
|
|
print("Keep dir1")
|
|
if pair.did2 in self.preferred_path:
|
|
print("Keep dir2")
|
|
print( f"which is a total of {len(self.per_path_dups)} set(s) of path dups to process" )
|
|
return
|