219 lines
11 KiB
Python
219 lines
11 KiB
Python
from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth, new_bill
|
|
from defines import END_YEAR
|
|
|
|
# give a bill dat in format YYYY-MM-DD, return quarter (1-4)
|
|
def qtr(d):
|
|
m = int(d[5:7])
|
|
return ( (m-1)//3 + 1 )
|
|
|
|
# missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions
|
|
# future only, so add ann_growth (based on drop-down) for each future year
|
|
# NOTE: only ever called when there is a need to add a new bill
|
|
def add_missing_annual_bill_in_yr( bill_type, bill_info, yr ):
|
|
mm_dd = bill_info[bill_type]['last_bill']['bill_date'][5:]
|
|
amt = bill_info[bill_type]['last_bill']['amount']
|
|
# okay the missing bill is before the first bill...
|
|
for i in range( bill_info[bill_type]['last_bill_year'], yr ):
|
|
amt += amt * bill_info[bill_type]['growth']/100
|
|
|
|
# last param is estimated (and this is an estimate for a future bill / not real)
|
|
new_bill( bill_type, amt, f'{yr}-{mm_dd}', 1 )
|
|
return
|
|
|
|
# missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year
|
|
# add growth (based on drop-down) for each future year
|
|
def add_missing_quarter_bills_in_yr( bill_type, bill_info, yr ):
|
|
print( f"*** add_missing_quarter_bills_in_yr( {bill_type}, bill_info, {yr} ): NOT YET" )
|
|
return
|
|
|
|
# missing monthly bills, find date based on DD and put in each missing month
|
|
# add growth (based on drop-down) for each future year
|
|
# NOTE: ALWAYS called for first year - don't always add bills/see below
|
|
def add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ):
|
|
|
|
# start date arithmetic from first bill (this is possibly an issue if monthly is not
|
|
# really perfectly the same each month, but its only for an estimate so should be ok
|
|
dd = bill_info[bill_type]['first_bill']['bill_date'][8:]
|
|
mm = bill_info[bill_type]['first_bill']['bill_date'][5:7]
|
|
lb_mm = bill_info[bill_type]['last_bill']['bill_date'][5:7]
|
|
|
|
# choose last bill from last amount to grow from as its most relevant
|
|
if not 'last_bill_amount' in bill_info[bill_type]:
|
|
bill_info[bill_type]['last_bill_amount']=bill_info[bill_type]['last_bill']['amount']
|
|
amt = bill_info[bill_type]['last_bill_amount']
|
|
|
|
#okay add monthly bills for the rest of this year if its the first year
|
|
if bill_info[bill_type]['first_bill_year'] == yr:
|
|
start_m=int(mm)
|
|
else:
|
|
start_m=0
|
|
|
|
# fill in rest of this year
|
|
for i in range( start_m+1, 13 ):
|
|
bill_found=False
|
|
new_date = f'{yr}-{i:02d}-{dd}'
|
|
if yr in bill_info[bill_type]['year']:
|
|
for b in bill_info[bill_type]['year'][yr]:
|
|
# this bill exists, skip adding it (this occurs when called to
|
|
# add bills as there are < 12 bills in first_year, BUT, we
|
|
# don't fill before first_bill so the < 12 ALWAYS triggers
|
|
if str(b['bill_date']) == new_date:
|
|
bill_found=True
|
|
break
|
|
if not bill_found:
|
|
# if this month is the same as the last bill month and as per above
|
|
# we don't have a bill for this date, then add annual grotwh
|
|
if i == int(lb_mm):
|
|
print(f"its month: {i} - time to add growth: {bill_info[bill_type]['growth']}" )
|
|
amt += amt * bill_info[bill_type]['growth']/100
|
|
bill_info[bill_type]['last_bill_amount']=amt
|
|
# last param is estimated (and this is an estimate for a future bill / not real)
|
|
new_bill( bill_type, amt, new_date, 1 )
|
|
return
|
|
|
|
# given the bill_type has a which_growth contain min/avg/max, return the corresponding growth number
|
|
def get_growth_value( bt, bill_type ):
|
|
for el in bt:
|
|
if el['id'] == bill_type:
|
|
which = el['which_growth']
|
|
break
|
|
|
|
if which == 'avg':
|
|
return el['ann_growth_avg']
|
|
elif which == 'min':
|
|
return el['ann_growth_min']
|
|
else:
|
|
return el['ann_growth_max']
|
|
|
|
|
|
# go through the bill data from the DB, put it into more friendly formats, then
|
|
# work out and then add missing bill data (might be b/c we have monthly bills,
|
|
# and I didn't want to input 12 of them at the same price), and it always
|
|
# occurs for future bills
|
|
def process_bill_data(bd, bt, bf):
|
|
# this maps a bill id to a freq id (e.g. bill #34 - has a frequency of #2 (which might be quarterly)
|
|
bt_id_freq = {row["id"]: row["freq"] for row in bt}
|
|
bt_id_ann_growth_avg = {row["id"]: row["ann_growth_avg"] for row in bt}
|
|
|
|
# this maps freq to bills per annum (e.g. id=2 to 4 bills per annum)
|
|
bf_id_num = {row["id"]: row["num_bills_per_annum"] for row in bf}
|
|
|
|
# want to proces all bill data into easier to maniuplate structure, so make
|
|
# a bill_info[bill_id] with first_bill, last_bill, [yr] with matching bills to process
|
|
bill_info={}
|
|
|
|
for bill in bd:
|
|
bill_type = bill['bill_type_id']
|
|
yr= int(bill['bill_date'][:4])
|
|
# new bill type
|
|
if not bill_type in bill_info:
|
|
bill_info[bill_type]={}
|
|
bill_info[bill_type]['growth'] = get_growth_value( bt, bill_type )
|
|
bill_info[bill_type]['num_ann_bills'] = bf_id_num[bt_id_freq[bill_type]]
|
|
bill_info[bill_type]['first_bill']={}
|
|
bill_info[bill_type]['last_bill']={}
|
|
# due to sql sorting, this first instance is the last bill
|
|
bill_info[bill_type]['last_bill']=bill
|
|
bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4])
|
|
if not bill['estimated']:
|
|
print( f"this bill is real, its the first so consider it the last bill paid - date is: {int(bill['bill_date'][:4])}" )
|
|
bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
|
|
bill_info[bill_type]['year']={}
|
|
bill_info[bill_type]['year_real']={}
|
|
if not yr in bill_info[bill_type]['year']:
|
|
bill_info[bill_type]['year'][yr]=[]
|
|
bill_info[bill_type]['year_real'][yr]=[]
|
|
|
|
# keep updating last to this matching bill
|
|
bill_info[bill_type]['first_bill']=bill
|
|
bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4])
|
|
if not 'last_real_bill_year' in bill_info[bill_type] and not bill['estimated']:
|
|
print( f"{bill_type}: seems we dont have a last_real_bill_year, set it to: {int(bill['bill_date'][:4])} ")
|
|
bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
|
|
# add this bill to list for this year
|
|
bill_info[bill_type]['year'][yr].append(bill)
|
|
if not bill['estimated']:
|
|
bill_info[bill_type]['year_real'][yr].append(bill)
|
|
|
|
# now process the bill_info from yr of first bill to yr of last bill
|
|
for bill_type in bill_info:
|
|
# find freq id based on bill_type id, then use that to find num bills by freq id
|
|
num = bf_id_num[bt_id_freq[bill_type]]
|
|
|
|
if 'last_bill' not in bill_info[bill_type]:
|
|
print("Cannot process bill_type={bill_type} - no bill info for it at all" )
|
|
# range of years to process (yr_min to yr_max)
|
|
yr_min=int(bill_info[bill_type]['first_bill']['bill_date'][:4])
|
|
yr_max=int(bill_info[bill_type]['last_bill']['bill_date'][:4])
|
|
|
|
# go from first_bill year until reach end year
|
|
for yr in range( yr_min, END_YEAR+1 ):
|
|
# we have all the bills needed for yr
|
|
if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == bill_info[bill_type]['num_ann_bills']:
|
|
continue
|
|
add_missing_bills_for_yr( bill_type, bill_info, yr )
|
|
derive_ann_growth( bill_type, bill_info )
|
|
|
|
################################################################################
|
|
# add_missing_bills_for_yr -- wrapper to call right func based on bill freq
|
|
################################################################################
|
|
def add_missing_bills_for_yr( bill_type, bill_info, yr ):
|
|
print(f"{bill_type}: add_missing_bills_for_yr( {bill_type}, bill_info, {yr} )")
|
|
num = bill_info[bill_type]['num_ann_bills']
|
|
if num == 1:
|
|
add_missing_annual_bill_in_yr( bill_type, bill_info, yr )
|
|
elif num == 4:
|
|
add_missing_quarter_bills_in_yr( bill_type, bill_info, yr )
|
|
elif num == 12:
|
|
add_missing_monthly_bills_in_yr( bill_type, bill_info, yr )
|
|
return
|
|
|
|
def derive_ann_growth( bill_type, bill_info ):
|
|
print(f"{bill_type}: Derive annual growth on bill_type: {bill_type} - fby={bill_info[bill_type]['first_bill_year']}, lby={bill_info[bill_type]['last_real_bill_year']} " )
|
|
|
|
total={}
|
|
for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_real_bill_year']+1):
|
|
# for monthly bills, 1 or 12 bills is enough to work with
|
|
if bill_info[bill_type]['num_ann_bills'] == 12:
|
|
if len(bill_info[bill_type]['year_real'][yr]) != 1 and len(bill_info[bill_type]['year_real'][yr]) != 12:
|
|
continue;
|
|
# okay annual or quarterly bills, only total them if we have all of them for the year
|
|
elif len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']:
|
|
continue;
|
|
|
|
# for monthlys add totals regardless (we only process total if there is 12 or 1 further below). Other bill_types, only total if there is all bills for year
|
|
if bill_info[bill_type]['num_ann_bills'] != 12 and len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']:
|
|
continue
|
|
total[yr] = 0
|
|
for b in bill_info[bill_type]['year'][yr]:
|
|
# ignore estimated bills, only use real bills to calc growth stats
|
|
if b['estimated']:
|
|
continue
|
|
total[yr] += b['amount']
|
|
|
|
# once we have all yr totals:
|
|
growth = {}
|
|
min_growth = 999
|
|
avg_growth = 0
|
|
max_growth = 0
|
|
count = 0
|
|
# start from year after first bill, so we can see annual growth from the following year onwards
|
|
for yr in range( bill_info[bill_type]['first_bill_year']+1, bill_info[bill_type]['last_bill_year']+1):
|
|
# if full data sets for consecutive years, work out annual growth stats
|
|
if yr-1 in total and yr in total:
|
|
growth = (total[yr] - total[yr-1]) / total[yr-1] * 100
|
|
avg_growth += growth
|
|
count += 1
|
|
if growth < min_growth:
|
|
min_growth = growth
|
|
if growth > max_growth:
|
|
max_growth = growth
|
|
if count:
|
|
print( f"{bill_type}: Min growth was: {min_growth}" )
|
|
print( f"{bill_type}: Avg growth is: {avg_growth/count}" )
|
|
print( f"{bill_type}: Max growth was: {max_growth}" )
|
|
set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth )
|
|
else:
|
|
# failsafe (just in case fill bills failed to add enough bills to average out)
|
|
print( f"{bill_type}: Unable to calculate growth!" )
|