From 5556b0ef1517949904635bf11fa8f1f5732331a7 Mon Sep 17 00:00:00 2001 From: Damien De Paoli Date: Wed, 20 Aug 2025 18:11:00 +1000 Subject: [PATCH] removed debugs, actually add new bills when needed for monthly and annual, support new growth fields, ensure growth only works on real bills not new estimated bills. --- bills.py | 95 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/bills.py b/bills.py index 06f0651..7ed772a 100644 --- a/bills.py +++ b/bills.py @@ -1,7 +1,6 @@ -from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth +from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth, new_bill from defines import END_YEAR - # give a bill dat in format YYYY-MM-DD, return quarter (1-4) def qtr(d): m = int(d[5:7]) @@ -9,27 +8,59 @@ def qtr(d): # missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions # future only, so add ann_growth (based on drop-down) for each future year +# NOTE: only ever called when there is a need to add a new bill def add_missing_annual_bill_in_yr( bill_type, bill_info, num, yr ): -# print( f"{bill_type}: Seems we are missing an annual bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" ) mm_dd = bill_info[bill_type]['last_bill']['bill_date'][5:] - l_amt = bill_info[bill_type]['last_bill']['amount'] -# print( f"{bill_type}: Should fake a bill into date={yr}-{mm_dd} of adjusted amount from base of {l_amt}" ) + amt = bill_info[bill_type]['last_bill']['amount'] # okay the missing bill is before the first bill... for i in range( bill_info[bill_type]['last_bill_year'], yr ): - l_amt += l_amt * 5.26/100 - print( f"{bill_type}: So should insert bill as: ${l_amt:.02f} on '{yr}-{mm_dd}'") + amt += amt * bill_info[bill_type]['growth']/100 + + # last param is estimated (and this is an estimate for a future bill / not real) + new_bill( bill_type, amt, f'{yr}-{mm_dd}', 1 ) return # missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year # add growth (based on drop-down) for each future year def add_missing_quarter_bills_in_yr( bill_type, bill_info, num, yr ): -# print( f"{bill_type}: Seems we are missing a quarterly bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" ) + print( f"*** add_missing_quarter_bills_in_yr( {bill_type}, bill_info, {num}, {yr} ): NOT YET" ) return # missing monthly bills, find date based on DD and put in each missing month # add growth (based on drop-down) for each future year +# NOTE: ALWAYS called for first year - don't always add bills/see below def add_missing_monthly_bills_in_yr( bill_type, bill_info, num, yr ): -# print( f"{bill_type}: Seems we are missing a monthly bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" ) + + # start date arithmetic from first bill (this is possibly an issue if monthly is not + # really perfectly the same each month, but its only for an estimate so should be ok + dd = bill_info[bill_type]['first_bill']['bill_date'][8:] + mm = bill_info[bill_type]['first_bill']['bill_date'][5:7] + + # choose last bill from last amount to grow from as its most relevant + amt = bill_info[bill_type]['last_bill']['amount'] + + growth=0 + #okay add monthly bills for the rest of this year if its the first year + if bill_info[bill_type]['first_bill_year'] == yr: + start_m=int(mm) + else: + start_m=0 + + # fill in rest of this year + for i in range( start_m+1, 13 ): + new_date = f'{yr}-{i:02d}-{dd}' + if yr in bill_info[bill_type]['year']: + for b in bill_info[bill_type]['year'][yr]: + # this bill exists, skip adding it (this occurs when called to + # add bilsl as there are < 12 bills in first_year, BUT, we + # don't fill before first_bill so the < 12 ALWAYS triggers + if b['bill_date'] == new_date: + continue + # grow if we are really adding one + amt += amt * growth/100 + # last param is estimated (and this is an estimate for a future bill / not real) + new_bill( bill_type, amt, new_date, 1 ) + return @@ -40,6 +71,7 @@ def add_missing_monthly_bills_in_yr( bill_type, bill_info, num, yr ): def process_bill_data(bd, bt, bf): # this maps a bill id to a freq id (e.g. bill #34 - has a frequency of #2 (which might be quarterly) bt_id_freq = {row["id"]: row["freq"] for row in bt} + bt_id_ann_growth_avg = {row["id"]: row["ann_growth_avg"] for row in bt} # this maps freq to bills per annum (e.g. id=2 to 4 bills per annum) bf_id_num = {row["id"]: row["num_bills_per_annum"] for row in bf} @@ -54,20 +86,26 @@ def process_bill_data(bd, bt, bf): # new bill type if not bill_type in bill_info: bill_info[bill_type]={} + bill_info[bill_type]['growth'] = bt_id_ann_growth_avg[bill_type] + bill_info[bill_type]['num_ann_bills'] = bf_id_num[bt_id_freq[bill_type]] bill_info[bill_type]['first_bill']={} bill_info[bill_type]['last_bill']={} # due to sql sorting, this first instance is the last bill bill_info[bill_type]['last_bill']=bill bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4]) bill_info[bill_type]['year']={} + bill_info[bill_type]['year_real']={} if not yr in bill_info[bill_type]['year']: bill_info[bill_type]['year'][yr]=[] + bill_info[bill_type]['year_real'][yr]=[] # keep updating last to this matching bill bill_info[bill_type]['first_bill']=bill bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4]) # add this bill to list for this year bill_info[bill_type]['year'][yr].append(bill) + if not bill['estimated']: + bill_info[bill_type]['year_real'][yr].append(bill) # now process the bill_info from yr of first bill to yr of last bill for bill_type in bill_info: @@ -82,21 +120,17 @@ def process_bill_data(bd, bt, bf): # go from first_bill year until reach end year for yr in range( yr_min, END_YEAR+1 ): - if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == num: -# print(f"{bill_type}: need {num} annual bills and found then for {yr}" ) + # we have all the bills needed for yr + if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == bill_info[bill_type]['num_ann_bills']: continue -# if yr not in bill_info[bill_type]['year']: -# print(f"{bill_type}: need {num} annual bills and 0 found for {yr}" ) -# else: -# print(f"{bill_type}: need {num} annual bills and only {len(bill_info[bill_type]['year'][yr])} found for {yr}" ) add_missing_bills_for_yr( bill_type, bill_info, num, yr ) - # now should have missing bills, calculate ann growth properly derive_ann_growth( bill_type, bill_info, num ) ################################################################################ # add_missing_bills_for_yr -- wrapper to call right func based on bill freq ################################################################################ def add_missing_bills_for_yr( bill_type, bill_info, num, yr ): + print(f"{bill_type}: add_missing_bills_for_yr( {bill_type}, bill_info, {num}, {yr} )") if num == 1: add_missing_annual_bill_in_yr( bill_type, bill_info, num, yr ) elif num == 4: @@ -106,16 +140,18 @@ def add_missing_bills_for_yr( bill_type, bill_info, num, yr ): return def derive_ann_growth( bill_type, bill_info, num ): - print(f"Derive annual growth on bill_type: {bill_type} " ) - # DDP: rewrite loop below to use bill_info more cleverly, start with type, then year, then use the data in there rather than in bd + print(f"{bill_type}: Derive annual growth on bill_type: {bill_type} " ) total={} for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_bill_year']+1): + if len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']: + continue total[yr] = 0 for b in bill_info[bill_type]['year'][yr]: - total[yr] += b['amount'] -# print( f"{yr} => {b['bill_date']} -- {b['amount']}" ) -# print( f"total for {bill_type} in {yr} is {total[yr]}" ) + # ignore estimated bills, only use real bills to calc growth stats + if b['estimated']: + continue + total[yr] += b['amount'] # once we have all yr totals: growth = {} @@ -124,12 +160,14 @@ def derive_ann_growth( bill_type, bill_info, num ): max_growth = 0 count = 0 for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_bill_year']+1): + # less than {num} bills in yr: {yr-1}, so can't use data if yr-1 in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr-1]) != num: -# print(f"less than {num} bills in yr: {yr-1}, so can't use data" ) continue + # less than {num} bills in yr: {yr-1}, so can't use data if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) != num: -# print(f"less than {num} bills in yr: {yr-1}, so can't use data" ) continue + + # we full data sets for consecutive years, work out annual growth stats if yr-1 in total and yr in total: growth = (total[yr] - total[yr-1]) / total[yr-1] * 100 avg_growth += growth @@ -138,12 +176,11 @@ def derive_ann_growth( bill_type, bill_info, num ): min_growth = growth if growth > max_growth: max_growth = growth -# print( f"growth from {yr} to {yr-1} = {growth}%") if count: - print( f"Min growth was: {min_growth}" ) - print( f"Avg growth is: {avg_growth/count}" ) - print( f"Max growth was: {max_growth}" ) - set_bill_type_growth( bill_type, avg_growth/count ) + print( f"{bill_type}: Min growth was: {min_growth}" ) + print( f"{bill_type}: Avg growth is: {avg_growth/count}" ) + print( f"{bill_type}: Max growth was: {max_growth}" ) + set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth ) else: # failsafe (just in case fill bills failed to add enough bills to average out) - print( f"Unable to calculate growth!" ) + print( f"{bill_type}: Unable to calculate growth!" )