from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth, new_bill from defines import END_YEAR # give a bill dat in format YYYY-MM-DD, return quarter (1-4) def qtr(d): m = int(d[5:7]) return ( (m-1)//3 + 1 ) def find_previous_bill( bill_type, bill_info, bill_date ): # print( f"{bill_type}: find_previous_bill -> {bill_date}" ) wanted_year = int(bill_date[:4]) wanted_mm = int(bill_date[5:7]) # if we don't have a bill before this date, no way to set price if int(wanted_year) < int(bill_info[bill_type]['first_bill_year']): # print(f"{bill_type}: find_previous_bill - failed. wanted_year={wanted_year} is older than our first_bill={bill_info[bill_type]['first_bill_year']}" ) return None # start loop from bill_date, go backwards and find which one it is (same year, should be month-based) # earlier year, then just last one from the year. yr_range=range( wanted_year, bill_info[bill_type]['first_bill_year'], -1 ) if wanted_year == int(bill_info[bill_type]['first_bill_year']): # range of this year with -1, does not return anything, so force this year. yr_range=[ wanted_year ] for yr in yr_range: # start with bills in the year wanted (if any) # must include 'estimated' bills to deal with growth of future years if yr in bill_info[bill_type]['year']: # okay, we have the previous billing year, and we wanted one for a year in the future, # just return the last one in this year as its the most recent if wanted_year > yr: # print("should be return last of {yr} - date={bill_info[bill_type]['year'][yr][-1]['bill_date']}" ) return bill_info[bill_type]['year'][yr][-1] else: # lets go through the newest to oldest of these bills for bill in bill_info[bill_type]['year'][wanted_year][::-1]: bill_mm = int(bill['bill_date'][5:7]) # reversing the bills, means we start with the 'most recent' in this year to the oldest # if the month we want is after the bill, we are done if wanted_mm > bill_mm: return bill # print(f"{bill_type}: find_previous_bill - failed. Seems our first bill = {bill_info[bill_type]['first_bill']['bill_date']} is in same year, but after wanted month={wanted_mm}, so no base to rely on" ) return None # missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions # future only, so add ann_growth (based on drop-down) for each future year # NOTE: only ever called when there is a need to add a new bill def add_missing_annual_bill_in_yr( bill_type, bill_info, yr ): mm_dd = bill_info[bill_type]['last_bill']['bill_date'][5:] amt = bill_info[bill_type]['last_bill']['amount'] # okay the missing bill is before the first bill... for i in range( bill_info[bill_type]['last_bill_year'], yr ): amt += amt * bill_info[bill_type]['growth']/100 # last param is estimated (and this is an estimate for a future bill / not real) new_bill( bill_type, amt, f'{yr}-{mm_dd}', 1 ) return # missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year # add growth (based on drop-down) for each future year def add_missing_quarter_bills_in_yr( bill_type, bill_info, yr ): print( f"*** add_missing_quarter_bills_in_yr( {bill_type}, bill_info, {yr} ): NOT YET" ) return # missing monthly bills, find date based on DD and put in each missing month # add growth (based on drop-down) for each future year # NOTE: ALWAYS called for first year - don't always add bills/see below def add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ): # start date arithmetic from first bill (this is possibly an issue if monthly is not # really perfectly the same each month, but its only for an estimate so should be ok dd = bill_info[bill_type]['first_bill']['bill_date'][8:] mm = bill_info[bill_type]['first_bill']['bill_date'][5:7] lb_mm = bill_info[bill_type]['last_bill']['bill_date'][5:7] #okay add monthly bills for the rest of this year if its the first year if bill_info[bill_type]['first_bill_year'] == yr: start_m=int(mm) else: start_m=0 # fill in rest of this year for i in range( start_m+1, 13 ): bill_found=False new_date = f'{yr}-{i:02d}-{dd}' if yr in bill_info[bill_type]['year']: for b in bill_info[bill_type]['year'][yr]: # this bill exists, skip adding it (this occurs when called to # add bills as there are < 12 bills in first_year, BUT, we # don't fill before first_bill so the < 12 ALWAYS triggers if str(b['bill_date']) == new_date: bill_found=True break if not bill_found: pb=find_previous_bill( bill_type, bill_info, new_date ) if not pb: print("Failed to find previous_bill, can't calculate missing bill - returning" ) return amt = pb['amount'] # if this month is the same as the last bill month and as per above # we don't have a bill for this date, then add annual grotwh if i == int(lb_mm): print(f"its month: {i} - time to add growth: {bill_info[bill_type]['growth']}" ) amt += amt * bill_info[bill_type]['growth']/100 bill_info[bill_type]['last_bill_amount']=amt # last param is estimated (and this is an estimate for a future bill / not real) new_bill( bill_type, amt, new_date, 1 ) if not yr in bill_info[bill_type]['year']: bill_info[bill_type]['year'][yr]=[] bill={} bill['bill_date']=new_date bill['amount']=amt bill['estimated']=1 # need this for find_previous_bill to work but only need the above 2 fields? bill_info[bill_type]['year'][yr].append(bill) return # given the bill_type has a which_growth contain min/avg/max, return the corresponding growth number def get_growth_value( bt, bill_type ): for el in bt: if el['id'] == bill_type: which = el['which_growth'] break if which == 'avg': return el['ann_growth_avg'] elif which == 'min': return el['ann_growth_min'] else: return el['ann_growth_max'] # go through the bill data from the DB, put it into more friendly formats, then # work out and then add missing bill data (might be b/c we have monthly bills, # and I didn't want to input 12 of them at the same price), and it always # occurs for future bills def process_bill_data(bd, bt, bf): # this maps a bill id to a freq id (e.g. bill #34 - has a frequency of #2 (which might be quarterly) bt_id_freq = {row["id"]: row["freq"] for row in bt} bt_id_ann_growth_avg = {row["id"]: row["ann_growth_avg"] for row in bt} # this maps freq to bills per annum (e.g. id=2 to 4 bills per annum) bf_id_num = {row["id"]: row["num_bills_per_annum"] for row in bf} # want to proces all bill data into easier to maniuplate structure, so make # a bill_info[bill_id] with first_bill, last_bill, [yr] with matching bills to process bill_info={} for bill in bd: bill_type = bill['bill_type_id'] yr= int(bill['bill_date'][:4]) # new bill type if not bill_type in bill_info: bill_info[bill_type]={} bill_info[bill_type]['growth'] = get_growth_value( bt, bill_type ) bill_info[bill_type]['num_ann_bills'] = bf_id_num[bt_id_freq[bill_type]] bill_info[bill_type]['first_bill']={} bill_info[bill_type]['last_bill']={} # due to sql sorting, this first instance is the last bill bill_info[bill_type]['last_bill']=bill bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4]) if not bill['estimated']: print( f"this bill is real, its the first so consider it the last bill paid - date is: {int(bill['bill_date'][:4])}" ) bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4]) bill_info[bill_type]['year']={} bill_info[bill_type]['year_real']={} if not yr in bill_info[bill_type]['year']: bill_info[bill_type]['year'][yr]=[] bill_info[bill_type]['year_real'][yr]=[] # keep updating last to this matching bill bill_info[bill_type]['first_bill']=bill bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4]) if not 'last_real_bill_year' in bill_info[bill_type] and not bill['estimated']: print( f"{bill_type}: seems we dont have a last_real_bill_year, set it to: {int(bill['bill_date'][:4])} ") bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4]) # add this bill to list for this year bill_info[bill_type]['year'][yr].append(bill) if not bill['estimated']: bill_info[bill_type]['year_real'][yr].append(bill) # now process the bill_info from yr of first bill to yr of last bill for bill_type in bill_info: # find freq id based on bill_type id, then use that to find num bills by freq id num = bf_id_num[bt_id_freq[bill_type]] if 'last_bill' not in bill_info[bill_type]: print("Cannot process bill_type={bill_type} - no bill info for it at all" ) # range of years to process (yr_min to yr_max) yr_min=int(bill_info[bill_type]['first_bill']['bill_date'][:4]) yr_max=int(bill_info[bill_type]['last_bill']['bill_date'][:4]) # go from first_bill year until reach end year for yr in range( yr_min, END_YEAR+1 ): # we have all the bills needed for yr if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == bill_info[bill_type]['num_ann_bills']: continue add_missing_bills_for_yr( bill_type, bill_info, yr ) derive_ann_growth( bill_type, bill_info ) ################################################################################ # add_missing_bills_for_yr -- wrapper to call right func based on bill freq ################################################################################ def add_missing_bills_for_yr( bill_type, bill_info, yr ): print(f"{bill_type}: add_missing_bills_for_yr( {bill_type}, bill_info, {yr} )") num = bill_info[bill_type]['num_ann_bills'] if num == 1: add_missing_annual_bill_in_yr( bill_type, bill_info, yr ) elif num == 4: add_missing_quarter_bills_in_yr( bill_type, bill_info, yr ) elif num == 12: add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ) return def derive_ann_growth( bill_type, bill_info ): print(f"{bill_type}: Derive annual growth on bill_type: {bill_type} - fby={bill_info[bill_type]['first_bill_year']}, lby={bill_info[bill_type]['last_real_bill_year']} " ) total={} for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_real_bill_year']+1): # for monthly bills, 1 or 12 bills is enough to work with if bill_info[bill_type]['num_ann_bills'] == 12: if len(bill_info[bill_type]['year_real'][yr]) != 1 and len(bill_info[bill_type]['year_real'][yr]) != 12: continue; # okay annual or quarterly bills, only total them if we have all of them for the year elif len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']: continue; # for monthlys add totals regardless (we only process total if there is 12 or 1 further below). Other bill_types, only total if there is all bills for year if bill_info[bill_type]['num_ann_bills'] != 12 and len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']: continue total[yr] = 0 for b in bill_info[bill_type]['year'][yr]: # ignore estimated bills, only use real bills to calc growth stats if b['estimated']: continue total[yr] += b['amount'] # once we have all yr totals: growth = {} min_growth = 999 avg_growth = 0 max_growth = 0 count = 0 # start from year after first bill, so we can see annual growth from the following year onwards for yr in range( bill_info[bill_type]['first_bill_year']+1, bill_info[bill_type]['last_bill_year']+1): # if full data sets for consecutive years, work out annual growth stats if yr-1 in total and yr in total: growth = (total[yr] - total[yr-1]) / total[yr-1] * 100 avg_growth += growth count += 1 if growth < min_growth: min_growth = growth if growth > max_growth: max_growth = growth if count: print( f"{bill_type}: Min growth was: {min_growth}" ) print( f"{bill_type}: Avg growth is: {avg_growth/count}" ) print( f"{bill_type}: Max growth was: {max_growth}" ) set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth ) else: # failsafe (just in case fill bills failed to add enough bills to average out) print( f"{bill_type}: Unable to calculate growth!" )