removed lots of debugs, added find_next_bill to allow better estimating of bills in the past. Growth now uses bills in the past (estimated included), which simplifies the code a bit. All working now except pragmatic missing quarterly bills not started

BUGS

@@ -1,2 +0,0 @@
-when I cancel items, the changes need to be reverted for:
-    new bill, new bill type, change bill

bills.py

@@ -1,11 +1,66 @@
 from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth, new_bill
 from defines import END_YEAR
+import datetime
 
 # give a bill dat in format YYYY-MM-DD, return quarter (1-4)
 def qtr(d):
     m = int(d[5:7])
     return ( (m-1)//3 + 1 )
    
+def find_next_bill( bill_type, bill_info, bill_date ):
+    wanted_year = int(bill_date[:4])
+    wanted_mm = int(bill_date[5:7])
+    # if we want a bill after our last year, just return None
+    if int(wanted_year) > int(bill_info[bill_type]['last_bill_year']):
+        return None
+
+    for yr in range( wanted_year, bill_info[bill_type]['last_bill_year']+1 ):
+        # start with bills in the year wanted (if any)
+        if yr in bill_info[bill_type]['year']:
+            for bill in bill_info[bill_type]['year'][yr][::-1]:
+                bill_mm = int(bill['bill_date'][5:7])
+                # if bill is in this year but later OR its a later year, return this bill
+                if (wanted_year == yr and bill_mm > wanted_mm) or wanted_year < yr:
+                    return bill
+    # failsafe
+    return None
+            
+
+def find_previous_bill( bill_type, bill_info, bill_date ):
+    wanted_year = int(bill_date[:4])
+    wanted_mm = int(bill_date[5:7])
+    # if we don't have a bill before this date, no way to set price
+    if int(wanted_year) < int(bill_info[bill_type]['first_bill_year']):
+        return None
+
+    # start loop from bill_date, go backwards and find which one it is (same year, should be month-based)
+    # earlier year, then just last one from the year.
+    yr_range=range( wanted_year, bill_info[bill_type]['first_bill_year'], -1 )
+    if wanted_year == int(bill_info[bill_type]['first_bill_year']):
+        # range of this year with -1, does not return anything, so force this year.
+        yr_range=[ wanted_year ]
+
+    for yr in yr_range:
+        # start with bills in the year wanted (if any)
+        # must include 'estimated' bills to deal with growth of future years
+        if yr in bill_info[bill_type]['year']:
+            # okay, we have the previous billing year, and we wanted one for a year in the future,
+            # just return the last one in this year as its the most recent
+            if wanted_year > yr:
+                return bill_info[bill_type]['year'][yr][-1]
+            else:
+                # lets go through the newest to oldest of these bills
+                for bill in bill_info[bill_type]['year'][yr][::-1]:
+                    bill_mm = int(bill['bill_date'][5:7])
+                    # reversing the bills, means we start with the 'most recent' in this year to the oldest
+                    # if the month we want is after the bill, we are done
+                    if wanted_mm > bill_mm:
+                        return bill
+
+    return None
+
+
+
 # missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions
 # future only, so add ann_growth (based on drop-down) for each future year
 # NOTE: only ever called when there is a need to add a new bill
@@ -37,11 +92,6 @@ def add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ):
     mm = bill_info[bill_type]['first_bill']['bill_date'][5:7]
     lb_mm = bill_info[bill_type]['last_bill']['bill_date'][5:7]
 
-    # choose last bill from last amount to grow from as its most relevant
-    if not 'last_bill_amount' in bill_info[bill_type]:
-        bill_info[bill_type]['last_bill_amount']=bill_info[bill_type]['last_bill']['amount']
-    amt = bill_info[bill_type]['last_bill_amount']
-
     #okay add monthly bills for the rest of this year if its the first year
     if bill_info[bill_type]['first_bill_year'] == yr:
         start_m=int(mm)
@@ -61,14 +111,32 @@ def add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ):
                     bill_found=True
                     break
         if not bill_found:
-            # if this month is the same as the last bill month and as per above
-            # we don't have a bill for this date, then add annual grotwh
-            if i == int(lb_mm):
-                print(f"its month: {i} - time to add growth: {bill_info[bill_type]['growth']}" )
-                amt += amt * bill_info[bill_type]['growth']/100
-                bill_info[bill_type]['last_bill_amount']=amt
+            pb=find_previous_bill( bill_type, bill_info, new_date )
+            nb=find_next_bill( bill_type, bill_info, new_date )
+            if not pb:
+                print("Failed to find previous_bill, can't calculate missing bill - returning" )
+                return
+
+            amt = pb['amount']
+            # if there is no next bill then use growth, otherwise, I am only putting in real bills
+            # where changes occur, so keep the pb amount 'unchanged'
+            if not nb:
+                # if this month is the same as the last bill month and as per above
+                # we don't have a bill for this date, then add annual grotwh
+                if i == int(lb_mm):
+                    amt += amt * bill_info[bill_type]['growth']/100
+                    bill_info[bill_type]['last_bill_amount']=amt
             # last param is estimated (and this is an estimate for a future bill / not real)
             new_bill( bill_type, amt, new_date, 1 )
+            if not yr in bill_info[bill_type]['year']:
+                bill_info[bill_type]['year'][yr]=[]
+            bill={}
+            bill['bill_date']=new_date
+            bill['amount']=amt
+            bill['estimated']=1
+            # need this for find_previous_bill to work but only need the above 2 fields?
+            bill_info[bill_type]['year'][yr].append(bill)
+
     return
 
 # given the bill_type has a which_growth contain min/avg/max, return the corresponding growth number
@@ -115,25 +183,15 @@ def process_bill_data(bd, bt, bf):
             # due to sql sorting, this first instance is the last bill
             bill_info[bill_type]['last_bill']=bill
             bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4])
-            if not bill['estimated']:
-                print( f"this bill is real, its the first so consider it the last bill paid - date is: {int(bill['bill_date'][:4])}" )
-                bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
             bill_info[bill_type]['year']={}
-            bill_info[bill_type]['year_real']={}
         if not yr in bill_info[bill_type]['year']:
             bill_info[bill_type]['year'][yr]=[]
-            bill_info[bill_type]['year_real'][yr]=[]
 
         # keep updating last to this matching bill
         bill_info[bill_type]['first_bill']=bill
         bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4])
-        if not 'last_real_bill_year' in bill_info[bill_type] and not bill['estimated']:
-            print( f"{bill_type}: seems we dont have a last_real_bill_year, set it to: {int(bill['bill_date'][:4])} ")
-            bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
         # add this bill to list for this year
         bill_info[bill_type]['year'][yr].append(bill)
-        if not bill['estimated']:
-            bill_info[bill_type]['year_real'][yr].append(bill)
 
     # now process the bill_info from yr of first bill to yr of last bill
     for bill_type in bill_info:
@@ -158,7 +216,6 @@ def process_bill_data(bd, bt, bf):
 # add_missing_bills_for_yr -- wrapper to call right func based on bill freq
 ################################################################################
 def add_missing_bills_for_yr( bill_type, bill_info, yr ):
-    print(f"{bill_type}:  add_missing_bills_for_yr( {bill_type}, bill_info, {yr} )")
     num = bill_info[bill_type]['num_ann_bills']
     if num == 1:
         add_missing_annual_bill_in_yr( bill_type, bill_info, yr )
@@ -169,26 +226,16 @@ def add_missing_bills_for_yr( bill_type, bill_info, yr ):
     return
 
 def derive_ann_growth( bill_type, bill_info ):
-    print(f"{bill_type}: Derive annual growth on bill_type: {bill_type} - fby={bill_info[bill_type]['first_bill_year']}, lby={bill_info[bill_type]['last_real_bill_year']} " )
-
+    # just do up to now so we stop earlier than looking at other estimated (just an optimisation)
+    now_yr = datetime.date.today().year
     total={}
-    for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_real_bill_year']+1):
-        # for monthly bills, 1 or 12 bills is enough to work with
-        if bill_info[bill_type]['num_ann_bills'] == 12:
-            if len(bill_info[bill_type]['year_real'][yr]) != 1 and len(bill_info[bill_type]['year_real'][yr]) != 12:
-                continue;
-        # okay annual or quarterly bills, only total them if we have all of them for the year
-        elif len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']:
+    for yr in range( bill_info[bill_type]['first_bill_year'], now_yr+1):
+        # if not enough bills in this year (or none), then try next year (first year might have not enough bills)
+        if yr not in bill_info[bill_type]['year'] or len(bill_info[bill_type]['year'][yr]) != bill_info[bill_type]['num_ann_bills']:
             continue;
 
-        # for monthlys add totals regardless (we only process total if there is 12 or 1 further below).  Other bill_types, only total if there is all bills for year
-        if bill_info[bill_type]['num_ann_bills'] != 12 and len(bill_info[bill_type]['year_real'][yr]) != bill_info[bill_type]['num_ann_bills']:
-            continue
         total[yr] = 0
         for b in bill_info[bill_type]['year'][yr]:
-            # ignore estimated bills, only use real bills to calc growth stats
-            if b['estimated']:
-                continue
             total[yr] += b['amount']
 
     # once we have all yr totals:
@@ -198,7 +245,7 @@ def derive_ann_growth( bill_type, bill_info ):
     max_growth = 0
     count = 0
     # start from year after first bill, so we can see annual growth from the following year onwards
-    for yr in range( bill_info[bill_type]['first_bill_year']+1, bill_info[bill_type]['last_bill_year']+1):
+    for yr in range( bill_info[bill_type]['first_bill_year']+1, now_yr+1):
         # if full data sets for consecutive years, work out annual growth stats
         if yr-1 in total and yr in total:
             growth = (total[yr] - total[yr-1]) / total[yr-1] * 100
@@ -209,9 +256,6 @@ def derive_ann_growth( bill_type, bill_info ):
             if growth > max_growth:
                 max_growth = growth
     if count:
-        print( f"{bill_type}: Min growth was: {min_growth}" )
-        print( f"{bill_type}: Avg growth is: {avg_growth/count}" )
-        print( f"{bill_type}: Max growth was: {max_growth}" )
         set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth )
     else:
         # failsafe (just in case fill bills failed to add enough bills to average out)

db.py

@@ -300,7 +300,7 @@ def get_bill_freqs():
 def new_bill( bill_type, amount, bill_date, estimated ):
     conn = connect_db(False)
     cur = conn.cursor()
-    cur.execute( f"insert into bill_data ( 'bill_type', 'amount', 'bill_date', 'estimated' ) values ( '{bill_type}', '{amount:.2f}', '{bill_date}', {estimated} )" )
+    cur.execute( f"insert into bill_data ( 'bill_type', 'amount', 'bill_date', 'estimated' ) values ( '{bill_type}', '{float(amount):.2f}', '{bill_date}', {estimated} )" )
     conn.commit()
     conn.close()
     return