noting multi-year growth figures are off - e.g. internet stays same price for 3 years

TODO

@@ -1,8 +1,23 @@
 UI:
 
 For bills:
-    * using frequency and known bills fill in missing gaps
-      [DONE] -- need to add to DB whether the bill was added manually or auto-flled in (so if we want, we can re-calc. based on growth calc change as we add more real bills over time?)
+    * growth for internet/monthly is a bit skewed.... Really think min/avg/max might need to be smarter
+        - at least max, its 114-134 BUT, not in 1 year, really that changed from 2022 to 2025, so 3 years... (or max = 18% over 3 years, or 6%)
+        -- so ann_growth_avg or max needs to factor in years of same prices
+      [DONE] -- Pragmatical growth before I bonkers:
+        [DONE] - monthly, prob. just flat by default (simple 1 or 12 bills for the year each year and I can get a growth rate from that)
+          [DONE] - and apply monthly growth - annually 12 + months from last bill each year
+        [DONE] - quarterly - should be able to take last qtr-1 ... qtr-4 and then grow them all by growth
+        [DONE]- annual easy
+    * once auto-filled bills exist:
+        [DONE]- calc growth
+        [DONE] - project out to I am 60 (A/Q/M) - A/Q done, M to go
+            [DONE] - probably need to allow a toggle to: allow show manual, show auto-filled past, show auto-filled future, show all
+        - remove bills from Living_Expenses (carefully - but by hand)
+        - fold future bills into calc so they are taken out in a more time and growth appropriate way
+        - inflation can then be put to a more realistic quarterly figure
+
+LONGER/HARDER:
     * need to work out 'first bill' and 'last bill' to auto-fill missing bills based on
       -- all missing bills follow varying growth models & its by choice -- therefore I need this in DB
          - ANN: flat, min, avg, max, manual
@@ -16,18 +31,6 @@ For bills:
            -- ANN: future only, so add ann_growth (based on drop-down) for each future year
            -- QTR: add growth (based on drop-down) for each future year
            -- MON: add growth (based on drop-down) for each future year 
-      -- Pragmatical growth before I bonkers:
-        [DONE] - monthly, prob. just flat by default (simple 1 or 12 bills for the year each year and I can get a growth rate from that)
-          [DONE] - and apply monthly growth - annually 12 + months from last bill each year
-        [DONE]- qty - manual only
-        [DONE]- annual easy
-    * once auto-filled bills exist:
-        [DONE]- calc growth
-        - project out to I am 60 (A/Q/M) - A/Q done, M to go
-            [DONE] - probably need to allow a toggle to: allow show manual, show auto-filled past, show auto-filled future, show all
-        - remove bills from Living_Expenses (carefully - but by hand)
-        - fold future bills into calc so they are taken out in a more time and growth appropriate way
-        - inflation can then be put to a more realistic quarterly figure
 
 MUCH LONGER/HARDER:
     potentially for each bill_type, there are unique extras - e.g. THIS feels too hard:

bills.py

@@ -60,6 +60,21 @@ def find_previous_bill( bill_type, bill_info, bill_date ):
     return None
 
 
+def new_estimated_bill( bill_info, yr, bill_type, amt, new_date ):
+    # add to DB
+    new_bill( bill_type, amt, new_date, 1 )
+
+    # patch this data back into bill_info so growth works in future
+    if not yr in bill_info[bill_type]['year']:
+        bill_info[bill_type]['year'][yr]=[]
+    bill={}
+    bill['bill_date']=new_date
+    bill['amount']=amt
+    bill['estimated']=1
+    # need this for find_previous_bill to work but only need the above 3 fields
+    bill_info[bill_type]['year'][yr].append(bill)
+    return
+
 
 # missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions
 # future only, so add ann_growth (based on drop-down) for each future year
@@ -72,13 +87,31 @@ def add_missing_annual_bill_in_yr( bill_type, bill_info, yr ):
         amt += amt * bill_info[bill_type]['growth']/100
 
     # last param is estimated (and this is an estimate for a future bill / not real)
-    new_bill( bill_type, amt, f'{yr}-{mm_dd}', 1 )
+    new_estimated_bill( bill_info, yr, bill_type, amt, f'{yr}-{mm_dd}' )
     return
 
 # missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year
 # add growth (based on drop-down) for each future year
 def add_missing_quarter_bills_in_yr( bill_type, bill_info, yr ):
-    print( f"*** add_missing_quarter_bills_in_yr( {bill_type}, bill_info, {yr} ): NOT YET" )
+    # okay we have data for this year but some missing (wouldnt be here otherwise)
+    # and data from previous year... lets fill in gaps
+    if yr in bill_info[bill_type]['year'] and yr-1 in bill_info[bill_type]['year']:
+        # per if above, ONLY get here if we have first few bills of {yr}, cannot be last few
+        have_q = qtr( bill_info[bill_type]['year'][yr][0]['bill_date'] )
+        for q in range(have_q+1,5):
+            # use 5-q, as bills are in descending order in bill_info, e.g. q4 is 1st, 
+            bill=bill_info[bill_type]['year'][yr-1][4-q]
+            mm_dd= bill['bill_date'][5:]
+            amt = bill['amount']*(1+bill_info[bill_type]['growth']/100)
+            new_date = f'{yr}-{mm_dd}'
+            new_estimated_bill( bill_info, yr, bill_type, amt, new_date )
+
+    # for now only add full new years based on last year with ann_growth (seasonal)
+    if yr not in bill_info[bill_type]['year'] and yr-1 in bill_info[bill_type]['year']:
+        for bill in bill_info[bill_type]['year'][yr-1]:
+            mm_dd= bill['bill_date'][5:]
+            amt = bill['amount']*(1+bill_info[bill_type]['growth']/100)
+            new_estimated_bill( bill_info, yr, bill_type, amt, f'{yr}-{mm_dd}' )
     return
 
 # missing monthly bills, find date based on DD and put in each missing month
@@ -127,16 +160,7 @@ def add_missing_monthly_bills_in_yr( bill_type, bill_info, yr ):
                     amt += amt * bill_info[bill_type]['growth']/100
                     bill_info[bill_type]['last_bill_amount']=amt
             # last param is estimated (and this is an estimate for a future bill / not real)
-            new_bill( bill_type, amt, new_date, 1 )
-            if not yr in bill_info[bill_type]['year']:
-                bill_info[bill_type]['year'][yr]=[]
-            bill={}
-            bill['bill_date']=new_date
-            bill['amount']=amt
-            bill['estimated']=1
-            # need this for find_previous_bill to work but only need the above 2 fields?
-            bill_info[bill_type]['year'][yr].append(bill)
-
+            new_estimated_bill( bill_info, yr, bill_type, amt, new_date )
     return
 
 # given the bill_type has a which_growth contain min/avg/max, return the corresponding growth number
@@ -183,6 +207,8 @@ def process_bill_data(bd, bt, bf):
             # due to sql sorting, this first instance is the last bill
             bill_info[bill_type]['last_bill']=bill
             bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4])
+            if not bill['estimated']:
+                bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
             bill_info[bill_type]['year']={}
         if not yr in bill_info[bill_type]['year']:
             bill_info[bill_type]['year'][yr]=[]
@@ -190,6 +216,8 @@ def process_bill_data(bd, bt, bf):
         # keep updating last to this matching bill
         bill_info[bill_type]['first_bill']=bill
         bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4])
+        if not 'last_real_bill_year' in bill_info[bill_type] and not bill['estimated']:
+            bill_info[bill_type]['last_real_bill_year']=int(bill['bill_date'][:4])
         # add this bill to list for this year
         bill_info[bill_type]['year'][yr].append(bill)
 
@@ -234,6 +262,15 @@ def derive_ann_growth( bill_type, bill_info ):
         if yr not in bill_info[bill_type]['year'] or len(bill_info[bill_type]['year'][yr]) != bill_info[bill_type]['num_ann_bills']:
             continue;
 
+        # just going to make sure we dont use estimated data in the last year of real data - can skew growths
+        if yr == bill_info[bill_type]['last_real_bill_year']:
+            skip_yr=False
+            for b in bill_info[bill_type]['year'][yr]:
+                if b['estimated']:
+                    skip_yr=True
+            if skip_yr:
+                continue
+
         total[yr] = 0
         for b in bill_info[bill_type]['year'][yr]:
             total[yr] += b['amount']

main.py

@@ -146,7 +146,6 @@ def DisplayBillData():
     bill_types = get_bill_types()
     bill_freqs = get_bill_freqs()
     bill_ui = get_bill_ui()
-    print( f"bu={bill_ui}" )
     process_bill_data(bill_data, bill_types, bill_freqs)
     bill_data = get_bill_data()
     return render_template('bills.html', bill_data=bill_data, bill_types=bill_types, bill_freqs=bill_freqs, bill_ui=bill_ui )