really should consider quarterly bill additions as seasonal <- more likely for elec, gas, etc

TODO

@@ -1,8 +1,27 @@
-For bills, there is too much to guess, need to ask:
+For bills:
-    frequency:
+    * using frequency and known bills fill in missing gaps
-        then I could work out growth rate
+      -- need to add to DB whether the bill was added manually or auto-flled in (so if we want, we can re-calc. based on growth calc change as we add more real bills over time?)
-
+    * need to work out 'first bill' and 'last bill' to auto-fill missing bills based on
-
+      -- all missing bills follow varying growth models & its by choice -- therefore I need this in DB
+         - ANN: flat, min, avg, max, manual
+         - QTR: flat, qtrly seasonal: min/avg/max/manual, qtrly simple: min/avg/max/manual, annual: min/avg/max/manual
+         - MON: flat, monthly: min/avg/max/manual, annual: min/avg/max/manual
+      -- use this logic to add missing bills (date):
+           -- ANN: missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions
+           -- QTR: missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year
+           -- MON: missing monthly bills, find date based on DD and put in each missing month 
+      -- use this logic to add missing bills (amount):
+           -- ANN: future only, so add ann_growth (based on drop-down) for each future year
+           -- QTR: add growth (based on drop-down) for each future year
+           -- MON: add growth (based on drop-down) for each future year 
+    * once auto-filled bills exist:
+        - calc growth [DONE]
+          -- WARNING: could be chicken/egg here, if not enough bills to calc growth - maybe only offer flat/manual until there are enough??
+        - project out to I am 60
+            - probably need to allow a toggle to: allow show manual, show auto-filled past, show auto-filled future, show all
+        - remove bills from Living_Expenses (carefully - but by hand)
+        - fold future bills into calc so they are taken out in a more time and growth appropriate way
+        - inflation can then be put to a more realistic quarterly figure
 
 MUCH LONGER/HARDER:
     potentially for each bill_type, there are unique extras - e.g. THIS feels too hard:

bills.py

@@ -1,41 +1,149 @@
-from db import get_bill_data, get_bill_types, set_bill_type_growth
+from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth
+from defines import END_YEAR
 
-def derive_bill_data():
-  bd=get_bill_data()
-  bt=get_bill_types()
-  water_id = None
-  for t in bt:
-    if t['name'] == "Water":
-        water_id = t['id']
 
-  if not water_id:
+# give a bill dat in format YYYY-MM-DD, return quarter (1-4)
+def qtr(d):
+    m = int(d[5:7])
+    return ( (m-1)//3 + 1 )
+   
+# missing annual bill, find date based on MM-DD and add new year - given we start with first_bill anyway, will only be used for future bill predictions
+# future only, so add ann_growth (based on drop-down) for each future year
+def add_missing_annual_bill_in_yr( bill_type, bill_info, num, yr ):
+#    print( f"{bill_type}: Seems we are missing an annual bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" )
+    mm_dd = bill_info[bill_type]['last_bill']['bill_date'][5:]
+    l_amt = bill_info[bill_type]['last_bill']['amount']
+#    print( f"{bill_type}: Should fake a bill into date={yr}-{mm_dd} of adjusted amount from base of {l_amt}" )
+    # okay the missing bill is before the first bill...
+    for i in range( bill_info[bill_type]['last_bill_year'], yr ):
+        l_amt += l_amt * 5.26/100
+    print( f"{bill_type}: So should insert bill as: ${l_amt:.02f} on '{yr}-{mm_dd}'")
     return
-  total={}
-  total[water_id]={}
-  for yr in [2022, 2023, 2024]:
-      print( f"water_id={water_id}")
-      total[water_id][yr] = 0
-      for b in bd:
-        if b['bill_type_id'] == water_id and str(yr) in b['bill_date']:
-            total[water_id][yr] += b['amount']
-            print( f"{yr} => {b['bill_date']} -- {b['amount']}" )
-      print( f"total for water in {yr} is {total[water_id][yr]}" )
 
-  # once we have all yr totals:
+# missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year
-  growth = {}
+# add growth (based on drop-down) for each future year
-  growth[water_id] = {}
+def add_missing_quarter_bills_in_yr( bill_type, bill_info, num, yr ):
-  max_growth = {}
+#    print( f"{bill_type}: Seems we are missing a quarterly bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" )
-  avg_growth = {}
+    return
-  max_growth[water_id] = 0
+
-  avg_growth[water_id] = 0
+# missing monthly bills, find date based on DD and put in each missing month
-  count = 0
+# add growth (based on drop-down) for each future year
-  for yr in [2023, 2024]:
+def add_missing_monthly_bills_in_yr( bill_type, bill_info, num, yr ):
-    growth[water_id][yr] = (total[water_id][yr] - total[water_id][yr-1]) / total[water_id][yr-1] * 100
+#    print( f"{bill_type}: Seems we are missing a monthly bill in {yr}, use first_bill={bill_info[bill_type]['first_bill']['bill_date']} to add one" )
-    avg_growth[water_id] += growth[water_id][yr]
+    return
-    count += 1
+
-    if growth[water_id][yr] > max_growth[water_id]:
+
-        max_growth[water_id] = growth[water_id][yr]
+# go through the bill data from the DB, put it into more friendly formats, then
-    print( f"growth from {yr} to {yr-1} = {growth}%")
+# work out and then add missing bill data (might be b/c we have monthly bills,
-  print( f"Max growth was: {max_growth[water_id]}" )
+# and I didn't want to input 12 of them at the same price), and it always
-  print( f"Avg growth is: {avg_growth[water_id]/count}" )
+# occurs for future bills
-  set_bill_type_growth( water_id, avg_growth[water_id]/count )
+def process_bill_data(bd, bt, bf):
+    # this maps a bill id to a freq id (e.g. bill #34 - has a frequency of #2 (which might be quarterly)
+    bt_id_freq = {row["id"]: row["freq"] for row in bt}
+
+    # this maps freq to bills per annum (e.g. id=2 to 4 bills per annum)
+    bf_id_num = {row["id"]: row["num_bills_per_annum"] for row in bf}
+
+    # want to proces all bill data into easier to maniuplate structure, so make
+    # a bill_info[bill_id] with first_bill, last_bill, [yr] with matching bills to process
+    bill_info={}
+    
+    for bill in bd:
+        bill_type = bill['bill_type_id']
+        yr= int(bill['bill_date'][:4])
+        # new bill type
+        if not bill_type in bill_info:
+            bill_info[bill_type]={}
+            bill_info[bill_type]['first_bill']={}
+            bill_info[bill_type]['last_bill']={}
+            # due to sql sorting, this first instance is the last bill
+            bill_info[bill_type]['last_bill']=bill
+            bill_info[bill_type]['last_bill_year']=int(bill['bill_date'][:4])
+            bill_info[bill_type]['year']={}
+        if not yr in bill_info[bill_type]['year']:
+            bill_info[bill_type]['year'][yr]=[]
+
+        # keep updating last to this matching bill
+        bill_info[bill_type]['first_bill']=bill
+        bill_info[bill_type]['first_bill_year']=int(bill['bill_date'][:4])
+        # add this bill to list for this year
+        bill_info[bill_type]['year'][yr].append(bill)
+
+    # now process the bill_info from yr of first bill to yr of last bill
+    for bill_type in bill_info:
+        # find freq id based on bill_type id, then use that to find num bills by freq id
+        num = bf_id_num[bt_id_freq[bill_type]]
+
+        if 'last_bill' not in bill_info[bill_type]:
+            print("Cannot process bill_type={bill_type} - no bill info for it at all" )
+        # range of years to process (yr_min to yr_max)
+        yr_min=int(bill_info[bill_type]['first_bill']['bill_date'][:4])
+        yr_max=int(bill_info[bill_type]['last_bill']['bill_date'][:4])
+
+        # go from first_bill year until reach end year
+        for yr in range( yr_min, END_YEAR+1 ):
+            if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == num:
+#                print(f"{bill_type}: need {num} annual bills and found then for {yr}" )
+                continue
+#           if yr not in bill_info[bill_type]['year']:
+#               print(f"{bill_type}: need {num} annual bills and 0 found for {yr}" )
+#           else:
+#               print(f"{bill_type}: need {num} annual bills and only {len(bill_info[bill_type]['year'][yr])} found for {yr}" )
+            add_missing_bills_for_yr( bill_type, bill_info, num, yr )
+        # now should have missing bills, calculate ann growth properly
+        derive_ann_growth( bill_type, bill_info, num )
+
+################################################################################
+# add_missing_bills_for_yr -- wrapper to call right func based on bill freq
+################################################################################
+def add_missing_bills_for_yr( bill_type, bill_info, num, yr ):
+    if num == 1:
+        add_missing_annual_bill_in_yr( bill_type, bill_info, num, yr )
+    elif num == 4:
+        add_missing_quarter_bills_in_yr( bill_type, bill_info, num, yr )
+    elif num == 12:
+        add_missing_monthly_bills_in_yr( bill_type, bill_info, num, yr )
+    return
+
+def derive_ann_growth( bill_type, bill_info, num ):
+    print(f"Derive annual growth on bill_type: {bill_type} " )
+    # DDP: rewrite loop below to use bill_info more cleverly, start with type, then year, then use the data in there rather than in bd
+
+    total={}
+    for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_bill_year']+1):
+        total[yr] = 0
+        for b in bill_info[bill_type]['year'][yr]:
+           total[yr] += b['amount']
+#           print( f"{yr} => {b['bill_date']} -- {b['amount']}" )
+#        print( f"total for {bill_type} in {yr} is {total[yr]}" )
+
+    # once we have all yr totals:
+    growth = {}
+    min_growth = 999
+    avg_growth = 0
+    max_growth = 0
+    count = 0
+    for yr in range( bill_info[bill_type]['first_bill_year'], bill_info[bill_type]['last_bill_year']+1):
+        if yr-1 in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr-1]) != num:
+#            print(f"less than {num} bills in yr: {yr-1}, so can't use data" )
+            continue
+        if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) != num:
+#            print(f"less than {num} bills in yr: {yr-1}, so can't use data" )
+            continue
+        if yr-1 in total and yr in total:
+            growth = (total[yr] - total[yr-1]) / total[yr-1] * 100
+            avg_growth += growth
+            count += 1
+            if growth < min_growth:
+                min_growth = growth
+            if growth > max_growth:
+                max_growth = growth
+#        print( f"growth from {yr} to {yr-1} = {growth}%")
+    if count:
+        print( f"Min growth was: {min_growth}" )
+        print( f"Avg growth is: {avg_growth/count}" )
+        print( f"Max growth was: {max_growth}" )
+        set_bill_type_growth( bill_type, avg_growth/count )
+    else:
+        # failsafe (just in case fill bills failed to add enough bills to average out)
+        print( f"Unable to calculate growth!" )

calc.py

@@ -1,5 +1,6 @@
 # calc.py
 from datetime import datetime, timedelta
+from defines import END_YEAR
 
 # GLOBAL CONSTANTS
 LEASE = 0
@@ -90,7 +91,7 @@ def calculate_savings_depletion(finance):
 
     # main loop range -- start from now, and simulate till D is 60 (April 2031)
     current_date = datetime.today()
-    end_date = datetime(2031, 4, 15)
+    end_date = datetime(END_YEAR, 4, 15)
 
     # Calculate daily living expenses
     daily_living_expenses = Living_Expenses / 365

db.py

@@ -94,7 +94,8 @@ def init_db():
 
     cur.execute('''CREATE TABLE IF NOT EXISTS bill_freq (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    name STRING
+                    name STRING,
+                    num_bills_per_annum INTEGER
                     )''')
 
     cur.execute('''CREATE TABLE IF NOT EXISTS bill_data (
@@ -121,9 +122,9 @@ def init_db():
                         Inflation, Mich_present, Overseas_trip, Mark_reno, D_leave_owed_in_days, D_TLS_shares, M_TLS_shares, D_CBA_shares, TLS_price, CBA_price, Overseas_trip_date, Mark_reno_date, Car_buyout_date, Sell_shares, compare_to, Ioniq6_future)
                         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', 
                     (4762.29, 10, 24000, 620, 2412, 45824.68, 83738.74, 80000, 424875.26, 4.75, 2.4, 10000, 50000, 10000, 76.85, 1000, 750, 1111, 4.52, 163.32, '2025-06-01', '2025-09-01', '2025-02-20', 4, 0, 0))
-        cur.execute( "INSERT INTO bill_freq values ( 1, 'Annual' )" )
+        cur.execute( "INSERT INTO bill_freq values ( 1, 'Annual', 1 )" )
-        cur.execute( "INSERT INTO bill_freq values ( 2, 'Quarterly' )" )
+        cur.execute( "INSERT INTO bill_freq values ( 2, 'Quarterly', 4 )" )
-        cur.execute( "INSERT INTO bill_freq values ( 3, 'Monthly' )" )
+        cur.execute( "INSERT INTO bill_freq values ( 3, 'Monthly', 12 )" )
     conn.commit()
     conn.close()
 

defines.py

@@ -0,0 +1 @@
+END_YEAR=2031

main.py

@@ -4,7 +4,7 @@ from calc import calculate_savings_depletion
 from db import init_db, get_finance_data, update_finance, get_budget_data, insert_cset, get_comp_set_data, get_comp_set_options, get_bill_freqs
 from db import get_bill_data, new_bill, update_bill_data, delete_bill
 from db import get_bill_types, insert_bill_type, update_bill_type, delete_bill_type
-from bills import derive_bill_data
+from bills import process_bill_data
 from collections import defaultdict, Counter
 from datetime import datetime
 import csv
@@ -144,9 +144,8 @@ def DisplayBillData():
     bill_data = get_bill_data()
     bill_types = get_bill_types()
     bill_freqs = get_bill_freqs()
-    now=datetime.today().strftime('%Y-%m-%d')
+    process_bill_data(bill_data, bill_types, bill_freqs)
-    derive_bill_data()
+    return render_template('bills.html', bill_data=bill_data, bill_types=bill_types, bill_freqs=bill_freqs )
-    return render_template('bills.html', now=now, bill_data=bill_data, bill_types=bill_types, bill_freqs=bill_freqs )
 
 @app.route('/newbilltype', methods=['POST'])
 def InsertBillType():