fixed a few bugs, annual growth was just broken, dont add another estimate bill when we have one for that year or in that quarter, removed lots of debugs, fixed a few bugs where the first data point in a new year/qtr would not have arrays initialised properly first, apportion quarterly data in future real bills - it happens with Rates

BUGS

@@ -0,0 +1,2 @@
+* added an electricity bill by accident for 2018, that kills lots :(
+    - something to do with missing year of data in quarterly bills - still an issue

TODO

@@ -1,4 +1,6 @@
 UI:
+    * when we choose a tab of bill_data -> set new bill select based on tab 
+        & vice-versa, add a bill of type, reload page to show the tab of those bills
 
 For bills:
     * gas bills are a mess and more than 4 per year... *SIGH* try this:

bills.py

@@ -1,4 +1,4 @@
-from db import get_bill_data, get_bill_types, get_bill_freqs, set_bill_type_growth, new_bill
+from db import set_bill_type_growth, new_bill
 from defines import END_YEAR
 import datetime
 from datetime import date, timedelta
@@ -51,6 +51,8 @@ def allocate_by_quarter( bill_info, bill_type, yr, prev_bill, bill):
                 bill_info[bill_type]['qtr'][q_start.year] = {}
                 for i in range(1,5):
                     bill_info[bill_type]['qtr'][q_start.year][i]=0
+            if q not in bill_info[bill_type]['qtr'][q_start.year]:
+                bill_info[bill_type]['qtr'][q_start.year][q]=0
             bill_info[bill_type]['qtr'][q_start.year][q] += days*cost_per_day
         # next quarter
         cur = q_end + timedelta(days=1)
@@ -139,14 +141,17 @@ def new_estimated_bill( bill_info, yr, bill_type, amt, new_date ):
 
     if bill_info[bill_type]['num_ann_bills'] == 4:
         q = qtr( new_date )
+        # new bill in this qtr of this year, so set arrays up
         if yr not in bill_info[bill_type]['qtr']:
             bill_info[bill_type]['qtr'][yr]={}
         pb = find_previous_bill( bill_type, bill_info, new_date )
         if pb['estimated'] == 0:
-            print( f"   FIXFIXFIX - have a prev real bill={pb['bill_date']} & this is first est - likely need to better apportion this bill into the quarters" )
             allocate_by_quarter( bill_info, bill_type, yr, pb, bill )
-
-        bill_info[bill_type]['qtr'][yr][q]=amt
+        else:
+            if not q in bill_info[bill_type]['qtr'][yr]:
+                # first in this year, just init it...
+                bill_info[bill_type]['qtr'][yr][q]=0
+            bill_info[bill_type]['qtr'][yr][q]+=amt
     return
 
 
@@ -155,12 +160,16 @@ def new_estimated_bill( bill_info, yr, bill_type, amt, new_date ):
 # NOTE: only ever called when there is a need to add a new bill
 def add_missing_annual_bill_in_yr( bill_type, bill_info, yr ):
     mm_dd = bill_info[bill_type]['last_bill']['bill_date'][5:]
+    new_date= f'{yr}-{mm_dd}'
+    pb=find_previous_bill( bill_type, bill_info, new_date )
+    if pb:
+        amt = pb['amount']
+    else:
         amt = bill_info[bill_type]['last_bill']['amount']
     # okay the missing bill is before the first bill...
-    for i in range( bill_info[bill_type]['last_bill_year'], yr ):
     amt += amt * bill_info[bill_type]['growth']/100
 
-    new_estimated_bill( bill_info, yr, bill_type, amt, f'{yr}-{mm_dd}' )
+    new_estimated_bill( bill_info, yr, bill_type, amt, new_date )
     return
 
 # missing quarterly bill, find date based on MM-DD and ??? - can have missing bilsl in first year
@@ -248,6 +257,8 @@ def get_growth_value( bt, bill_type ):
         return el['ann_growth_avg']
     elif which == 'min':
         return el['ann_growth_min']
+    elif which == 'simple':
+        return el['ann_growth_simple']
     else:
         return el['ann_growth_max']
 
@@ -313,7 +324,8 @@ def process_bill_data(bd, bt, bf):
         # go from first_bill year until reach end year
         for yr in range( yr_min, END_YEAR+1 ):
             # we have all the bills needed for yr - but dont be cute with qtrly, gas bills suck can have missing with 4 bills 
-            if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) == bill_info[bill_type]['num_ann_bills'] and bill_info[bill_type]['num_ann_bills'] !=4:
+            # > can occur when we add a real bill "on top of" an estimate.
+            if yr in bill_info[bill_type]['year'] and len(bill_info[bill_type]['year'][yr]) >= bill_info[bill_type]['num_ann_bills'] and bill_info[bill_type]['num_ann_bills'] !=4:
                 continue
             add_missing_bills_for_yr( bill_type, bill_info, yr )
         derive_ann_growth( bill_type, bill_info )
@@ -333,7 +345,7 @@ def add_missing_bills_for_yr( bill_type, bill_info, yr ):
 
 ################################################################################
 # Takes qtrly bills and start from 2nd year of bills (so we can estimate growth)
-# and go through each bill allocating hte proportion of each bill to each
+# and go through each bill allocating the proportion of each bill to each
 # relevant quarter - to build more accurate totals.   Would be mostly marginal
 # accept when Gas qtrly bills have 6 per year, and we need to guess say qtr4 in
 # the future, we can't easily find corresponding bill form previous year, so
@@ -344,7 +356,8 @@ def ProportionQtrlyData( bill_type, bill_info ):
     now_yr = datetime.date.today().year
     # FIX UP CRAPPY QUARTERLY BILLING PROPORTIONS (only useful as some gas bills are 6 / year!)
     if bill_info[bill_type]['num_ann_bills']==4:
-        for yr in range( bill_info[bill_type]['first_bill_year'], now_yr+1):
+        for yr in range( bill_info[bill_type]['first_bill_year'], END_YEAR+1):
+            if yr in bill_info[bill_type]['year']:
                 for b in bill_info[bill_type]['year'][yr]:
                     pb = find_previous_bill( bill_type, bill_info, b['bill_date'] )
                     if not pb:
@@ -368,7 +381,7 @@ def derive_ann_growth( bill_type, bill_info ):
             continue;
 
         # just going to make sure we dont use estimated data in the last year of real data - can skew growths
-        if yr == bill_info[bill_type]['last_real_bill_year']:
+        if yr == bill_info[bill_type]['last_real_bill_year'] or bill_info[bill_type]['num_ann_bills'] ==1:
             skip_yr=False
             for b in bill_info[bill_type]['year'][yr]:
                 if b['estimated']:
@@ -397,24 +410,37 @@ def derive_ann_growth( bill_type, bill_info ):
     avg_growth = 0
     max_growth = 0
     count = 0
+    simple_first_yr=0
+    simple_last_yr=0
     # start from year after first bill, so we can see annual growth from the following year onwards
     for yr in range( bill_info[bill_type]['first_bill_year']+1, now_yr+1):
         # if full data sets for consecutive years, work out annual growth stats
         if yr-1 in total and yr in total:
+            if simple_first_yr==0:
+                simple_first_yr=yr
             growth = (total[yr] - total[yr-1]) / total[yr-1] * 100
             avg_growth += growth
             count += 1
+            simple_last_yr=yr
             if growth < min_growth:
                 min_growth = growth
             if growth > max_growth:
                 max_growth = growth
+    # data to work with
     if count:
-        ##  print( f"Before sanity check, min={min_growth}, avg={avg_growth/count}, max_growth={max_growth}" )
-        # HACK FOR SANITY SAKE NOW - bills wont decrease normally, and 10% is unlikely for sustained growth
-        if min_growth< 0: min_growth=0
-        if avg_growth< 0 or avg_growth > 10: avg_growth = 3*count
-        if max_growth>10 : max_growth = 9.99
-        set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth )
+        if simple_first_yr != simple_last_yr:
+            # calculate a simple growth with full year consecutive totals -> last - first / years
+            simple_growth=( ((total[simple_last_yr]-total[simple_first_yr])/(simple_last_yr-simple_first_yr)) / total[simple_first_yr] )*100.0
+        else:
+            # calculate a simple growth based on last - first / years - only 1 consecutive year I guess, so can't use it, use real first/last
+            if bill_info[bill_type]['first_bill_year'] != bill_info[bill_type]['last_real_bill_year']:
+                simple_growth=( ((total[bill_info[bill_type]['last_real_bill_year']]-total[bill_info[bill_type]['first_bill_year']])/(bill_info[bill_type]['last_real_bill_year']-bill_info[bill_type]['first_bill_year'])) / total[bill_info[bill_type]['first_bill_year']] )*100.0
+        set_bill_type_growth( bill_type, min_growth, avg_growth/count, max_growth, simple_growth )
+    else:
+        # okay use last - first / years to get a simple_growth, just need bills from different years
+        if bill_info[bill_type]['first_bill_year'] != bill_info[bill_type]['last_real_bill_year']:
+            simple_growth=( ((total[bill_info[bill_type]['last_real_bill_year']]-total[bill_info[bill_type]['first_bill_year']])/(bill_info[bill_type]['last_real_bill_year']-bill_info[bill_type]['first_bill_year'])) / total[bill_info[bill_type]['first_bill_year']] )*100.0
+            set_bill_type_growth( bill_type, 0, 0, 0, simple_growth )
         else:
             # failsafe (just in case fill bills failed to add enough bills to average out)
             print( f"{bill_type}: Unable to calculate growth!" )

db.py

@@ -91,6 +91,7 @@ def init_db():
                     ann_growth_min REAL,
                     ann_growth_avg REAL,
                     ann_growth_max REAL,
+                    ann_growth_simple REAL,
                     FOREIGN KEY(freq) REFERENCES bill_freq(id)
                     )''')
 
@@ -148,8 +149,8 @@ def get_finance_data():
     return dict(finance)
 
 def get_budget_data(finance_data):
-    # annual bills - health ins (5k), rates (2.4), electricity (1.5), gas (2), internet (1.6), car insurance (.7), rego (.8), house insurance (2.4), GFC (2.2), phones (.5), melb. pollen (.03), nabu casa (.1), eweka (.1)  --- noting phone is elevated presuming I also go onto Aldi plan, but that there is no family discount
-    bills = 19330
+    # annual bills - health ins (5k), rates (2.4), electricity (1.5), gas (2), internet (1.6), car insurance (.7), rego (.8), house insurance (2.4), GFC (2.2), water (1.2), eweka (.e (.7)), phones (.5), melb. pollen (.03), nabu casa (.1), eweka (.1)  --- noting phone is elevated presuming I also go onto Aldi plan, but that there is no family discount
+    bills = 21330
     BUDGET=[]
     BUDGET.append( ('Bills', f"${bills:,.2f}") )
     BUDGET.append( ('Buffer', f"${finance_data['CBA']*finance_data['CBA_price']+finance_data['TLS']*finance_data['TLS_price']:,.2f}") )
@@ -308,6 +309,9 @@ def get_bill_freqs():
 def new_bill( bill_type, amount, bill_date, estimated ):
     conn = connect_db(False)
     cur = conn.cursor()
+    # force delete estimates as new bill will potentially change them/growth, etc.
+    if not estimated:
+        cur.execute( f"delete from bill_data where estimated=1" )
     cur.execute( f"insert into bill_data ( 'bill_type', 'amount', 'bill_date', 'estimated' ) values ( '{bill_type}', '{float(amount):.2f}', '{bill_date}', {estimated} )" )
     conn.commit()
     conn.close()
@@ -325,7 +329,7 @@ def insert_bill_type( bt, fq ):
     conn = connect_db(False)
     cur = conn.cursor()
     print( f"fq={fq}" )
-    cur.execute( f"insert into bill_type ( 'name', 'freq', 'ann_growth_min', 'ann_growth_avg', 'ann_growth_max' ) values ( '{bt}', {fq}, 0, 0, 0 )" )
+    cur.execute( f"insert into bill_type ( 'name', 'freq', 'ann_growth_min', 'ann_growth_avg', 'ann_growth_max', 'ann_growth_simple' ) values ( '{bt}', {fq}, 0, 0, 0, 0 )" )
     conn.commit()
     conn.close()
     return
@@ -354,10 +358,10 @@ def delete_bill_type( id ):
     conn.close()
     return
 
-def set_bill_type_growth( id, min_g, avg_g, max_g ):
+def set_bill_type_growth( id, min_g, avg_g, max_g, simple_g ):
     conn = connect_db(False)
     cur = conn.cursor()
-    cur.execute( f"update bill_type set ann_growth_min='{min_g}', ann_growth_avg ='{avg_g}', ann_growth_max='{max_g}'  where id = {id}" )
+    cur.execute( f"update bill_type set ann_growth_min={min_g}, ann_growth_avg ={avg_g}, ann_growth_max={max_g}, ann_growth_simple= {simple_g} where id = {id}" )
     conn.commit()
     conn.close()
     return

templates/bills.html

@@ -35,10 +35,10 @@
             <button id="canc-bill-type" class="new-bill-type-class px-0 col-1 btn btn-danger bg-danger-subtle text-danger d-none" onClick="CancelNewBillType()"><span class="bi bi-x"> Cancel</span></button>
         </div>
         <div class="row">
-            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0 h-100"><br>Name</ ></div>
-            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0 h-100"><br>Frequency</ ></div>
-            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0">Annual Growth Est (min/avg/max)</ ></div>
-            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0 h-100"><br>Actions</ ></div>
+            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0">Name</ ></div>
+            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0">Frequency</ ></div>
+            <div class="px-0 col-4"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0">Annual Growth Est (min/avg/max/simple)</ ></div>
+            <div class="px-0 col-2"><label class="form-control text-center border-0 fw-bold bg-body-tertiary rounded-0">Actions</ ></div>
         </div>
         {% for bt in bill_types %}
             <div class="row">
@@ -52,26 +52,32 @@
                     </select>
                 </div>
                 <script>$('#bill-type-freq-{{bt.id}}').val( {{bt.freq}} );</script>
-                <div class="px-0 col-2">
+                <div class="px-0 col-4">
                     <div class="btn-group w-100" role="group">
                         <input type="radio" class="btn-check" name="growth-{{bt.id}}" id="min-{{bt.id}}" autocomplete="off"
                             onChange="UseGrowth({{bt.id}}, 'min')" {% if bt.which_growth == 'min' %}checked{% endif %}>
-                        <label class="btn btn-outline-secondary" for="min-{{bt.id}}">
-                        {% if bt.ann_growth_min < 10 %} &nbsp; {% endif %}
-                        {{'%.2f'|format(bt.ann_growth_min)}}
+                        <label class="btn btn-outline-secondary font-monospace d-inline-block text-end" for="min-{{bt.id}}" style="width: 6ch;">
+                        {% if bt.ann_growth_min> 0 and bt.ann_growth_min < 10 %} &nbsp; {% endif %}
+                        {{'%5.2f'|format(bt.ann_growth_min)}}
                         </label>
                         <input type="radio" class="btn-check" name="growth-{{bt.id}}" id="avg-{{bt.id}}" autocomplete="off"
                             onChange="UseGrowth({{bt.id}}, 'avg')" {% if bt.which_growth == 'avg' %}checked{% endif %}>
-                        <label class="btn btn-outline-secondary" for="avg-{{bt.id}}">
+                        <label class="btn btn-outline-secondary font-monospace d-inline-block text-end" for="avg-{{bt.id}}" style="width: 6ch;">
                         {% if bt.ann_growth_avg < 10 %} &nbsp; {% endif %}
                         {{'%.2f'|format(bt.ann_growth_avg)}}
                         </label>
                         <input type="radio" class="btn-check" name="growth-{{bt.id}}" id="max-{{bt.id}}" autocomplete="off"
                             onChange="UseGrowth({{bt.id}}, 'max')" {% if bt.which_growth == 'max' %}checked{% endif %}>
-                        <label class="btn btn-outline-secondary" for="max-{{bt.id}}">
+                        <label class="btn btn-outline-secondary font-monospace d-inline-block text-end" for="max-{{bt.id}}" style="width: 6ch;">
                         {% if bt.ann_growth_max < 10 %} &nbsp; {% endif %}
                         {{'%.2f'|format(bt.ann_growth_max)}}
                         </label>
+                        <input type="radio" class="btn-check" name="growth-{{bt.id}}" id="simple-{{bt.id}}" autocomplete="off"
+                            onChange="UseGrowth({{bt.id}}, 'simple')" {% if bt.which_growth == 'simple' %}checked{% endif %}>
+                        <label class="btn btn-outline-secondary font-monospace d-inline-block text-end" for="simple-{{bt.id}}" style="width: 6ch;">
+                        {% if bt.ann_growth_simple < 10 %} &nbsp; {% endif %}
+                        {{'%.2f'|format(bt.ann_growth_simple)}}
+                        </label>
                     </div>
                 </div>
                 <button id="bill-type-chg-{{bt.id}}" class="px-0 col-1 btn btn-success bg-success-subtle text-success" onClick="StartUpdateBillType( {{bt.id}} )"><span class="bi bi-pencil-square"> Change</button>
@@ -80,6 +86,20 @@
                 <button id="bill-type-canc-{{bt.id}}" class="px-0 col-1 btn btn-danger bg-danger-subtle text-danger d-none" onClick="CancelUpdateBillType({{bt.id}}, '{{bt.name}}')"><span class="bi bi-x"> Cancel</button>
             </div>
         {% endfor %}
+        {% set total=namespace( sum=0 ) %}
+        {% for bd in bill_data %}
+            {% if '2025' in bd['bill_date'] %}
+                {% set total.sum = total.sum + bd['amount'] %}
+            {% endif %}
+        {% endfor %}
+        <div class="row">
+            <div class="pt-4 col text-end display-6">
+                Total bills in 2025:
+            </div>
+            <div class="pt-4 col display-6 text-primary">
+                ${{total.sum}}
+            </div>
+        </div>
     </div>
 
     <!-- right-hand-side, bill types (e.g. gas, phone, etc.) -->

templates/index.html

@@ -259,6 +259,10 @@
             // Highcharts configuration
             Highcharts.chart('container', {
                 chart: { type: 'line' },
+                colors: [
+                    'orange', // Custom color 1
+                    'cyan', // Custom color 2
+                ],
                 title: { text: 'Savings Over Time' },
                 xAxis: {
                     type: 'datetime',