Additional cache columns on balances for activity view breakdowns (#2505)

* Initial schema iteration

* Add new balance components

* Add existing data migrator to backfill components

* Update calculator test assertions for new balance components

* Update flow assertions for forward calculator

* Update reverse calculator flows assumptions

* Forward calculator tests passing

* Get all calculator tests passing

* Assert flows factor

Author: zachgoll

app/data_migrations/balance_component_migrator.rb

@@ -0,0 +1,59 @@
+class BalanceComponentMigrator
+  def self.run
+    ActiveRecord::Base.transaction do
+      # Step 1: Update flows factor
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances SET
+          flows_factor = CASE WHEN a.classification = 'asset' THEN 1 ELSE -1 END
+        FROM accounts a
+        WHERE a.id = balances.account_id
+      SQL
+
+      # Step 2: Set start values using LOCF (Last Observation Carried Forward)
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances b1
+        SET
+          start_cash_balance = COALESCE(prev.cash_balance, 0),
+          start_non_cash_balance = COALESCE(prev.balance - prev.cash_balance, 0)
+        FROM balances b1_inner
+        LEFT JOIN LATERAL (
+          SELECT
+            b2.cash_balance,
+            b2.balance
+          FROM balances b2
+          WHERE b2.account_id = b1_inner.account_id
+          AND b2.currency = b1_inner.currency
+          AND b2.date < b1_inner.date
+          ORDER BY b2.date DESC
+          LIMIT 1
+        ) prev ON true
+        WHERE b1.id = b1_inner.id
+      SQL
+
+      # Step 3: Calculate net inflows
+      # A slight workaround to the fact that we can't easily derive inflows/outflows from our current data model, and
+      # the tradeoff not worth it since each new sync will fix it. So instead, we sum up *net* flows, and throw the signed
+      # amount in the "inflows" column, and zero-out the "outflows" column so our math works correctly with incomplete data.
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances SET
+          cash_inflows = (cash_balance - start_cash_balance) * flows_factor,
+          cash_outflows = 0,
+          non_cash_inflows = ((balance - cash_balance) - start_non_cash_balance) * flows_factor,
+          non_cash_outflows = 0,
+          net_market_flows = 0
+      SQL
+
+      # Verify data integrity
+      # All end_balance values should match the original balance
+      invalid_count = ActiveRecord::Base.connection.select_value(<<~SQL)
+        SELECT COUNT(*)
+        FROM balances b
+        WHERE ABS(b.balance - b.end_balance) > 0.0001
+      SQL
+
+      if invalid_count > 0
+        raise "Data migration failed validation: #{invalid_count} balances have incorrect end_balance values"
+      end
+    end
+  end
+end

app/models/balance.rb

@@ -2,8 +2,16 @@ class Balance < ApplicationRecord
   include Monetizable
 
   belongs_to :account
+
   validates :account, :date, :balance, presence: true
-  monetize :balance, :cash_balance
+  validates :flows_factor, inclusion: { in: [ -1, 1 ] }
+
+  monetize :balance, :cash_balance,
+           :start_cash_balance, :start_non_cash_balance, :start_balance,
+           :cash_inflows, :cash_outflows, :non_cash_inflows, :non_cash_outflows, :net_market_flows,
+           :cash_adjustments, :non_cash_adjustments,
+           :end_cash_balance, :end_non_cash_balance, :end_balance
+
   scope :in_period, ->(period) { period.nil? ? all : where(date: period.date_range) }
   scope :chronological, -> { order(:date) }
 end

app/models/balance/base_calculator.rb

@@ -15,8 +15,8 @@ def sync_cache
     end
 
     def holdings_value_for_date(date)
-      holdings = sync_cache.get_holdings(date)
-      holdings.sum(&:amount)
+      @holdings_value_for_date ||= {}
+      @holdings_value_for_date[date] ||= sync_cache.get_holdings(date).sum(&:amount)
     end
 
     def derive_cash_balance_on_date_from_total(total_balance:, date:)
@@ -29,6 +29,67 @@ def derive_cash_balance_on_date_from_total(total_balance:, date:)
       end
     end
 
+    def cash_adjustments_for_date(start_cash, net_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type != :non_cash
+
+      valuation.amount - start_cash - net_cash_flows
+    end
+
+    def non_cash_adjustments_for_date(start_non_cash, non_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type == :non_cash
+
+      valuation.amount - start_non_cash - non_cash_flows
+    end
+
+    # If holdings value goes from $100 -> $200 (change_holdings_value is $100)
+    # And non-cash flows (i.e. "buys") for day are +$50 (net_buy_sell_value is $50)
+    # That means value increased by $100, where $50 of that is due to the change in holdings value, and $50 is due to the buy/sell
+    def market_value_change_on_date(date, flows)
+      return 0 unless account.balance_type == :investment
+
+      start_of_day_holdings_value = holdings_value_for_date(date.prev_day)
+      end_of_day_holdings_value = holdings_value_for_date(date)
+
+      change_holdings_value = end_of_day_holdings_value - start_of_day_holdings_value
+      net_buy_sell_value = flows[:non_cash_inflows] - flows[:non_cash_outflows]
+
+      change_holdings_value - net_buy_sell_value
+    end
+
+    def flows_for_date(date)
+      entries = sync_cache.get_entries(date)
+
+      cash_inflows = 0
+      cash_outflows = 0
+      non_cash_inflows = 0
+      non_cash_outflows = 0
+
+      txn_inflow_sum = entries.select { |e| e.amount < 0 && e.transaction? }.sum(&:amount)
+      txn_outflow_sum = entries.select { |e| e.amount >= 0 && e.transaction? }.sum(&:amount)
+
+      trade_cash_inflow_sum = entries.select { |e| e.amount < 0 && e.trade? }.sum(&:amount)
+      trade_cash_outflow_sum = entries.select { |e| e.amount >= 0 && e.trade? }.sum(&:amount)
+
+      if account.balance_type == :non_cash && account.accountable_type == "Loan"
+        non_cash_inflows = txn_inflow_sum.abs
+        non_cash_outflows = txn_outflow_sum
+      elsif account.balance_type != :non_cash
+        cash_inflows = txn_inflow_sum.abs + trade_cash_inflow_sum.abs
+        cash_outflows = txn_outflow_sum + trade_cash_outflow_sum
+
+        # Trades are inverse (a "buy" is outflow of cash, but "inflow" of non-cash, aka "holdings")
+        non_cash_outflows = trade_cash_inflow_sum.abs
+        non_cash_inflows = trade_cash_outflow_sum
+      end
+
+      {
+        cash_inflows: cash_inflows,
+        cash_outflows: cash_outflows,
+        non_cash_inflows: non_cash_inflows,
+        non_cash_outflows: non_cash_outflows
+      }
+    end
+
     def derive_cash_balance(cash_balance, date)
       entries = sync_cache.get_entries(date)
 
@@ -57,13 +118,23 @@ def signed_entry_flows(entries)
       raise NotImplementedError, "Directional calculators must implement this method"
     end
 
-    def build_balance(date:, cash_balance:, non_cash_balance:)
+    def build_balance(date:, **args)
       Balance.new(
         account_id: account.id,
+        currency: account.currency,
         date: date,
-        balance: non_cash_balance + cash_balance,
-        cash_balance: cash_balance,
-        currency: account.currency
+        balance: args[:balance],
+        cash_balance: args[:cash_balance],
+        start_cash_balance: args[:start_cash_balance] || 0,
+        start_non_cash_balance: args[:start_non_cash_balance] || 0,
+        cash_inflows: args[:cash_inflows] || 0,
+        cash_outflows: args[:cash_outflows] || 0,
+        non_cash_inflows: args[:non_cash_inflows] || 0,
+        non_cash_outflows: args[:non_cash_outflows] || 0,
+        cash_adjustments: args[:cash_adjustments] || 0,
+        non_cash_adjustments: args[:non_cash_adjustments] || 0,
+        net_market_flows: args[:net_market_flows] || 0,
+        flows_factor: account.classification == "asset" ? 1 : -1
       )
     end
 end

app/models/balance/forward_calculator.rb

@@ -2,13 +2,13 @@ class Balance::ForwardCalculator < Balance::BaseCalculator
   def calculate
     Rails.logger.tagged("Balance::ForwardCalculator") do
       start_cash_balance = derive_cash_balance_on_date_from_total(
-        total_balance: account.opening_anchor_balance,
+        total_balance: 0,
         date: account.opening_anchor_date
       )
-      start_non_cash_balance = account.opening_anchor_balance - start_cash_balance
+      start_non_cash_balance = 0
 
       calc_start_date.upto(calc_end_date).map do |date|
-        valuation = sync_cache.get_reconciliation_valuation(date)
+        valuation = sync_cache.get_valuation(date)
 
         if valuation
           end_cash_balance = derive_cash_balance_on_date_from_total(
@@ -21,10 +21,22 @@ def calculate
           end_non_cash_balance = derive_end_non_cash_balance(start_non_cash_balance: start_non_cash_balance, date: date)
         end
 
+        flows = flows_for_date(date)
+        market_value_change = market_value_change_on_date(date, flows)
+
         output_balance = build_balance(
           date: date,
+          balance: end_cash_balance + end_non_cash_balance,
           cash_balance: end_cash_balance,
-          non_cash_balance: end_non_cash_balance
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          cash_adjustments: cash_adjustments_for_date(start_cash_balance, flows[:cash_inflows] - flows[:cash_outflows], valuation),
+          non_cash_adjustments: non_cash_adjustments_for_date(start_non_cash_balance, flows[:non_cash_inflows] - flows[:non_cash_outflows], valuation),
+          net_market_flows: market_value_change
         )
 
         # Set values for the next iteration

app/models/balance/reverse_calculator.rb

@@ -11,6 +11,8 @@ def calculate
 
       # Calculates in reverse-chronological order (End of day -> Start of day)
       account.current_anchor_date.downto(account.opening_anchor_date).map do |date|
+        flows = flows_for_date(date)
+
         if use_opening_anchor_for_date?(date)
           end_cash_balance = derive_cash_balance_on_date_from_total(
             total_balance: account.opening_anchor_balance,
@@ -20,29 +22,30 @@ def calculate
 
           start_cash_balance = end_cash_balance
           start_non_cash_balance = end_non_cash_balance
-
-          build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
+          market_value_change = 0
         else
           start_cash_balance = derive_start_cash_balance(end_cash_balance: end_cash_balance, date: date)
           start_non_cash_balance = derive_start_non_cash_balance(end_non_cash_balance: end_non_cash_balance, date: date)
+          market_value_change = market_value_change_on_date(date, flows)
+        end
 
-          # Even though we've just calculated "start" balances, we set today equal to end of day, then use those
-          # in our next iteration (slightly confusing, but just the nature of a "reverse" sync)
-          output_balance = build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
+        output_balance = build_balance(
+          date: date,
+          balance: end_cash_balance + end_non_cash_balance,
+          cash_balance: end_cash_balance,
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          net_market_flows: market_value_change
+        )
 
-          end_cash_balance = start_cash_balance
-          end_non_cash_balance = start_non_cash_balance
+        end_cash_balance = start_cash_balance
+        end_non_cash_balance = start_non_cash_balance
 
-          output_balance
-        end
+        output_balance
       end
     end
   end
@@ -58,13 +61,6 @@ def signed_entry_flows(entries)
       account.asset? ? entry_flows : -entry_flows
     end
 
-    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
-    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
-    # explanation, see the test suite.
-    def use_opening_anchor_for_date?(date)
-      account.has_opening_anchor? && date == account.opening_anchor_date
-    end
-
     # Alias method, for algorithmic clarity
     # Derives cash balance, starting from the end-of-day, applying entries in reverse to get the start-of-day balance
     def derive_start_cash_balance(end_cash_balance:, date:)
@@ -76,4 +72,11 @@ def derive_start_cash_balance(end_cash_balance:, date:)
     def derive_start_non_cash_balance(end_non_cash_balance:, date:)
       derive_non_cash_balance(end_non_cash_balance, date, direction: :reverse)
     end
+
+    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
+    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
+    # explanation, see the test suite.
+    def use_opening_anchor_for_date?(date)
+      account.has_opening_anchor? && date == account.opening_anchor_date
+    end
 end

app/models/balance/sync_cache.rb

@@ -3,8 +3,8 @@ def initialize(account)
     @account = account
   end
 
-  def get_reconciliation_valuation(date)
-    converted_entries.find { |e| e.date == date && e.valuation? && e.valuation.reconciliation? }
+  def get_valuation(date)
+    converted_entries.find { |e| e.date == date && e.valuation? }
   end
 
   def get_holdings(date)

db/migrate/20250719121103_add_start_end_columns_to_balances.rb

@@ -0,0 +1,72 @@
+class AddStartEndColumnsToBalances < ActiveRecord::Migration[7.2]
+  def up
+    # Add new columns for balance tracking
+    add_column :balances, :start_cash_balance, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+    add_column :balances, :start_non_cash_balance, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+
+    # Flow tracking columns (absolute values)
+    add_column :balances, :cash_inflows, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+    add_column :balances, :cash_outflows, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+    add_column :balances, :non_cash_inflows, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+    add_column :balances, :non_cash_outflows, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+
+    # Market value changes
+    add_column :balances, :net_market_flows, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+
+    # Manual adjustments from valuations
+    add_column :balances, :cash_adjustments, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+    add_column :balances, :non_cash_adjustments, :decimal, precision: 19, scale: 4, null: false, default: 0.0
+
+    # Flows factor determines *how* the flows affect the balance.
+    # Inflows increase asset accounts, while inflows decrease liability accounts (reducing debt via "payment")
+    add_column :balances, :flows_factor, :integer, null: false, default: 1
+
+    # Add generated columns
+    change_table :balances do |t|
+      t.virtual :start_balance, type: :decimal, precision: 19, scale: 4, stored: true,
+        as: "start_cash_balance + start_non_cash_balance"
+
+      t.virtual :end_cash_balance, type: :decimal, precision: 19, scale: 4, stored: true,
+        as: "start_cash_balance + ((cash_inflows - cash_outflows) * flows_factor) + cash_adjustments"
+
+      t.virtual :end_non_cash_balance, type: :decimal, precision: 19, scale: 4, stored: true,
+        as: "start_non_cash_balance + ((non_cash_inflows - non_cash_outflows) * flows_factor) + net_market_flows + non_cash_adjustments"
+
+      # Postgres doesn't support generated columns depending on other generated columns,
+      # but we want the integrity of the data to happen at the DB level, so this is the full formula.
+      # Formula: (cash components) + (non-cash components)
+      t.virtual :end_balance, type: :decimal, precision: 19, scale: 4, stored: true,
+        as: <<~SQL.squish
+          (
+            start_cash_balance +
+            ((cash_inflows - cash_outflows) * flows_factor) +
+            cash_adjustments
+          ) + (
+            start_non_cash_balance +
+            ((non_cash_inflows - non_cash_outflows) * flows_factor) +
+            net_market_flows +
+            non_cash_adjustments
+          )
+        SQL
+    end
+  end
+
+  def down
+    # Remove generated columns first (PostgreSQL requirement)
+    remove_column :balances, :start_balance
+    remove_column :balances, :end_cash_balance
+    remove_column :balances, :end_non_cash_balance
+    remove_column :balances, :end_balance
+
+    # Remove new columns
+    remove_column :balances, :start_cash_balance
+    remove_column :balances, :start_non_cash_balance
+    remove_column :balances, :cash_inflows
+    remove_column :balances, :cash_outflows
+    remove_column :balances, :non_cash_inflows
+    remove_column :balances, :non_cash_outflows
+    remove_column :balances, :net_market_flows
+    remove_column :balances, :cash_adjustments
+    remove_column :balances, :non_cash_adjustments
+  end
+end