Collect account balances, optimized.

app/Http/Controllers/DebugController.php

@@ -157,6 +157,10 @@ class DebugController extends Controller
         return view('debug', compact('table', 'now', 'logContent'));
     }
 
+    public function apiTest() {
+        return view('test.api-test');
+    }
+
     private function generateTable(): string
     {
         // system information:

app/Support/JsonApi/Enrichments/AccountEnrichment.php

@@ -51,7 +51,7 @@ use Override;
  */
 class AccountEnrichment implements EnrichmentInterface
 {
-    private array               $accountIds;
+    private array $ids;
     private array $accountTypeIds;
     private array               $accountTypes;
     private Collection          $collection;
@@ -66,13 +66,14 @@ class AccountEnrichment implements EnrichmentInterface
     private array               $lastActivities;
     private ?Carbon             $date             = null;
     private bool                $convertToPrimary = false;
+    private array $balances = [];
 
     /**
      * TODO The account enricher must do conversion from and to the primary currency.
      */
     public function __construct()
     {
-        $this->accountIds       = [];
+        $this->ids             = [];
         $this->openingBalances = [];
         $this->currencies       = [];
         $this->accountTypeIds   = [];
@@ -105,7 +106,7 @@ class AccountEnrichment implements EnrichmentInterface
 
         // prep local fields
         $this->collection = $collection;
-        $this->collectAccountIds();
+        $this->collectIds();
         $this->getAccountTypes();
         $this->collectMetaData();
         $this->collectNotes();
@@ -118,14 +119,14 @@ class AccountEnrichment implements EnrichmentInterface
         return $this->collection;
     }
 
-    private function collectAccountIds(): void
+    private function collectIds(): void
     {
         /** @var Account $account */
         foreach ($this->collection as $account) {
-            $this->accountIds[]     = (int) $account->id;
+            $this->ids[]            = (int) $account->id;
             $this->accountTypeIds[] = (int) $account->account_type_id;
         }
-        $this->accountIds     = array_unique($this->accountIds);
+        $this->ids            = array_unique($this->ids);
         $this->accountTypeIds = array_unique($this->accountTypeIds);
     }
 
@@ -142,7 +143,7 @@ class AccountEnrichment implements EnrichmentInterface
     private function collectMetaData(): void
     {
         $set                 = AccountMeta::whereIn('name', ['is_multi_currency', 'include_net_worth', 'currency_id', 'account_role', 'account_number', 'BIC', 'liability_direction', 'interest', 'interest_period', 'current_debt'])
-            ->whereIn('account_id', $this->accountIds)
+            ->whereIn('account_id', $this->ids)
             ->get(['account_meta.id', 'account_meta.account_id', 'account_meta.name', 'account_meta.data'])->toArray()
         ;
 
@@ -167,7 +168,7 @@ class AccountEnrichment implements EnrichmentInterface
 
     private function collectNotes(): void
     {
-        $notes = Note::query()->whereIn('noteable_id', $this->accountIds)
+        $notes = Note::query()->whereIn('noteable_id', $this->ids)
             ->whereNotNull('notes.text')
             ->where('notes.text', '!=', '')
             ->where('noteable_type', Account::class)->get(['notes.noteable_id', 'notes.text'])->toArray()
@@ -180,7 +181,7 @@ class AccountEnrichment implements EnrichmentInterface
 
     private function collectLocations(): void
     {
-        $locations = Location::query()->whereIn('locatable_id', $this->accountIds)
+        $locations = Location::query()->whereIn('locatable_id', $this->ids)
             ->where('locatable_type', Account::class)->get(['locations.locatable_id', 'locations.latitude', 'locations.longitude', 'locations.zoom_level'])->toArray()
         ;
         foreach ($locations as $location) {
@@ -234,13 +235,10 @@ class AccountEnrichment implements EnrichmentInterface
 
     private function appendCollectedData(): void
     {
-        $notes            = $this->notes;
+        $this->collection = $this->collection->map(function  (Account $item) {
-        $openingBalances  = $this->openingBalances;
+            $id = (int) $item->id;
-        $locations        = $this->locations;
-        $lastActivities   = $this->lastActivities;
-        $this->collection = $this->collection->map(function (Account $item) use ($notes, $openingBalances, $locations, $lastActivities) {
             $item->full_account_type = $this->accountTypes[(int) $item->account_type_id] ?? null;
-            $accountMeta             = [
+            $meta             = [
                 'currency'             => null,
                 'location'             => [
                     'latitude'   => null,
@@ -248,45 +246,50 @@ class AccountEnrichment implements EnrichmentInterface
                     'zoom_level' => null,
                 ],
                 'opening_balance_date' => null,
+                'opening_balance_amount' => null,
+                'account_number'=> null,
+                'notes' => $notes[$id] ?? null,
+                'last_activity' => $this->lastActivities[$id] ?? null,
             ];
-            if (array_key_exists((int) $item->id, $this->meta)) {
+
-                foreach ($this->meta[(int) $item->id] as $name => $value) {
+            // if location, add location:
-                    $accountMeta[$name] = $value;
+            if (array_key_exists($id, $this->locations)) {
+                $meta['location'] = $this->locations[$id];
+            }
+            if (array_key_exists($id, $this->meta)) {
+                foreach ($this->meta[$id] as $name => $value) {
+                    $meta[$name] = $value;
                 }
             }
             // also add currency, if present.
-            if (array_key_exists('currency_id', $accountMeta)) {
+            if (array_key_exists('currency_id', $meta)) {
-                $currencyId              = (int) $accountMeta['currency_id'];
+                $currencyId              = (int) $meta['currency_id'];
-                $accountMeta['currency'] = $this->currencies[$currencyId];
+                $meta['currency'] = $this->currencies[$currencyId];
             }
 
-            // if notes, add notes.
+            if (array_key_exists($id, $this->openingBalances)) {
-            if (array_key_exists($item->id, $notes)) {
+                $meta['opening_balance_date']   = $this->openingBalances[$id]['date'];
-                $accountMeta['notes'] = $notes[$item->id];
+                $meta['opening_balance_amount'] = $this->openingBalances[$id]['amount'];
-            }
-            // if opening balance, add opening balance
-            if (array_key_exists($item->id, $openingBalances)) {
-                $accountMeta['opening_balance_date']   = $openingBalances[$item->id]['date'];
-                $accountMeta['opening_balance_amount'] = $openingBalances[$item->id]['amount'];
             }
 
             // add balances
             // get currencies:
             $currency                = $this->primaryCurrency; // assume primary currency
-            if (null !== $accountMeta['currency']) {
+            if (null !== $meta['currency']) {
-                $currency = $accountMeta['currency'];
+                $currency = $meta['currency'];
             }
 
             // get the current balance:
             $date                    = $this->getDate();
-            $finalBalance            = Steam::finalAccountBalance($item, $date, $this->primaryCurrency, $this->convertToPrimary);
+            //$finalBalance            = Steam::finalAccountBalance($item, $date, $this->primaryCurrency, $this->convertToPrimary);
+            $finalBalance = $this->balances[$id];
             Log::debug(sprintf('Call finalAccountBalance(%s) with date/time "%s"', var_export($this->convertToPrimary, true), $date->toIso8601String()), $finalBalance);
 
             // collect current balances:
             $currentBalance          = Steam::bcround($finalBalance[$currency->code] ?? '0', $currency->decimal_places);
-            $openingBalance          = Steam::bcround($accountMeta['opening_balance_amount'] ?? '0', $currency->decimal_places);
+            $openingBalance          = Steam::bcround($meta['opening_balance_amount'] ?? '0', $currency->decimal_places);
             $virtualBalance          = Steam::bcround($account->virtual_balance ?? '0', $currency->decimal_places);
-            $debtAmount              = $accountMeta['current_debt'] ?? null;
+            $debtAmount              = $meta['current_debt'] ?? null;
 
             // set some pc_ default values to NULL:
             $pcCurrentBalance        = null;
@@ -311,11 +314,11 @@ class AccountEnrichment implements EnrichmentInterface
             }
 
             // set opening balance(s) to NULL if the date is null
-            if (null === $accountMeta['opening_balance_date']) {
+            if (null === $meta['opening_balance_date']) {
                 $openingBalance   = null;
                 $pcOpeningBalance = null;
             }
-            $accountMeta['balances'] = [
+            $meta['balances'] = [
                 'current_balance'    => $currentBalance,
                 'pc_current_balance' => $pcCurrentBalance,
                 'opening_balance'    => $openingBalance,
@@ -326,16 +329,7 @@ class AccountEnrichment implements EnrichmentInterface
                 'pc_debt_amount'     => $pcDebtAmount,
             ];
             // end add balances
-
+            $item->meta              = $meta;
-
-            // if location, add location:
-            if (array_key_exists($item->id, $locations)) {
-                $accountMeta['location'] = $locations[$item->id];
-            }
-            if (array_key_exists($item->id, $lastActivities)) {
-                $accountMeta['last_activity'] = $lastActivities[$item->id];
-            }
-            $item->meta              = $accountMeta;
 
             return $item;
         });
@@ -343,10 +337,12 @@ class AccountEnrichment implements EnrichmentInterface
 
     private function collectLastActivities(): void
     {
-        $this->lastActivities = Steam::getLastActivities($this->accountIds);
+        $this->lastActivities = Steam::getLastActivities($this->ids);
     }
 
-    private function collectBalances(): void {}
+    private function collectBalances(): void {
+        $this->balances = Steam::finalAccountsBalanceOptimized($this->collection, $this->getDate(), $this->primaryCurrency, $this->convertToPrimary);
+    }
 
     public function setDate(?Carbon $date): void
     {

app/Support/Steam.php

@@ -24,8 +24,10 @@ declare(strict_types=1);
 namespace FireflyIII\Support;
 
 use Carbon\Carbon;
+use Exception;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\Account;
+use FireflyIII\Models\AccountMeta;
 use FireflyIII\Models\Transaction;
 use FireflyIII\Models\TransactionCurrency;
 use FireflyIII\Support\Facades\Amount;
@@ -34,11 +36,9 @@ use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
-use Exception;
 use ValueError;
-
-use function Safe\preg_replace;
 use function Safe\parse_url;
+use function Safe\preg_replace;
 
 /**
  * Class Steam.
@@ -264,8 +264,7 @@ class Steam
                              'transactions.transaction_currency_id',
                              DB::raw('SUM(transactions.amount) AS sum_of_day'),
                            ]
-            )
+                       );
-        ;
 
         $currentBalance = $startBalance;
         $converter      = new ExchangeRateConverter();
@@ -318,6 +317,70 @@ class Steam
         return $balances;
     }
 
+    public function finalAccountsBalanceOptimized(Collection $accounts, Carbon $date, ?TransactionCurrency $primary = null, ?bool $convertToPrimary = null): array
+    {
+        $result           = [];
+        $convertToPrimary = $convertToPrimary ?? Amount::convertToPrimary();
+        $primary          = $primary ?? Amount::getPrimaryCurrency();
+        $currencies       = $this->getCurrencies($accounts);
+
+        // balance(s) in all currencies for ALL accounts.
+        $array = Transaction::whereIn('account_id', $accounts->pluck('id')->toArray())
+                            ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
+                            ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id')
+                            ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s'))
+                            ->get(['transactions.account_id', 'transaction_currencies.code', 'transactions.amount'])->toArray();
+
+        /** @var Account $account */
+        foreach ($accounts as $account) {
+            // filter array back to this account:
+            $filtered = array_filter($array, function ($item) use ($account) {
+                return (int)$item['account_id'] === $account->id;
+            });
+            $currency = $currencies[$account->id];
+            // this array is PER account, so we wait a bit before we change code here.
+            $return = [
+                'pc_balance' => '0',
+                'balance'    => '0', // this key is overwritten right away, but I must remember it is always created.
+            ];
+
+            // balance(s) in all currencies.
+            $others = $this->groupAndSumTransactions($filtered, 'code', 'amount');
+            // Log::debug('All balances are (joined)', $others);
+            // if there is no request to convert, take this as "balance" and "pc_balance".
+            $return['balance'] = $others[$currency->code] ?? '0';
+            if (!$convertToPrimary) {
+                unset($return['pc_balance']);
+                // Log::debug(sprintf('Set balance to %s, unset pc_balance', $return['balance']));
+            }
+            // if there is a request to convert, convert to "pc_balance" and use "balance" for whichever amount is in the primary currency.
+            if ($convertToPrimary) {
+                $return['pc_balance'] = $this->convertAllBalances($others, $primary, $date); // todo sum all and convert.
+                // Log::debug(sprintf('Set pc_balance to %s', $return['pc_balance']));
+            }
+
+            // either way, the balance is always combined with the virtual balance:
+            $virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance);
+
+            if ($convertToPrimary) {
+                // the primary currency balance is combined with a converted virtual_balance:
+                $converter            = new ExchangeRateConverter();
+                $pcVirtualBalance     = $converter->convert($currency, $primary, $date, $virtualBalance);
+                $return['pc_balance'] = bcadd($pcVirtualBalance, $return['pc_balance']);
+                // Log::debug(sprintf('Primary virtual balance makes the primary total %s', $return['pc_balance']));
+            }
+            if (!$convertToPrimary) {
+                // if not, also increase the balance + primary balance for consistency.
+                $return['balance'] = bcadd($return['balance'], $virtualBalance);
+                // Log::debug(sprintf('Virtual balance makes the (primary currency) total %s', $return['balance']));
+            }
+            $final                = array_merge($return, $others);
+            $result[$account->id] = $final;
+            // Log::debug('Final balance is', $final);
+        }
+        return $result;
+    }
+
     /**
      * Returns smaller than or equal to, so be careful with END OF DAY.
      *
@@ -340,7 +403,7 @@ class Steam
         if ($cache->has()) {
             Log::debug(sprintf('CACHED finalAccountBalance(#%d, %s)', $account->id, $date->format('Y-m-d H:i:s')));
 
-            return $cache->get();
+            //return $cache->get();
         }
         // Log::debug(sprintf('finalAccountBalance(#%d, %s)', $account->id, $date->format('Y-m-d H:i:s')));
         if (null === $convertToPrimary) {
@@ -369,8 +432,7 @@ class Steam
                           ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
                           ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id')
                           ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s'))
-            ->get(['transaction_currencies.code', 'transactions.amount'])->toArray()
+                          ->get(['transaction_currencies.code', 'transactions.amount'])->toArray();
-        ;
         $others = $this->groupAndSumTransactions($array, 'code', 'amount');
         // Log::debug('All balances are (joined)', $others);
         // if there is no request to convert, take this as "balance" and "pc_balance".
@@ -689,4 +751,42 @@ class Steam
 
         return $amount;
     }
+
+    private function getCurrencies(Collection $accounts): array
+    {
+        $currencies         = [];
+        $accountCurrencies  = [];
+        $accountPreferences = [];
+        $primary            = Amount::getPrimaryCurrency();
+
+        $ids    = $accounts->pluck('id')->toArray();
+        $result = AccountMeta::whereIn('account_id', $ids)->where('name', 'currency_id')->get();
+        /** @var AccountMeta $item */
+        foreach ($result as $item) {
+            $accountPreferences[(int)$item->account_id] = (int)$item->data;
+        }
+        // collect those currencies.
+        $set = TransactionCurrency::whereIn('id', $accountPreferences)->get();
+        foreach ($set as $item) {
+            $currencies[$item->id] = $item;
+        }
+
+        /** @var Account $account */
+        foreach ($accounts as $account) {
+            $accountId       = $account->id;
+            $currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency'];
+            if ($currencyPresent) {
+                $currencyId                    = $account->meta['currency']->id;
+                $currencies[$currencyId]       ??= $account->meta['currency'];
+                $accountCurrencies[$accountId] = $account->meta['currency'];
+            }
+            if (!$currencyPresent && !array_key_exists($account->id, $accountPreferences)) {
+                $accountCurrencies[$accountId] = $primary;
+            }
+            if (!$currencyPresent && array_key_exists($account->id, $accountPreferences)) {
+                $accountCurrencies[$account->id] = $currencies[$accountPreferences[$account->id]];
+            }
+        }
+        return $accountCurrencies;
+    }
 }

app/Transformers/AccountTransformer.php

@@ -143,7 +143,7 @@ class AccountTransformer extends AbstractTransformer
             'notes'                           => $account->meta['notes'] ?? null,
             'monthly_payment_date'            => $monthlyPaymentDate,
             'credit_card_type'                => $creditCardType,
-            'account_number'                  => $account->meta['account_number'] ?? null,
+            'account_number'                  => $account->meta['account_number'],
             'iban'                            => '' === $account->iban ? null : $account->iban,
             'bic'                             => $account->meta['BIC'] ?? null,
             'opening_balance_date'            => $openingBalanceDate,
@@ -155,7 +155,7 @@ class AccountTransformer extends AbstractTransformer
             'longitude'                       => $longitude,
             'latitude'                        => $latitude,
             'zoom_level'                      => $zoomLevel,
-            'last_activity'                   => array_key_exists('last_activity', $account->meta) ? $account->meta['last_activity']->toAtomString() : null,
+            'last_activity'                   => $account->meta['last_activity']?->toAtomString(),
             'links'                           => [
                 [
                     'rel' => 'self',

resources/views/test/api-test.twig

@@ -0,0 +1,33 @@
+<html>
+<head>
+    <title>Firefly III API test</title>
+    <base href="{{ route('index', null, true) }}/">
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="csrf-token" content="{{ csrf_token() }}">
+</head>
+<body>
+<p>
+    Hi there,
+</p>
+<p>
+    This page is created to do some basic API testing. It's not very exciting, is it?
+</p>
+<script src="v1/js/app.js?v={{ FF_VERSION }}" type="text/javascript" nonce="{{ JS_NONCE }}"></script>
+<script type="text/javascript" nonce="{{ JS_NONCE }}">
+    $(function () {
+        "use strict";
+        console.log('Hello from the API test page!');
+        $.ajax({
+                   url: 'api/v1/accounts?size=50&date=2025-08-06',
+                   type: 'GET',
+                   headers: {
+                       'Content-Type': 'application/json',
+                       'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content'),
+                   },
+               });
+    });
+
+</script>
+</body>
+</html>

resources/views/test/test.twig

@@ -1,2 +0,0 @@
-list-length: {{ listLength }}
-user-email: {{ Auth.user.email }}

routes/web.php

@@ -120,6 +120,7 @@ Route::group(
         Route::get('flush', ['uses' => 'DebugController@flush', 'as' => 'flush']);
         Route::get('routes', ['uses' => 'DebugController@routes', 'as' => 'routes']);
         Route::get('debug', 'DebugController@index')->name('debug');
+        Route::get('debug/api-test', 'DebugController@apiTest')->name('api-test');
     }
 );