From 28962007c1c345d7f7c0fe00859590e711d8d24a Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 12 Aug 2016 09:27:09 +0200 Subject: [PATCH] More code for new importer --- app/Import/Converter/CurrencyCode.php | 2 +- app/Import/Converter/Date.php | 2 +- app/Import/Converter/INGDebetCredit.php | 2 +- app/Import/Converter/OpposingAccountIban.php | 2 +- app/Import/Converter/OpposingAccountName.php | 5 +- app/Import/ImportEntry.php | 33 +++++++-- app/Import/Importer/CsvImporter.php | 59 ++++++++------- app/Import/Specifics/AbnAmroDescription.php | 10 +++ app/Import/Specifics/RabobankDescription.php | 34 ++++++++- app/Import/Specifics/SpecificInterface.php | 7 ++ storage/database/seed.import-test.json | 76 ++++++++------------ 11 files changed, 146 insertions(+), 86 deletions(-) diff --git a/app/Import/Converter/CurrencyCode.php b/app/Import/Converter/CurrencyCode.php index 9a448eacf5..3e78e80d80 100644 --- a/app/Import/Converter/CurrencyCode.php +++ b/app/Import/Converter/CurrencyCode.php @@ -30,7 +30,7 @@ class CurrencyCode extends BasicConverter implements ConverterInterface */ public function convert($value): TransactionCurrency { - Log::debug('Going to convert ', ['value' => $value]); + Log::debug('Going to convert currency code', ['value' => $value]); /** @var CurrencyRepositoryInterface $repository */ $repository = app(CurrencyRepositoryInterface::class); diff --git a/app/Import/Converter/Date.php b/app/Import/Converter/Date.php index 134d141e10..e5ccc28e57 100644 --- a/app/Import/Converter/Date.php +++ b/app/Import/Converter/Date.php @@ -32,7 +32,7 @@ class Date extends BasicConverter implements ConverterInterface */ public function convert($value): Carbon { - Log::debug('Going to convert ', ['value' => $value]); + Log::debug('Going to convert date', ['value' => $value]); Log::debug('Format: ', ['format' => $this->config['date-format']]); try { $date = Carbon::createFromFormat($this->config['date-format'], $value); diff --git a/app/Import/Converter/INGDebetCredit.php b/app/Import/Converter/INGDebetCredit.php index a6c4da9d69..378fad1130 100644 --- a/app/Import/Converter/INGDebetCredit.php +++ b/app/Import/Converter/INGDebetCredit.php @@ -29,7 +29,7 @@ class INGDebetCredit extends BasicConverter implements ConverterInterface */ public function convert($value) { - Log::debug('Going to convert ', ['value' => $value]); + Log::debug('Going to convert ing debet credit', ['value' => $value]); if ($value === 'Af') { Log::debug('Return -1'); diff --git a/app/Import/Converter/OpposingAccountIban.php b/app/Import/Converter/OpposingAccountIban.php index 27487b4542..bf227412da 100644 --- a/app/Import/Converter/OpposingAccountIban.php +++ b/app/Import/Converter/OpposingAccountIban.php @@ -31,7 +31,7 @@ class OpposingAccountIban extends BasicConverter implements ConverterInterface public function convert($value): Account { $value = trim($value); - Log::debug('Going to convert ', ['value' => $value]); + Log::debug('Going to convert opposing IBAN', ['value' => $value]); if (strlen($value) === 0) { $this->setCertainty(0); diff --git a/app/Import/Converter/OpposingAccountName.php b/app/Import/Converter/OpposingAccountName.php index bf872bec82..973a8b4da5 100644 --- a/app/Import/Converter/OpposingAccountName.php +++ b/app/Import/Converter/OpposingAccountName.php @@ -31,10 +31,11 @@ class OpposingAccountName extends BasicConverter implements ConverterInterface public function convert($value): Account { $value = trim($value); - Log::debug('Going to convert ', ['value' => $value]); + Log::debug('Going to convert opposing account name', ['value' => $value]); if (strlen($value) === 0) { - $value = '(empty account name)'; + $this->setCertainty(0); + return new Account; } /** @var AccountCrudInterface $repository */ diff --git a/app/Import/ImportEntry.php b/app/Import/ImportEntry.php index 29a49effae..b982e806e0 100644 --- a/app/Import/ImportEntry.php +++ b/app/Import/ImportEntry.php @@ -27,12 +27,18 @@ class ImportEntry { /** @var array */ public $certain = []; + /** @var string */ + public $externalID; /** @var array */ public $fields = []; /** @var User */ public $user; /** @var bool */ public $valid = true; + + /** @var int */ + private $amountMultiplier = 0; + /** @var array */ private $validFields = ['amount', @@ -74,16 +80,13 @@ class ImportEntry /** * @param string $role - * @param string $value * @param int $certainty * @param $convertedValue * * @throws FireflyException */ - public function importValue(string $role, string $value, int $certainty, $convertedValue) + public function importValue(string $role, int $certainty, $convertedValue) { - Log::debug('Going to import', ['role' => $role, 'value' => $value, 'certainty' => $certainty]); - switch ($role) { default: Log::error('Import entry cannot handle object.', ['role' => $role]); @@ -96,6 +99,8 @@ class ImportEntry */ $this->setFloat('amount', $convertedValue, $certainty); + $this->applyMultiplier('amount'); // if present. + return; case 'account-id': case 'account-iban': @@ -139,6 +144,9 @@ class ImportEntry case 'date-process': $this->setDate('date-process', $convertedValue, $certainty); break; + case 'sepa-ct-id': + case 'sepa-db': + case 'sepa-ct-op': case'description': $this->setAppendableString('description', $convertedValue); break; @@ -147,12 +155,13 @@ class ImportEntry case 'ing-debet-credit': case 'rabo-debet-credit': $this->manipulateFloat('amount', 'multiply', $convertedValue); + $this->applyMultiplier('amount'); // if present. break; case 'tags-comma': case 'tags-space': $this->appendCollection('tags', $convertedValue); case 'external-id': - // ignore for now. + $this->externalID = $convertedValue; break; } @@ -178,6 +187,16 @@ class ImportEntry $this->fields[$field] = $this->fields[$field]->merge($convertedValue); } + /** + * @param string $field + */ + private function applyMultiplier(string $field) + { + if ($this->fields[$field] != 0 && $this->amountMultiplier != 0) { + $this->fields[$field] = $this->fields[$field] * $this->amountMultiplier; + } + } + /** * @param string $field * @param string $action @@ -191,8 +210,8 @@ class ImportEntry default: Log::error('Cannot handle manipulateFloat', ['field' => $field, 'action' => $action]); throw new FireflyException('Cannot manipulateFloat with action ' . $action); - case'multiply': - $this->fields[$field] = $this->fields[$field] * $convertedValue; + case 'multiply': + $this->amountMultiplier = $convertedValue; break; } } diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 38ec2d272c..ff9845dd5c 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -11,10 +11,9 @@ declare(strict_types = 1); namespace FireflyIII\Import\Importer; -use FireflyIII\Crud\Account\AccountCrud; use FireflyIII\Import\Converter\ConverterInterface; use FireflyIII\Import\ImportEntry; -use FireflyIII\Models\Account; +use FireflyIII\Import\Specifics\SpecificInterface; use FireflyIII\Models\ImportJob; use Illuminate\Support\Collection; use League\Csv\Reader; @@ -27,9 +26,19 @@ use Log; */ class CsvImporter implements ImporterInterface { + /** @var Collection */ + public $collection; /** @var ImportJob */ public $job; + /** + * CsvImporter constructor. + */ + public function __construct() + { + $this->collection = new Collection; + } + /** * Run the actual import * @@ -41,22 +50,21 @@ class CsvImporter implements ImporterInterface $content = $this->job->uploadFileContents(); // create CSV reader. - $reader = Reader::createFromString($content); + $reader = Reader::createFromString($content); $reader->setDelimiter($config['delimiter']); - $start = $config['has-headers'] ? 1 : 0; - $results = $reader->fetch(); - $collection = new Collection; + $start = $config['has-headers'] ? 1 : 0; + $results = $reader->fetch(); foreach ($results as $index => $row) { if ($index >= $start) { Log::debug('----- import entry build start --'); Log::debug(sprintf('Now going to import row %d.', $index)); $importEntry = $this->importSingleRow($index, $row); - $collection->put($index, $importEntry); + $this->collection->put($index, $importEntry); } } - Log::debug(sprintf('Import collection contains %d entries', $collection->count())); + Log::debug(sprintf('Import collection contains %d entries', $this->collection->count())); - return $collection; + return $this->collection; } /** @@ -75,19 +83,28 @@ class CsvImporter implements ImporterInterface */ private function importSingleRow(int $index, array $row): ImportEntry { - // create import object: + // create import object. This is where each entry ends up. $object = new ImportEntry; // set some vars: $object->setUser($this->job->user); $config = $this->job->configuration; - foreach ($row as $index => $value) { + // and this is the point where the specifix go to work. + foreach ($config['specifics'] as $name => $enabled) { + /** @var SpecificInterface $specific */ + $specific = app('FireflyIII\Import\Specifics\\' . $name); + + // it returns the row, possibly modified: + $row = $specific->run($row); + } + + foreach ($row as $rowIndex => $value) { // find the role for this column: - $role = $config['column-roles'][$index] ?? '_ignore'; - $doMap = $config['column-do-mapping'][$index] ?? false; + $role = $config['column-roles'][$rowIndex] ?? '_ignore'; + $doMap = $config['column-do-mapping'][$rowIndex] ?? false; $converterClass = config('csv.import_roles.' . $role . '.converter'); - $mapping = $config['column-mapping-config'][$index] ?? []; + $mapping = $config['column-mapping-config'][$rowIndex] ?? []; /** @var ConverterInterface $converter */ $converter = app('FireflyIII\\Import\\Converter\\' . $converterClass); // set some useful values for the converter: @@ -101,21 +118,15 @@ class CsvImporter implements ImporterInterface $certainty = $converter->getCertainty(); // log it. - Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]); + Log::debug('Value ', ['index' => $rowIndex, 'value' => $value, 'role' => $role]); // store in import entry: - $object->importValue($role, $value, $certainty, $convertedValue); + Log::debug('Going to import', ['role' => $role, 'value' => $value, 'certainty' => $certainty]); + $object->importValue($role, $certainty, $convertedValue); } + return $object; - // $result = $object->import(); - // if ($result->failed()) { - // Log::error(sprintf('Import of row %d has failed.', $index), $result->errors->toArray()); - // } - // - // exit; - // - // return true; } } \ No newline at end of file diff --git a/app/Import/Specifics/AbnAmroDescription.php b/app/Import/Specifics/AbnAmroDescription.php index a3d3f058fa..f6787e11f5 100644 --- a/app/Import/Specifics/AbnAmroDescription.php +++ b/app/Import/Specifics/AbnAmroDescription.php @@ -34,4 +34,14 @@ class AbnAmroDescription implements SpecificInterface { return 'Fixes possible problems with ABN Amro descriptions.'; } + + /** + * @param array $row + * + * @return array + */ + public function run(array $row): array + { + return $row; + } } \ No newline at end of file diff --git a/app/Import/Specifics/RabobankDescription.php b/app/Import/Specifics/RabobankDescription.php index 09fe209d47..7372ba3dd0 100644 --- a/app/Import/Specifics/RabobankDescription.php +++ b/app/Import/Specifics/RabobankDescription.php @@ -11,6 +11,8 @@ declare(strict_types = 1); namespace FireflyIII\Import\Specifics; +use Log; + /** * Class RabobankDescription * @@ -18,6 +20,14 @@ namespace FireflyIII\Import\Specifics; */ class RabobankDescription implements SpecificInterface { + /** + * @return string + */ + static public function getDescription(): string + { + return 'Fixes possible problems with Rabobank descriptions.'; + } + /** * @return string */ @@ -27,10 +37,28 @@ class RabobankDescription implements SpecificInterface } /** - * @return string + * @param array $row + * + * @return array */ - static public function getDescription(): string + public function run(array $row): array { - return 'Fixes possible problems with Rabobank descriptions.'; + $oppositeAccount = trim($row[5]); + $oppositeName = trim($row[6]); + $alternateName = trim($row[10]); + if (strlen($oppositeAccount) < 1 && strlen($oppositeName) < 1) { + Log::debug( + sprintf( + 'Rabobank specific: Opposite account and opposite name are' . + ' both empty. Will use "%s" (from description) instead', $alternateName + ) + ); + $row[6] = $alternateName; + $row[10] = ''; + } else { + Log::debug('Rabobank specific: either opposite account or name are filled.'); + } + + return $row; } } \ No newline at end of file diff --git a/app/Import/Specifics/SpecificInterface.php b/app/Import/Specifics/SpecificInterface.php index 52015ab01e..847e930af9 100644 --- a/app/Import/Specifics/SpecificInterface.php +++ b/app/Import/Specifics/SpecificInterface.php @@ -28,4 +28,11 @@ interface SpecificInterface */ static public function getDescription(): string; + /** + * @param array $row + * + * @return array + */ + public function run(array $row): array; + } \ No newline at end of file diff --git a/storage/database/seed.import-test.json b/storage/database/seed.import-test.json index bbdd6fe38c..20ea6708ec 100644 --- a/storage/database/seed.import-test.json +++ b/storage/database/seed.import-test.json @@ -100,63 +100,40 @@ "has-headers": false, "date-format": "Ymd", "delimiter": ",", - "import-account": 0, + "import-account": 1, "specifics": { - "RabobankDescription": 1, - "AbnAmroDescription": 1 + "RabobankDescription": 1 }, - "column-count": 30, + "column-count": 19, "column-roles": [ - "amount", - "account-id", "account-iban", - "account-name", - "opposing-number", - "bill-id", - "bill-name", - "budget-id", - "budget-name", - "category-id", - "category-name", "currency-code", - "currency-id", - "currency-symbol", - "currency-name", - "date-transaction", + "date-interest", + "rabo-debet-credit", + "amount", + "opposing-iban", + "opposing-name", + "date-book", "description", "_ignore", - "ing-debet-credit", - "opposing-iban", - "opposing-id", - "opposing-name", - "opposing-number", - "rabo-debet-credit", - "tags-comma", - "tags-space", - "date-interest", - "date-book", - "date-process", - "external-id" + "description", + "description", + "description", + "description", + "description", + "description", + "sepa-ct-id", + "sepa-ct-op", + "sepa-db" ], "column-do-mapping": [ + true, + true, false, false, false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, + true, + true, false, false, false, @@ -171,7 +148,14 @@ false ], "column-roles-complete": false, - "column-mapping-config": {}, + "column-mapping-config": { + "0": [], + "1": { + "EUR": 1 + }, + "5": [], + "6": [] + }, "column-mapping-complete": false } }