diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index c230ee241f..eacd89d09b 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -20,6 +20,7 @@ use FireflyIII\Import\ImportProcedureInterface; use FireflyIII\Import\Routine\ImportRoutine; use FireflyIII\Import\Storage\ImportStorage; use FireflyIII\Models\ImportJob; +use FireflyIII\Models\Tag; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\Repositories\User\UserRepositoryInterface; @@ -188,11 +189,12 @@ class ImportController extends Controller if ($job->extended_status['total_steps'] !== 0) { $percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0); } - if ($job->status === 'complete') { - $tagId = $job->extended_status['importTag']; - /** @var TagRepositoryInterface $repository */ - $repository = app(TagRepositoryInterface::class); - $tag = $repository->find($tagId); + if ($job->status === 'finished') { +// $tagId = $job->extended_status['importTag']; +// /** @var TagRepositoryInterface $repository */ +// $repository = app(TagRepositoryInterface::class); +// $tag = $repository->find($tagId); + $tag = new Tag; $result['finished'] = true; $result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]); } @@ -201,6 +203,7 @@ class ImportController extends Controller $result['started'] = true; $result['running'] = true; $result['percentage'] = $percentage; + $result['showPercentage'] = true; } return Response::json($result); @@ -241,7 +244,7 @@ class ImportController extends Controller $routine = new ImportRoutine($job); $routine->run(); - return 'ok dan'; + return 'done!'; } /** diff --git a/app/Import/FileProcessor/CsvProcessor.php b/app/Import/FileProcessor/CsvProcessor.php index 92329c980e..56b196ee8a 100644 --- a/app/Import/FileProcessor/CsvProcessor.php +++ b/app/Import/FileProcessor/CsvProcessor.php @@ -64,8 +64,6 @@ class CsvProcessor implements FileProcessorInterface */ public function run(): bool { - $this->job->status = 'running'; - $this->job->save(); Log::debug('Now in CsvProcessor run(). Job is now running...'); $entries = $this->getImportArray(); @@ -84,9 +82,9 @@ class CsvProcessor implements FileProcessorInterface * 3. Store journal. * 4. Run rules. */ - $this->job->addTotalSteps(4); $this->job->addStepsDone(1); $count++; + sleep(1); } return true; diff --git a/app/Import/Routine/ImportRoutine.php b/app/Import/Routine/ImportRoutine.php index 9d055adc2c..78e167d028 100644 --- a/app/Import/Routine/ImportRoutine.php +++ b/app/Import/Routine/ImportRoutine.php @@ -21,10 +21,10 @@ use Log; class ImportRoutine { - /** @var Collection */ - public $journals; /** @var Collection */ public $errors; + /** @var Collection */ + public $journals; /** @var int */ public $lines = 0; /** @var ImportJob */ @@ -39,7 +39,7 @@ class ImportRoutine { $this->job = $job; $this->journals = new Collection; - $this->errors = new Collection; + $this->errors = new Collection; } /** @@ -60,8 +60,14 @@ class ImportRoutine /** @var FileProcessorInterface $processor */ $processor = app($class); $processor->setJob($this->job); + set_time_limit(0); if ($this->job->status == 'configured') { + + // set job as "running"... + $this->job->status = 'running'; + $this->job->save(); + Log::debug('Job is configured, start with run()'); $processor->run(); $objects = $processor->getObjects(); @@ -81,7 +87,10 @@ class ImportRoutine $this->job->save(); $this->journals = $storage->journals; - $this->errors = $storage->errors; + $this->errors = $storage->errors; + + // run rules: + Log::debug(sprintf('Done with import job %s', $this->job->key)); diff --git a/app/Import/Storage/ImportStorage.php b/app/Import/Storage/ImportStorage.php index d21f02707b..68b9cb09fa 100644 --- a/app/Import/Storage/ImportStorage.php +++ b/app/Import/Storage/ImportStorage.php @@ -17,10 +17,12 @@ use FireflyIII\Import\Object\ImportJournal; use FireflyIII\Models\Account; use FireflyIII\Models\AccountType; use FireflyIII\Models\ImportJob; +use FireflyIII\Models\Rule; use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionCurrency; use FireflyIII\Models\TransactionJournal; use FireflyIII\Models\TransactionType; +use FireflyIII\Rules\Processor; use Illuminate\Support\Collection; use Illuminate\Support\MessageBag; use Log; @@ -45,6 +47,8 @@ class ImportStorage private $job; /** @var Collection */ private $objects; + /** @var Collection */ + private $rules; /** * ImportStorage constructor. @@ -54,6 +58,8 @@ class ImportStorage $this->objects = new Collection; $this->journals = new Collection; $this->errors = new Collection; + $this->rules = $this->getUserRules(); + } /** @@ -80,7 +86,6 @@ class ImportStorage $this->objects = $objects; } - /** * Do storage of import objects */ @@ -88,6 +93,8 @@ class ImportStorage { $this->defaultCurrency = Amount::getDefaultCurrencyByUser($this->job->user); + + // routine below consists of 3 steps. /** * @var int $index * @var ImportJournal $object @@ -130,6 +137,8 @@ class ImportStorage $transactionType = TransactionType::whereType(TransactionType::TRANSFER)->first(); } + $this->job->addStepsDone(1); + $journal = new TransactionJournal; $journal->user_id = $this->job->user_id; $journal->transaction_type_id = $transactionType->id; @@ -174,6 +183,9 @@ class ImportStorage } Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id)); + $this->job->addStepsDone(1); + sleep(1); + // category $category = $object->category->getCategory(); if (!is_null($category->id)) { @@ -210,11 +222,67 @@ class ImportStorage if (strlen($object->notes) > 0) { $journal->setMeta('notes', $object->notes); } + $this->job->addStepsDone(1); + + // run rules: + $this->applyRules($journal); + $this->job->addStepsDone(1); + $this->journals->push($journal); $this->errors->push($errors); + + + sleep(1); } } + /** + * @param TransactionJournal $journal + * + * @return bool + */ + protected function applyRules(TransactionJournal $journal): bool + { + if ($this->rules->count() > 0) { + + /** @var Rule $rule */ + foreach ($this->rules as $rule) { + Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id)); + $processor = Processor::make($rule); + $processor->handleTransactionJournal($journal); + + if ($rule->stop_processing) { + return true; + } + } + } + + return true; + } + + /** + * @return Collection + */ + private function getUserRules(): Collection + { + $set = Rule::distinct() + ->where('rules.user_id', $this->job->user->id) + ->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id') + ->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id') + ->where('rule_groups.active', 1) + ->where('rule_triggers.trigger_type', 'user_action') + ->where('rule_triggers.trigger_value', 'store-journal') + ->where('rules.active', 1) + ->orderBy('rule_groups.order', 'ASC') + ->orderBy('rules.order', 'ASC') + ->get(['rules.*', 'rule_groups.order']); + Log::debug(sprintf('Found %d user rules.', $set->count())); + + return $set; + + } + + } \ No newline at end of file diff --git a/app/Models/Tag.php b/app/Models/Tag.php index 30018b5e87..19573b2b29 100644 --- a/app/Models/Tag.php +++ b/app/Models/Tag.php @@ -133,6 +133,9 @@ class Tag extends Model */ public function getTagAttribute($value) { + if(is_null($value)) { + return null; + } return Crypt::decrypt($value); } diff --git a/app/Support/Import/Configuration/Csv/Map.php b/app/Support/Import/Configuration/Csv/Map.php index 9184cf0374..3286acd753 100644 --- a/app/Support/Import/Configuration/Csv/Map.php +++ b/app/Support/Import/Configuration/Csv/Map.php @@ -92,6 +92,12 @@ class Map implements ConfigurationInterface foreach ($this->data as $index => $entry) { $this->data[$index]['values'] = array_unique($this->data[$index]['values']); } + // save number of rows, thus number of steps, in job: + $steps = $rowIndex * 5; + $extended = $this->job->extended_status; + $extended['total_steps'] = $steps; + $this->job->extended_status = $extended; + $this->job->save(); return $this->data; diff --git a/public/js/ff/import/status.js b/public/js/ff/import/status.js index 75671c2971..a2fc1e123f 100644 --- a/public/js/ff/import/status.js +++ b/public/js/ff/import/status.js @@ -14,12 +14,17 @@ var displayStatus = 'initial'; var timeOutId; -var startedImport = false; +// var startedImport = false; var startInterval = 2000; var interval = 500; -var timeoutLimit = 5000; -var currentLimit = 0; -var stepCount = 0; +// var timeoutLimit = 5000; +// var currentLimit = 0; +// var stepCount = 0; + +// after this many tries, stop checking when the job is not running anyway. +var maxNotRunningCount = 20; +var notRunningCount = 0; + $(function () { "use strict"; @@ -37,6 +42,12 @@ $(function () { function startJob() { console.log('Job started.'); $.post(jobStartUrl); + + // reset not running thing + notRunningCount = 0; + // check status, every 500 ms. + timeOutId = setTimeout(checkImportStatus, startInterval); + return false; } @@ -45,12 +56,23 @@ function checkImportStatus() { $.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport); } +function reportToConsole(data) { + console.log('status: ' + data.status + ', steps: ' + data.steps + ', stepsDone: ' + data.stepsDone); + + //console.log('more status: ' + data); +} + function reportOnJobImport(data) { "use strict"; + if (data.running == false) { + notRunningCount++; + } + displayCorrectBox(data.status); - //updateBar(data); + reportToConsole(data); + updateBar(data); //reportErrors(data); - //reportStatus(data); + reportStatus(data); //updateTimeout(data); //if (importJobFinished(data)) { @@ -72,36 +94,47 @@ function reportOnJobImport(data) { //} // trigger another check. - //timeOutId = setTimeout(checkImportStatus, interval); + if (notRunningCount < maxNotRunningCount && data.finished === false) { + timeOutId = setTimeout(checkImportStatus, interval); + } + if (notRunningCount >= maxNotRunningCount && data.finished === false) { + console.error('Job still not running, stop checking for it.'); + } + if(data.finished === true) { + console.log('Job is done'); + } } function displayCorrectBox(status) { console.log('Current job state is ' + status); - if(status === 'configured' && displayStatus === 'initial') { + if (status === 'configured' && displayStatus === 'initial') { // hide some boxes: $('.status_initial').hide(); return; } + if (status === 'running') { + // hide some boxes: + $('.status_initial').hide(); + $('.status_running').show(); + $('.status_configured').hide(); + + + return; + } + + if(status === 'finished') { + $('.status_initial').hide(); + $('.status_running').hide(); + $('.status_configured').hide(); + $('.status_finished').show(); + } + + console.error('CANNOT HANDLE CURRENT STATE'); } - - - - - - - - - - - - - - - function importComplete() { "use strict"; var bar = $('#import-status-bar'); @@ -110,8 +143,10 @@ function importComplete() { function updateBar(data) { "use strict"; + var bar = $('#import-status-bar'); if (data.showPercentage) { + console.log('Going to update bar with percentage.'); bar.addClass('progress-bar-success').removeClass('progress-bar-info'); bar.attr('aria-valuenow', data.percentage); bar.css('width', data.percentage + '%'); @@ -123,92 +158,95 @@ function updateBar(data) { } return; } + console.log('Going to update bar without percentage.'); // dont show percentage: bar.removeClass('progress-bar-success').addClass('progress-bar-info'); bar.attr('aria-valuenow', 100); bar.css('width', '100%'); } - -function reportErrors(data) { - "use strict"; - if (data.errors.length === 1) { - $('#import-status-error-intro').text(langImportSingleError); - //'An error has occured during the import. The import can continue, however.' - } - if (data.errors.length > 1) { - // 'Errors have occured during the import. The import can continue, however.' - $('#import-status-error-intro').text(langImportMultiError); - } - - // fill the list with error texts - $('#import-status-error-list').empty(); - for (var i = 0; i < data.errors.length; i++) { - var item = $('
  • ').html(data.errors[i]); - $('#import-status-error-list').append(item); - } -} - +// +// function reportErrors(data) { +// "use strict"; +// if (data.errors.length === 1) { +// $('#import-status-error-intro').text(langImportSingleError); +// //'An error has occured during the import. The import can continue, however.' +// } +// if (data.errors.length > 1) { +// // 'Errors have occured during the import. The import can continue, however.' +// $('#import-status-error-intro').text(langImportMultiError); +// } +// +// // fill the list with error texts +// $('#import-status-error-list').empty(); +// for (var i = 0; i < data.errors.length; i++) { +// var item = $('
  • ').html(data.errors[i]); +// $('#import-status-error-list').append(item); +// } +// } +// function reportStatus(data) { "use strict"; + console.log('Going to report status: ' + data.statusText); $('#import-status-txt').removeClass('text-danger').text(data.statusText); } - -function kickStartJob() { - "use strict"; - $.post(jobStartUrl); - startedTheImport(); - startedImport = true; -} - -function updateTimeout(data) { - "use strict"; - if (data.stepsDone !== stepCount) { - stepCount = data.stepsDone; - currentLimit = 0; - return; - } - - currentLimit = currentLimit + interval; -} - -function timeoutError() { - "use strict"; - // set status - $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError); - - // remove progress bar. - $('#import-status-holder').hide(); - -} - -function importJobFinished(data) { - "use strict"; - return data.finished; -} - -function finishedJob(data) { - "use strict"; - // "There was an error during the import routine. Please check the log files. The error seems to be: '" - $('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished); - - // remove progress bar. - $('#import-status-holder').hide(); - - // show info: - $('#import-status-intro').show(); - $('#import-status-more-info').html(data.finishedText).show(); - -} - - - -function startedTheImport() { - "use strict"; - setTimeout(checkImportStatus, interval); -} +// +// function kickStartJob() { +// "use strict"; +// $.post(jobStartUrl); +// startedTheImport(); +// startedImport = true; +// } +// +// function updateTimeout(data) { +// "use strict"; +// if (data.stepsDone !== stepCount) { +// stepCount = data.stepsDone; +// currentLimit = 0; +// return; +// } +// +// currentLimit = currentLimit + interval; +// } +// +// function timeoutError() { +// "use strict"; +// // set status +// $('#import-status-txt').addClass('text-danger').text(langImportTimeOutError); +// +// // remove progress bar. +// $('#import-status-holder').hide(); +// +// } +// +// function importJobFinished(data) { +// "use strict"; +// return data.finished; +// } +// +// function finishedJob(data) { +// "use strict"; +// // "There was an error during the import routine. Please check the log files. The error seems to be: '" +// $('#import-status-txt').removeClass('text-danger').addClass('text-success').text(langImportFinished); +// +// // remove progress bar. +// $('#import-status-holder').hide(); +// +// // show info: +// $('#import-status-intro').show(); +// $('#import-status-more-info').html(data.finishedText).show(); +// +// } +// +// +// +// function startedTheImport() { +// "use strict"; +// setTimeout(checkImportStatus, interval); +// } function failedJobImport(jqxhr, textStatus, error) { "use strict"; + console.error('Job status failed!'); // set status // "There was an error during the import routine. Please check the log files. The error seems to be: '" $('#import-status-txt').addClass('text-danger').text(langImportFatalError + ' ' + textStatus + ' ' + error); diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 2755ae6f58..cd1b366763 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -987,7 +987,7 @@ return [ 'import_file_help' => 'Select your file', 'import_status_settings_complete' => 'The import is ready to start.', 'import_status_import_complete' => 'The import has completed.', - 'import_status_import_running' => 'The import is currently running. Please be patient.', + 'import_status_running' => 'The import is currently running. Please be patient.', 'import_status_header' => 'Import status and progress', 'import_status_errors' => 'Import errors', 'import_status_report' => 'Import report', diff --git a/resources/views/import/status.twig b/resources/views/import/status.twig index fab2dacb3f..575c3fd744 100644 --- a/resources/views/import/status.twig +++ b/resources/views/import/status.twig @@ -7,7 +7,7 @@ {# Initial display. Will refresh (and disappear almost immediately. #} -
    +