From bbed5d0701cf90d03ab90c8402cd45a487f62a97 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 13 Aug 2016 21:51:01 +0200 Subject: [PATCH] First version of a web-based import status thing. --- app/Console/Commands/Import.php | 38 +------- app/Http/Controllers/ImportController.php | 63 ++++++++++++- app/Http/routes.php | 2 +- app/Import/ImportProcedure.php | 77 ++++++++++++++++ app/Import/ImportStorage.php | 14 ++- app/Import/ImportValidator.php | 16 ++++ app/Import/Importer/CsvImporter.php | 3 + app/Models/ImportJob.php | 50 +++++++++- .../ImportJob/ImportJobRepository.php | 7 +- app/Support/Migration/TestData.php | 15 +-- public/js/ff/import/status.js | 91 +++++++++++++++++++ resources/lang/en_US/firefly.php | 5 + resources/views/import/status.twig | 47 ++++++++++ storage/database/seed.import-test.json | 4 + 14 files changed, 383 insertions(+), 49 deletions(-) create mode 100644 app/Import/ImportProcedure.php create mode 100644 public/js/ff/import/status.js create mode 100644 resources/views/import/status.twig diff --git a/app/Console/Commands/Import.php b/app/Console/Commands/Import.php index 6109cab5e2..e2e68ea4ed 100644 --- a/app/Console/Commands/Import.php +++ b/app/Console/Commands/Import.php @@ -13,6 +13,7 @@ namespace FireflyIII\Console\Commands; use FireflyIII\Crud\Account\AccountCrud; use FireflyIII\Import\Importer\ImporterInterface; +use FireflyIII\Import\ImportProcedure; use FireflyIII\Import\ImportResult; use FireflyIII\Import\ImportStorage; use FireflyIII\Import\ImportValidator; @@ -71,47 +72,15 @@ class Import extends Command return; } - $job->status = 'import_running'; - $job->save(); $this->line('Going to import job with key "' . $job->key . '" of type ' . $job->file_type); - $valid = array_keys(config('firefly.import_formats')); - $class = 'INVALID'; - if (in_array($job->file_type, $valid)) { - $class = config('firefly.import_formats.' . $job->file_type); - } - /** @var ImporterInterface $importer */ - $importer = app($class); - $importer->setJob($job); - // intercept logging by importer. + // intercept log entries and print them on the command line $monolog = Log::getMonolog(); $handler = new CommandHandler($this); - $monolog->pushHandler($handler); - // create import entries - $collection = $importer->createImportEntries(); - - // validate / clean collection: - $validator = new ImportValidator($collection); - $validator->setUser($job->user); - if ($job->configuration['import-account'] != 0) { - $repository = app(AccountCrud::class, [$job->user]); - $validator->setDefaultImportAccount($repository->find($job->configuration['import-account'])); - } - - $cleaned = $validator->clean(); - - // then import collection: - $storage = new ImportStorage($cleaned); - $storage->setUser($job->user); - - // and run store routine: - $result = $storage->store(); - - $job->status = 'import_complete'; - $job->save(); + $result = ImportProcedure::run($job); /** * @var int $index @@ -128,5 +97,6 @@ class Import extends Command $this->line('The import has completed.'); + } } diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index 2d10dc6c1f..ba58c4d113 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -11,13 +11,14 @@ namespace FireflyIII\Http\Controllers; use Crypt; use FireflyIII\Exceptions\FireflyException; -use FireflyIII\Http\Requests; use FireflyIII\Http\Requests\ImportUploadRequest; +use FireflyIII\Import\ImportProcedure; use FireflyIII\Import\Setup\SetupInterface; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use Illuminate\Http\Request; use Log; +use Response; use SplFileObject; use Storage; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -139,6 +140,35 @@ class ImportController extends Controller return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType')); } + /** + * @param ImportJob $job + * + * @return \Illuminate\Http\JsonResponse + */ + public function json(ImportJob $job) + { + $result = [ + 'showPercentage' => false, + 'status' => $job->status, + 'key' => $job->key, + 'started' => false, + 'completed' => false, + 'running' => false, + 'percentage' => 0, + 'steps' => $job->extended_status['total_steps'], + 'stepsDone' => $job->extended_status['steps_done'], + 'statusText' => trans('firefly.import_status_' . $job->status), + ]; + if ($job->status === 'import_running') { + $result['started'] = true; + $result['running'] = true; + $result['showPercentage'] = true; + $result['percentage'] = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0); + } + + return Response::json($result); + } + /** * Step 4. Save the configuration. * @@ -237,6 +267,35 @@ class ImportController extends Controller // depends of course on the data in the job. } + /** + * @param ImportJob $job + */ + public function start(ImportJob $job) + { + if ($job->status == "settings_complete") { + ImportProcedure::run($job); + } + } + + /** + * This is the last step before the import starts. + * + * @param ImportJob $job + * + * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View + */ + public function status(ImportJob $job) + { + Log::debug('Now in status()', ['job' => $job->key]); + if (!$this->jobInCorrectStep($job, 'status')) { + return $this->redirectToCorrectStep($job); + } + $subTitle = trans('firefy.import_status'); + $subTitleIcon = 'fa-star'; + + return view('import.status', compact('job', 'subTitle', 'subTitleIcon')); + } + /** * This is step 2. It creates an Import Job. Stores the import. * @@ -308,6 +367,8 @@ class ImportController extends Controller return $job->status === 'import_configuration_saved'; case 'complete': return $job->status === 'settings_complete'; + case 'status': + return ($job->status === 'settings_complete') || ($job->status === 'import_running'); } return false; diff --git a/app/Http/routes.php b/app/Http/routes.php index 6516c1d5d3..a29f331a7f 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -234,7 +234,7 @@ Route::group( Route::get('/import/status/{importJob}', ['uses' => 'ImportController@status', 'as' => 'import.status']); Route::get('/import/json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'import.json']); - Route::get('/import/start/{importJob}', ['uses' => 'ImportController@run', 'as' => 'import.start']); + Route::post('/import/start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'import.start']); /** diff --git a/app/Import/ImportProcedure.php b/app/Import/ImportProcedure.php new file mode 100644 index 0000000000..058c1e3571 --- /dev/null +++ b/app/Import/ImportProcedure.php @@ -0,0 +1,77 @@ +status = 'import_running'; + $job->save(); + + // create Importer + $valid = array_keys(config('firefly.import_formats')); + $class = 'INVALID'; + if (in_array($job->file_type, $valid)) { + $class = config('firefly.import_formats.' . $job->file_type); + } + + /** @var ImporterInterface $importer */ + $importer = app($class); + $importer->setJob($job); + + // create import entries + $collection = $importer->createImportEntries(); + + // validate / clean collection: + $validator = new ImportValidator($collection); + $validator->setUser($job->user); + $validator->setJob($job); + if ($job->configuration['import-account'] != 0) { + $repository = app(AccountCrud::class, [$job->user]); + $validator->setDefaultImportAccount($repository->find($job->configuration['import-account'])); + } + + $cleaned = $validator->clean(); + + // then import collection: + $storage = new ImportStorage($cleaned); + $storage->setJob($job); + $storage->setUser($job->user); + + // and run store routine: + $result = $storage->store(); + + $job->status = 'import_complete'; + $job->save(); + + return $result; + } + +} \ No newline at end of file diff --git a/app/Import/ImportStorage.php b/app/Import/ImportStorage.php index 28497a39d0..5a2f37a03e 100644 --- a/app/Import/ImportStorage.php +++ b/app/Import/ImportStorage.php @@ -12,6 +12,7 @@ declare(strict_types = 1); namespace FireflyIII\Import; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Models\ImportJob; use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionJournal; use FireflyIII\Models\TransactionType; @@ -29,7 +30,8 @@ class ImportStorage /** @var Collection */ public $entries; - + /** @var ImportJob */ + public $job; /** @var User */ public $user; @@ -44,6 +46,14 @@ class ImportStorage } + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job) + { + $this->job = $job; + } + /** * @param User $user */ @@ -62,6 +72,8 @@ class ImportStorage foreach ($this->entries as $index => $entry) { Log::debug(sprintf('--- import store start for row %d ---', $index)); $result = $this->storeSingle($index, $entry); + $this->job->addStepsDone(1); + sleep(1); $collection->put($index, $result); } Log::notice(sprintf('Finished storing %d entry(ies).', $collection->count())); diff --git a/app/Import/ImportValidator.php b/app/Import/ImportValidator.php index 9724bde290..35e0d08883 100644 --- a/app/Import/ImportValidator.php +++ b/app/Import/ImportValidator.php @@ -15,6 +15,7 @@ use Carbon\Carbon; use FireflyIII\Crud\Account\AccountCrudInterface; use FireflyIII\Models\Account; use FireflyIII\Models\AccountType; +use FireflyIII\Models\ImportJob; use FireflyIII\Models\TransactionType; use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface; use FireflyIII\User; @@ -35,6 +36,19 @@ class ImportValidator /** @var User */ protected $user; + /** @var ImportJob */ + public $job; + + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job) + { + $this->job = $job; + } + + + /** * ImportValidator constructor. * @@ -71,6 +85,8 @@ class ImportValidator $entry = $this->setTransactionCurrency($entry); $newCollection->put($index, $entry); + $this->job->addStepsDone(1); + sleep(1); } Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count())); diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 75f07b6b50..9732bb24fa 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -64,6 +64,9 @@ class CsvImporter implements ImporterInterface Log::debug(sprintf('Now going to import row %d.', $index)); $importEntry = $this->importSingleRow($index, $row); $this->collection->put($line, $importEntry); + $this->job->addTotalSteps(3); + $this->job->addStepsDone(1); + sleep(1); } } Log::debug(sprintf('Import collection contains %d entries', $this->collection->count())); diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 3673dae624..bb1bd9ab0e 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -28,7 +28,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; * @property string $key * @property string $file_type * @property string $status - * @property array $configuration + * @property array $configuration * @property-read \FireflyIII\User $user * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereId($value) * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereCreatedAt($value) @@ -39,7 +39,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereStatus($value) * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereConfiguration($value) * @mixin \Eloquent - * @property string $extended_status + * @property string $extended_status * @method static \Illuminate\Database\Query\Builder|\FireflyIII\Models\ImportJob whereExtendedStatus($value) */ class ImportJob extends Model @@ -62,6 +62,30 @@ class ImportJob extends Model throw new NotFoundHttpException; } + /** + * @param int $count + */ + public function addStepsDone(int $count) + { + $status = $this->extended_status; + $status['steps_done'] += $count; + $this->extended_status = $status; + $this->save(); + + } + + /** + * @param int $count + */ + public function addTotalSteps(int $count) + { + $status = $this->extended_status; + $status['total_steps'] += $count; + $this->extended_status = $status; + $this->save(); + + } + /** * @param $status */ @@ -85,6 +109,20 @@ class ImportJob extends Model return json_decode($value, true); } + /** + * @param $value + * + * @return mixed + */ + public function getExtendedStatusAttribute($value) + { + if (strlen($value) == 0) { + return []; + } + + return json_decode($value, true); + } + /** * @param $value */ @@ -93,6 +131,14 @@ class ImportJob extends Model $this->attributes['configuration'] = json_encode($value); } + /** + * @param $value + */ + public function setExtendedStatusAttribute($value) + { + $this->attributes['extended_status'] = json_encode($value); + } + /** * @return string */ diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index 8badf97189..ae96b61115 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -49,9 +49,10 @@ class ImportJobRepository implements ImportJobRepositoryInterface if (is_null($existing->id)) { $importJob = new ImportJob; $importJob->user()->associate($this->user); - $importJob->file_type = $fileType; - $importJob->key = Str::random(12); - $importJob->status = 'import_status_never_started'; + $importJob->file_type = $fileType; + $importJob->key = Str::random(12); + $importJob->status = 'import_status_never_started'; + $importJob->extended_status = ['total_steps' => 0, 'steps_done' => 0,]; $importJob->save(); // breaks the loop: diff --git a/app/Support/Migration/TestData.php b/app/Support/Migration/TestData.php index 3d39cc3215..cb38a63fc7 100644 --- a/app/Support/Migration/TestData.php +++ b/app/Support/Migration/TestData.php @@ -271,13 +271,14 @@ class TestData $insert = []; foreach ($this->data['import-jobs'] as $job) { $insert[] = [ - 'created_at' => $this->time, - 'updated_at' => $this->time, - 'user_id' => $job['user_id'], - 'file_type' => $job['file_type'], - 'key' => $job['key'], - 'status' => $job['status'], - 'configuration' => json_encode($job['configuration']), + 'created_at' => $this->time, + 'updated_at' => $this->time, + 'user_id' => $job['user_id'], + 'file_type' => $job['file_type'], + 'key' => $job['key'], + 'status' => $job['status'], + 'extended_status' => json_encode($job['extended_status']), + 'configuration' => json_encode($job['configuration']), ]; } DB::table('import_jobs')->insert($insert); diff --git a/public/js/ff/import/status.js b/public/js/ff/import/status.js new file mode 100644 index 0000000000..fa7ce2d9b7 --- /dev/null +++ b/public/js/ff/import/status.js @@ -0,0 +1,91 @@ +/* + * status.js + * Copyright (C) 2016 thegrumpydictator@gmail.com + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ +/* globals $, jobImportUrl, jobStartUrl, token */ + + +var startedImport = false; +var interval = 500; +$(function () { + "use strict"; + + // check status, every 500 ms. + setTimeout(checkImportStatus, 500); + +}); + + +function checkImportStatus() { + "use strict"; + $.getJSON(jobImportUrl).success(reportOnJobImport).fail(failedJobImport); +} + +function reportOnJobImport(data) { + "use strict"; + console.log('Now in reportOnJobImport'); + + // update bar if it's a percentage or not: + + var bar = $('#import-status-bar'); + if (data.showPercentage) { + console.log('Has percentage.'); + bar.addClass('progress-bar-success').removeClass('progress-bar-info'); + bar.attr('aria-valuenow', data.percentage); + bar.css('width', data.percentage + '%'); + $('#import-status-bar').text(data.stepsDone + '/' + data.steps); + + + if (data.percentage >= 100) { + console.log('Now import complete!'); + bar.removeClass('active'); + return; + } + + } else { + $('#import-status-more-info').text(''); + console.log('Has no percentage.'); + bar.removeClass('progress-bar-success').addClass('progress-bar-info'); + bar.attr('aria-valuenow', 100); + bar.css('width', '100%'); + } + + // update the message: + $('#import-status-txt').removeClass('text-danger').text(data.statusText); + + // if the job has not actually started, do so now: + if (!data.started && !startedImport) { + console.log('Will now start job.'); + $.post(jobStartUrl, {_token: token}); + startedTheImport(); + startedImport = true; + } else { + // trigger another check. + setTimeout(checkImportStatus, 500); + } +} + +function startedTheImport() { + "use strict"; + console.log('Started the import. Now starting over again.'); + setTimeout(checkImportStatus, 500); +} + +function failedJobImport(jqxhr, textStatus, error) { + "use strict"; + + // set status + $('#import-status-txt').addClass('text-danger').text( + "There was an error during the import routine. Please check the log files. The error seems to be: '" + textStatus + ' ' + error + "'." + ); + + // remove progress bar. + $('#import-status-holder').hide(); + console.log('failedJobImport'); + console.log(textStatus); + console.log(error); + +} \ No newline at end of file diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 446a452766..40d60e76ef 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -767,6 +767,8 @@ return [ 'configure_import' => 'Further configure your import', 'import_finish_configuration' => 'Finish configuration', 'settings_for_import' => 'Settings', + 'import_status' => 'Import status', + 'import_status_text' => 'The import is currently running, or will start momentarily.', 'import_complete' => 'Import configuration complete!', 'import_complete_text' => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.', 'import_download_config' => 'Download configuration', @@ -777,4 +779,7 @@ return [ 'import' => 'Import', 'import_intro_text' => 'Welcome to the Firefly III data import routine. At the moment, this routine can help you import files into Firefly. To do so, you must download or export transactions from other systems or software, and upload them here. The next steps will let you help Firefly III determin what the content is of your file, and how to handle it. Please select a file, and read all instructions carefully.', 'import_file_help' => 'Select your file', + 'import_status_settings_complete' => 'The import is ready to start.', + 'import_status_import_complete' => 'The import has completed.', + 'import_status_import_running' => 'The import is currently running. Please be patient. An apparent lack of progress may be a trick of the light.', ]; diff --git a/resources/views/import/status.twig b/resources/views/import/status.twig new file mode 100644 index 0000000000..37510ca6f1 --- /dev/null +++ b/resources/views/import/status.twig @@ -0,0 +1,47 @@ +{% extends "./layout/default.twig" %} + +{% block breadcrumbs %} + {{ Breadcrumbs.renderIfExists }} +{% endblock %} +{% block content %} +
+
+
+
+

{{ 'import_status'|_ }}

+
+
+

+ {{ 'import_status_text'|_ }} +

+
+ +
+
+
+
+
+

+

+
+
+ + +
+
+
+
+{% endblock %} +{% block scripts %} + + +{% endblock %} +{% block styles %} +{% endblock %} diff --git a/storage/database/seed.import-test.json b/storage/database/seed.import-test.json index 20ea6708ec..1bff69943c 100644 --- a/storage/database/seed.import-test.json +++ b/storage/database/seed.import-test.json @@ -96,6 +96,10 @@ "key": "testImport", "file_type": "csv", "status": "settings_complete", + "extended_status": { + "steps_done": 0, + "total_steps": 0 + }, "configuration": { "has-headers": false, "date-format": "Ymd",