diff --git a/application/Espo/Controllers/Import.php b/application/Espo/Controllers/Import.php index 6a3067381a..65a448482e 100644 --- a/application/Espo/Controllers/Import.php +++ b/application/Espo/Controllers/Import.php @@ -127,8 +127,48 @@ class Import extends \Espo\Core\Controllers\Record throw new BadRequest(); } + if (!isset($data['fieldDelimiter'])) { + throw new BadRequest(); + } + + if (!isset($data['textQualifier'])) { + throw new BadRequest(); + } + + if (!isset($data['dateFormat'])) { + throw new BadRequest(); + } + + if (!isset($data['timeFormat'])) { + throw new BadRequest(); + } + + if (!isset($data['personNameFormat'])) { + throw new BadRequest(); + } + + if (!isset($data['decimalMark'])) { + throw new BadRequest(); + } + + if (!isset($data['defaultValues'])) { + throw new BadRequest(); + } + + if (!isset($data['action'])) { + throw new BadRequest(); + } + + if (!isset($data['attachmentId'])) { + throw new BadRequest(); + } + + if (!isset($data['entityType'])) { + throw new BadRequest(); + } + $importParams = array( - 'headerRow' => $data['headerRow'], + 'headerRow' => !empty($data['headerRow']), 'fieldDelimiter' => $data['fieldDelimiter'], 'textQualifier' => $data['textQualifier'], 'dateFormat' => $data['dateFormat'], @@ -138,7 +178,8 @@ class Import extends \Espo\Core\Controllers\Record 'currency' => $data['currency'], 'defaultValues' => $data['defaultValues'], 'action' => $data['action'], - 'skipDuplicateChecking' => $data['skipDuplicateChecking'] + 'skipDuplicateChecking' => !empty($data['skipDuplicateChecking']), + 'idleMode' => !empty($data['idleMode']) ); if (array_key_exists('updateBy', $data)) { diff --git a/application/Espo/Resources/i18n/en_US/Import.json b/application/Espo/Resources/i18n/en_US/Import.json index 9663d38d27..130f70902d 100644 --- a/application/Espo/Resources/i18n/en_US/Import.json +++ b/application/Espo/Resources/i18n/en_US/Import.json @@ -54,13 +54,22 @@ }, "messages": { "utf8": "Should be UTF-8 encoded", - "duplicatesRemoved": "Duplicates removed" + "duplicatesRemoved": "Duplicates removed", + "inIdle": "Execute in idle (for big data; via cron)" }, "fields": { "file": "File", "entityType": "Entity Type", "imported": "Imported Records", "duplicates": "Duplicate Records", - "updated": "Updated Records" + "updated": "Updated Records", + "status": "Status" + }, + "options": { + "status": { + "Failed": "Failed", + "In Process": "In Process", + "Complete": "Complete" + } } } diff --git a/application/Espo/Resources/layouts/Import/detail.json b/application/Espo/Resources/layouts/Import/detail.json index cfdf561011..160c4feb78 100644 --- a/application/Espo/Resources/layouts/Import/detail.json +++ b/application/Espo/Resources/layouts/Import/detail.json @@ -6,6 +6,9 @@ ], [ {"name": "file"} + ], + [ + {"name": "status"} ] ] } diff --git a/application/Espo/Resources/metadata/entityDefs/Import.json b/application/Espo/Resources/metadata/entityDefs/Import.json index 97ac14fe52..4f5d1875d2 100644 --- a/application/Espo/Resources/metadata/entityDefs/Import.json +++ b/application/Espo/Resources/metadata/entityDefs/Import.json @@ -3,11 +3,18 @@ "entityType": { "type": "enum", "translation": "Global.scopeNames", - "required": true + "required": true, + "readOnly": true + }, + "status": { + "type": "enum", + "options": ["In Process", "Complete", "Failed"], + "readOnly": true }, "file": { "type": "file", - "required": true + "required": true, + "readOnly": true }, "importedCount": { "type": "int", diff --git a/application/Espo/Services/Import.php b/application/Espo/Services/Import.php index 0000ca7f1e..f261cf2ad0 100644 --- a/application/Espo/Services/Import.php +++ b/application/Espo/Services/Import.php @@ -268,7 +268,20 @@ class Import extends \Espo\Services\Record return true; } - public function import($scope, array $importFieldList, $attachmentId, array $params = array()) + public function runIdleImport($data) + { + $entityType = $data['entityType']; + $params = json_decode(json_encode($data['params']), true); + + $importFieldList = $data['importFieldList']; + $attachmentId = $data['attachmentId']; + + $importId = $data['importId']; + + $this->import($entityType, $importFieldList, $attachmentId, $params, $importId); + } + + public function import($scope, array $importFieldList, $attachmentId, array $params = array(), $importId = null) { $delimiter = ','; if (!empty($params['fieldDelimiter'])) { @@ -284,63 +297,104 @@ class Import extends \Espo\Services\Record throw new Error('Import error'); } - $import = $this->getEntityManager()->getEntity('Import'); - $import->set(array( - 'entityType' => $scope, - 'fileId' => $attachmentId - )); + if ($importId) { + $import = $this->getEntityManager()->getEntity('Import', $importId); + if (!$import) { + throw new Error('Import error: Could not find import record.'); + } + } else { + $import = $this->getEntityManager()->getEntity('Import'); + $import->set(array( + 'entityType' => $scope, + 'fileId' => $attachmentId + )); + $import->set('status', 'In Process'); + } + $this->getEntityManager()->saveEntity($import); - $pdo = $this->getEntityManager()->getPDO(); + if (!empty($params['idleMode'])) { + $params['idleMode'] = false; + $job = $this->getEntityManager()->getEntity('Job'); + $job->set(array( + 'serviceName' => 'Import', + 'method' => 'runIdleImport', + 'data' => array( + 'entityType' => $scope, + 'params' => $params, + 'attachmentId' => $attachmentId, + 'importFieldList' => $importFieldList, + 'importId' => $import->id + ) + )); + $this->getEntityManager()->saveEntity($job); - $result = array( - 'importedIds' => array(), - 'updatedIds' => array(), - 'duplicateIds' => array(), - ); - $i = -1; - - $contents = str_replace("\r\n", "\n", $contents); - - while ($arr = $this->readCsvString($contents, $delimiter, $enclosure)) { - $i++; - if ($i == 0 && !empty($params['headerRow'])) { - continue; - } - if (count($arr) == 1 && empty($arr[0])) { - continue; - } - $r = $this->importRow($scope, $importFieldList, $arr, $params); - if (empty($r)) { - continue; - } - if (!empty($r['isImported'])) { - $result['importedIds'][] = $r['id']; - } - if (!empty($r['isUpdated'])) { - $result['updatedIds'][] = $r['id']; - } - if (!empty($r['isDuplicate'])) { - $result['duplicateIds'][] = $r['id']; - } - $sql = " - INSERT INTO import_entity - (entity_type, entity_id, import_id, is_imported, is_updated, is_duplicate) - VALUES - (:entityType, :entityId, :importId, :isImported, :isUpdated, :isDuplicate) - "; - $sth = $pdo->prepare($sql); - $sth->bindValue(':entityType', $scope); - $sth->bindValue(':entityId', $r['id']); - $sth->bindValue(':importId', $import->id); - $sth->bindValue(':isImported', !empty($r['isImported']), \PDO::PARAM_BOOL); - $sth->bindValue(':isUpdated', !empty($r['isUpdated']), \PDO::PARAM_BOOL); - $sth->bindValue(':isDuplicate', !empty($r['isDuplicate']), \PDO::PARAM_BOOL); - - $sth->execute(); - + return array( + 'id' => $import->id, + 'countCreated' => 0, + 'countUpdated' => 0 + ); } + + try { + $pdo = $this->getEntityManager()->getPDO(); + + $result = array( + 'importedIds' => array(), + 'updatedIds' => array(), + 'duplicateIds' => array() + ); + $i = -1; + + $contents = str_replace("\r\n", "\n", $contents); + + while ($arr = $this->readCsvString($contents, $delimiter, $enclosure)) { + $i++; + if ($i == 0 && !empty($params['headerRow'])) { + continue; + } + if (count($arr) == 1 && empty($arr[0])) { + continue; + } + $r = $this->importRow($scope, $importFieldList, $arr, $params); + if (empty($r)) { + continue; + } + if (!empty($r['isImported'])) { + $result['importedIds'][] = $r['id']; + } + if (!empty($r['isUpdated'])) { + $result['updatedIds'][] = $r['id']; + } + if (!empty($r['isDuplicate'])) { + $result['duplicateIds'][] = $r['id']; + } + $sql = " + INSERT INTO import_entity + (entity_type, entity_id, import_id, is_imported, is_updated, is_duplicate) + VALUES + (:entityType, :entityId, :importId, :isImported, :isUpdated, :isDuplicate) + "; + $sth = $pdo->prepare($sql); + $sth->bindValue(':entityType', $scope); + $sth->bindValue(':entityId', $r['id']); + $sth->bindValue(':importId', $import->id); + $sth->bindValue(':isImported', !empty($r['isImported']), \PDO::PARAM_BOOL); + $sth->bindValue(':isUpdated', !empty($r['isUpdated']), \PDO::PARAM_BOOL); + $sth->bindValue(':isDuplicate', !empty($r['isDuplicate']), \PDO::PARAM_BOOL); + + $sth->execute(); + } + } catch (\Exception $e) { + $GLOBALS['log']->error('Import Error: '. $e->getMessage()); + $import->set('status', 'Failed'); + } + + $import->set('status', 'Complete'); + + $this->getEntityManager()->saveEntity($import); + return array( 'id' => $import->id, 'countCreated' => count($result['importedIds']), @@ -360,7 +414,6 @@ class Import extends \Espo\Services\Record return; } - if (in_array($action, ['createAndUpdate', 'update'])) { if (!empty($params['updateBy']) && is_array($params['updateBy'])) { $updateByFieldList = []; diff --git a/client/res/templates/import/step-1.tpl b/client/res/templates/import/step-1.tpl index 8383c30c77..16c44c1b9a 100644 --- a/client/res/templates/import/step-1.tpl +++ b/client/res/templates/import/step-1.tpl @@ -111,6 +111,12 @@
+
+ +
+ +
+
diff --git a/client/src/views/import/record/detail.js b/client/src/views/import/record/detail.js index 6990e63578..72ad7d3419 100644 --- a/client/src/views/import/record/detail.js +++ b/client/src/views/import/record/detail.js @@ -32,7 +32,48 @@ Espo.define('views/import/record/detail', 'views/record/detail', function (Dep) readOnly: true, - returnUrl: '#Import/list' + returnUrl: '#Import/list', + + setup: function () { + Dep.prototype.setup.call(this); + + if (this.model.get('status') === 'In Process') { + setTimeout(this.runChecking.bind(this), 3000); + this.on('remove', function () { + this.stopChecking = true; + }, this); + } + }, + + runChecking: function () { + if (this.stopChecking) return; + + this.model.fetch().done(function () { + + var bottomView = this.getView('bottom'); + if (bottomView) { + var importedView = bottomView.getView('imported'); + if (importedView && importedView.collection) { + importedView.collection.fetch(); + } + + var duplicatesView = bottomView.getView('duplicates'); + if (duplicatesView && duplicatesView.collection) { + duplicatesView.collection.fetch(); + } + + var updatedView = bottomView.getView('updated'); + if (updatedView && updatedView.collection) { + updatedView.collection.fetch(); + } + } + + if (this.model.get('status') !== 'In Process') { + return; + } + setTimeout(this.runChecking.bind(this), 5000); + }.bind(this)); + } }); diff --git a/client/src/views/import/step1.js b/client/src/views/import/step1.js index 5481c142df..b97b383b6c 100644 --- a/client/src/views/import/step1.js +++ b/client/src/views/import/step1.js @@ -110,6 +110,7 @@ Espo.define('views/import/step1', 'view', function (Dep) { this.formData.currency = $('#import-currency').val(); this.formData.personNameFormat = $('#import-person-name-format').val(); this.formData.skipDuplicateChecking = $('#skip-duplicate-checking').get(0).checked; + this.formData.idleMode = $('#import-idle-mode').get(0).checked; this.getParentView().formData = this.formData; this.getParentView().changeStep(2); @@ -118,6 +119,8 @@ Espo.define('views/import/step1', 'view', function (Dep) { setupFormData: function () { $('#import-header-row').get(0).checked = this.formData.headerRow || false; + $('#import-idle-mode').get(0).checked = this.formData.idleMode || false; + $('#skip-duplicate-checking').get(0).checked = this.formData.skipDuplicateChecking || false; if (this.formData.entityType) {