viscous/espocrm
1
0
Fork 0
mirror of https://github.com/espocrm/espocrm.git synced 2026-06-28 06:56:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

import: idle mode

Browse Source
This commit is contained in:
yuri
2016-11-07 16:01:52 +02:00
parent d2cd098ab0
commit 5c345fb4d9
8 changed files with 224 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
application/Espo/Controllers/Import.php
View File

@@ -127,8 +127,48 @@ class Import extends \Espo\Core\Controllers\Record
throw new BadRequest();
}
if (!isset($data['fieldDelimiter'])) {
throw new BadRequest();
}
if (!isset($data['textQualifier'])) {
throw new BadRequest();
}
if (!isset($data['dateFormat'])) {
throw new BadRequest();
}
if (!isset($data['timeFormat'])) {
throw new BadRequest();
}
if (!isset($data['personNameFormat'])) {
throw new BadRequest();
}
if (!isset($data['decimalMark'])) {
throw new BadRequest();
}
if (!isset($data['defaultValues'])) {
throw new BadRequest();
}
if (!isset($data['action'])) {
throw new BadRequest();
}
if (!isset($data['attachmentId'])) {
throw new BadRequest();
}
if (!isset($data['entityType'])) {
throw new BadRequest();
}
$importParams = array(
'headerRow' => $data['headerRow'],
'headerRow' => !empty($data['headerRow']),
'fieldDelimiter' => $data['fieldDelimiter'],
'textQualifier' => $data['textQualifier'],
'dateFormat' => $data['dateFormat'],
@@ -138,7 +178,8 @@ class Import extends \Espo\Core\Controllers\Record
'currency' => $data['currency'],
'defaultValues' => $data['defaultValues'],
'action' => $data['action'],
'skipDuplicateChecking' => $data['skipDuplicateChecking']
'skipDuplicateChecking' => !empty($data['skipDuplicateChecking']),
'idleMode' => !empty($data['idleMode'])
);
if (array_key_exists('updateBy', $data)) {

13
application/Espo/Resources/i18n/en_US/Import.json
View File

@@ -54,13 +54,22 @@
},
"messages": {
"utf8": "Should be UTF-8 encoded",
"duplicatesRemoved": "Duplicates removed"
"duplicatesRemoved": "Duplicates removed",
"inIdle": "Execute in idle (for big data; via cron)"
},
"fields": {
"file": "File",
"entityType": "Entity Type",
"imported": "Imported Records",
"duplicates": "Duplicate Records",
"updated": "Updated Records"
"updated": "Updated Records",
"status": "Status"
},
"options": {
"status": {
"Failed": "Failed",
"In Process": "In Process",
"Complete": "Complete"
}
}
}

3
application/Espo/Resources/layouts/Import/detail.json
View File

@@ -6,6 +6,9 @@
],
[
{"name": "file"}
],
[
{"name": "status"}
]
]
}

11
application/Espo/Resources/metadata/entityDefs/Import.json
View File

@@ -3,11 +3,18 @@
"entityType": {
"type": "enum",
"translation": "Global.scopeNames",
"required": true
"required": true,
"readOnly": true
},
"status": {
"type": "enum",
"options": ["In Process", "Complete", "Failed"],
"readOnly": true
},
"file": {
"type": "file",
"required": true
"required": true,
"readOnly": true
},
"importedCount": {
"type": "int",

161
application/Espo/Services/Import.php
View File

@@ -268,7 +268,20 @@ class Import extends \Espo\Services\Record
return true;
}
public function import($scope, array $importFieldList, $attachmentId, array $params = array())
public function runIdleImport($data)
{
$entityType = $data['entityType'];
$params = json_decode(json_encode($data['params']), true);
$importFieldList = $data['importFieldList'];
$attachmentId = $data['attachmentId'];
$importId = $data['importId'];
$this->import($entityType, $importFieldList, $attachmentId, $params, $importId);
}
public function import($scope, array $importFieldList, $attachmentId, array $params = array(), $importId = null)
{
$delimiter = ',';
if (!empty($params['fieldDelimiter'])) {
@@ -284,63 +297,104 @@ class Import extends \Espo\Services\Record
throw new Error('Import error');
}
$import = $this->getEntityManager()->getEntity('Import');
$import->set(array(
'entityType' => $scope,
'fileId' => $attachmentId
));
if ($importId) {
$import = $this->getEntityManager()->getEntity('Import', $importId);
if (!$import) {
throw new Error('Import error: Could not find import record.');
}
} else {
$import = $this->getEntityManager()->getEntity('Import');
$import->set(array(
'entityType' => $scope,
'fileId' => $attachmentId
));
$import->set('status', 'In Process');
}
$this->getEntityManager()->saveEntity($import);
$pdo = $this->getEntityManager()->getPDO();
if (!empty($params['idleMode'])) {
$params['idleMode'] = false;
$job = $this->getEntityManager()->getEntity('Job');
$job->set(array(
'serviceName' => 'Import',
'method' => 'runIdleImport',
'data' => array(
'entityType' => $scope,
'params' => $params,
'attachmentId' => $attachmentId,
'importFieldList' => $importFieldList,
'importId' => $import->id
)
));
$this->getEntityManager()->saveEntity($job);
$result = array(
'importedIds' => array(),
'updatedIds' => array(),
'duplicateIds' => array(),
);
$i = -1;
$contents = str_replace("\r\n", "\n", $contents);
while ($arr = $this->readCsvString($contents, $delimiter, $enclosure)) {
$i++;
if ($i == 0 && !empty($params['headerRow'])) {
continue;
}
if (count($arr) == 1 && empty($arr[0])) {
continue;
}
$r = $this->importRow($scope, $importFieldList, $arr, $params);
if (empty($r)) {
continue;
}
if (!empty($r['isImported'])) {
$result['importedIds'][] = $r['id'];
}
if (!empty($r['isUpdated'])) {
$result['updatedIds'][] = $r['id'];
}
if (!empty($r['isDuplicate'])) {
$result['duplicateIds'][] = $r['id'];
}
$sql = "
INSERT INTO import_entity
(entity_type, entity_id, import_id, is_imported, is_updated, is_duplicate)
VALUES
(:entityType, :entityId, :importId, :isImported, :isUpdated, :isDuplicate)
";
$sth = $pdo->prepare($sql);
$sth->bindValue(':entityType', $scope);
$sth->bindValue(':entityId', $r['id']);
$sth->bindValue(':importId', $import->id);
$sth->bindValue(':isImported', !empty($r['isImported']), \PDO::PARAM_BOOL);
$sth->bindValue(':isUpdated', !empty($r['isUpdated']), \PDO::PARAM_BOOL);
$sth->bindValue(':isDuplicate', !empty($r['isDuplicate']), \PDO::PARAM_BOOL);
$sth->execute();
return array(
'id' => $import->id,
'countCreated' => 0,
'countUpdated' => 0
);
}
try {
$pdo = $this->getEntityManager()->getPDO();
$result = array(
'importedIds' => array(),
'updatedIds' => array(),
'duplicateIds' => array()
);
$i = -1;
$contents = str_replace("\r\n", "\n", $contents);
while ($arr = $this->readCsvString($contents, $delimiter, $enclosure)) {
$i++;
if ($i == 0 && !empty($params['headerRow'])) {
continue;
}
if (count($arr) == 1 && empty($arr[0])) {
continue;
}
$r = $this->importRow($scope, $importFieldList, $arr, $params);
if (empty($r)) {
continue;
}
if (!empty($r['isImported'])) {
$result['importedIds'][] = $r['id'];
}
if (!empty($r['isUpdated'])) {
$result['updatedIds'][] = $r['id'];
}
if (!empty($r['isDuplicate'])) {
$result['duplicateIds'][] = $r['id'];
}
$sql = "
INSERT INTO import_entity
(entity_type, entity_id, import_id, is_imported, is_updated, is_duplicate)
VALUES
(:entityType, :entityId, :importId, :isImported, :isUpdated, :isDuplicate)
";
$sth = $pdo->prepare($sql);
$sth->bindValue(':entityType', $scope);
$sth->bindValue(':entityId', $r['id']);
$sth->bindValue(':importId', $import->id);
$sth->bindValue(':isImported', !empty($r['isImported']), \PDO::PARAM_BOOL);
$sth->bindValue(':isUpdated', !empty($r['isUpdated']), \PDO::PARAM_BOOL);
$sth->bindValue(':isDuplicate', !empty($r['isDuplicate']), \PDO::PARAM_BOOL);
$sth->execute();
}
} catch (\Exception $e) {
$GLOBALS['log']->error('Import Error: '. $e->getMessage());
$import->set('status', 'Failed');
}
$import->set('status', 'Complete');
$this->getEntityManager()->saveEntity($import);
return array(
'id' => $import->id,
'countCreated' => count($result['importedIds']),
@@ -360,7 +414,6 @@ class Import extends \Espo\Services\Record
return;
}
if (in_array($action, ['createAndUpdate', 'update'])) {
if (!empty($params['updateBy']) && is_array($params['updateBy'])) {
$updateByFieldList = [];

6
client/res/templates/import/step-1.tpl
View File

@@ -111,6 +111,12 @@
</div>
</div>
<div class="row">
<div class="col-sm-4 form-group">
<label class="control-label">{{translate 'inIdle' scope='Import' category='messages'}}</label>
<div>
<input type="checkbox" id="import-idle-mode">
</div>
</div>
<div class="col-sm-4 form-group">
<label class="control-label">{{translate 'Skip searching for duplicates' scope='Import'}}</label>
<div>

43
client/src/views/import/record/detail.js
View File

@@ -32,7 +32,48 @@ Espo.define('views/import/record/detail', 'views/record/detail', function (Dep)
readOnly: true,
returnUrl: '#Import/list'
returnUrl: '#Import/list',
setup: function () {
Dep.prototype.setup.call(this);
if (this.model.get('status') === 'In Process') {
setTimeout(this.runChecking.bind(this), 3000);
this.on('remove', function () {
this.stopChecking = true;
}, this);
}
},
runChecking: function () {
if (this.stopChecking) return;
this.model.fetch().done(function () {
var bottomView = this.getView('bottom');
if (bottomView) {
var importedView = bottomView.getView('imported');
if (importedView && importedView.collection) {
importedView.collection.fetch();
}
var duplicatesView = bottomView.getView('duplicates');
if (duplicatesView && duplicatesView.collection) {
duplicatesView.collection.fetch();
}
var updatedView = bottomView.getView('updated');
if (updatedView && updatedView.collection) {
updatedView.collection.fetch();
}
}
if (this.model.get('status') !== 'In Process') {
return;
}
setTimeout(this.runChecking.bind(this), 5000);
}.bind(this));
}
});

3
client/src/views/import/step1.js
View File

@@ -110,6 +110,7 @@ Espo.define('views/import/step1', 'view', function (Dep) {
this.formData.currency = $('#import-currency').val();
this.formData.personNameFormat = $('#import-person-name-format').val();
this.formData.skipDuplicateChecking = $('#skip-duplicate-checking').get(0).checked;
this.formData.idleMode = $('#import-idle-mode').get(0).checked;
this.getParentView().formData = this.formData;
this.getParentView().changeStep(2);
@@ -118,6 +119,8 @@ Espo.define('views/import/step1', 'view', function (Dep) {
setupFormData: function () {
$('#import-header-row').get(0).checked = this.formData.headerRow || false;
$('#import-idle-mode').get(0).checked = this.formData.idleMode || false;
$('#skip-duplicate-checking').get(0).checked = this.formData.skipDuplicateChecking || false;
if (this.formData.entityType) {