diff --git a/application/Espo/Acl/Webhook.php b/application/Espo/Acl/Webhook.php new file mode 100644 index 0000000000..666d12e30c --- /dev/null +++ b/application/Espo/Acl/Webhook.php @@ -0,0 +1,73 @@ +id === $entity->get('userId') && $user->isApi(); + } + + public function checkEntityCreate(EntityUser $user, Entity $entity, $data) + { + if ($user->isAdmin()) return true; + if (!$data) return false; + if ($user->isApi() && $user->id === $entity->get('userId')) return true; + return false; + } + + public function checkEntityRead(EntityUser $user, Entity $entity, $data) + { + if ($user->isAdmin()) return true; + if (!$data) return false; + if ($user->isApi() && $user->id === $entity->get('userId')) return true; + return false; + } + + public function checkEntityDelete(EntityUser $user, Entity $entity, $data) + { + if ($user->isAdmin()) return true; + if (!$data) return false; + if ($user->isApi() && $user->id === $entity->get('userId')) return true; + return false; + } + + public function checkEntityEdit(EntityUser $user, Entity $entity, $data) + { + if ($user->isAdmin()) return true; + if (!$data) return false; + if ($user->isApi() && $user->id === $entity->get('userId')) return true; + return false; + } +} diff --git a/application/Espo/Controllers/Webhook.php b/application/Espo/Controllers/Webhook.php new file mode 100644 index 0000000000..7599716502 --- /dev/null +++ b/application/Espo/Controllers/Webhook.php @@ -0,0 +1,50 @@ +getUser()->isAdmin() && !$this->getUser()->isApi()) { + throw new Forbidden(); + } + } + + public function actionCreate($params, $data, $request, $response = null) + { + $result = parent::actionCreate($params, $data, $request, $response); + if ($response) $response->setStatus(201); + return $result; + } +} diff --git a/application/Espo/Core/Loaders/WebhookManager.php b/application/Espo/Core/Loaders/WebhookManager.php new file mode 100644 index 0000000000..a92abfa1fa --- /dev/null +++ b/application/Espo/Core/Loaders/WebhookManager.php @@ -0,0 +1,43 @@ +getContainer()->get('config'), + $this->getContainer()->get('fileManager'), + $this->getContainer()->get('entityManager'), + $this->getContainer()->get('fieldManagerUtil') + ); + } +} diff --git a/application/Espo/Core/Webhook/Manager.php b/application/Espo/Core/Webhook/Manager.php new file mode 100644 index 0000000000..2b910f1561 --- /dev/null +++ b/application/Espo/Core/Webhook/Manager.php @@ -0,0 +1,233 @@ +config = $config; + $this->fileManager = $fileManager; + $this->entityManager = $entityManager; + $this->fieldManager = $fieldManager; + + $this->loadData(); + } + + private function loadData() + { + if ($this->config->get('useCache')) { + if ($this->fileManager->isFile($this->cacheFile)) { + if (file_exists($this->cacheFile)) { + $this->data = $this->fileManager->getPhpContents($this->cacheFile); + } + } + } + + if (is_null($this->data)) { + $this->data = $this->buildData(); + } + + if ($this->config->get('useCache')) { + $this->storeDataToCache(); + } + } + + private function storeDataToCache() + { + $this->fileManager->putPhpContents($this->cacheFile, $this->data); + } + + private function buildData() + { + $data = []; + + $list = $this->entityManager->getRepository('Webhook') + ->select(['event']) + ->groupBy(['event']) + ->where([ + 'isActive' => true, + 'event!=' => null, + ]) + ->find(); + + foreach ($list as $e) { + $data[$e->get('event')] = true; + } + + return $data; + } + + public function addEvent(string $event) + { + $this->data[$event] = true; + + if ($this->config->get('useCache')) { + $this->storeDataToCache(); + } + } + + public function removeEvent(string $event) + { + $notExists = !$this->entityManager->getRepository('Webhook')->select(['id'])->where([ + 'event' => $event, + 'isActive' => true, + ])->findOne(); + + if ($notExists) { + unset($this->data[$event]); + if ($this->config->get('useCache')) { + $this->storeDataToCache(); + } + } + } + + protected function eventExists(string $event) : bool + { + return isset($this->data[$event]); + } + + protected function logDebugEvent(string $event, Entity $entity) + { + $GLOBALS['log']->debug("Webhook: {$event} on record {$entity->id}."); + } + + public function processCreate(Entity $entity) + { + $event = $entity->getEntityType() . '.create'; + + if (!$this->eventExists($event)) return; + + $this->entityManager->createEntity('WebhookEventQueueItem', [ + 'event' => $event, + 'targetType' => $entity->getEntityType(), + 'targetId' => $entity->id, + 'data' => $entity->getValueMap(), + ]); + + $this->logDebugEvent($event, $entity); + } + + public function processDelete(Entity $entity) + { + $event = $entity->getEntityType() . '.delete'; + + if (!$this->eventExists($event)) return; + + $this->entityManager->createEntity('WebhookEventQueueItem', [ + 'event' => $event, + 'targetType' => $entity->getEntityType(), + 'targetId' => $entity->id, + 'data' => (object) [ + 'id' => $entity->id, + ], + ]); + + $this->logDebugEvent($event, $entity); + } + + public function processUpdate(Entity $entity) + { + $event = $entity->getEntityType() . '.update'; + + $data = (object) []; + foreach ($entity->getAttributeList() as $attribute) { + if (in_array($attribute, $this->skipAttributeList)) continue; + if ($entity->isAttributeChanged($attribute)) { + $data->$attribute = $entity->get($attribute); + } + } + + if (!count(get_object_vars($data))) return; + + $data->id = $entity->id; + + if ($this->eventExists($event)) { + $this->entityManager->createEntity('WebhookEventQueueItem', [ + 'event' => $event, + 'targetType' => $entity->getEntityType(), + 'targetId' => $entity->id, + 'data' => $data, + ]); + $this->logDebugEvent($event, $entity); + } + + foreach ($this->fieldManager->getEntityTypeFieldList($entity->getEntityType()) as $field) { + $itemEvent = $entity->getEntityType() . '.fieldUpdate.' . $field; + if (!$this->eventExists($itemEvent)) continue; + + $attributeList = $this->fieldManager->getActualAttributeList($entity->getEntityType(), $field); + $isChanged = false; + foreach ($attributeList as $attribute) { + if (in_array($attribute, $this->skipAttributeList)) continue; + if (property_exists($data, $attribute)) { + $isChanged = true; + break; + } + } + + if ($isChanged) { + $itemData = (object) []; + $itemData->id = $entity->id; + $attributeList = $this->fieldManager->getAttributeList($entity->getEntityType(), $field); + foreach ($attributeList as $attribute) { + if (in_array($attribute, $this->skipAttributeList)) continue; + $itemData->$attribute = $entity->get($attribute); + } + + $this->entityManager->createEntity('WebhookEventQueueItem', [ + 'event' => $itemEvent, + 'targetType' => $entity->getEntityType(), + 'targetId' => $entity->id, + 'data' => $itemData, + ]); + + $this->logDebugEvent($itemEvent, $entity); + } + } + } +} diff --git a/application/Espo/Core/Webhook/Queue.php b/application/Espo/Core/Webhook/Queue.php new file mode 100644 index 0000000000..7c55becfc2 --- /dev/null +++ b/application/Espo/Core/Webhook/Queue.php @@ -0,0 +1,291 @@ +sender = $sender; + $this->config = $config; + $this->entityManager = $entityManager; + $this->aclManager = $aclManager; + } + + public function process() + { + $this->processEvents(); + $this->processSending(); + } + + protected function processEvents() + { + $portionSize = $this->config->get('webhookQueueEventPortionSize', self::EVENT_PORTION_SIZE); + + $itemList = $this->entityManager->getRepository('WebhookEventQueueItem')->where([ + 'isProcessed' => false, + ])->order('number')->limit(0, $portionSize)->find(); + + foreach ($itemList as $item) { + $this->createQueueFromEvent($item); + $item->set([ + 'isProcessed' => true, + ]); + $this->entityManager->saveEntity($item); + } + } + + protected function createQueueFromEvent(\Espo\Entities\WebhookEventQueueItem $item) + { + $webhookList = $this->entityManager->getRepository('Webhook')->where([ + 'event' => $item->get('event'), + 'isActive' => true, + ])->order('createdAt')->find(); + + foreach ($webhookList as $webhook) { + $this->entityManager->createEntity('WebhookQueueItem', [ + 'webhookId' => $webhook->id, + 'event' => $item->get('event'), + 'targetId' => $item->get('targetId'), + 'targetType' => $item->get('targetType'), + 'status' => 'Pending', + 'data' => $item->get('data'), + 'attempts' => 0, + ]); + } + } + + protected function processSending() + { + $portionSize = $this->config->get('webhookQueuePortionSize', self::PORTION_SIZE); + $batchSize = $this->config->get('webhookBatchSize', self::BATCH_SIZE); + + $groupedItemList = $this->entityManager->getRepository('WebhookQueueItem')->where([ + 'status' => 'Pending', + 'OR' => [ + ['processAt' => null], + ['processAt<=' => DateTime::getSystemNowString()], + ], + ])->order('number')->limit(0, $portionSize)->groupBy(['webhookId'])->find(); + + foreach ($groupedItemList as $group) { + $webhookId = $group->get('webhookId'); + + $itemList = $this->entityManager->getRepository('WebhookQueueItem')->where([ + 'webhookId' => $webhookId, + 'status' => 'Pending', + 'OR' => [ + ['processAt' => null], + ['processAt<=' => DateTime::getSystemNowString()], + ], + ])->order('number')->limit(0, $batchSize)->find(); + + $webhook = $this->entityManager->getEntity('Webhook', $webhookId); + if (!$webhook || !$webhook->get('isActive')) { + foreach ($itemList as $item) { + $this->deleteQueueItem($item); + } + } + + $forbiddenAttributeList = []; + $user = null; + if ($webhook->get('userId')) { + $user = $this->entityManager->getEntity('User', $webhook->get('userId')); + if (!$user) { + foreach ($itemList as $item) { + $this->deleteQueueItem($item); + } + continue; + } else { + $forbiddenAttributeList = $this->aclManager->getScopeForbiddenAttributeList($user, $webhook->get('entityType')); + } + } + + $actualItemList = []; + + $dataList = []; + foreach ($itemList as $item) { + $targetType = $item->get('targetType'); + $target = null; + if ($this->entityManager->hasRepository($targetType)) { + $target = $this->entityManager->getRepository($targetType)->where([ + 'id' => $item->get('targetId') + ])->findOne(['withDeleted' => true]); + } + if (!$target) { + $this->deleteQueueItem($item); + continue; + } + + if ($user) { + if (!$this->aclManager->check($user, $target)) { + $this->deleteQueueItem($item); + continue; + } + } + + $data = $item->get('data') ?? (object) []; + $data = clone $data; + + foreach ($forbiddenAttributeList as $attribute) { + unset($data->$attribute); + } + + $actualItemList[] = $item; + $dataList[] = $data; + } + if (empty($dataList)) continue; + + $this->send($webhook, $dataList, $actualItemList); + } + } + + protected function send(Webhook $webhook, array $dataList, array $itemList) + { + try { + $code = $this->sender->send($webhook, $dataList); + } catch (\Exception $e) { + $this->failQueueItemList($itemList, true); + $GLOBALS['log']->error("Webhook Queue: Webhook {$webhook->id} sending failed. Error: " . $e->getMessage()); + return; + } + + if ($code >= 200 && $code < 400) { + $this->succeedQueueItemList($itemList); + } else if ($code === 410) { + $this->dropWebhook($webhook); + } else if (in_array($code, [0, 401, 403, 404, 405, 408, 500, 503])) { + $this->failQueueItemList($itemList); + } else if ($code >= 400 && $code < 500) { + $this->failQueueItemList($itemList, true); + } else { + $this->failQueueItemList($itemList, true); + } + + $this->logSending($webhook, $code); + } + + protected function logSending(Webhook $webhook, int $code) + { + $GLOBALS['log']->debug("Webhook Queue: Webhook {$webhook->id} sent, response code: {$code}."); + } + + protected function failQueueItemList(array $itemList, bool $force = false) + { + foreach ($itemList as $item) { + $this->failQueueItem($item, $force); + } + } + + protected function succeedQueueItemList(array $itemList) + { + foreach ($itemList as $item) { + $this->succeedQueueItem($item); + } + } + + protected function deleteQueueItem(WebhookQueueItem $item) + { + $this->entityManager->getRepository('WebhookQueueItem')->deleteFromDb($item->id); + } + + protected function dropWebhook(Webhook $webhook) + { + $itemList = $this->entityManager->getRepository('WebhookQueueItem')->where([ + 'status' => 'Pending', + 'webhookId' => $webhook->id, + ])->order('number')->find(); + + foreach ($itemList as $item) { + $this->deleteQueueItem($item); + } + + $this->entityManager->removeEntity($webhook); + } + + protected function succeedQueueItem(WebhookQueueItem $item) + { + $item->set([ + 'attempts' => $item->get('attempts') + 1, + 'status' => 'Success', + 'processedAt' => DateTime::getSystemNowString(), + ]); + + $this->entityManager->saveEntity($item); + } + + protected function failQueueItem(WebhookQueueItem $item, bool $force = false) + { + $attempts = $item->get('attempts') + 1; + $maxAttemptsNumber = $this->config->get('webhookMaxAttemptNumber', self::MAX_ATTEMPT_NUMBER); + $period = $this->config->get('webhookFailAttemptPeriod', self::FAIL_ATTEMPT_PERIOD); + + if ($force) $maxAttemptsNumber = 0; + + $dt = new \DateTime(); + $dt->modify($period); + + $item->set([ + 'attempts' => $attempts, + 'processAt' => $dt->format(DateTime::$systemDateTimeFormat), + ]); + + if ($attempts >= $maxAttemptsNumber) { + $item->set('status', 'Failed'); + $item->set('processAt', null); + } + + $this->entityManager->saveEntity($item); + } +} diff --git a/application/Espo/Core/Webhook/Sender.php b/application/Espo/Core/Webhook/Sender.php new file mode 100644 index 0000000000..ae8094e1c6 --- /dev/null +++ b/application/Espo/Core/Webhook/Sender.php @@ -0,0 +1,100 @@ +config = $config; + } + + public function send(Webhook $webhook, array $dataList) : int + { + $payload = json_encode($dataList); + + $signature = null; + $secretKey = $webhook->get('secretKey'); + if ($secretKey) { + $signature = $this->buildSignature($webhook, $payload, $secretKey); + } + + $connectTimeout = $this->config->get('webhookConnectTimeout', self::CONNECT_TIMEOUT); + $timeout = $this->config->get('webhookTimeout', self::TIMEOUT); + + $headerList = []; + $headerList[] = 'Content-Type: application/json'; + $headerList[] = 'Content-Length: ' . strlen($payload); + if ($signature) { + $headerList[] = 'X-Signature: ' . $signature; + } + + $handler = curl_init($webhook->get('url')); + curl_setopt($handler, \CURLOPT_RETURNTRANSFER, true); + curl_setopt($handler, \CURLOPT_FOLLOWLOCATION, true); + curl_setopt($handler, \CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($handler, \CURLOPT_HEADER, true); + curl_setopt($handler, \CURLOPT_CUSTOMREQUEST, 'POST'); + curl_setopt($handler, \CURLOPT_CONNECTTIMEOUT, $connectTimeout); + curl_setopt($handler, \CURLOPT_TIMEOUT, $timeout); + curl_setopt($handler, \CURLOPT_HTTPHEADER, $headerList); + curl_setopt($handler, \CURLOPT_POSTFIELDS, $payload); + + curl_exec($handler); + + $code = curl_getinfo($handler, \CURLINFO_HTTP_CODE); + + if (!is_numeric($code)) $code = 0; + if (!is_int($code)) $code = intval($code); + + if ($errorNumber = curl_errno($handler)) { + if (in_array($errorNumber, [\CURLE_OPERATION_TIMEDOUT, \CURLE_OPERATION_TIMEOUTED])) { + $code = 408; + } + } + + curl_close($handler); + + return $code; + } + + protected function buildSignature(Webhook $webhook, string $payload, string $secretKey) + { + return base64_encode($webhook->id . ':' . hash_hmac('sha256', $payload, $secretKey, true)); + } +} diff --git a/application/Espo/Entities/Webhook.php b/application/Espo/Entities/Webhook.php new file mode 100644 index 0000000000..8f049eb70e --- /dev/null +++ b/application/Espo/Entities/Webhook.php @@ -0,0 +1,35 @@ +addDependency('metadata'); + $this->addDependency('container'); + } + + protected function getMetadata() + { + return $this->getInjection('metadata'); + } + + protected function getWebhookManager() + { + return $this->getInjection('container')->get('webhookManager'); + } + + public function afterSave(Entity $entity, array $options = []) + { + if (!empty($options['silent'])) return; + if (!$this->getMetadata()->get(['scopes', $entity->getEntityType(), 'object'])) return; + + if ($entity->isNew()) { + $this->getWebhookManager()->processCreate($entity); + } else { + $this->getWebhookManager()->processUpdate($entity); + } + } + + public function afterRemove(Entity $entity, array $options = []) + { + if (!empty($options['silent'])) return; + if (!$this->getMetadata()->get(['scopes', $entity->getEntityType(), 'object'])) return; + + $this->getWebhookManager()->processDelete($entity); + } +} diff --git a/application/Espo/Jobs/Cleanup.php b/application/Espo/Jobs/Cleanup.php index 2c9dbe368d..4e0f2627fa 100644 --- a/application/Espo/Jobs/Cleanup.php +++ b/application/Espo/Jobs/Cleanup.php @@ -53,6 +53,8 @@ class Cleanup extends \Espo\Core\Jobs\Base protected $cleanupDeletedRecordsPeriod = '3 months'; + protected $cleanupWebhookQueuePeriod = '10 days'; + public function run() { $this->cleanupJobs(); @@ -66,6 +68,7 @@ class Cleanup extends \Espo\Core\Jobs\Base $this->cleanupUpgradeBackups(); $this->cleanupUniqueIds(); $this->cleanupDeletedRecords(); + $this->cleanupWebhookQueue(); } protected function cleanupJobs() @@ -459,4 +462,28 @@ class Cleanup extends \Espo\Core\Jobs\Base } } } + + protected function cleanupWebhookQueue() + { + $pdo = $this->getEntityManager()->getPDO(); + + $period = '-' . $this->getConfig()->get('cleanupWebhookQueuePeriod', $this->cleanupWebhookQueuePeriod); + $datetime = new \DateTime(); + $datetime->modify($period); + $from = $datetime->format('Y-m-d H:i:s'); + + $query = " + DELETE FROM `webhook_queue_item` + WHERE + DATE(created_at) < ".$pdo->quote($from)." AND + (status <> 'Pending' OR deleted = 1) + "; + $pdo->query($query); + + $query = " + DELETE FROM `webhook_event_queue_item` + WHERE DATE(created_at) < ".$pdo->quote($from)." AND (is_processed = 1 OR deleted = 1) + "; + $pdo->query($query); + } } diff --git a/application/Espo/Jobs/ProcessWebhookQueue.php b/application/Espo/Jobs/ProcessWebhookQueue.php new file mode 100644 index 0000000000..bfebdf42f1 --- /dev/null +++ b/application/Espo/Jobs/ProcessWebhookQueue.php @@ -0,0 +1,51 @@ +getContainer()->get('config') + ); + + $webhookQueue = new \Espo\Core\Webhook\Queue( + $sender, + $this->getContainer()->get('config'), + $this->getContainer()->get('entityManager'), + $this->getContainer()->get('aclManager') + ); + + $webhookQueue->process(); + } +} diff --git a/application/Espo/Repositories/Webhook.php b/application/Espo/Repositories/Webhook.php new file mode 100644 index 0000000000..c96ef9cc4b --- /dev/null +++ b/application/Espo/Repositories/Webhook.php @@ -0,0 +1,87 @@ +isNew()) { + $this->fillSecretKey($entity); + } + parent::beforeSave($entity); + $this->processSettingAdditionalFields($entity); + } + + protected function fillSecretKey(Entity $entity) + { + $secretKey = \Espo\Core\Utils\Util::generateKey(); + $entity->set('secretKey', $secretKey); + } + + protected function processSettingAdditionalFields(Entity $entity) + { + $event = $entity->get('event'); + if (!$event) return; + + $arr = explode('.', $event); + if (count($arr) !== 2 && count($arr) !== 3) return; + + $arr = explode('.', $event); + $entityType = $arr[0]; + $type = $arr[1]; + + $entity->set('entityType', $entityType); + $entity->set('type', $type); + + $field = null; + + if (!$entityType) return; + + if ($type === 'fieldUpdate') { + if (count($arr) == 3) { + $field = $arr[2]; + } + $entity->set('field', $field); + } else { + $entity->set('field', null); + } + } +} diff --git a/application/Espo/Resources/i18n/en_US/Admin.json b/application/Espo/Resources/i18n/en_US/Admin.json index 59b55724da..f6ff3cdd2f 100644 --- a/application/Espo/Resources/i18n/en_US/Admin.json +++ b/application/Espo/Resources/i18n/en_US/Admin.json @@ -38,6 +38,7 @@ "Currency": "Currency", "Integrations": "Integrations", "Extensions": "Extensions", + "Webhooks": "Webhooks", "Upload": "Upload", "Installing...": "Installing...", "Upgrading...": "Upgrading...", @@ -243,6 +244,7 @@ "attachments": "All file attachments stored in the system.", "systemRequirements": "System Requirements for EspoCRM.", "apiUsers": "Separate users for integration purposes.", + "webhooks": "Manage webhooks.", "pdfTemplates": "Templates for printing to PDF." }, "options": { diff --git a/application/Espo/Resources/i18n/en_US/Global.json b/application/Espo/Resources/i18n/en_US/Global.json index 92b3f9a4e1..a379e66da9 100644 --- a/application/Espo/Resources/i18n/en_US/Global.json +++ b/application/Espo/Resources/i18n/en_US/Global.json @@ -48,7 +48,8 @@ "AuthFailLogRecord": "Auth Fail Log Record", "LeadCapture": "Lead Capture Entry Point", "LeadCaptureLogRecord": "Lead Capture Log Record", - "ArrayValue": "Array Value" + "ArrayValue": "Array Value", + "Webhook": "Webhook" }, "scopeNamesPlural": { "Email": "Emails", @@ -86,7 +87,8 @@ "AuthFailLogRecord": "Auth Fail Log", "LeadCapture": "Lead Capture", "LeadCaptureLogRecord": "Lead Capture Log", - "ArrayValue": "Array Values" + "ArrayValue": "Array Values", + "Webhook": "Webhooks" }, "labels": { "Misc": "Misc", diff --git a/application/Espo/Resources/i18n/en_US/ScheduledJob.json b/application/Espo/Resources/i18n/en_US/ScheduledJob.json index 27ca26ffe0..36a6f62e25 100644 --- a/application/Espo/Resources/i18n/en_US/ScheduledJob.json +++ b/application/Espo/Resources/i18n/en_US/ScheduledJob.json @@ -19,7 +19,8 @@ "SendEmailReminders": "Send Email Reminders", "AuthTokenControl": "Auth Token Control", "SendEmailNotifications": "Send Email Notifications", - "CheckNewVersion": "Check for New Version" + "CheckNewVersion": "Check for New Version", + "ProcessWebhookQueue": "Process Webhook Queue" }, "cronSetup": { "linux": "Note: Add this line to the crontab file to run Espo Scheduled Jobs:", diff --git a/application/Espo/Resources/i18n/en_US/User.json b/application/Espo/Resources/i18n/en_US/User.json index 2a321be645..641a4a8b1d 100644 --- a/application/Espo/Resources/i18n/en_US/User.json +++ b/application/Espo/Resources/i18n/en_US/User.json @@ -112,6 +112,7 @@ }, "presetFilters": { "active": "Active", - "activePortal": "Portal Active" + "activePortal": "Portal Active", + "activeApi": "API Active" } } diff --git a/application/Espo/Resources/i18n/en_US/Webhook.json b/application/Espo/Resources/i18n/en_US/Webhook.json new file mode 100644 index 0000000000..b0a2b64c70 --- /dev/null +++ b/application/Espo/Resources/i18n/en_US/Webhook.json @@ -0,0 +1,17 @@ +{ + "labels": { + "Create Webhook": "Create Webhook" + }, + "fields": { + "event": "Event", + "url": "URL", + "isActive": "Is Active", + "user": "API User", + "entityType": "Entity Type", + "field": "Field", + "secretKey": "Secret Key" + }, + "links": { + "user": "User" + } +} diff --git a/application/Espo/Resources/layouts/Webhook/detail.json b/application/Espo/Resources/layouts/Webhook/detail.json new file mode 100644 index 0000000000..2ef62facd0 --- /dev/null +++ b/application/Espo/Resources/layouts/Webhook/detail.json @@ -0,0 +1,18 @@ +[ + { + "rows": [ + [ + {"name": "event"}, + {"name": "isActive"} + ], + [ + {"name": "url"}, + {"name": "user"} + ], + [ + {"name": "secretKey"}, + false + ] + ] + } +] diff --git a/application/Espo/Resources/layouts/Webhook/detailSmall.json b/application/Espo/Resources/layouts/Webhook/detailSmall.json new file mode 100644 index 0000000000..12211c3060 --- /dev/null +++ b/application/Espo/Resources/layouts/Webhook/detailSmall.json @@ -0,0 +1,19 @@ +[ + { + "rows": [ + [ + {"name": "event"}, + {"name": "isActive"} + ], + [ + {"name": "url", "fullWidth": true} + ], + [ + {"name": "user", "fullWidth": true} + ], + [ + {"name": "secretKey", "fullWidth": true} + ] + ] + } +] diff --git a/application/Espo/Resources/layouts/Webhook/filters.json b/application/Espo/Resources/layouts/Webhook/filters.json new file mode 100644 index 0000000000..9fafc8129e --- /dev/null +++ b/application/Espo/Resources/layouts/Webhook/filters.json @@ -0,0 +1,5 @@ +[ + "type", + "entityType", + "user" +] \ No newline at end of file diff --git a/application/Espo/Resources/layouts/Webhook/list.json b/application/Espo/Resources/layouts/Webhook/list.json new file mode 100644 index 0000000000..c6259457f9 --- /dev/null +++ b/application/Espo/Resources/layouts/Webhook/list.json @@ -0,0 +1,18 @@ +[ + { + "name": "event", + "width": 22, + "link": true + }, + { + "name": "isActive", + "width": 14 + }, + { + "name": "user", + "width": 22 + }, + { + "name": "url" + } +] \ No newline at end of file diff --git a/application/Espo/Resources/metadata/app/acl.json b/application/Espo/Resources/metadata/app/acl.json index 3ef7c43887..6a36477db5 100644 --- a/application/Espo/Resources/metadata/app/acl.json +++ b/application/Espo/Resources/metadata/app/acl.json @@ -87,7 +87,8 @@ "read": "all", "edit": "no" }, - "Import": false + "Import": false, + "Webhook": false }, "fieldLevel": { }, @@ -103,7 +104,8 @@ "read": "own", "edit": "no" }, - "Import": false + "Import": false, + "Webhook": false }, "fieldLevel": { }, diff --git a/application/Espo/Resources/metadata/app/adminPanel.json b/application/Espo/Resources/metadata/app/adminPanel.json index 03f2a45de1..0f8331cdca 100644 --- a/application/Espo/Resources/metadata/app/adminPanel.json +++ b/application/Espo/Resources/metadata/app/adminPanel.json @@ -206,6 +206,13 @@ "url": "#Template", "label": "PDF Templates", "description": "pdfTemplates" + + }, + { + + "url": "#Webhook", + "label": "Webhooks", + "description": "webhooks" }, { "url": "#Attachment", diff --git a/application/Espo/Resources/metadata/clientDefs/Webhook.json b/application/Espo/Resources/metadata/clientDefs/Webhook.json new file mode 100644 index 0000000000..f053c45b4f --- /dev/null +++ b/application/Espo/Resources/metadata/clientDefs/Webhook.json @@ -0,0 +1,27 @@ +{ + "controller": "controllers/record", + "dynamicLogic": { + "fields": { + "event": { + "readOnly": { + "conditionGroup": [ + { + "type": "isNotEmpty", + "attribute": "id" + } + ] + } + }, + "secretKey": { + "visible": { + "conditionGroup": [ + { + "type": "isNotEmpty", + "attribute": "id" + } + ] + } + } + } + } +} diff --git a/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json b/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json index 50a34a86a8..2035508c01 100644 --- a/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json +++ b/application/Espo/Resources/metadata/entityDefs/ScheduledJob.json @@ -73,7 +73,8 @@ "SendEmailReminders": "*/2 * * * *", "Cleanup": "1 1 * * 0", "AuthTokenControl": "*/6 * * * *", - "SendEmailNotifications": "*/2 * * * *" + "SendEmailNotifications": "*/2 * * * *", + "ProcessWebhookQueue": "*/5 * * * *" }, "jobs": { "Dummy": { diff --git a/application/Espo/Resources/metadata/entityDefs/Webhook.json b/application/Espo/Resources/metadata/entityDefs/Webhook.json new file mode 100644 index 0000000000..a0f2bb4a7c --- /dev/null +++ b/application/Espo/Resources/metadata/entityDefs/Webhook.json @@ -0,0 +1,95 @@ +{ + "fields": { + "event": { + "type": "varchar", + "maxLength": 100, + "required": true + }, + "url": { + "type": "varchar", + "maxLength": 512, + "required": true + }, + "isActive": { + "type": "bool", + "default": true + }, + "user": { + "type": "link", + "view": "views/webhook/fields/user" + }, + "entityType": { + "type": "varchar", + "readOnly": true, + "view": "views/fields/entity-type" + }, + "type": { + "type": "enum", + "options": [ + "create", + "update", + "fieldUpdate", + "delete" + ], + "readOnly": true + }, + "field": { + "type": "varchar", + "readOnly": true + }, + "secretKey": { + "type": "varchar", + "maxLength": 100, + "readOnly": true, + "layoutMassUpdateDisabled": true, + "layoutFiltersDisabled": true, + "layoutListDisabled": true + }, + "createdAt": { + "type": "datetime", + "readOnly": true + }, + "modifiedAt": { + "type": "datetime", + "readOnly": true + }, + "createdBy": { + "type": "link", + "readOnly": true + }, + "modifiedBy": { + "type": "link", + "readOnly": true + } + }, + "links": { + "user": { + "type": "belongsTo", + "entity": "User" + }, + "createdBy": { + "type": "belongsTo", + "entity": "User" + }, + "modifiedBy": { + "type": "belongsTo", + "entity": "User" + } + }, + "collection": { + "orderBy": "createdAt", + "order": "desc", + "textFilterFields": ["event"] + }, + "indexes": { + "event": { + "columns": ["event"] + }, + "entityTypeType": { + "columns": ["entityType", "type"] + }, + "entityTypeField": { + "columns": ["entityType", "field"] + } + } +} diff --git a/application/Espo/Resources/metadata/entityDefs/WebhookEventQueueItem.json b/application/Espo/Resources/metadata/entityDefs/WebhookEventQueueItem.json new file mode 100644 index 0000000000..4a0ea657c5 --- /dev/null +++ b/application/Espo/Resources/metadata/entityDefs/WebhookEventQueueItem.json @@ -0,0 +1,32 @@ +{ + "fields": { + "number": { + "type": "autoincrement", + "dbType": "bigint" + }, + "event": { + "type": "varchar", + "maxLength": 100, + "required": true + }, + "target": { + "type": "linkParent" + }, + "data": { + "type": "jsonObject" + }, + "createdAt": { + "type": "datetime", + "readOnly": true + }, + "isProcessed": { + "type": "bool" + } + }, + "links": { + }, + "collection": { + "orderBy": "number", + "order": "desc" + } +} diff --git a/application/Espo/Resources/metadata/entityDefs/WebhookQueueItem.json b/application/Espo/Resources/metadata/entityDefs/WebhookQueueItem.json new file mode 100644 index 0000000000..24062d44c8 --- /dev/null +++ b/application/Espo/Resources/metadata/entityDefs/WebhookQueueItem.json @@ -0,0 +1,52 @@ +{ + "fields": { + "number": { + "type": "autoincrement", + "dbType": "bigint" + }, + "event": { + "type": "varchar", + "maxLength": 100, + "required": true + }, + "webhook": { + "type": "link" + }, + "target": { + "type": "linkParent" + }, + "data": { + "type": "jsonObject" + }, + "createdAt": { + "type": "datetime" + }, + "status": { + "type": "enum", + "options": ["Pending", "Success", "Failed"], + "default": "Pending", + "maxLength": 15 + }, + "processedAt": { + "type": "datetime" + }, + "attempts": { + "type": "int", + "default": 0 + }, + "processAt": { + "type": "datetime" + } + }, + "links": { + "webhook": { + "type": "belongsTo", + "entity": "Webhook", + "foreignName": "id" + } + }, + "collection": { + "orderBy": "number", + "order": "desc" + } +} diff --git a/application/Espo/Resources/metadata/scopes/Webhook.json b/application/Espo/Resources/metadata/scopes/Webhook.json new file mode 100644 index 0000000000..d53680616c --- /dev/null +++ b/application/Espo/Resources/metadata/scopes/Webhook.json @@ -0,0 +1,4 @@ +{ + "entity": true, + "acl": "boolean" +} diff --git a/application/Espo/Resources/metadata/scopes/WebhookEventQueueItem.json b/application/Espo/Resources/metadata/scopes/WebhookEventQueueItem.json new file mode 100644 index 0000000000..7ec928c601 --- /dev/null +++ b/application/Espo/Resources/metadata/scopes/WebhookEventQueueItem.json @@ -0,0 +1,3 @@ +{ + "entity": true +} diff --git a/application/Espo/Resources/metadata/scopes/WebhookQueueItem.json b/application/Espo/Resources/metadata/scopes/WebhookQueueItem.json new file mode 100644 index 0000000000..7ec928c601 --- /dev/null +++ b/application/Espo/Resources/metadata/scopes/WebhookQueueItem.json @@ -0,0 +1,3 @@ +{ + "entity": true +} diff --git a/application/Espo/SelectManagers/Webhook.php b/application/Espo/SelectManagers/Webhook.php new file mode 100644 index 0000000000..1e3465c07d --- /dev/null +++ b/application/Espo/SelectManagers/Webhook.php @@ -0,0 +1,48 @@ +getUser()->isAdmin() && !$this->getUser()->isApi()) { + $result['whereClause'][] = [ + 'id' => null + ]; + } + + if ($this->getUser()->isApi()) { + $result['whereClause'][] = [ + 'userId' => $this->getUser()->id + ]; + } + } +} diff --git a/application/Espo/Services/Webhook.php b/application/Espo/Services/Webhook.php new file mode 100644 index 0000000000..71ba40ead0 --- /dev/null +++ b/application/Espo/Services/Webhook.php @@ -0,0 +1,194 @@ +addDependencyList([ + 'webhookManager', + ]); + } + + public function populateDefaults(Entity $entity, $data) + { + parent::populateDefaults($entity, $data); + + if ($this->getUser()->isApi()) { + $entity->set('userId', $this->getUser()->id); + } + } + + protected function filtetInput($data) + { + parent::filtetInput($data); + + unset($data->entityType); + unset($data->field); + unset($data->type); + } + + protected function filterUpdateInput($data) + { + if (!$this->getUser()->isAdmin()) { + unset($data->event); + } + } + + protected function beforeCreateEntity(Entity $entity, $data) + { + $this->checkEntityUserIsApi($entity); + $this->processEntityEventData($entity); + + if (!$this->getUser()->isAdmin()) { + $this->checkMaxCount(); + } + } + + protected function checkMaxCount() + { + $maxCount = $this->getConfig()->get('webhookMaxCountPerUser', self::WEBHOOK_MAX_COUNT_PER_USER); + + $count = $this->getEntityManager()->getRepository('Webhook')->where([ + 'userId' => $this->getUser()->id, + ])->count(); + + if ($maxCount && $count >= $maxCount) { + throw new Forbidden("Webhook number per user exceeded the limit."); + } + } + + protected function beforeUpdateEntity(Entity $entity, $data) + { + $this->checkEntityUserIsApi($entity); + $this->processEntityEventData($entity); + } + + protected function checkEntityUserIsApi(Entity $entity) + { + $userId = $entity->get('userId'); + if (!$userId) return; + + $user = $this->getEntityManager()->getEntity('User', $userId); + if (!$user || !$user->isApi()) throw new Forbidden("User must be an API User."); + } + + protected function processEntityEventData(Entity $entity) + { + $event = $entity->get('event'); + if (!$event) throw new Forbidden("Event is empty."); + + if (!$entity->isNew()) { + if ($entity->isAttributeChanged('event')) { + throw new Forbidden("Event can't be changed."); + } + } + + $arr = explode('.', $event); + if (count($arr) !== 2 && count($arr) !== 3) throw new Forbidden("Not supported event."); + + $arr = explode('.', $event); + $entityType = $arr[0]; + $type = $arr[1]; + + $entity->set('entityType', $entityType); + $entity->set('type', $type); + + $field = null; + + if (!$entityType) throw new Forbidden("Entity Type is empty."); + if (!$this->getMetadata()->get(['scopes', $entityType, 'object'])) throw new Forbidden("Entity type is not available for Webhooks."); + if (!$this->getEntityManager()->hasRepository($entityType)) throw new Forbidden("Not existing Entity Type."); + if (!$this->getAcl()->checkScope($entityType, 'read')) throw new Forbidden("Entity type is forbidden."); + + if (!in_array($type, $this->eventTypeList)) throw new Forbidden("Not supported event."); + + if ($type === 'fieldUpdate') { + if (count($arr) == 3) { + $field = $arr[2]; + } + $entity->set('field', $field); + + if (!$field) throw new Forbidden("Field is empty."); + $forbiddenFieldList = $this->getAcl()->getScopeForbiddenFieldList($entityType); + if (in_array($field, $forbiddenFieldList)) throw new Forbidden("Field is forbidden."); + + if (!$this->getMetadata()->get(['entityDefs', $entityType, 'fields', $field])) throw new Forbidden("Field does not exist."); + } else { + $entity->set('field', null); + } + } + + protected function afterCreateEntity(Entity $entity, $data) + { + if ($entity->get('isActive')) { + $this->getInjection('webhookManager')->addEvent($entity->get('event')); + } + } + + protected function afterDeleteEntity(Entity $entity) + { + if ($entity->get('isActive')) { + $this->getInjection('webhookManager')->removeEvent($entity->get('event')); + } + } + + protected function afterUpdateEntity(Entity $entity, $data) + { + if (isset($data->isActive)) { + if ($entity->get('isActive')) { + $this->getInjection('webhookManager')->addEvent($entity->get('event')); + } else { + $this->getInjection('webhookManager')->removeEvent($entity->get('event')); + } + } + } +} diff --git a/client/src/views/webhook/fields/user.js b/client/src/views/webhook/fields/user.js new file mode 100644 index 0000000000..7d67ae522c --- /dev/null +++ b/client/src/views/webhook/fields/user.js @@ -0,0 +1,36 @@ +/************************************************************************ + * This file is part of EspoCRM. + * + * EspoCRM - Open Source CRM application. + * Copyright (C) 2014-2019 Yuri Kuznetsov, Taras Machyshyn, Oleksiy Avramenko + * Website: https://www.espocrm.com + * + * EspoCRM is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * EspoCRM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with EspoCRM. If not, see http://www.gnu.org/licenses/. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "EspoCRM" word. + ************************************************************************/ + +define('views/webhook/fields/user', 'views/fields/link', function (Dep) { + + return Dep.extend({ + + selectPrimaryFilterName: 'activeApi', + + }); +}); diff --git a/install/core/afterInstall/records.php b/install/core/afterInstall/records.php index 88fc632ec5..e1ad6bf04a 100644 --- a/install/core/afterInstall/records.php +++ b/install/core/afterInstall/records.php @@ -84,6 +84,12 @@ return array( 'job' => 'ControlKnowledgeBaseArticleStatus', 'status' => 'Active', 'scheduling' => '10 1 * * *' - ) + ), + [ + 'name' => 'Process Webhook Queue', + 'job' => 'ProcessWebhookQueue', + 'status' => 'Active', + 'scheduling' => '*/5 * * * *', + ], ) ); \ No newline at end of file diff --git a/tests/integration/Espo/Webhook/AclTest.php b/tests/integration/Espo/Webhook/AclTest.php new file mode 100644 index 0000000000..85d30c8a98 --- /dev/null +++ b/tests/integration/Espo/Webhook/AclTest.php @@ -0,0 +1,167 @@ +createUser( + [ + 'userName' => 'test', + 'password' => '1', + ], + [ + 'data' => [ + 'Webhook' => true, + 'Account' => ['create'=> true, 'read' => 'own'], + ], + ] + ); + + $this->auth('test', '1'); + + $app = $this->createApplication(); + + $controllerManager = $app->getContainer()->get('controllerManager'); + + $params = []; + $data = '{"event":"Account.create"}'; + + $this->expectException(\Espo\Core\Exceptions\Forbidden::class); + + $request = $this->createRequest('POST', $params, ['CONTENT_TYPE' => 'application/json']); + $result = $controllerManager->process('Webhook', 'create', $params, $data, $request); + } + + public function testApiUserNoAccess1() + { + $this->createUser( + [ + 'userName' => 'api', + 'type' => 'api', + 'authMethod' => 'ApiKey', + 'apiKey' => 'test-key', + ], + [ + 'data' => [ + 'Webhook' => false, + ], + ] + ); + + $this->auth('test-key', null, null, 'ApiKey'); + + $app = $this->createApplication(); + + $controllerManager = $app->getContainer()->get('controllerManager'); + + $data = '{"event":"Account.create", "url": "https://test"}'; + + $params = []; + + $this->expectException(\Espo\Core\Exceptions\Forbidden::class); + + $request = $this->createRequest('POST', $params, ['CONTENT_TYPE' => 'application/json']); + $result = $controllerManager->process('Webhook', 'create', $params, $data, $request); + } + + public function testApiUserNoAccess2() + { + $this->createUser( + [ + 'userName' => 'api', + 'type' => 'api', + 'authMethod' => 'ApiKey', + 'apiKey' => 'test-key', + ], + [ + 'data' => [ + 'Webhook' => false, + 'Account' => false, + ], + ] + ); + + $this->auth('test-key', null, null, 'ApiKey'); + + $app = $this->createApplication(); + + $controllerManager = $app->getContainer()->get('controllerManager'); + + $data = '{"event":"Account.create", "url": "https://test"}'; + + $params = []; + + $this->expectException(\Espo\Core\Exceptions\Forbidden::class); + + $request = $this->createRequest('POST', $params, ['CONTENT_TYPE' => 'application/json']); + $result = $controllerManager->process('Webhook', 'create', $params, $data, $request); + } + + public function testApiUserHasAccess1() + { + $this->createUser( + [ + 'userName' => 'api', + 'type' => 'api', + 'authMethod' => 'ApiKey', + 'apiKey' => 'test-key', + ], + [ + 'data' => [ + 'Webhook' => true, + 'Account' => ['create'=> true, 'read' => 'own'], + ], + ] + ); + + $this->auth('test-key', null, null, 'ApiKey'); + + $app = $this->createApplication(); + + $controllerManager = $app->getContainer()->get('controllerManager'); + + $data = '{"event":"Account.create", "url": "https://test"}'; + + $params = []; + + $request = $this->createRequest('POST', $params, ['CONTENT_TYPE' => 'application/json']); + $result = $controllerManager->process('Webhook', 'create', $params, $data, $request); + + $this->assertTrue(!empty($result)); + } +}