From b982ab9daf00007edd5e45e7d8b4d23ec62f845a Mon Sep 17 00:00:00 2001 From: Yuri Kuznetsov Date: Sat, 3 Feb 2024 10:13:39 +0200 Subject: [PATCH] ref --- .../Crm/Entities/CampaignTrackingUrl.php | 10 + .../Modules/Crm/Entities/EmailQueueItem.php | 32 + .../Espo/Modules/Crm/Entities/MassEmail.php | 7 + .../Crm/Tools/MassEmail/SendingProcessor.php | 611 +++++++++++------- 4 files changed, 419 insertions(+), 241 deletions(-) diff --git a/application/Espo/Modules/Crm/Entities/CampaignTrackingUrl.php b/application/Espo/Modules/Crm/Entities/CampaignTrackingUrl.php index 7cb005e95d..55fc1f6415 100644 --- a/application/Espo/Modules/Crm/Entities/CampaignTrackingUrl.php +++ b/application/Espo/Modules/Crm/Entities/CampaignTrackingUrl.php @@ -30,6 +30,7 @@ namespace Espo\Modules\Crm\Entities; use Espo\Core\ORM\Entity; +use LogicException; class CampaignTrackingUrl extends Entity { @@ -57,6 +58,15 @@ class CampaignTrackingUrl extends Entity return $this->get('url'); } + public function getUrlToUse(): string + { + if (!$this->id) { + throw new LogicException(); + } + + return $this->get('urlToUse'); + } + protected function _getUrlToUse(): string { return '{trackingUrl:' . $this->id . '}'; diff --git a/application/Espo/Modules/Crm/Entities/EmailQueueItem.php b/application/Espo/Modules/Crm/Entities/EmailQueueItem.php index b05497324f..d2de48326d 100644 --- a/application/Espo/Modules/Crm/Entities/EmailQueueItem.php +++ b/application/Espo/Modules/Crm/Entities/EmailQueueItem.php @@ -30,6 +30,7 @@ namespace Espo\Modules\Crm\Entities; use Espo\Core\ORM\Entity; +use Espo\Core\Utils\DateTime as DateTimeUtil; class EmailQueueItem extends Entity { @@ -86,4 +87,35 @@ class EmailQueueItem extends Entity { return $this->get('emailAddress'); } + + public function setStatus(string $status): self + { + $this->set('status', $status); + + return $this; + } + + public function setSentAtNow(): self + { + $this->set('sentAt', date(DateTimeUtil::SYSTEM_DATE_TIME_FORMAT)); + + return $this; + } + + public function setEmailAddress(string $emailAddress): self + { + $this->set('emailAddress', $emailAddress); + + return $this; + } + + public function incrementAttemptCount(): self + { + $attemptCount = $this->getAttemptCount(); + $attemptCount++; + + $this->set('attemptCount', $attemptCount); + + return $this; + } } diff --git a/application/Espo/Modules/Crm/Entities/MassEmail.php b/application/Espo/Modules/Crm/Entities/MassEmail.php index b2e411c06e..17750e218d 100644 --- a/application/Espo/Modules/Crm/Entities/MassEmail.php +++ b/application/Espo/Modules/Crm/Entities/MassEmail.php @@ -90,4 +90,11 @@ class MassEmail extends Entity { return (bool) $this->get('optOutEntirely'); } + + public function setStatus(string $status): self + { + $this->set('status', $status); + + return $this; + } } diff --git a/application/Espo/Modules/Crm/Tools/MassEmail/SendingProcessor.php b/application/Espo/Modules/Crm/Tools/MassEmail/SendingProcessor.php index b325ba2e05..8513f8f874 100644 --- a/application/Espo/Modules/Crm/Tools/MassEmail/SendingProcessor.php +++ b/application/Espo/Modules/Crm/Tools/MassEmail/SendingProcessor.php @@ -29,6 +29,7 @@ namespace Espo\Modules\Crm\Tools\MassEmail; +use Espo\Tools\EmailTemplate\Result; use Laminas\Mail\Message; use Espo\Core\Mail\Account\GroupAccount\AccountFactory; @@ -86,64 +87,13 @@ class SendingProcessor */ public function process(MassEmail $massEmail, bool $isTest = false): void { - $hourMaxSize = $this->config->get('massEmailMaxPerHourCount', self::MAX_PER_HOUR_COUNT); - $batchMaxSize = $this->config->get('massEmailMaxPerBatchCount'); - - if (!$isTest) { - $threshold = new DateTime(); - $threshold->modify('-1 hour'); - - $sentLastHourCount = $this->entityManager - ->getRDBRepositoryByClass(EmailQueueItem::class) - ->where([ - 'status' => EmailQueueItem::STATUS_SENT, - 'sentAt>' => $threshold->format(DateTimeUtil::SYSTEM_DATE_TIME_FORMAT), - ]) - ->count(); - - if ($sentLastHourCount >= $hourMaxSize) { - return; - } - - $hourMaxSize = $hourMaxSize - $sentLastHourCount; - } - - $maxSize = $hourMaxSize; - - if ($batchMaxSize) { - $maxSize = min($batchMaxSize, $maxSize); - } - - $queueItemList = $this->entityManager - ->getRDBRepositoryByClass(EmailQueueItem::class) - ->sth() - ->where([ - 'status' => EmailQueueItem::STATUS_PENDING, - 'massEmailId' => $massEmail->getId(), - 'isTest' => $isTest, - ]) - ->limit(0, $maxSize) - ->find(); - - $templateId = $massEmail->getEmailTemplateId(); - - if (!$templateId) { - $this->setFailed($massEmail); + $maxSize = 0; + if ($this->handleMaxSize($isTest, $maxSize)) { return; } - $campaign = null; - - $campaignId = $massEmail->getCampaignId(); - - if ($campaignId) { - $campaign = $this->entityManager->getEntityById(Campaign::ENTITY_TYPE, $campaignId); - } - - $emailTemplate = $this->entityManager - ->getRDBRepositoryByClass(EmailTemplate::class) - ->getById($templateId); + $emailTemplate = $this->getEmailTemplate($massEmail); if (!$emailTemplate) { $this->setFailed($massEmail); @@ -151,37 +101,10 @@ class SendingProcessor return; } - /** @var Collection $attachmentList */ - $attachmentList = $this->entityManager - ->getRDBRepositoryByClass(EmailTemplate::class) - ->getRelation($emailTemplate, 'attachments') - ->find(); - - $smtpParams = null; - $senderParams = SenderParams::create(); - - $inboundEmailId = $massEmail->getInboundEmailId(); - - if ($inboundEmailId) { - $account = $this->accountFactory->create($inboundEmailId); - - $smtpParams = $account->getSmtpParams(); - - if ( - !$account->isAvailableForSending() || - !$account->getEntity()->smtpIsForMassEmail() || - !$smtpParams - ) { - throw new Error("Mass Email: Group email account $inboundEmailId can't be used for mass email."); - } - - if ($account->getEntity()->getReplyToAddress()) { - $senderParams = $senderParams - ->withReplyToAddress( - $account->getEntity()->getReplyToAddress() - ); - } - } + $campaign = $this->getCampaign($massEmail); + $attachmentList = $this->getAttachments($emailTemplate); + [$smtpParams, $senderParams] = $this->getSenderParams($massEmail); + $queueItemList = $this->getQueueItems($massEmail, $isTest, $maxSize); foreach ($queueItemList as $queueItem) { $this->sendQueueItem( @@ -200,20 +123,11 @@ class SendingProcessor return; } - $countLeft = $this->entityManager - ->getRDBRepositoryByClass(EmailQueueItem::class) - ->where([ - 'status' => EmailQueueItem::STATUS_PENDING, - 'massEmailId' => $massEmail->getId(), - 'isTest' => false, - ]) - ->count(); - - if ($countLeft === 0) { - $massEmail->set('status', MassEmail::STATUS_COMPLETE); - - $this->entityManager->saveEntity($massEmail); + if ($this->getCountLeft($massEmail) !== 0) { + return; } + + $this->setComplete($massEmail); } /** @@ -242,52 +156,7 @@ class SendingProcessor ->withParent($target) ); - $body = $emailData->getBody(); - - $optOutUrl = $this->getSiteUrl() . '?entryPoint=unsubscribe&id=' . $queueItem->getId(); - - $optOutLink = - '' . - $this->defaultLanguage->translateLabel('Unsubscribe', 'labels', Campaign::ENTITY_TYPE) . - ''; - - $body = str_replace('{optOutUrl}', $optOutUrl, $body); - $body = str_replace('{optOutLink}', $optOutLink, $body); - - foreach ($trackingUrlList as $trackingUrl) { - $url = $this->getSiteUrl() . - '?entryPoint=campaignUrl&id=' . $trackingUrl->getId() . '&queueItemId=' . $queueItem->getId(); - - $body = str_replace($trackingUrl->get('urlToUse'), $url, $body); - } - - if ( - !$this->config->get('massEmailDisableMandatoryOptOutLink') && - stripos($body, '?entryPoint=unsubscribe&id') === false - ) { - if ($emailData->isHtml()) { - $body .= "

" . $optOutLink; - } - else { - $body .= "\n\n" . $optOutUrl; - } - } - - $trackImageAlt = $this->defaultLanguage->translateLabel('Campaign', 'scopeNames'); - - $trackOpenedUrl = $this->getSiteUrl() . '?entryPoint=campaignTrackOpened&id=' . $queueItem->getId(); - - $trackOpenedHtml = - '' . $trackImageAlt . ''; - - if ( - $massEmail->getCampaignId() && - $this->config->get('massEmailOpenTracking') - ) { - if ($emailData->isHtml()) { - $body .= '
' . $trackOpenedHtml; - } - } + $body = $this->prepareBody($emailData, $queueItem, $trackingUrlList, $massEmail); /** @var Email $email */ $email = $this->entityManager @@ -341,7 +210,7 @@ class SendingProcessor private function setFailed(MassEmail $massEmail): void { - $massEmail->set('status', MassEmail::STATUS_FAILED); + $massEmail->setStatus(MassEmail::STATUS_FAILED); $this->entityManager->saveEntity($massEmail); @@ -354,9 +223,7 @@ class SendingProcessor ->find(); foreach ($queueItemList as $queueItem) { - $queueItem->set('status', EmailQueueItem::STATUS_FAILED); - - $this->entityManager->saveEntity($queueItem); + $this->setItemFailed($queueItem); } } @@ -374,19 +241,11 @@ class SendingProcessor SenderParams $senderParams ): void { - /** @var ?EmailQueueItem $queueItemFetched */ - $queueItemFetched = $this->entityManager->getEntityById($queueItem->getEntityType(), $queueItem->getId()); - - if ( - !$queueItemFetched || - $queueItemFetched->getStatus() !== EmailQueueItem::STATUS_PENDING - ) { + if ($this->isNotPending($queueItem)) { return; } - $queueItem->set('status', EmailQueueItem::STATUS_SENDING); - - $this->entityManager->saveEntity($queueItem); + $this->setItemSending($queueItem); $target = $this->entityManager->getEntityById($queueItem->getTargetType(), $queueItem->getTargetId()); @@ -397,48 +256,302 @@ class SendingProcessor !$target->hasId() || !$emailAddress ) { - $queueItem->set('status', EmailQueueItem::STATUS_FAILED); - - $this->entityManager->saveEntity($queueItem); + $this->setItemFailed($queueItem); return; } - /** @var EmailAddressRepository $emailAddressRepository */ - $emailAddressRepository = $this->entityManager->getRepository(EmailAddress::ENTITY_TYPE); - - $emailAddressRecord = $emailAddressRepository->getByAddress($emailAddress); + $emailAddressRecord = $this->getEmailAddressRepository()->getByAddress($emailAddress); if ( - $emailAddressRecord && ( - $emailAddressRecord->isInvalid() || - $emailAddressRecord->isOptedOut() - ) + $emailAddressRecord && + ($emailAddressRecord->isInvalid() || $emailAddressRecord->isOptedOut()) ) { - $queueItem->set('status', EmailQueueItem::STATUS_FAILED); - - $this->entityManager->saveEntity($queueItem); + $this->setItemFailed($queueItem); return; } - /** @var CampaignTrackingUrl[] $trackingUrlList */ - $trackingUrlList = []; - - if ($campaign) { - /** @var Collection $trackingUrlList */ - $trackingUrlList = $this->entityManager - ->getRDBRepositoryByClass(Campaign::class) - ->getRelation($campaign, 'trackingUrls') - ->find(); - } - - $email = $this->getPreparedEmail($queueItem, $massEmail, $emailTemplate, $target, $trackingUrlList); + $email = $this->getPreparedEmail( + $queueItem, + $massEmail, + $emailTemplate, + $target, + $this->getTrackingUrls($campaign) + ); if (!$email) { return; } + $senderParams = $this->prepareItemSenderParams($email, $senderParams, $campaign, $massEmail); + + $queueItem->incrementAttemptCount(); + + $sender = $this->emailSender->create(); + + if ($smtpParams) { + $sender->withSmtpParams($smtpParams); + } + + $message = new Message(); + + try { + $this->prepareQueueItemMessage($queueItem, $sender, $message, $senderParams); + + $sender + ->withParams($senderParams) + ->withMessage($message) + ->withAttachments($attachmentList) + ->send($email); + } + catch (Exception $e) { + $this->processException($queueItem, $e); + + return; + } + + $emailObject = $emailTemplate; + + if ($massEmail->storeSentEmails() && !$isTest) { + $this->entityManager->saveEntity($email); + + $emailObject = $email; + } + + $this->setItemSent($queueItem, $emailAddress); + + if ($campaign) { + $this->campaignService->logSent($campaign->getId(), $queueItem, $emailObject); + } + } + + private function getSiteUrl(): string + { + return + $this->config->get('massEmailSiteUrl') ?? + $this->config->get('siteUrl'); + } + + /** + * @throws Error + * @throws NoSmtp + * @return array{?SmtpParams, SenderParams} + */ + private function getSenderParams(MassEmail $massEmail): array + { + $smtpParams = null; + $senderParams = SenderParams::create(); + + $inboundEmailId = $massEmail->getInboundEmailId(); + + if (!$inboundEmailId) { + return [$smtpParams, $senderParams]; + } + + $account = $this->accountFactory->create($inboundEmailId); + + $smtpParams = $account->getSmtpParams(); + + if ( + !$account->isAvailableForSending() || + !$account->getEntity()->smtpIsForMassEmail() || + !$smtpParams + ) { + throw new Error("Mass Email: Group email account $inboundEmailId can't be used for mass email."); + } + + if ($account->getEntity()->getReplyToAddress()) { + $senderParams = $senderParams + ->withReplyToAddress($account->getEntity()->getReplyToAddress()); + } + + return [$smtpParams, $senderParams]; + } + + private function getCountLeft(MassEmail $massEmail): int + { + return $this->entityManager + ->getRDBRepositoryByClass(EmailQueueItem::class) + ->where([ + 'status' => EmailQueueItem::STATUS_PENDING, + 'massEmailId' => $massEmail->getId(), + 'isTest' => false, + ]) + ->count(); + } + + private function processException(EmailQueueItem $queueItem, Exception $e): void + { + $maxAttemptCount = $this->config->get('massEmailMaxAttemptCount', self::MAX_ATTEMPT_COUNT); + + $queueItem->getAttemptCount() >= $maxAttemptCount ? + $queueItem->setStatus(EmailQueueItem::STATUS_FAILED) : + $queueItem->setStatus(EmailQueueItem::STATUS_PENDING); + + $this->entityManager->saveEntity($queueItem); + + $this->log->error("Mass Email, send item: {$e->getCode()}, {$e->getMessage()}"); + } + + private function getEmailAddressRepository(): EmailAddressRepository + { + /** @var EmailAddressRepository */ + return $this->entityManager->getRepository(EmailAddress::ENTITY_TYPE); + } + + private function isNotPending(EmailQueueItem $queueItem): bool + { + /** @var ?EmailQueueItem $queueItemFetched */ + $queueItemFetched = $this->entityManager->getEntityById(EmailQueueItem::ENTITY_TYPE, $queueItem->getId()); + + if (!$queueItemFetched) { + return true; + } + + return $queueItemFetched->getStatus() !== EmailQueueItem::STATUS_PENDING; + } + + /** + * @return Collection + */ + private function getQueueItems(MassEmail $massEmail, bool $isTest, int $maxSize): Collection + { + return $this->entityManager + ->getRDBRepositoryByClass(EmailQueueItem::class) + ->sth() + ->where([ + 'status' => EmailQueueItem::STATUS_PENDING, + 'massEmailId' => $massEmail->getId(), + 'isTest' => $isTest, + ]) + ->limit(0, $maxSize) + ->find(); + } + + private function getCampaign(MassEmail $massEmail): ?Campaign + { + $campaignId = $massEmail->getCampaignId(); + + if (!$campaignId) { + return null; + } + + return $this->entityManager->getRDBRepositoryByClass(Campaign::class)->getById($campaignId); + } + + /** + * @return Collection + */ + private function getAttachments(EmailTemplate $emailTemplate): Collection + { + /** @var Collection */ + return $this->entityManager + ->getRDBRepositoryByClass(EmailTemplate::class) + ->getRelation($emailTemplate, 'attachments') + ->find(); + } + + /** + * @return bool Whether to skip. + */ + private function handleMaxSize(bool $isTest, int &$maxSize): bool + { + $hourMaxSize = $this->config->get('massEmailMaxPerHourCount', self::MAX_PER_HOUR_COUNT); + $batchMaxSize = $this->config->get('massEmailMaxPerBatchCount'); + + if (!$isTest) { + $threshold = new DateTime(); + $threshold->modify('-1 hour'); + + $sentLastHourCount = $this->entityManager + ->getRDBRepositoryByClass(EmailQueueItem::class) + ->where([ + 'status' => EmailQueueItem::STATUS_SENT, + 'sentAt>' => $threshold->format(DateTimeUtil::SYSTEM_DATE_TIME_FORMAT), + ]) + ->count(); + + if ($sentLastHourCount >= $hourMaxSize) { + return true; + } + + $hourMaxSize = $hourMaxSize - $sentLastHourCount; + } + + $maxSize = $hourMaxSize; + + if ($batchMaxSize) { + $maxSize = min($batchMaxSize, $maxSize); + } + + return false; + } + + private function getEmailTemplate(MassEmail $massEmail): ?EmailTemplate + { + $id = $massEmail->getEmailTemplateId(); + + if (!$id) { + return null; + } + + return $this->entityManager->getRDBRepositoryByClass(EmailTemplate::class)->getById($id); + } + + private function setComplete(MassEmail $massEmail): void + { + $massEmail->setStatus(MassEmail::STATUS_COMPLETE); + + $this->entityManager->saveEntity($massEmail); + } + + private function setItemSending(EmailQueueItem $queueItem): void + { + $queueItem->setStatus(EmailQueueItem::STATUS_SENDING); + + $this->entityManager->saveEntity($queueItem); + } + + private function setItemFailed(EmailQueueItem $queueItem): void + { + $queueItem->setStatus(EmailQueueItem::STATUS_FAILED); + + $this->entityManager->saveEntity($queueItem); + } + + private function setItemSent(EmailQueueItem $queueItem, string $emailAddress): void + { + $queueItem->setEmailAddress($emailAddress); + $queueItem->setStatus(EmailQueueItem::STATUS_SENT); + $queueItem->setSentAtNow(); + + $this->entityManager->saveEntity($queueItem); + } + + /** + * @return iterable + */ + private function getTrackingUrls(?Campaign $campaign): iterable + { + if (!$campaign) { + return []; + } + + /** @var Collection */ + return $this->entityManager + ->getRDBRepositoryByClass(Campaign::class) + ->getRelation($campaign, 'trackingUrls') + ->find(); + } + + private function prepareItemSenderParams( + Email $email, + SenderParams $senderParams, + ?Campaign $campaign, + MassEmail $massEmail + ): SenderParams { + if ($email->get('replyToAddress')) { // @todo Revise. $senderParams = $senderParams->withReplyToAddress(null); } @@ -460,65 +573,81 @@ class SendingProcessor $senderParams = $senderParams->withReplyToName($massEmail->getReplyToName()); } - try { - $attemptCount = $queueItem->getAttemptCount(); - $attemptCount++; - - $queueItem->set('attemptCount', $attemptCount); - - $sender = $this->emailSender->create(); - - if ($smtpParams) { - $sender->withSmtpParams($smtpParams); - } - - $message = new Message(); - - $this->prepareQueueItemMessage($queueItem, $sender, $message, $senderParams); - - $sender - ->withParams($senderParams) - ->withMessage($message) - ->withAttachments($attachmentList) - ->send($email); - } - catch (Exception $e) { - $maxAttemptCount = $this->config->get('massEmailMaxAttemptCount', self::MAX_ATTEMPT_COUNT); - - $queueItem->getAttemptCount() >= $maxAttemptCount ? - $queueItem->set('status', EmailQueueItem::STATUS_FAILED) : - $queueItem->set('status', EmailQueueItem::STATUS_PENDING); - - $this->entityManager->saveEntity($queueItem); - - $this->log->error("Mass Email, send item: {$e->getCode()}, " . $e->getMessage()); - - return; - } - - $emailObject = $emailTemplate; - - if ($massEmail->storeSentEmails() && !$isTest) { - $this->entityManager->saveEntity($email); - - $emailObject = $email; - } - - $queueItem->set('emailAddress', $target->get('emailAddress')); - $queueItem->set('status', EmailQueueItem::STATUS_SENT); - $queueItem->set('sentAt', date(DateTimeUtil::SYSTEM_DATE_TIME_FORMAT)); - - $this->entityManager->saveEntity($queueItem); - - if ($campaign) { - $this->campaignService->logSent($campaign->getId(), $queueItem, $emailObject); - } + return $senderParams; } - private function getSiteUrl(): string + private function getOptOutUrl(EmailQueueItem $queueItem): string { - return - $this->config->get('massEmailSiteUrl') ?? - $this->config->get('siteUrl'); + return "{$this->getSiteUrl()}?entryPoint=unsubscribe&id={$queueItem->getId()}"; + } + + private function getOptOutLink(string $optOutUrl): string + { + $label = $this->defaultLanguage->translateLabel('Unsubscribe', 'labels', Campaign::ENTITY_TYPE); + + return "$label"; + } + + private function getTrackUrl(mixed $trackingUrl, EmailQueueItem $queueItem): string + { + $siteUrl = $this->getSiteUrl(); + $id1 = $trackingUrl->getId(); + $id2 = $queueItem->getId(); + + return "$siteUrl?entryPoint=campaignUrl&id=$id1&queueItemId=$id2"; + } + + /** + * @param iterable $trackingUrlList + */ + private function prepareBody( + Result $emailData, + EmailQueueItem $queueItem, + $trackingUrlList, + MassEmail $massEmail + ): string { + + $body = $emailData->getBody(); + + $optOutUrl = $this->getOptOutUrl($queueItem); + $optOutLink = $this->getOptOutLink($optOutUrl); + + $body = str_replace('{optOutUrl}', $optOutUrl, $body); + $body = str_replace('{optOutLink}', $optOutLink, $body); + + foreach ($trackingUrlList as $trackingUrl) { + $url = $this->getTrackUrl($trackingUrl, $queueItem); + + $body = str_replace($trackingUrl->getUrlToUse(), $url, $body); + } + + if ( + !$this->config->get('massEmailDisableMandatoryOptOutLink') && + stripos($body, '?entryPoint=unsubscribe&id') === false + ) { + if ($emailData->isHtml()) { + $body .= "

" . $optOutLink; + } else { + $body .= "\n\n" . $optOutUrl; + } + } + + $trackImageAlt = $this->defaultLanguage->translateLabel('Campaign', 'scopeNames'); + + $trackOpenedUrl = $this->getSiteUrl() . '?entryPoint=campaignTrackOpened&id=' . $queueItem->getId(); + + /** @noinspection HtmlDeprecatedAttribute */ + $trackOpenedHtml = "\"$trackImageAlt\""; + + if ( + $massEmail->getCampaignId() && + $this->config->get('massEmailOpenTracking') + ) { + if ($emailData->isHtml()) { + $body .= '
' . $trackOpenedHtml; + } + } + + return $body; } }