From e1355acda1f76483fc498984bd975de72d3df264 Mon Sep 17 00:00:00 2001
From: Yuri Kuznetsov <yurak@list.ru>
Date: Thu, 28 Aug 2014 17:15:44 +0300
Subject: [PATCH] email account 5

---
 application/Espo/Core/Mail/Importer.php       |  55 +++++++---
 application/Espo/Core/Mail/Sender.php         |   6 +
 application/Espo/Core/Mail/Storage/Imap.php   |   8 +-
 application/Espo/Core/defaults/config.php     |   1 +
 .../layouts/InboundEmail/detail.json          |   7 +-
 .../metadata/entityDefs/InboundEmail.json     |   7 +-
 .../Modules/Crm/Services/InboundEmail.php     | 103 +++++++++++-------
 .../Espo/Resources/i18n/en_US/Email.json      |   3 +-
 .../Resources/i18n/en_US/EmailAccount.json    |   3 +-
 .../layouts/EmailAccount/detail.json          |   3 +
 .../Resources/metadata/entityDefs/Email.json  |   4 +
 .../metadata/entityDefs/EmailAccount.json     |   7 +-
 application/Espo/Services/EmailAccount.php    |  75 ++++++++++---
 13 files changed, 197 insertions(+), 85 deletions(-)

diff --git a/application/Espo/Core/Mail/Importer.php b/application/Espo/Core/Mail/Importer.php
index 681d226f1b..a6ae800231 100644
--- a/application/Espo/Core/Mail/Importer.php
+++ b/application/Espo/Core/Mail/Importer.php
@@ -8,9 +8,12 @@ class Importer
 {	
 	private $entityManager;
 	
-	public function __construct($entityManager)
+	private $fileManager;
+	
+	public function __construct($entityManager, $fileManager)
 	{
 		$this->entityManager = $entityManager;
+		$this->fileManager = $fileManager;
 	}
 	
 	protected function getEntityManager()
@@ -18,6 +21,11 @@ class Importer
 		return $this->entityManager;
 	}
 	
+	protected function getFileManager()
+	{
+		return $this->fileManager;
+	}
+	
 	public function importMessage($message, $userId, $teamsIds = array())
 	{
 		try {
@@ -49,10 +57,19 @@ class Importer
 				return false;
 			}			
 
-			$dt = new \DateTime($message->date);
-			if ($dt) {
-				$dateSent = $dt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s');		
-				$email->set('dateSent', $dateSent);
+			if (isset($message->date)) {
+				$dt = new \DateTime($message->date);
+				if ($dt) {
+					$dateSent = $dt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s');		
+					$email->set('dateSent', $dateSent);
+				}
+			}
+			if (isset($message->deliveryDate)) {
+				$dt = new \DateTime($message->deliveryDate);
+				if ($dt) {
+					$deliveryDate = $dt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s');		
+					$email->set('deliveryDate', $deliveryDate);
+				}
 			}
 			
 			$inlineIds = array();
@@ -111,8 +128,6 @@ class Importer
 			$type = strtok($part->contentType, ';');
 			$encoding = null;
 			
-			// TODO do not import if size exceeds limit
-			
 			switch ($type) {
 				case 'text/plain':					
 					$content = $this->getContentFromPart($part);				
@@ -153,17 +168,23 @@ class Importer
 					$attachment = $this->getEntityManager()->getEntity('Attachment');
 					$attachment->set('name', $fileName);							
 					$attachment->set('type', $type);
-					$attachment->set('role', 'Inline Attachment');
-							
-					$this->getEntityManager()->saveEntity($attachment);
-												
-					$path = 'data/upload/' . $attachment->id;
-							
+					
+					if ($disposition == 'inline') {
+						$attachment->set('role', 'Inline Attachment');
+					} else {
+						$attachment->set('role', 'Attachment');
+					}
+					
 					if ($encoding == 'base64') {
 						$content = base64_decode($content);
 					}
-					// TODO store size
-					$this->getFileManager()->putContents($path, $content);
+					
+					$attachment->set('size', strlen($content));
+							
+					$this->getEntityManager()->saveEntity($attachment);
+												
+					$path = 'data/upload/' . $attachment->id;					
+					$this->getFileManager()->putContents($path, $content);					
 					
 					if ($disposition == 'attachment') {
 						$attachmentsIds = $email->get('attachmentsIds');
@@ -173,9 +194,7 @@ class Importer
 						$inlineIds[$contentId] = $attachment->id;
 					}		
 			}
-		} catch (\Exception $e){
-			// TODO log	
-		}		
+		} catch (\Exception $e) {}		
 	}
 	
 	protected function getContentFromPart($part)
diff --git a/application/Espo/Core/Mail/Sender.php b/application/Espo/Core/Mail/Sender.php
index 5a1d90105e..777c97a595 100644
--- a/application/Espo/Core/Mail/Sender.php
+++ b/application/Espo/Core/Mail/Sender.php
@@ -256,6 +256,12 @@ class Sender
 
 		try {
 			$this->transport->send($message);
+			
+			$headers = $message->getHeaders();
+			if ($headers->has('messageId')) {			
+				$email->set('messageId', $headers->get('messageId')->getId());
+			}
+
 			$email->set('status', 'Sent');
 			$email->set('dateSent', date("Y-m-d H:i:s"));
 		} catch (\Exception $e) {
diff --git a/application/Espo/Core/Mail/Storage/Imap.php b/application/Espo/Core/Mail/Storage/Imap.php
index d74062715e..65a9ded66b 100644
--- a/application/Espo/Core/Mail/Storage/Imap.php
+++ b/application/Espo/Core/Mail/Storage/Imap.php
@@ -6,8 +6,14 @@ class Imap extends \Zend\Mail\Storage\Imap
 {	
 	public function getIdsFromUID($uid)
 	{
-		$uid = intval($lastUID) + 1;
+		$uid = intval($uid) + 1;
 		return $this->protocol->search(array('UID ' . $uid . ':*'));
 	}
+	
+	public function getIdsFromDate($date)
+	{		
+		return $this->protocol->search(array('SINCE "' . $date . '"'));
+	}
+	
 }
 
diff --git a/application/Espo/Core/defaults/config.php b/application/Espo/Core/defaults/config.php
index e2093e6d42..43a0feff96 100644
--- a/application/Espo/Core/defaults/config.php
+++ b/application/Espo/Core/defaults/config.php
@@ -91,6 +91,7 @@ return array (
 	'disableExport' => false,
 	'assignmentEmailNotifications' => false,
 	'assignmentEmailNotificationsEntityList' => array('Lead', 'Opportunity', 'Task', 'Case'),
+	'emailMessageMaxSize' => 10,
 	'isInstalled' => false,
 );
 
diff --git a/application/Espo/Modules/Crm/Resources/layouts/InboundEmail/detail.json b/application/Espo/Modules/Crm/Resources/layouts/InboundEmail/detail.json
index 02a18609e1..6b2be07cfb 100644
--- a/application/Espo/Modules/Crm/Resources/layouts/InboundEmail/detail.json
+++ b/application/Espo/Modules/Crm/Resources/layouts/InboundEmail/detail.json
@@ -22,13 +22,10 @@
 				{"name":"host"},{"name":"ssl"}
 			],
 			[
-				{"name":"port"},false
+				{"name":"port"},{"name":"username"}
 			],
 			[
-				{"name":"monitoredFolders"},{"name":"username"}
-			],
-			[
-				{"name":"trashFolder"},{"name":"password"}
+				{"name":"monitoredFolders"},{"name":"password"}
 			]
 		]
 	},
diff --git a/application/Espo/Modules/Crm/Resources/metadata/entityDefs/InboundEmail.json b/application/Espo/Modules/Crm/Resources/metadata/entityDefs/InboundEmail.json
index 6016e38c8d..4ed9bc3e08 100644
--- a/application/Espo/Modules/Crm/Resources/metadata/entityDefs/InboundEmail.json
+++ b/application/Espo/Modules/Crm/Resources/metadata/entityDefs/InboundEmail.json
@@ -33,10 +33,9 @@
             "default": "INBOX",
             "view": "Crm:InboundEmail.Fields.Folders"
         },
-        "trashFolder": {
-            "type": "varchar",
-            "required": true,
-            "view": "Crm:InboundEmail.Fields.Folder"
+        "fetchData": {
+        	"type": "text",
+            "readOnly": true
         },
         "assignToUser": {
             "type": "link",
diff --git a/application/Espo/Modules/Crm/Services/InboundEmail.php b/application/Espo/Modules/Crm/Services/InboundEmail.php
index 7402d66cc6..ba0decf886 100644
--- a/application/Espo/Modules/Crm/Services/InboundEmail.php
+++ b/application/Espo/Modules/Crm/Services/InboundEmail.php
@@ -72,17 +72,7 @@ class InboundEmail extends \Espo\Services\Record
 	protected function getMailSender()
 	{
 		return $this->injections['mailSender'];
-	}
-	
-	protected function findFolder($storage, $path)
-	{
-		$arr = explode('/', $path);
-		$pointer = $storage->getFolders();
-		foreach ($arr as $folderName) {
-			$pointer = $pointer->$folderName;
-		}
-		return $pointer;
-	}
+	}	
 	
 	public function getFolders($params)
 	{		
@@ -123,7 +113,9 @@ class InboundEmail extends \Espo\Services\Record
 			throw new Error();
 		}
 		
-		$importer = new \Espo\Core\Mail\Importer($this->getEntityManager());
+		$importer = new \Espo\Core\Mail\Importer($this->getEntityManager(), $this->getFileManager());
+		
+		$maxSize = $this->getConfig()->get('emailMessageMaxSize');
 		
 		$teamId = $inboundEmail->get('teamId');
 		$userId = $this->getUser()->id;		
@@ -131,6 +123,17 @@ class InboundEmail extends \Espo\Services\Record
 			$userId = $inboundEmail->get('assignToUserId');
 		}
 		
+		$fetchData = json_decode($inboundEmail->get('fetchData'), true);
+		if (empty($fetchData)) {
+			$fetchData = array();			
+		}
+		if (!array_key_exists('lastUID', $fetchData)) {
+			$fetchData['lastUID'] = array();
+		}
+		if (!array_key_exists('lastUID', $fetchData)) {
+			$fetchData['lastDate'] = array();
+		}
+		
 		$imapParams = array(
 			'host' => $inboundEmail->get('host'),
 			'port' => $inboundEmail->get('port'),
@@ -142,18 +145,7 @@ class InboundEmail extends \Espo\Services\Record
 			$imapParams['ssl'] = 'SSL';
 		}
 		
-		$storage = new \Zend\Mail\Storage\Imap($imapParams);
-		
-		$trash = null;
-		$trashFolder = $inboundEmail->get('trashFolder');
-		if (empty($trashFolder)) {
-			$trashFolder = 'INBOX.Trash';
-		}
-		try {
-			$trash = $this->findFolder($storage, $trashFolder);
-		} catch (\Exception $e) {
-			throw new Error("No trash folder '{$trashFolder}' found for Inbound Email {$id}");
-		}
+		$storage = new \Espo\Core\Mail\Storage\Imap($imapParams);
 		
 		$monitoredFolders = $inboundEmail->get('monitoredFolders');
 		if (empty($monitoredFolders)) {
@@ -161,15 +153,41 @@ class InboundEmail extends \Espo\Services\Record
 		}
 		
 		$monitoredFoldersArr = explode(',', $monitoredFolders);				
-		foreach ($monitoredFoldersArr as $path) {
-			$toRemove = array();
-			$path = trim($path);			
+		foreach ($monitoredFoldersArr as $folder) {
+			$folder = trim($folder);		
+			$storage->selectFolder($folder);
 			
-			$folder = $this->findFolder($storage, $path);			
-			$storage->selectFolder($folder);			 
+			$lastUID = 0;
+			$lastDate = 0;
+			if (!empty($fetchData['lastUID'][$folder])) {
+				$lastUID = $fetchData['lastUID'][$folder];
+			}
+			if (!empty($fetchData['lastDate'][$folder])) {
+				$lastDate = $fetchData['lastDate'][$folder];
+			}
+
+			$ids = $storage->getIdsFromUID($lastUID);
+			
+			if ((count($ids) == 1) && !empty($lastUID)) {
+				if ($storage->getUniqueId($ids[0]) == $lastUID) {
+					continue;
+				}
+			}
+			
+			$k = 0;	
+			foreach ($ids as $i => $id) {
+				if ($k == count($ids) - 1) {
+					$lastUID = $storage->getUniqueId($id);
+				}
+				
+				if ($maxSize) {
+					if ($storage->getSize($id) > $maxSize * 1024 * 1024) {
+						continue;
+					}
+				}
+				
+				$message = $storage->getMessage($id);
 			
-			$k = 0;			
-			foreach ($storage as $number => $message) {
 				$email = $importer->importMessage($message, $userId, array($teamId));
 				
 				if ($email) {
@@ -183,19 +201,30 @@ class InboundEmail extends \Espo\Services\Record
 					}
 				}
 				
+				if ($k == count($ids) - 1) {
+					if ($message) {
+						$dt = new \DateTime($message->date);
+						if ($dt) {
+							$dateSent = $dt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s');		
+							$lastDate = $dateSent;
+						}
+					}
+				}
+				
 				if ($k == self::PORTION_LIMIT - 1) {
+					$lastUID = $storage->getUniqueId($id);
 					break;
 				}
 				$k++;						
 			}
 			
-			if ($trash) {
-				while ($k) {				
-					$storage->moveMessage(1, $trash);
-					$k--;
-				}
-			}
+			$fetchData['lastUID'][$folder] = $lastUID;
+			$fetchData['lastDate'][$folder] = $lastDate;
+			
+			$inboundEmail->set('fetchData', json_encode($fetchData));
+			$this->getEntityManager()->saveEntity($inboundEmail);
 		}
+		
 		return true;
 	}
 	
diff --git a/application/Espo/Resources/i18n/en_US/Email.json b/application/Espo/Resources/i18n/en_US/Email.json
index 6674a99dc6..6a278f5dc3 100644
--- a/application/Espo/Resources/i18n/en_US/Email.json
+++ b/application/Espo/Resources/i18n/en_US/Email.json
@@ -15,7 +15,8 @@
 		"selectTemplate": "Select Template",
 		"fromEmailAddress": "From Address",
 		"toEmailAddresses": "To Address",
-		"emailAddress": "Email Address"
+		"emailAddress": "Email Address",
+		"deliveryDate": "Delivery Date"
 	},
 	"links": {
 	},
diff --git a/application/Espo/Resources/i18n/en_US/EmailAccount.json b/application/Espo/Resources/i18n/en_US/EmailAccount.json
index a14e1dec30..b0775ba57a 100644
--- a/application/Espo/Resources/i18n/en_US/EmailAccount.json
+++ b/application/Espo/Resources/i18n/en_US/EmailAccount.json
@@ -7,7 +7,8 @@
 		"password": "Password",
 		"port": "Port",
 		"monitoredFolders": "Monitored Folders",
-		"ssl": "SSL"
+		"ssl": "SSL",
+		"fetchSince": "Fetch Since"
 	},
 	"links": {
 	},
diff --git a/application/Espo/Resources/layouts/EmailAccount/detail.json b/application/Espo/Resources/layouts/EmailAccount/detail.json
index 880bbbc722..da91a57c8e 100644
--- a/application/Espo/Resources/layouts/EmailAccount/detail.json
+++ b/application/Espo/Resources/layouts/EmailAccount/detail.json
@@ -5,6 +5,9 @@
 			[
 				{"name":"name"},
 				{"name":"status"}
+			],
+			[
+				{"name":"fetchSince"}
 			]
 		]
 	},
diff --git a/application/Espo/Resources/metadata/entityDefs/Email.json b/application/Espo/Resources/metadata/entityDefs/Email.json
index c6eac7e5d6..a54a912086 100644
--- a/application/Espo/Resources/metadata/entityDefs/Email.json
+++ b/application/Espo/Resources/metadata/entityDefs/Email.json
@@ -105,6 +105,10 @@
         "dateSent": {
             "type": "datetime"
         },
+        "deliveryDate": {
+            "type": "datetime",
+            "readOnly": true
+        },        
         "createdAt": {
             "type": "datetime",
             "readOnly": true
diff --git a/application/Espo/Resources/metadata/entityDefs/EmailAccount.json b/application/Espo/Resources/metadata/entityDefs/EmailAccount.json
index 2b6509724c..de747916c7 100644
--- a/application/Espo/Resources/metadata/entityDefs/EmailAccount.json
+++ b/application/Espo/Resources/metadata/entityDefs/EmailAccount.json
@@ -33,8 +33,13 @@
             "default": "INBOX",
             "view": "EmailAccount.Fields.Folders"
         },
+        "fetchSince": {
+        	"type": "date",
+        	"required": true
+        },
         "fetchData": {
-        	"type": "text"
+        	"type": "text",
+            "readOnly": true
         },
         "createdAt": {
             "type": "datetime",
diff --git a/application/Espo/Services/EmailAccount.php b/application/Espo/Services/EmailAccount.php
index 8708b549ab..477e145def 100644
--- a/application/Espo/Services/EmailAccount.php
+++ b/application/Espo/Services/EmailAccount.php
@@ -35,6 +35,16 @@ class EmailAccount extends Record
 	
 	const PORTION_LIMIT = 10;
 	
+	protected function init()
+	{
+		$this->dependencies[] = 'fileManager';
+	}
+	
+	protected function getFileManager()
+	{
+		return $this->injections['fileManager'];
+	}
+	
 	public function getFolders($params)
 	{		
 		$password = $params['password'];
@@ -74,7 +84,9 @@ class EmailAccount extends Record
 			throw new Error();
 		}
 		
-		$importer = new \Espo\Core\Mail\Importer($this->getEntityManager());
+		$importer = new \Espo\Core\Mail\Importer($this->getEntityManager(), $this->getFileManager());
+		
+		$maxSize = $this->getConfig()->get('emailMessageMaxSize');
 		
 		$user = $this->getEntityManager()->getEntity('User', $emailAccount->get('assignedUserId'));
 		
@@ -83,11 +95,11 @@ class EmailAccount extends Record
 		}
 		
 		$userId = $user->id;
-		$teamId = $user->get('defaultTeam');
-		
+		$teamId = $user->get('defaultTeam');		
+	
 		$fetchData = json_decode($emailAccount->get('fetchData'), true);
 		if (empty($fetchData)) {
-			$fetchData = array();
+			$fetchData = array();			
 		}
 		if (!array_key_exists('lastUID', $fetchData)) {
 			$fetchData['lastUID'] = array();
@@ -107,7 +119,7 @@ class EmailAccount extends Record
 			$imapParams['ssl'] = 'SSL';
 		}
 		
-		$storage = new \Espo\Core\Mail\Storage\Imap($imapParams);		
+		$storage = new \Espo\Core\Mail\Storage\Imap($imapParams);
 		
 		$monitoredFolders = $emailAccount->get('monitoredFolders');		
 		if (empty($monitoredFolders)) {
@@ -116,12 +128,9 @@ class EmailAccount extends Record
 		
 		$monitoredFoldersArr = explode(',', $monitoredFolders);				
 		foreach ($monitoredFoldersArr as $folder) {
-			$folder = trim($folder);
-			
+			$folder = trim($folder);			
 			$storage->selectFolder($folder);
 			
-			// TODO fetch from last date
-			
 			$lastUID = 0;
 			$lastDate = 0;
 			if (!empty($fetchData['lastUID'][$folder])) {
@@ -131,26 +140,58 @@ class EmailAccount extends Record
 				$lastDate = $fetchData['lastDate'][$folder];
 			}
 
-			$ids = $storage->getIdsFromUID();
+			if (!empty($lastUID)) {
+				$ids = $storage->getIdsFromUID($lastUID);
+			} else {
+				$dt = new \DateTime($emailAccount->get('fetchSince'));
+				if ($dt) {
+					$ids = $storage->getIdsFromDate($dt->format('d-M-Y'));
+				} else {
+					return false;
+				}
+			}
 			
-			$k = 0;
-	
-			foreach ($ids as $i => $id) {
-				$message = $storage->getMessage($id);												
+			if ((count($ids) == 1) && !empty($lastUID)) {
+				if ($storage->getUniqueId($ids[0]) == $lastUID) {
+					continue;
+				}
+			}
+			
+			$k = 0;	
+			foreach ($ids as $i => $id) {			
+				if ($k == count($ids) - 1) {
+					$lastUID = $storage->getUniqueId($id);
+				}
+								
+				if ($maxSize) {
+					if ($storage->getSize($id) > $maxSize * 1024 * 1024) {
+						continue;
+					}
+				}
+			
+				$message = $storage->getMessage($id);
 				
-				$importer->importMessage($message, $userId, array($teamId));
+				$email = $importer->importMessage($message, $userId, array($teamId));
 								
 				if ($k == count($ids) - 1) {
 					$lastUID = $storage->getUniqueId($id);
-					$lastDate = $message->date;
+					
+					if ($message) {
+						$dt = new \DateTime($message->date);
+						if ($dt) {
+							$dateSent = $dt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s');		
+							$lastDate = $dateSent;
+						}
+					}
 				}
 				
 				if ($k == self::PORTION_LIMIT - 1) {
+					$lastUID = $storage->getUniqueId($id);
 					break;
 				}
 				$k++;
 			}		
-									
+			
 			$fetchData['lastUID'][$folder] = $lastUID;
 			$fetchData['lastDate'][$folder] = $lastDate;