Merge branch 'release/v1.6.8'

* Bugfix: Don't panic on mails without from line

* Feature: Add allowlist to filter recipients before relaying messages

.github/workflows/release-build.yml

@@ -33,7 +33,7 @@ jobs:
     - run: npm run package
 
     # build the binaries
-    - uses: wangyoucao577/go-release-action@v1.37
+    - uses: wangyoucao577/go-release-action@v1.38
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}

CHANGELOG.md

@@ -2,6 +2,60 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.6.8]
+
+### Bugfix
+- Fix Date display when message doesn't contain a Date header
+
+### Feature
+- Add allowlist to filter recipients before relaying messages ([#109](https://github.com/axllent/mailpit/issues/109))
+- Add `-S` short flag for sendmail `--smtp-addr`
+
+
+## [v1.6.7]
+
+### Bugfix
+- Fix auto-deletion cron
+
+
+## [v1.6.6]
+
+### API
+- Set Access-Control-Allow-Headers when --api-cors is set
+- Include correct start value in search reponse
+
+### Feature
+- Option to ignore duplicate Message-IDs
+
+### Libs
+- Update node modules
+- Update Go modules
+
+### Swagger
+- Update swagger field descriptions
+
+### UI
+- Style Undisclosed recipients in message view
+
+
+## [v1.6.5]
+
+### Feature
+- Add Access-Control-Allow-Methods methods when CORS origin is set
+
+
+## [v1.6.4]
+
+### Bugfix
+- Fix UI images not displaying when multiple cid names overlap
+
+
+## [v1.6.3]
+
+### Feature
+- Display clickable toast notifications for new messages
+
+
 ## [v1.6.2]
 
 ### Bugfix

README.md

@@ -28,7 +28,7 @@ Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster.
 - Configurable automatic email pruning (default keeps the most recent 500 emails)
 - Email storage either in a temporary or persistent database ([see wiki](https://github.com/axllent/mailpit/wiki/Email-storage))
 - Fast SMTP processing & storing - approximately 70-100 emails per second depending on CPU, network speed & email size, easily handling tens of thousands of emails
-- SMTP relaying / message release - relay messages via a different SMTP server ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-relay))
+- SMTP relaying / message release - relay messages via a different SMTP server including an optional allowlist of accepted recipients ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-relay))
 - Optional SMTP with STARTTLS & SMTP authentication, including an "accept anything" mode ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-with-STARTTLS-and-authentication))
 - Optional HTTPS for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/HTTPS))
 - Optional basic authentication for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
@@ -72,20 +72,14 @@ See [Docker instructions](https://github.com/axllent/mailpit/wiki/Docker-images)
 To build Mailpit from source see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source).
 
 
+### Testing Mailpit
+
+Please refer to [the documentation](https://github.com/axllent/mailpit/wiki/Testing-Mailpit) of how to easily test email delivery to Mailpit.
+
+
 ### Configuring sendmail
 
-There are several different options available:
-
-You can use `mailpit sendmail` as your sendmail configuration in `php.ini`:
-```
-sendmail_path = /usr/local/bin/mailpit sendmail
-```
-
-If Mailpit is found on the same host as sendmail, you can symlink the Mailpit binary to sendmail, eg: `ln -s /usr/local/bin/mailpit /usr/sbin/sendmail`  (only if Mailpit is running on default 1025 port).
-
-You can use your default system `sendmail` binary to route directly to port `1025` (configurable) by calling `/usr/sbin/sendmail -S localhost:1025`.
-
-You can build a Mailpit-specific sendmail binary from source (see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source)).
+Mailpit's SMTP server (by default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. A common MTA (Mail Transfer Agent) that delivers system emails to a SMTP server is `sendmail`, used by many applications including PHP. Mailpit can also act as substitute for sendmail. For instructions of how to set this up, please refer to the [sendmail documentation](https://github.com/axllent/mailpit/wiki/Configuring-sendmail).
 
 
 ## Why rewrite MailHog?

cmd/root.go

@@ -87,6 +87,7 @@ func init() {
 	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
 	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
 	rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates")
+	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)")
 
 	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI authentication")
 	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-tls-cert", config.UITLSCert, "TLS certificate for web UI (HTTPS) - requires ui-tls-key")
@@ -97,11 +98,11 @@ func init() {
 	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key")
 	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert")
 	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication")
-	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 
 	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
 	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
 
+	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
 	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
 
@@ -201,8 +202,8 @@ func initConfigFromEnv() {
 	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
 		config.UseMessageDates = true
 	}
-	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
-		config.UseMessageDates = true
+	if getEnabledFromEnv("MP_IGNORE_DUPLICATE_IDS") {
+		config.IgnoreDuplicateIDs = true
 	}
 	if getEnabledFromEnv("MP_QUIET") {
 		logger.QuietLogging = true

cmd/sendmail.go

@@ -27,7 +27,7 @@ func init() {
 
 	// these are simply repeated for cli consistency
 	sendmailCmd.Flags().StringVarP(&fromAddr, "from", "f", fromAddr, "SMTP sender")
-	sendmailCmd.Flags().StringVar(&smtpAddr, "smtp-addr", smtpAddr, "SMTP server address")
+	sendmailCmd.Flags().StringVarP(&smtpAddr, "smtp-addr", "S", smtpAddr, "SMTP server address")
 	sendmailCmd.Flags().BoolVarP(&sendmail.Verbose, "verbose", "v", false, "Verbose mode (sends debug output to stderr)")
 	sendmailCmd.Flags().BoolP("long-b", "b", false, "Ignored. This flag exists for sendmail compatibility.")
 	sendmailCmd.Flags().BoolP("long-i", "i", false, "Ignored. This flag exists for sendmail compatibility.")

config/config.go

@@ -65,6 +65,9 @@ var (
 	// SMTPAuthAcceptAny accepts any username/password including none
 	SMTPAuthAcceptAny bool
 
+	// IgnoreDuplicateIDs will skip messages with the same ID
+	IgnoreDuplicateIDs bool
+
 	// SMTPCLITags is used to map the CLI args
 	SMTPCLITags string
 
@@ -108,15 +111,17 @@ type AutoTag struct {
 
 // SMTPRelayConfigStruct struct for parsing yaml & storing variables
 type smtpRelayConfigStruct struct {
-	Host          string `yaml:"host"`
-	Port          int    `yaml:"port"`
-	STARTTLS      bool   `yaml:"starttls"`
-	AllowInsecure bool   `yaml:"allow-insecure"`
-	Auth          string `yaml:"auth"`        // none, plain, cram-md5
-	Username      string `yaml:"username"`    // plain & cram-md5
-	Password      string `yaml:"password"`    // plain
-	Secret        string `yaml:"secret"`      // cram-md5
-	ReturnPath    string `yaml:"return-path"` // allows overriding the boune address
+	Host                     string `yaml:"host"`
+	Port                     int    `yaml:"port"`
+	STARTTLS                 bool   `yaml:"starttls"`
+	AllowInsecure            bool   `yaml:"allow-insecure"`
+	Auth                     string `yaml:"auth"`                // none, plain, cram-md5
+	Username                 string `yaml:"username"`            // plain & cram-md5
+	Password                 string `yaml:"password"`            // plain
+	Secret                   string `yaml:"secret"`              // cram-md5
+	ReturnPath               string `yaml:"return-path"`         // allows overriding the boune address
+	RecipientAllowlist       string `yaml:"recipient-allowlist"` // regex, if set needs to match for mails to be relayed
+	RecipientAllowlistRegexp *regexp.Regexp
 }
 
 // VerifyConfig wil do some basic checking
@@ -293,6 +298,18 @@ func parseRelayConfig(c string) error {
 
 	logger.Log().Infof("[smtp] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
 
+	allowlistRegexp, err := regexp.Compile(SMTPRelayConfig.RecipientAllowlist)
+
+	if SMTPRelayConfig.RecipientAllowlist != "" {
+		if err != nil {
+			return fmt.Errorf("failed to compile recipient allowlist regexp: %e", err)
+		}
+
+		SMTPRelayConfig.RecipientAllowlistRegexp = allowlistRegexp
+		logger.Log().Infof("[smtp] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist)
+
+	}
+
 	return nil
 }
 

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/tg123/go-htpasswd v1.2.1
 	golang.org/x/text v0.9.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.21.2
+	modernc.org/sqlite v1.22.1
 )
 
 require (
@@ -54,7 +54,7 @@ require (
 	lukechampine.com/uint128 v1.3.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.4 // indirect
+	modernc.org/libc v1.22.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
 	modernc.org/memory v1.5.0 // indirect
 	modernc.org/opt v0.1.3 // indirect

go.sum

@@ -196,6 +196,8 @@ modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
 modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
 modernc.org/libc v1.22.4 h1:wymSbZb0AlrjdAVX3cjreCHTPCpPARbQXNz6BHPzdwQ=
 modernc.org/libc v1.22.4/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
 modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
 modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
@@ -204,9 +206,13 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sqlite v1.21.2 h1:ixuUG0QS413Vfzyx6FWx6PYTmHaOegTY+hjzhn7L+a0=
 modernc.org/sqlite v1.21.2/go.mod h1:cxbLkB5WS32DnQqeH4h4o1B0eMr8W/y8/RGuxQ3JsC0=
+modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE=
+modernc.org/sqlite v1.22.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
 modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
 modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
 modernc.org/tcl v1.15.1 h1:mOQwiEK4p7HruMZcwKTZPw/aqtGM4aY00uzWhlKKYws=
+modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
 modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
+modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=

sendmail/cmd/cmd.go

@@ -56,7 +56,7 @@ func Run() {
 
 	// override defaults from cli flags
 	flag.StringVarP(&fromAddr, "from", "f", fromAddr, "SMTP sender address")
-	flag.StringVar(&smtpAddr, "smtp-addr", smtpAddr, "SMTP server address")
+	flag.StringVarP(&smtpAddr, "smtp-addr", "S", smtpAddr, "SMTP server address")
 	flag.BoolVarP(&Verbose, "verbose", "v", false, "Verbose mode (sends debug output to stderr)")
 	flag.BoolP("long-b", "b", false, "Ignored. This flag exists for sendmail compatibility.")
 	flag.BoolP("long-i", "i", false, "Ignored. This flag exists for sendmail compatibility.")

server/apiv1/api.go

@@ -34,13 +34,13 @@ func GetMessages(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: start
 	//	    in: query
-	//	    description: pagination offset
+	//	    description: Pagination offset
 	//	    required: false
 	//	    type: integer
 	//	    default: 0
 	//	  + name: limit
 	//	    in: query
-	//	    description: limit results
+	//	    description: Limit results
 	//	    required: false
 	//	    type: integer
 	//	    default: 50
@@ -88,12 +88,12 @@ func Search(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: query
 	//	    in: query
-	//	    description: search query
+	//	    description: Search query
 	//	    required: true
 	//	    type: string
 	//	  + name: limit
 	//	    in: query
-	//	    description: limit results
+	//	    description: Limit results
 	//	    required: false
 	//	    type: integer
 	//	    default: 50
@@ -119,7 +119,7 @@ func Search(w http.ResponseWriter, r *http.Request) {
 
 	var res MessagesSummary
 
-	res.Start = 0
+	res.Start = start
 	res.Messages = messages
 	res.Count = len(messages)
 	res.Total = stats.Total
@@ -147,7 +147,7 @@ func GetMessage(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -188,12 +188,12 @@ func DownloadAttachment(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//	  + name: PartID
 	//	    in: path
-	//	    description: attachment part id
+	//	    description: Attachment part ID
 	//	    required: true
 	//	    type: string
 	//
@@ -237,7 +237,7 @@ func GetHeaders(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -284,7 +284,7 @@ func DownloadRaw(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -330,7 +330,7 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to delete
+	//	    description: Message IDs to delete
 	//	    required: false
 	//	    type: DeleteRequest
 	//
@@ -381,7 +381,7 @@ func SetReadStatus(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to update
+	//	    description: Message IDs to update
 	//	    required: false
 	//	    type: SetReadStatusRequest
 	//
@@ -459,7 +459,7 @@ func SetTags(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to update
+	//	    description: Message IDs to update
 	//	    required: true
 	//	    type: SetTagsRequest
 	//
@@ -515,10 +515,10 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
-	//		+ name: To
+	//		+ name: to
 	//	    in: body
 	//	    description: Array of email addresses to release message to
 	//	    required: true
@@ -554,10 +554,17 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, to := range tos {
-		if _, err := mail.ParseAddress(to); err != nil {
+		address, err := mail.ParseAddress(to)
+
+		if err != nil {
 			httpError(w, "Invalid email address: "+to)
 			return
 		}
+
+		if config.SMTPRelayConfig.RecipientAllowlistRegexp != nil && !config.SMTPRelayConfig.RecipientAllowlistRegexp.MatchString(address.Address) {
+			httpError(w, "Mail address does not match allowlist: "+to)
+			return
+		}
 	}
 
 	reader := bytes.NewReader(msg)

server/apiv1/webui.go

@@ -20,6 +20,8 @@ type webUIConfiguration struct {
 		SMTPServer string
 		// Enforced Return-Path (if set) for relay bounces
 		ReturnPath string
+		// Allowlist of accepted recipients
+		RecipientAllowlist string
 	}
 }
 
@@ -45,6 +47,7 @@ func WebUIConfig(w http.ResponseWriter, r *http.Request) {
 	if config.ReleaseEnabled {
 		conf.MessageRelay.SMTPServer = fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
 		conf.MessageRelay.ReturnPath = config.SMTPRelayConfig.ReturnPath
+		conf.MessageRelay.RecipientAllowlist = config.SMTPRelayConfig.RecipientAllowlist
 	}
 
 	bytes, _ := json.Marshal(conf)

server/server.go

@@ -122,6 +122,8 @@ func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 
 		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
 			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
+			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
 		}
 
 		if config.UIAuthFile != "" {
@@ -159,6 +161,8 @@ func middlewareHandler(h http.Handler) http.Handler {
 
 		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
 			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
+			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
 		}
 
 		if config.UIAuthFile != "" {

server/smtpd/smtp.go

@@ -2,14 +2,48 @@ package smtpd
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"net/mail"
 	"net/smtp"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/utils/logger"
 )
 
+func allowedRecipients(to []string) []string {
+	if config.SMTPRelayConfig.RecipientAllowlistRegexp == nil {
+		return to
+	}
+
+	var ar []string
+
+	for _, recipient := range to {
+		address, err := mail.ParseAddress(recipient)
+
+		if err != nil {
+			logger.Log().Warnf("ignoring invalid email address: %s", recipient)
+			continue
+		}
+
+		if !config.SMTPRelayConfig.RecipientAllowlistRegexp.MatchString(address.Address) {
+			logger.Log().Debugf("[smtp] not allowed to relay to %s: does not match the allowlist %s", recipient, config.SMTPRelayConfig.RecipientAllowlist)
+		} else {
+			ar = append(ar, recipient)
+		}
+	}
+
+	return ar
+}
+
 // Send will connect to a pre-configured SMTP server and send a message to one or more recipients.
 func Send(from string, to []string, msg []byte) error {
+	recipients := allowedRecipients(to)
+
+	if len(recipients) == 0 {
+		return errors.New("no valid recipients")
+	}
+
 	addr := fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
 
 	c, err := smtp.Dial(addr)
@@ -48,7 +82,7 @@ func Send(from string, to []string, msg []byte) error {
 		return err
 	}
 
-	for _, addr := range to {
+	for _, addr := range recipients {
 		if err = c.Rcpt(addr); err != nil {
 			return err
 		}

server/smtpd/smtpd.go

@@ -24,18 +24,25 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 		return err
 	}
 
+	messageID := strings.Trim(msg.Header.Get("Message-Id"), "<>")
+
 	// add a message ID if not set
-	if msg.Header.Get("Message-Id") == "" {
+	if messageID == "" {
 		// generate unique ID
-		uid := uuid.NewV4().String() + "@mailpit"
+		messageID = uuid.NewV4().String() + "@mailpit"
 		// add unique ID
-		data = append([]byte("Message-Id: <"+uid+">\r\n"), data...)
+		data = append([]byte("Message-Id: <"+messageID+">\r\n"), data...)
+	} else if config.IgnoreDuplicateIDs {
+		if storage.MessageIDExists(messageID) {
+			logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID)
+			return nil
+		}
 	}
 
 	// if enabled, this will route the email 1:1 through to the preconfigured smtp server
 	if config.SMTPRelayAllIncoming {
 		if err := Send(from, to, data); err != nil {
-			logger.Log().Errorf("[smtp] error relaying message: %s", err.Error())
+			logger.Log().Warnf("[smtp] error relaying message: %s", err.Error())
 		} else {
 			logger.Log().Debugf("[smtp] relayed message from %s via %s:%d", from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
 		}
@@ -81,7 +88,8 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
 	}
 
-	if _, err := storage.Store(data); err != nil {
+	_, err = storage.Store(data)
+	if err != nil {
 		logger.Log().Errorf("[db] error storing message: %d", err.Error())
 
 		return err

server/ui-src/App.vue

@@ -3,6 +3,7 @@ import commonMixins from './mixins.js';
 import Message from './templates/Message.vue';
 import MessageSummary from './templates/MessageSummary.vue';
 import MessageRelease from './templates/MessageRelease.vue';
+import MessageToast from './templates/MessageToast.vue';
 import moment from 'moment';
 import Tinycon from 'tinycon';
 
@@ -12,7 +13,8 @@ export default {
 	components: {
 		Message,
 		MessageSummary,
-		MessageRelease
+		MessageRelease,
+		MessageToast
 	},
 
 	data() {
@@ -41,6 +43,7 @@ export default {
 			lastLoaded: false,
 			relayConfig: {},
 			releaseAddresses: false,
+			toastMessage: false,
 		}
 	},
 
@@ -205,11 +208,11 @@ export default {
 			let self = this;
 			self.selected = [];
 			self.releaseAddresses = false;
+			self.toastMessage = false;
 			self.existingTags = JSON.parse(JSON.stringify(self.tags));
 
 			let uri = 'api/v1/message/' + self.currentPath
 			self.get(uri, false, function (response) {
-
 				for (let i in self.items) {
 					if (self.items[i].ID == self.currentPath) {
 						if (!self.items[i].Read) {
@@ -218,51 +221,56 @@ export default {
 						}
 					}
 				}
+
 				let d = response.data;
-				// replace inline images
+
+				// replace inline images embedded as inline attachments
 				if (d.HTML && d.Inline) {
 					for (let i in d.Inline) {
 						let a = d.Inline[i];
 						if (a.ContentID != '') {
 							d.HTML = d.HTML.replace(
-								new RegExp('cid:' + a.ContentID, 'g'),
-								window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID
+								new RegExp('(=["\']?)(cid:' + a.ContentID + ')(["|\'|\s|\/|>|;])', 'g'),
+								'$1' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '$3'
 							);
 						}
 						if (a.FileName.match(/^[a-zA-Z0-9\_\-\.]+$/)) {
 							// some old email clients use the filename
 							d.HTML = d.HTML.replace(
-								new RegExp('src=(\'|")' + a.FileName + '(\'|")', 'g'),
-								'src="' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '"'
+								new RegExp('(=["\']?)(' + a.FileName + ')(["|\'|\s|\/|>|;])', 'g'),
+								'$1' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '$3'
 							);
 						}
 					}
 				}
-				// replace inline images
+
+				// replace inline images embedded as regular attachments
 				if (d.HTML && d.Attachments) {
 					for (let i in d.Attachments) {
 						let a = d.Attachments[i];
 						if (a.ContentID != '') {
 							d.HTML = d.HTML.replace(
-								new RegExp('cid:' + a.ContentID, 'g'),
-								window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID
+								new RegExp('(=["\']?)(cid:' + a.ContentID + ')(["|\'|\s|\/|>|;])', 'g'),
+								'$1' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '$3'
 							);
 						}
 						if (a.FileName.match(/^[a-zA-Z0-9\_\-\.]+$/)) {
 							// some old email clients use the filename
 							d.HTML = d.HTML.replace(
-								new RegExp('src=(\'|")' + a.FileName + '(\'|")', 'g'),
-								'src="' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '"'
+								new RegExp('(=["\']?)(' + a.FileName + ')(["|\'|\s|\/|>|;])', 'g'),
+								'$1' + window.location.origin + window.location.pathname + 'api/v1/message/' + d.ID + '/part/' + a.PartID + '$3'
 							);
 						}
 					}
 				}
 
 				self.message = d;
+
 				// generate the prev/next links based on current message list
 				self.messagePrev = false;
 				self.messageNext = false;
 				let found = false;
+
 				for (let i in self.items) {
 					if (self.items[i].ID == self.message.ID) {
 						found = true;
@@ -396,10 +404,16 @@ export default {
 				if (response.Type == "new" && response.Data) {
 					if (!self.searching) {
 						if (self.start < 1) {
+							// first page
 							self.items.unshift(response.Data);
 							if (self.items.length > self.limit) {
 								self.items.pop();
 							}
+
+							// first message was open, set messagePrev
+							if (!self.messagePrev) {
+								self.messagePrev = response.Data.ID;
+							}
 						} else {
 							self.start++;
 						}
@@ -416,6 +430,7 @@ export default {
 
 					let from = response.Data.From != null ? response.Data.From.Address : '[unknown]';
 					self.browserNotify("New mail from: " + from, response.Data.Subject);
+					self.setMessageToast(response.Data);
 				} else if (response.Type == "prune") {
 					// messages have been deleted, reload messages to adjust
 					self.scrollInPlace = true;
@@ -591,6 +606,18 @@ export default {
 					document.querySelector('#ReleaseModal input[role="combobox"]').focus()
 				}, 500);
 			}, 300);
+		},
+
+		setMessageToast: function (m) {
+			if (this.toastMessage) {
+				return;
+			}
+
+			this.toastMessage = m;
+		},
+
+		clearMessageToast: function () {
+			this.toastMessage = false;
 		}
 	}
 }
@@ -1014,4 +1041,6 @@ export default {
 		<MessageRelease v-if="releaseAddresses" :message="message" :relayConfig="relayConfig"
 			:releaseAddresses="releaseAddresses"></MessageRelease>
 	</div>
+
+	<MessageToast v-if="toastMessage" :message="toastMessage" @clearMessageToast="clearMessageToast"></MessageToast>
 </template>

server/ui-src/assets/bootstrap.scss

@@ -32,7 +32,7 @@
 // @import "../../../node_modules/bootstrap/scss/progress";
 @import "../../../node_modules/bootstrap/scss/list-group";
 @import "../../../node_modules/bootstrap/scss/close";
-// @import "../../../node_modules/bootstrap/scss/toasts";
+@import "../../../node_modules/bootstrap/scss/toasts";
 @import "../../../node_modules/bootstrap/scss/modal";
 // @import "../../../node_modules/bootstrap/scss/tooltip";
 // @import "../../../node_modules/bootstrap/scss/popover";

server/ui-src/templates/Message.vue

@@ -195,7 +195,7 @@ export default {
 									<template v-if="i > 0">, </template>
 									<span class="text-nowrap">{{ t.Name + " &lt;" + t.Address + "&gt;" }}</span>
 								</span>
-								<span v-else>Undisclosed recipients</span>
+								<span v-else class="text-muted">[Undisclosed recipients]</span>
 							</td>
 						</tr>
 						<tr v-if="message.Cc && message.Cc.length" class="small">
@@ -272,28 +272,28 @@ export default {
 					role="tab" aria-controls="nav-html" aria-selected="true" v-if="message.HTML"
 					v-on:click="showMobileBtns = true; resizeIframes()">HTML</button>
 				<button class="nav-link" id="nav-html-source-tab" data-bs-toggle="tab" data-bs-target="#nav-html-source"
-					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if="message.HTML"
-					v-on:click="showMobileBtns = false">
+					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if=" message.HTML "
+					v-on:click=" showMobileBtns = false ">
 					HTML <span class="d-sm-none">Src</span><span class="d-none d-sm-inline">Source</span>
 				</button>
 				<button class="nav-link" id="nav-plain-text-tab" data-bs-toggle="tab" data-bs-target="#nav-plain-text"
 					type="button" role="tab" aria-controls="nav-plain-text" aria-selected="false"
-					:class="message.HTML == '' ? 'show' : ''" v-on:click="showMobileBtns = false">Text</button>
+					:class=" message.HTML == '' ? 'show' : '' " v-on:click=" showMobileBtns = false ">Text</button>
 				<button class="nav-link" id="nav-headers-tab" data-bs-toggle="tab" data-bs-target="#nav-headers"
 					type="button" role="tab" aria-controls="nav-headers" aria-selected="false"
-					v-on:click="showMobileBtns = false">
+					v-on:click=" showMobileBtns = false ">
 					<span class="d-sm-none">Hdrs</span><span class="d-none d-sm-inline">Headers</span>
 				</button>
 				<button class="nav-link" id="nav-raw-tab" data-bs-toggle="tab" data-bs-target="#nav-raw" type="button"
 					role="tab" aria-controls="nav-raw" aria-selected="false"
-					v-on:click="showMobileBtns = false">Raw</button>
+					v-on:click=" showMobileBtns = false ">Raw</button>
 
-				<div class="d-none d-lg-block ms-auto me-2" v-if="showMobileBtns">
-					<template v-for="vals, key in responsiveSizes">
-						<button class="btn" :class="scaleHTMLPreview == key ? 'btn-outline-primary' : ''"
-							:disabled="scaleHTMLPreview == key" :title="'Switch to ' + key + ' view'"
-							v-on:click="scaleHTMLPreview = key">
-							<i class="bi" :class="'bi-' + key"></i>
+				<div class="d-none d-lg-block ms-auto me-2" v-if=" showMobileBtns ">
+					<template v-for="  vals, key   in   responsiveSizes  ">
+						<button class="btn" :class=" scaleHTMLPreview == key ? 'btn-outline-primary' : '' "
+							:disabled=" scaleHTMLPreview == key " :title=" 'Switch to ' + key + ' view' "
+							v-on:click=" scaleHTMLPreview = key ">
+							<i class="bi" :class=" 'bi-' + key "></i>
 						</button>
 					</template>
 				</div>
@@ -301,31 +301,31 @@ export default {
 		</nav>
 
 		<div class="tab-content mb-5" id="nav-tabContent">
-			<div v-if="message.HTML != ''" class="tab-pane fade show" id="nav-html" role="tabpanel"
+			<div v-if=" message.HTML != '' " class="tab-pane fade show" id="nav-html" role="tabpanel"
 				aria-labelledby="nav-html-tab" tabindex="0">
-				<div id="responsive-view" :class="scaleHTMLPreview" :style="responsiveSizes[scaleHTMLPreview]">
-					<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc="message.HTML"
-						v-on:load="resizeIframe" seamless frameborder="0" style="width: 100%; height: 100%;">
+				<div id="responsive-view" :class=" scaleHTMLPreview " :style=" responsiveSizes[scaleHTMLPreview] ">
+					<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc=" message.HTML "
+						v-on:load=" resizeIframe " seamless frameborder="0" style="width: 100%; height: 100%;">
 					</iframe>
 				</div>
-				<Attachments v-if="allAttachments(message).length" :message="message"
-					:attachments="allAttachments(message)"></Attachments>
+				<Attachments v-if=" allAttachments(message).length " :message=" message "
+					:attachments=" allAttachments(message) "></Attachments>
 			</div>
 			<div class="tab-pane fade" id="nav-html-source" role="tabpanel" aria-labelledby="nav-html-source-tab"
-				tabindex="0" v-if="message.HTML">
+				tabindex="0" v-if=" message.HTML ">
 				<pre><code class="language-html">{{ message.HTML }}</code></pre>
 			</div>
 			<div class="tab-pane fade" id="nav-plain-text" role="tabpanel" aria-labelledby="nav-plain-text-tab" tabindex="0"
-				:class="message.HTML == '' ? 'show' : ''">
+				:class=" message.HTML == '' ? 'show' : '' ">
 				<div class="text-view">{{ message.Text }}</div>
-				<Attachments v-if="allAttachments(message).length" :message="message"
-					:attachments="allAttachments(message)"></Attachments>
+				<Attachments v-if=" allAttachments(message).length " :message=" message "
+					:attachments=" allAttachments(message) "></Attachments>
 			</div>
 			<div class="tab-pane fade" id="nav-headers" role="tabpanel" aria-labelledby="nav-headers-tab" tabindex="0">
-				<Headers v-if="loadHeaders" :message="message"></Headers>
+				<Headers v-if=" loadHeaders " :message=" message "></Headers>
 			</div>
 			<div class="tab-pane fade" id="nav-raw" role="tabpanel" aria-labelledby="nav-raw-tab" tabindex="0">
-				<iframe v-if="srcURI" :src="srcURI" v-on:load="resizeIframe" seamless frameborder="0"
+				<iframe v-if=" srcURI " :src=" srcURI " v-on:load=" resizeIframe " seamless frameborder="0"
 					style="width: 100%; height: 300px;" id="message-src"></iframe>
 			</div>
 		</div>

server/ui-src/templates/MessageRelease.vue

@@ -83,6 +83,11 @@ export default {
 						<div class="invalid-feedback">Invalid email address</div>
 					</div>
 				</div>
+				<div class="form-text text-center" v-if="relayConfig.MessageRelay.RecipientAllowlist != ''">
+					Note: A recipient allowlist has been configured. Any mail address not matching it will be rejected.
+					<br class="d-none d-md-inline">
+					Configured allowlist: <b>{{ relayConfig.MessageRelay.RecipientAllowlist }}</b>
+				</div>
 				<div class="form-text text-center">
 					Note: For testing purposes, a unique Message-Id will be generated on send.
 					<br class="d-none d-md-inline">

server/ui-src/templates/MessageToast.vue

@@ -0,0 +1,44 @@
+<script>
+import { Toast } from 'bootstrap';
+
+export default {
+	props: {
+		message: Object
+	},
+
+	mounted() {
+		let self = this;
+		let el = document.getElementById('messageToast');
+		if (el) {
+			el.addEventListener('hidden.bs.toast', () => {
+				self.$emit("clearMessageToast");
+			})
+
+			let b = Toast.getOrCreateInstance(el);
+			b.show();
+		}
+	}
+}
+</script>
+
+<template>
+	<div class="toast-container position-fixed bottom-0 end-0 p-3">
+		<div id="messageToast" class="toast" role="alert" aria-live="assertive" aria-atomic="true">
+			<div class="toast-header">
+				<i class="bi bi-envelope-exclamation-fill me-2"></i>
+				<strong class="me-auto"><a :href="'#' + message.ID">New message</a></strong>
+				<small class="text-body-secondary">now</small>
+				<button type="button" class="btn-close" data-bs-dismiss="toast" aria-label="Close"></button>
+			</div>
+
+			<div class="toast-body">
+				<div>
+					<a :href="'#' + message.ID" class="d-block text-truncate text-muted">
+						<template v-if="message.Subject != ''">{{ message.Subject }}</template>
+						<template v-else>[ no subject ]</template>
+					</a>
+				</div>
+			</div>
+		</div>
+	</div>
+</template>

server/ui/api/v1/swagger.json

@@ -66,7 +66,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -103,7 +103,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -142,14 +142,14 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
           },
           {
             "type": "string",
-            "description": "attachment part id",
+            "description": "Attachment part ID",
             "name": "PartID",
             "in": "path",
             "required": true
@@ -224,7 +224,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -261,14 +261,14 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
           },
           {
             "description": "Array of email addresses to release message to",
-            "name": "To",
+            "name": "to",
             "in": "body",
             "required": true,
             "schema": {
@@ -307,14 +307,14 @@
           {
             "type": "integer",
             "default": 0,
-            "description": "pagination offset",
+            "description": "Pagination offset",
             "name": "start",
             "in": "query"
           },
           {
             "type": "integer",
             "default": 50,
-            "description": "limit results",
+            "description": "Limit results",
             "name": "limit",
             "in": "query"
           }
@@ -347,11 +347,11 @@
         "operationId": "SetReadStatus",
         "parameters": [
           {
-            "description": "Message ids to update",
+            "description": "Message IDs to update",
             "name": "ids",
             "in": "body",
             "schema": {
-              "description": "Message ids to update",
+              "description": "Message IDs to update",
               "type": "object",
               "$ref": "#/definitions/SetReadStatusRequest"
             }
@@ -385,11 +385,11 @@
         "operationId": "Delete",
         "parameters": [
           {
-            "description": "Message ids to delete",
+            "description": "Message IDs to delete",
             "name": "ids",
             "in": "body",
             "schema": {
-              "description": "Message ids to delete",
+              "description": "Message IDs to delete",
               "type": "object",
               "$ref": "#/definitions/DeleteRequest"
             }
@@ -423,7 +423,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "search query",
+            "description": "Search query",
             "name": "query",
             "in": "query",
             "required": true
@@ -431,7 +431,7 @@
           {
             "type": "integer",
             "default": 50,
-            "description": "limit results",
+            "description": "Limit results",
             "name": "limit",
             "in": "query"
           }
@@ -466,12 +466,12 @@
         "operationId": "SetTags",
         "parameters": [
           {
-            "description": "Message ids to update",
+            "description": "Message IDs to update",
             "name": "ids",
             "in": "body",
             "required": true,
             "schema": {
-              "description": "Message ids to update",
+              "description": "Message IDs to update",
               "type": "object",
               "$ref": "#/definitions/SetTagsRequest"
             }
@@ -568,23 +568,23 @@
       "type": "object",
       "properties": {
         "ContentID": {
-          "description": "content id",
+          "description": "Content ID",
           "type": "string"
         },
         "ContentType": {
-          "description": "content type",
+          "description": "Content type",
           "type": "string"
         },
         "FileName": {
-          "description": "file name",
+          "description": "File name",
           "type": "string"
         },
         "PartID": {
-          "description": "attachment part id",
+          "description": "Attachment part ID",
           "type": "string"
         },
         "Size": {
-          "description": "size in bytes",
+          "description": "Size in bytes",
           "type": "integer",
           "format": "int64"
         }
@@ -645,7 +645,7 @@
           "type": "string"
         },
         "ID": {
-          "description": "Unique message database id",
+          "description": "Database ID",
           "type": "string"
         },
         "Inline": {
@@ -655,6 +655,10 @@
             "$ref": "#/definitions/Attachment"
           }
         },
+        "MessageID": {
+          "description": "Message ID",
+          "type": "string"
+        },
         "Read": {
           "description": "Read status",
           "type": "boolean"
@@ -667,7 +671,7 @@
           }
         },
         "ReturnPath": {
-          "description": "ReturnPath is the Return-Path",
+          "description": "Return-Path",
           "type": "string"
         },
         "Size": {
@@ -744,7 +748,7 @@
           "$ref": "#/definitions/Address"
         },
         "ID": {
-          "description": "Unique message database id",
+          "description": "Database ID",
           "type": "string"
         },
         "Read": {

storage/database.go

@@ -72,20 +72,55 @@ var (
 			Script: `ALTER TABLE mailbox ADD COLUMN Tags Text  NOT NULL DEFAULT '[]';
 			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
 		},
+		{
+			Version:     1.2,
+			Description: "Creating new mailbox format",
+			Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
+				Created INTEGER NOT NULL,
+				ID TEXT NOT NULL,
+				MessageID TEXT NOT NULL,
+				Subject TEXT NOT NULL,
+				Metadata TEXT,
+				Size INTEGER NOT NULL,
+				Inline INTEGER NOT NULL,
+				Attachments INTEGER NOT NULL,
+				Read INTEGER,
+				Tags TEXT,
+				SearchText TEXT
+			);
+			INSERT INTO mailboxtmp 
+				(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+			SELECT 
+				Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+				json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+				Search, Read, Tags
+			FROM mailbox;
+
+			DROP TABLE IF EXISTS mailbox;
+			ALTER TABLE mailboxtmp RENAME TO mailbox;
+			CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
+			CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
+			CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
+			CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
+			CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
 	}
 )
 
 // DBMailSummary struct for storing mail summary
 type DBMailSummary struct {
-	Created     time.Time
-	From        *mail.Address
-	To          []*mail.Address
-	Cc          []*mail.Address
-	Bcc         []*mail.Address
-	Subject     string
-	Size        int
-	Inline      int
-	Attachments int
+	From *mail.Address
+	To   []*mail.Address
+	Cc   []*mail.Address
+	Bcc  []*mail.Address
+	// Subject     string
+	// Size        int
+	// Inline      int
+	// Attachments int
 }
 
 // InitDB will initialise the database
@@ -144,6 +179,8 @@ func InitDB() error {
 	// auto-prune & delete
 	go dbCron()
 
+	go dataMigrations()
+
 	return nil
 }
 
@@ -189,22 +226,21 @@ func Store(body []byte) (string, error) {
 		from = &mail.Address{Name: env.GetHeader("From")}
 	}
 
+	messageID := strings.Trim(env.Root.Header.Get("Message-ID"), "<>")
+
 	obj := DBMailSummary{
-		Created:     time.Now(),
-		From:        from,
-		To:          addressToSlice(env, "To"),
-		Cc:          addressToSlice(env, "Cc"),
-		Bcc:         addressToSlice(env, "Bcc"),
-		Subject:     env.GetHeader("Subject"),
-		Size:        len(body),
-		Inline:      len(env.Inlines),
-		Attachments: len(env.Attachments),
+		From: from,
+		To:   addressToSlice(env, "To"),
+		Cc:   addressToSlice(env, "Cc"),
+		Bcc:  addressToSlice(env, "Bcc"),
 	}
 
+	created := time.Now()
+
 	// use message date instead of created date
 	if config.UseMessageDates {
 		if mDate, err := env.Date(); err == nil {
-			obj.Created = mDate
+			created = mDate
 		}
 	}
 
@@ -237,8 +273,14 @@ func Store(body []byte) (string, error) {
 	// roll back if it fails
 	defer tx.Rollback()
 
+	subject := env.GetHeader("Subject")
+	size := len(body)
+	inline := len(env.Inlines)
+	attachments := len(env.Attachments)
+
 	// insert mail summary data
-	_, err = tx.Exec("INSERT INTO mailbox(ID, Data, Search, Tags, Read) values(?,?,?,?,0)", id, string(summaryJSON), searchText, string(tagJSON))
+	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)",
+		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON))
 	if err != nil {
 		return "", err
 	}
@@ -259,9 +301,12 @@ func Store(body []byte) (string, error) {
 		return "", err
 	}
 
-	c.Tags = tagData
-
+	c.Created = created
 	c.ID = id
+	c.Attachments = attachments
+	c.Subject = subject
+	c.Size = size
+	c.Tags = tagData
 
 	websockets.Broadcast("new", c)
 
@@ -276,24 +321,28 @@ func List(start, limit int) ([]MessageSummary, error) {
 	results := []MessageSummary{}
 
 	q := sqlf.From("mailbox").
-		Select(`ID, Data, Tags, Read`).
-		OrderBy("Sort DESC").
+		Select(`Created, ID, Subject, Metadata, Size, Attachments, Read, Tags`).
+		OrderBy("Created DESC").
 		Limit(limit).
 		Offset(start)
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
 		var id string
-		var summary string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
 		var tags string
 		var read int
 		em := MessageSummary{}
 
-		if err := row.Scan(&id, &summary, &tags, &read); err != nil {
+		if err := row.Scan(&created, &id, &subject, &metadata, &size, &attachments, &read, &tags); err != nil {
 			logger.Log().Error(err)
 			return
 		}
 
-		if err := json.Unmarshal([]byte(summary), &em); err != nil {
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -303,11 +352,17 @@ func List(start, limit int) ([]MessageSummary, error) {
 			return
 		}
 
+		em.Created = time.UnixMilli(created)
 		em.ID = id
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
 		em.Read = read == 1
 
 		results = append(results, em)
 
+		// logger.PrettyPrint(em)
+
 	}); err != nil {
 		return results, err
 	}
@@ -342,19 +397,23 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 	q := searchParser(args, start, limit)
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
 		var id string
-		var summary string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
 		var tags string
 		var read int
 		var ignore string
 		em := MessageSummary{}
 
-		if err := row.Scan(&id, &summary, &tags, &read, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+		if err := row.Scan(&created, &id, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
 			logger.Log().Error(err)
 			return
 		}
 
-		if err := json.Unmarshal([]byte(summary), &em); err != nil {
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -364,7 +423,11 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 			return
 		}
 
+		em.Created = time.UnixMilli(created)
 		em.ID = id
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
 		em.Read = read == 1
 
 		results = append(results, em)
@@ -404,8 +467,10 @@ func GetMessage(id string) (*Message, error) {
 		from = &mail.Address{Name: env.GetHeader("From")}
 	}
 
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
 	returnPath := strings.Trim(env.GetHeader("Return-Path"), "<>")
-	if returnPath == "" {
+	if returnPath == "" && from != nil {
 		returnPath = from.Address
 	}
 
@@ -413,27 +478,20 @@ func GetMessage(id string) (*Message, error) {
 	if err != nil {
 		// return received datetime when message does not contain a date header
 		q := sqlf.From("mailbox").
-			Select(`Data`).
-			OrderBy("Sort DESC").
+			Select(`Created`).
 			Where(`ID = ?`, id)
 
 		if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
-			var summary string
-			em := MessageSummary{}
+			var created int64
 
-			if err := row.Scan(&summary); err != nil {
-				logger.Log().Error(err)
-				return
-			}
-
-			if err := json.Unmarshal([]byte(summary), &em); err != nil {
+			if err := row.Scan(&created); err != nil {
 				logger.Log().Error(err)
 				return
 			}
 
 			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
 
-			date = em.Created
+			date = time.UnixMilli(created)
 		}); err != nil {
 			logger.Log().Error(err)
 		}
@@ -441,6 +499,7 @@ func GetMessage(id string) (*Message, error) {
 
 	obj := Message{
 		ID:         id,
+		MessageID:  messageID,
 		Read:       true,
 		From:       from,
 		Date:       date,
@@ -821,3 +880,16 @@ func IsUnread(id string) bool {
 
 	return unread == 1
 }
+
+// MessageIDExists blaah
+func MessageIDExists(id string) bool {
+	var total int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		Where("MessageID = ?", id)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return total != 0
+}

storage/migrationTasks.go

@@ -0,0 +1,200 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/utils/logger"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+)
+
+func dataMigrations() {
+	updateOrderByCreatedTask()
+	assignMessageIDsTask()
+}
+
+// Update Created column using Created metadata datetime <= v1.6.5
+// Migration task implemented 05/2023 - can be removed end 2023
+func updateOrderByCreatedTask() {
+	q := sqlf.From("mailbox").
+		Select("ID").
+		Select(`json_extract(Metadata, '$.Created') as Created`).
+		Where("Created < ?", 1155000600)
+
+	toUpdate := make(map[string]int64)
+	p := message.NewPrinter(language.English)
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var id string
+		var ts sql.NullString
+		if err := row.Scan(&id, &ts); err != nil {
+			logger.Log().Error("[migration]", err)
+			return
+		}
+
+		if !ts.Valid {
+			logger.Log().Errorf("[migration] cannot get Created timestamp from %s", id)
+			return
+		}
+
+		t, _ := time.Parse(time.RFC3339Nano, ts.String)
+		toUpdate[id] = t.UnixMilli()
+	}); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	total := len(toUpdate)
+
+	if total == 0 {
+		return
+	}
+
+	logger.Log().Infof("[migration] updating timestamp for %s messages", p.Sprintf("%d", len(toUpdate)))
+
+	// begin a transaction
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	var blockTime = time.Now()
+
+	count := 0
+	for id, ts := range toUpdate {
+		count++
+		_, err := tx.Exec(`UPDATE mailbox SET Created = ? WHERE ID = ?`, ts, id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+		}
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] updated timestamp for 1,000 messages [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	logger.Log().Infof("[migration] commit %s changes", p.Sprintf("%d", count))
+
+	if err := tx.Commit(); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	logger.Log().Infof("[migration] complete")
+}
+
+// Find any messages without a stored Message-ID and update it <= v1.6.5
+// Migration task implemented 05/2023 - can be removed end 2023
+func assignMessageIDsTask() {
+	if !config.IgnoreDuplicateIDs {
+		return
+	}
+
+	q := sqlf.From("mailbox").
+		Select("ID").
+		Where("MessageID = ''")
+
+	missingIDS := make(map[string]string)
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var id string
+		if err := row.Scan(&id); err != nil {
+			logger.Log().Error("[migration]", err)
+			return
+		}
+		missingIDS[id] = ""
+	}); err != nil {
+		logger.Log().Error("[migration]", err)
+	}
+
+	if len(missingIDS) == 0 {
+		return
+	}
+
+	var count int
+	var blockTime = time.Now()
+	p := message.NewPrinter(language.English)
+
+	total := len(missingIDS)
+
+	logger.Log().Infof("[migration] extracting Message-IDs for %s messages", p.Sprintf("%d", total))
+
+	for id := range missingIDS {
+		raw, err := GetMessageRaw(id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+			continue
+		}
+
+		r := bytes.NewReader(raw)
+
+		env, err := enmime.ReadEnvelope(r)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+			continue
+		}
+
+		messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
+		missingIDS[id] = messageID
+
+		count++
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] extracted 1,000 Message-IDs [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	// begin a transaction
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	count = 0
+
+	for id, mid := range missingIDS {
+		_, err = tx.Exec(`UPDATE mailbox SET MessageID = ? WHERE ID = ?`, mid, id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+		}
+
+		count++
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] stored 1,000 Message-IDs [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	logger.Log().Infof("[migration] commit %s changes", p.Sprintf("%d", count))
+
+	if err := tx.Commit(); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	logger.Log().Infof("[migration] complete")
+}

storage/search.go

@@ -14,15 +14,13 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 	}
 
 	q := sqlf.From("mailbox").
-		Select(`ID, Data, Tags, Read,
-			json_extract(Data, '$.To') as ToJSON,
-			json_extract(Data, '$.From') as FromJSON,
-			IFNULL(json_extract(Data, '$.Cc'), '{}') as CcJSON,
-			IFNULL(json_extract(Data, '$.Bcc'), '{}') as BccJSON,
-			json_extract(Data, '$.Subject') as Subject,
-			json_extract(Data, '$.Attachments') as Attachments
+		Select(`Created, ID, Subject, Metadata, Size, Attachments, Read, Tags,
+			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
+			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
+			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
+			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON
 		`).
-		OrderBy("Sort DESC").
+		OrderBy("Created DESC").
 		Limit(limit).
 		Offset(start)
 
@@ -92,6 +90,15 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 					q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%")
 				}
 			}
+		} else if strings.HasPrefix(w, "message-id:") {
+			w = cleanString(w[11:])
+			if w != "" {
+				if exclude {
+					q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
 		} else if strings.HasPrefix(w, "tag:") {
 			w = cleanString(w[4:])
 			if w != "" {
@@ -122,9 +129,9 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 		} else {
 			// search text
 			if exclude {
-				q.Where("search NOT LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
+				q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
 			} else {
-				q.Where("search LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
+				q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
 			}
 		}
 	}

storage/structs.go

@@ -11,8 +11,10 @@ import (
 //
 // swagger:model Message
 type Message struct {
-	// Unique message database id
+	// Database ID
 	ID string
+	// Message ID
+	MessageID string
 	// Read status
 	Read bool
 	// From address
@@ -25,7 +27,7 @@ type Message struct {
 	Bcc []*mail.Address
 	// ReplyTo addresses
 	ReplyTo []*mail.Address
-	// ReturnPath is the Return-Path
+	// Return-Path
 	ReturnPath string
 	// Message subject
 	Subject string
@@ -49,15 +51,15 @@ type Message struct {
 //
 // swagger:model Attachment
 type Attachment struct {
-	// attachment part id
+	// Attachment part ID
 	PartID string
-	// file name
+	// File name
 	FileName string
-	// content type
+	// Content type
 	ContentType string
-	// content id
+	// Content ID
 	ContentID string
-	// size in bytes
+	// Size in bytes
 	Size int
 }
 
@@ -65,7 +67,7 @@ type Attachment struct {
 //
 // swagger:model MessageSummary
 type MessageSummary struct {
-	// Unique message database id
+	// Database ID
 	ID string
 	// Read status
 	Read bool

storage/utils.go

@@ -39,7 +39,7 @@ func createSearchText(env *enmime.Envelope) string {
 	b.WriteString(env.GetHeader("Bcc") + " ")
 	h := strings.TrimSpace(
 		html2text.HTML2TextWithOptions(
-			env.HTML, 
+			env.HTML,
 			html2text.WithLinksInnerText(),
 		),
 	)
@@ -92,7 +92,7 @@ func dbCron() {
 		if config.MaxMessages > 0 {
 			q := sqlf.Select("ID").
 				From("mailbox").
-				OrderBy("Sort DESC").
+				OrderBy("Created DESC").
 				Limit(5000).
 				Offset(config.MaxMessages)