Merge branch 'release/0.1.0'

Resolves #4

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "develop" ]
+  schedule:
+    - cron: '34 23 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

CHANGELOG.md

@@ -3,6 +3,24 @@
 Notable changes to Mailpit will be documented in this file.
 
 
+## 0.1.0
+
+### Feature
+- SMTP STARTTLS & SMTP authentication support
+
+
+## 0.0.9
+
+### Bugfix
+- Include read status in search results
+
+### Feature
+- HTTPS option for web UI
+
+### Testing
+- Memory & physical database tests
+
+
 ## 0.0.8
 
 ### Bugfix

README.md

@@ -8,23 +8,25 @@ Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster.
 
 ![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/screenshot.png)
 
+
 ## Features
 
 - Runs completely on a single binary
 - SMTP server (default `0.0.0.0:1025`)
 - Web UI to view emails (HTML format, text, source and MIME attachments, default `0.0.0.0:8025`)
 - Real-time web UI updates using web sockets for new mail
-- Optional basic authentication for web UI (see [wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
-- Email storage in either memory or disk (using [CloverDB](https://github.com/ostafen/clover)) - note that in-memory has a physical limit of 1MB per email
 - Configurable automatic email pruning (default keeps the most recent 500 emails)
+- Email storage either in memory or disk ([see wiki](https://github.com/axllent/mailpit/wiki/Email-storage))
 - Fast SMTP processing & storing - approximately 300-600 emails per second depending on CPU, network speed & email size
 - Can handle tens of thousands of emails
-- Multi-arch [Docker images](https://github.com/axllent/mailpit/wiki/Docker-images)
+- Optional SMTP with STARTTLS & SMTP authentication ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-with-STARTTLS-and-authentication))
+- Optional HTTPS for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/HTTPS))
+- Optional basic authentication for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
+- Multi-architecture [Docker images](https://github.com/axllent/mailpit/wiki/Docker-images)
 
 
 ## Planned features
 
-- Optional HTTPS for web UI
 - Browser notifications for new mail (HTTPS only)
 
 
@@ -48,7 +50,7 @@ If Mailpit is found on the same host as sendmail, you can symlink the Mailpit bi
 
 You can use your default system `sendmail` binary to route directly to port `1025` (configurable) by calling `/usr/sbin/sendmail -S localhost:1025`.
 
-You can build a Mailpit-specific sendmail binary from source ( see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source)).
+You can build a Mailpit-specific sendmail binary from source (see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source)).
 
 
 ## Why rewrite MailHog?

cmd/root.go

@@ -85,13 +85,52 @@ func init() {
 		config.MaxMessages, _ = strconv.Atoi(os.Getenv("MP_MAX_MESSAGES"))
 	}
 	if len(os.Getenv("MP_AUTH_FILE")) > 0 {
-		config.AuthFile = os.Getenv("MP_AUTH_FILE")
+		config.UIAuthFile = os.Getenv("MP_AUTH_FILE")
+	}
+	if len(os.Getenv("MP_UI_AUTH_FILE")) > 0 {
+		config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
+	}
+	if len(os.Getenv("MP_SMTP_AUTH_FILE")) > 0 {
+		config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
+	}
+	// deprecated 2022/08/06
+	if len(os.Getenv("MP_SSL_CERT")) > 0 {
+		config.UISSLCert = os.Getenv("MP_SSL_CERT")
+	}
+	// deprecated 2022/08/06
+	if len(os.Getenv("MP_SSL_KEY")) > 0 {
+		config.UISSLKey = os.Getenv("MP_SSL_KEY")
+	}
+	if len(os.Getenv("MP_UI_SSL_CERT")) > 0 {
+		config.UISSLCert = os.Getenv("MP_UI_SSL_CERT")
+	}
+	if len(os.Getenv("MP_UISSL_KEY")) > 0 {
+		config.UISSLKey = os.Getenv("MP_UI_SSL_KEY")
 	}
 
 	rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store peristent data")
 	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
 	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
 	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
-	rootCmd.Flags().StringVarP(&config.AuthFile, "auth-file", "a", config.AuthFile, "A password file for authentication (see wiki)")
+
+	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI authentication")
+	rootCmd.Flags().StringVar(&config.UISSLCert, "ui-ssl-cert", config.UISSLCert, "SSL certificate for web UI - requires ui-ssl-key")
+	rootCmd.Flags().StringVar(&config.UISSLKey, "ui-ssl-key", config.UISSLKey, "SSL key for web UI - requires ui-ssl-cert")
+
+	rootCmd.Flags().StringVar(&config.SMTPAuthFile, "smtp-auth-file", config.SMTPAuthFile, "A password file for SMTP authentication")
+	rootCmd.Flags().StringVar(&config.SMTPSSLCert, "smtp-ssl-cert", config.SMTPSSLCert, "SSL certificate for SMTP - requires smtp-ssl-key")
+	rootCmd.Flags().StringVar(&config.SMTPSSLKey, "smtp-ssl-key", config.SMTPSSLKey, "SSL key for SMTP - requires smtp-ssl-cert")
+
 	rootCmd.Flags().BoolVarP(&config.VerboseLogging, "verbose", "v", false, "Verbose logging")
+
+	// deprecated 2022/08/06
+	rootCmd.Flags().StringVarP(&config.UIAuthFile, "auth-file", "a", config.UIAuthFile, "A password file for web UI authentication")
+	rootCmd.Flags().StringVar(&config.UISSLCert, "ssl-cert", config.UISSLCert, "SSL certificate - requires ssl-key")
+	rootCmd.Flags().StringVar(&config.UISSLKey, "ssl-key", config.UISSLKey, "SSL key - requires ssl-cert")
+	rootCmd.Flags().Lookup("auth-file").Hidden = true
+	rootCmd.Flags().Lookup("auth-file").Deprecated = "use --ui-auth-file"
+	rootCmd.Flags().Lookup("ssl-cert").Hidden = true
+	rootCmd.Flags().Lookup("ssl-cert").Deprecated = "use --ui-ssl-cert"
+	rootCmd.Flags().Lookup("ssl-key").Hidden = true
+	rootCmd.Flags().Lookup("ssl-key").Deprecated = "use --ui-ssl-key"
 }

config/config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"errors"
+	"fmt"
+	"os"
 	"regexp"
 
 	"github.com/tg123/go-htpasswd"
@@ -26,16 +28,29 @@ var (
 	// NoLogging for tests
 	NoLogging = false
 
-	// SSLCert @TODO
-	SSLCert string
-	// SSLKey @TODO
-	SSLKey string
+	// UISSLCert file
+	UISSLCert string
 
-	// AuthFile for basic authentication
-	AuthFile string
+	// UISSLKey file
+	UISSLKey string
 
-	// Auth used for euthentication
-	Auth *htpasswd.File
+	// UIAuthFile for basic authentication
+	UIAuthFile string
+
+	// UIAuth used for euthentication
+	UIAuth *htpasswd.File
+
+	// SMTPSSLCert file
+	SMTPSSLCert string
+
+	// SMTPSSLKey file
+	SMTPSSLKey string
+
+	// SMTPAuthFile for SMTP authentication
+	SMTPAuthFile string
+
+	// SMTPAuth used for euthentication
+	SMTPAuth *htpasswd.File
 )
 
 // VerifyConfig wil do some basic checking
@@ -48,13 +63,71 @@ func VerifyConfig() error {
 		return errors.New("HTTP bind should be in the format of <ip>:<port>")
 	}
 
-	if AuthFile != "" {
-		a, err := htpasswd.New(AuthFile, htpasswd.DefaultSystems, nil)
+	if UIAuthFile != "" {
+		if !isFile(UIAuthFile) {
+			return fmt.Errorf("HTTP password file not found: %s", UIAuthFile)
+		}
+
+		a, err := htpasswd.New(UIAuthFile, htpasswd.DefaultSystems, nil)
 		if err != nil {
 			return err
 		}
-		Auth = a
+		UIAuth = a
+	}
+
+	if UISSLCert != "" && UISSLKey == "" || UISSLCert == "" && UISSLKey != "" {
+		return errors.New("you must provide both a UI SSL certificate and a key")
+	}
+
+	if UISSLCert != "" {
+		if !isFile(UISSLCert) {
+			return fmt.Errorf("SSL certificate not found: %s", UISSLCert)
+		}
+
+		if !isFile(UISSLKey) {
+			return fmt.Errorf("SSL key not found: %s", UISSLKey)
+		}
+	}
+
+	if SMTPSSLCert != "" && SMTPSSLKey == "" || SMTPSSLCert == "" && SMTPSSLKey != "" {
+		return errors.New("you must provide both an SMTP SSL certificate and a key")
+	}
+
+	if SMTPSSLCert != "" {
+		if !isFile(SMTPSSLCert) {
+			return fmt.Errorf("SMTP SSL certificate not found: %s", SMTPSSLCert)
+		}
+
+		if !isFile(SMTPSSLKey) {
+			return fmt.Errorf("SMTP SSL key not found: %s", SMTPSSLKey)
+		}
+	}
+
+	if SMTPAuthFile != "" {
+		if !isFile(SMTPAuthFile) {
+			return fmt.Errorf("SMTP password file not found: %s", SMTPAuthFile)
+		}
+
+		if SMTPSSLCert == "" {
+			return errors.New("SMTP authentication requires SMTP encryption")
+		}
+
+		a, err := htpasswd.New(SMTPAuthFile, htpasswd.DefaultSystems, nil)
+		if err != nil {
+			return err
+		}
+		SMTPAuth = a
 	}
 
 	return nil
 }
+
+// IsFile returns if a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}

server/server.go

@@ -47,9 +47,13 @@ func Listen() {
 	r.PathPrefix("/").Handler(middlewareHandler(http.FileServer(http.FS(serverRoot))))
 	http.Handle("/", r)
 
-	if config.SSLCert != "" && config.SSLKey != "" {
+	if config.UIAuthFile != "" {
+		logger.Log().Info("[http] enabling web UI basic authentication")
+	}
+
+	if config.UISSLCert != "" && config.UISSLKey != "" {
 		logger.Log().Infof("[http] starting secure server on https://%s", config.HTTPListen)
-		log.Fatal(http.ListenAndServeTLS(config.HTTPListen, config.SSLCert, config.SSLKey, nil))
+		log.Fatal(http.ListenAndServeTLS(config.HTTPListen, config.UISSLCert, config.UISSLKey, nil))
 	} else {
 		logger.Log().Infof("[http] starting server on http://%s", config.HTTPListen)
 		log.Fatal(http.ListenAndServe(config.HTTPListen, nil))
@@ -76,7 +80,7 @@ func (w gzipResponseWriter) Write(b []byte) (int, error) {
 // and gzip compression.
 func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if config.AuthFile != "" {
+		if config.UIAuthFile != "" {
 			user, pass, ok := r.BasicAuth()
 
 			if !ok {
@@ -84,7 +88,7 @@ func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 				return
 			}
 
-			if !config.Auth.Match(user, pass) {
+			if !config.UIAuth.Match(user, pass) {
 				basicAuthResponse(w)
 				return
 			}
@@ -107,7 +111,7 @@ func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 func middlewareHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 
-		if config.AuthFile != "" {
+		if config.UIAuthFile != "" {
 			user, pass, ok := r.BasicAuth()
 
 			if !ok {
@@ -115,7 +119,7 @@ func middlewareHandler(h http.Handler) http.Handler {
 				return
 			}
 
-			if !config.Auth.Match(user, pass) {
+			if !config.UIAuth.Match(user, pass) {
 				basicAuthResponse(w)
 				return
 			}

server/websockets/client.go

@@ -99,8 +99,8 @@ func (c *Client) writePump() {
 
 // ServeWs handles websocket requests from the peer.
 func ServeWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
-	if config.AuthFile != "" {
-		if config.AuthFile != "" {
+	if config.UIAuthFile != "" {
+		if config.UIAuthFile != "" {
 			user, pass, ok := r.BasicAuth()
 
 			if !ok {
@@ -108,7 +108,7 @@ func ServeWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			if !config.Auth.Match(user, pass) {
+			if !config.UIAuth.Match(user, pass) {
 				basicAuthResponse(w)
 				return
 			}

smtpd/smtpd.go

@@ -9,7 +9,7 @@ import (
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/logger"
 	"github.com/axllent/mailpit/storage"
-	s "github.com/mhale/smtpd"
+	"github.com/mhale/smtpd"
 )
 
 func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
@@ -37,12 +37,45 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 	return nil
 }
 
+func authHandler(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error) {
+	return config.SMTPAuth.Match(string(username), string(password)), nil
+}
+
 // Listen starts the SMTPD server
 func Listen() error {
-	logger.Log().Infof("[smtp] starting on %s", config.SMTPListen)
-	if err := s.ListenAndServe(config.SMTPListen, mailHandler, "Mailpit", ""); err != nil {
-		return err
+	if config.SMTPSSLCert != "" {
+		logger.Log().Info("[smtp] enabling TLS")
+	}
+	if config.SMTPAuthFile != "" {
+		logger.Log().Info("[smtp] enabling authentication")
 	}
 
-	return nil
+	logger.Log().Infof("[smtp] starting on %s", config.SMTPListen)
+
+	return listenAndServe(config.SMTPListen, mailHandler, authHandler)
+}
+
+func listenAndServe(addr string, handler smtpd.Handler, authHandler smtpd.AuthHandler) error {
+	srv := &smtpd.Server{
+		Addr:         addr,
+		Handler:      handler,
+		Appname:      "Mailpit",
+		Hostname:     "",
+		AuthHandler:  nil,
+		AuthRequired: false,
+	}
+
+	if config.SMTPAuthFile != "" {
+		srv.AuthHandler = authHandler
+		srv.AuthRequired = true
+	}
+
+	if config.SMTPSSLCert != "" {
+		err := srv.ConfigureTLS(config.SMTPSSLCert, config.SMTPSSLKey)
+		if err != nil {
+			return err
+		}
+	}
+
+	return srv.ListenAndServe()
 }

storage/database.go

@@ -327,12 +327,12 @@ func Search(mailbox, search string, start, limit int) ([]data.Summary, error) {
 	results := []data.Summary{}
 
 	for _, d := range q {
-		cs := &CloverStore{}
+		cs := &data.Summary{}
 		if err := d.Unmarshal(cs); err != nil {
 			return nil, err
 		}
-
-		results = append(results, cs.Summary(d.ObjectId()))
+		cs.ID = d.ObjectId()
+		results = append(results, *cs)
 	}
 
 	return results, nil
@@ -351,25 +351,6 @@ func CountUnread(mailbox string) (int, error) {
 	)
 }
 
-// Summary generated a message summary. ID must be supplied
-// as this is not stored within the CloverStore but rather the
-// *clover.Document
-func (c *CloverStore) Summary(id string) data.Summary {
-	s := data.Summary{
-		ID:          id,
-		From:        c.From,
-		To:          c.To,
-		Cc:          c.Cc,
-		Bcc:         c.Bcc,
-		Subject:     c.Subject,
-		Created:     c.Created,
-		Size:        c.Size,
-		Attachments: c.Attachments,
-	}
-
-	return s
-}
-
 // GetMessage returns a data.Message generated from the {mailbox}_data collection.
 // ID must be supplied as this is not stored within the CloverStore but rather the
 // *clover.Document

storage/database_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
+	"path"
 	"testing"
 	"time"
 
@@ -18,8 +20,10 @@ var (
 )
 
 func TestTextEmailInserts(t *testing.T) {
-	setup()
+	setup(false)
+	t.Log("Testing memory storage")
 
+RepeatTest:
 	start := time.Now()
 	for i := 0; i < 1000; i++ {
 		if _, err := Store(DefaultMailbox, testTextEmail); err != nil {
@@ -55,11 +59,20 @@ func TestTextEmailInserts(t *testing.T) {
 	t.Logf("deleted 1,000 text emails in %s\n", time.Since(delStart))
 
 	db.Close()
+	if config.DataDir == "" {
+		setup(true)
+		t.Logf("Testing physical storage to %s", config.DataDir)
+		defer os.RemoveAll(config.DataDir)
+		goto RepeatTest
+	}
+
 }
 
 func TestMimeEmailInserts(t *testing.T) {
-	setup()
+	setup(false)
+	t.Log("Testing memory storage")
 
+RepeatTest:
 	start := time.Now()
 	for i := 0; i < 1000; i++ {
 		if _, err := Store(DefaultMailbox, testMimeEmail); err != nil {
@@ -95,11 +108,19 @@ func TestMimeEmailInserts(t *testing.T) {
 	t.Logf("deleted 1,000 mime emails in %s\n", time.Since(delStart))
 
 	db.Close()
+	if config.DataDir == "" {
+		setup(true)
+		t.Logf("Testing physical storage to %s", config.DataDir)
+		defer os.RemoveAll(config.DataDir)
+		goto RepeatTest
+	}
 }
 
 func TestRetrieveMimeEmail(t *testing.T) {
-	setup()
+	setup(false)
+	t.Log("Testing memory storage")
 
+RepeatTest:
 	id, err := Store(DefaultMailbox, testMimeEmail)
 	if err != nil {
 		t.Log("error ", err)
@@ -128,11 +149,20 @@ func TestRetrieveMimeEmail(t *testing.T) {
 	assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
 
 	db.Close()
+
+	if config.DataDir == "" {
+		setup(true)
+		t.Logf("Testing physical storage to %s", config.DataDir)
+		defer os.RemoveAll(config.DataDir)
+		goto RepeatTest
+	}
 }
 
 func TestSearch(t *testing.T) {
-	setup()
+	setup(false)
+	t.Log("Testing memory storage")
 
+RepeatTest:
 	for i := 0; i < 1000; i++ {
 		msg := enmime.Builder().
 			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
@@ -198,10 +228,17 @@ func TestSearch(t *testing.T) {
 	assertEqual(t, len(summaries), 200, "200 search results expected")
 
 	db.Close()
+
+	if config.DataDir == "" {
+		setup(true)
+		t.Logf("Testing physical storage to %s", config.DataDir)
+		defer os.RemoveAll(config.DataDir)
+		goto RepeatTest
+	}
 }
 
 func BenchmarkImportText(b *testing.B) {
-	setup()
+	setup(false)
 
 	for i := 0; i < b.N; i++ {
 		if _, err := Store(DefaultMailbox, testTextEmail); err != nil {
@@ -214,7 +251,7 @@ func BenchmarkImportText(b *testing.B) {
 }
 
 func BenchmarkImportMime(b *testing.B) {
-	setup()
+	setup(false)
 
 	for i := 0; i < b.N; i++ {
 		if _, err := Store(DefaultMailbox, testMimeEmail); err != nil {
@@ -225,9 +262,16 @@ func BenchmarkImportMime(b *testing.B) {
 	db.Close()
 }
 
-func setup() {
+func setup(dataDir bool) {
 	config.NoLogging = true
 	config.MaxMessages = 0
+
+	if dataDir {
+		config.DataDir = fmt.Sprintf("%s-%d", path.Join(os.TempDir(), "mailpit-tests"), time.Now().UnixNano())
+	} else {
+		config.DataDir = ""
+	}
+
 	if err := InitDB(); err != nil {
 		panic(err)
 	}
@@ -243,7 +287,6 @@ func setup() {
 	if err != nil {
 		panic(err)
 	}
-
 }
 
 func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {