Merge branch 'release/v1.23.2'

Using goquery sometimes resulted in incorrect partial matches, eg `transform:` matching `text-transform:`. This refactor switches to regex matches which should prevent this, and allow more accurate detection.

.chglog/config.yml

@@ -17,6 +17,24 @@ options:
       fix: Fix
     #   perf: Performance Improvements
     #   refactor: Code Refactoring
+    sort_by: Custom
+    title_order:
+      - Feature
+      - Chore
+      - UI
+      - API
+      - Libs
+      - Docker
+      - Security
+      - Fix
+      - Bugfix
+      - Docs
+      - Swagger
+      - Build
+      - Testing
+      - Test
+      - Tests
+      - Pull Requests
   header:
     pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$"
     pattern_maps:

.github/workflows/build-docker-edge.yml

@@ -0,0 +1,45 @@
+on:
+  push:
+    branches: [ develop ]
+
+name: Build docker edge images
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log into Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+    - name: Log into GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+
+    - uses: benjlevesque/short-sha@v3.0
+      id: short-sha
+
+    - name: Build and push
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        platforms: linux/386,linux/amd64,linux/arm64
+        build-args: |
+          "VERSION=edge-${{ steps.short-sha.outputs.sha }}"
+        push: true
+        tags: |
+          axllent/mailpit:edge
+          ghcr.io/${{ github.repository }}:edge

.github/workflows/build-docker.yml

@@ -8,20 +8,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v2
+    - name: Log into Docker Hub
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
+    - name: Log into GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+
     - name: Parse semver
       id: semver_parser 
       uses: booxmedialtd/ws-action-parse-semver@v1.4.7
@@ -30,10 +37,9 @@ jobs:
         version_extractor_regex: 'v(.*)$'
 
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v6
       with:
         context: .
-        # platforms: linux/386,linux/amd64,linux/arm,linux/arm64
         platforms: linux/386,linux/amd64,linux/arm64
         build-args: |
           "VERSION=${{ github.ref_name }}"
@@ -42,3 +48,6 @@ jobs:
           axllent/mailpit:latest
           axllent/mailpit:${{ github.ref_name }}
           axllent/mailpit:v${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}
+          ghcr.io/${{ github.repository }}:${{ github.ref_name }}
+          ghcr.io/${{ github.repository }}:v${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}
+          ghcr.io/${{ github.repository }}:latest

.github/workflows/close-stale-issues.yml

@@ -10,14 +10,14 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@v8.0.0
+      - uses: actions/stale@v9.1.0
         with:
-          days-before-issue-stale: 21
-          days-before-issue-close: 7
-          exempt-issue-labels: "enhancement,bug,javascript,docker"
+          days-before-issue-stale: 7
+          days-before-issue-close: 3
+          exempt-issue-labels: "enhancement,bug,awaiting feedback"
           stale-issue-label: "stale"
-          stale-issue-message: "This issue is stale because it has been open for 21 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 7 days since being marked as stale."
+          stale-issue-message: "This issue has been marked as stale because it has been open for 7 days with no activity."
+          close-issue-message: "This issue was closed because there has been no activity since being marked as stale."
           days-before-pr-stale: -1
           days-before-pr-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +69,4 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/release-build.yml

@@ -5,7 +5,7 @@ on:
 name: Build & release
 jobs:
   releases-matrix:
-    name: Release Go Binary
+    name: Build
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -21,10 +21,10 @@ jobs:
           - goarch: arm
             goos: windows
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # build the assets
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
         node-version: 18
         cache: 'npm'
@@ -33,7 +33,7 @@ jobs:
     - run: npm run package
 
     # build the binaries
-    - uses: wangyoucao577/go-release-action@v1.38
+    - uses: wangyoucao577/go-release-action@v1
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}

.github/workflows/tests.yml

@@ -1,22 +1,24 @@
 name: Tests
 on:
   pull_request:
-    branches: [ develop ]
+    branches: [ develop, 'feature/**' ]
   push:
     branches: [ develop, 'feature/**' ]
 jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.18.x]
-        os: [ubuntu-latest]
+        go-version: ['1.23']
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/setup-go@v4
+    - uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+        cache: false
+    - uses: actions/checkout@v4
+    - name: Run Go tests
+      uses: actions/cache@v4
       with:
         path: |
           ~/.cache/go-build
@@ -24,13 +26,24 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test ./storage ./server -v
-    - run: go test ./storage -bench=.
+    - run: go test -p 1 ./internal/storage ./server ./internal/smtpd ./internal/pop3 ./internal/tools ./internal/html2text ./internal/htmlcheck ./internal/linkcheck -v
+    - run: go test -p 1 ./internal/storage ./internal/html2text -bench=.
     
     # build the assets
-    - uses: actions/setup-node@v3
+    - name: Build web UI
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: actions/setup-node@v4
       with:
         node-version: 18
         cache: 'npm'
-    - run: npm install
-    - run: npm run package
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm install
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm run package
+
+    # validate the swagger file
+    - name: Validate OpenAPI definition
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: swaggerexpert/swagger-editor-validate@v1
+      with:
+        definition-file: server/ui/api/v1/swagger.json

.gitignore

@@ -4,5 +4,6 @@
 /server/ui/dist
 /Makefile
 /mailpit*
+/.idea
 *.old
 *.db

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:alpine as builder
+FROM golang:alpine AS builder
 
 ARG VERSION=dev
 
@@ -6,16 +6,25 @@ COPY . /app
 
 WORKDIR /app
 
-RUN apk add --no-cache git npm && \
+RUN  apk upgrade && apk add git npm && \
 npm install && npm run package && \
 CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/axllent/mailpit/config.Version=${VERSION}" -o /mailpit
 
 FROM alpine:latest
 
+LABEL org.opencontainers.image.title="Mailpit" \
+  org.opencontainers.image.description="An email and SMTP testing tool with API for developers" \
+  org.opencontainers.image.source="https://github.com/axllent/mailpit" \
+  org.opencontainers.image.url="https://mailpit.axllent.org" \
+  org.opencontainers.image.documentation="https://mailpit.axllent.org/docs/" \
+  org.opencontainers.image.licenses="MIT"
+
 COPY --from=builder /mailpit /mailpit
 
-RUN apk add --no-cache tzdata
+RUN apk upgrade --no-cache && apk add --no-cache tzdata
 
-EXPOSE 1025/tcp 8025/tcp
+EXPOSE 1025/tcp 1110/tcp 8025/tcp
+
+HEALTHCHECK --interval=15s --start-period=10s --start-interval=1s CMD ["/mailpit", "readyz"]
 
 ENTRYPOINT ["/mailpit"]

README.md

@@ -1,51 +1,71 @@
-# Mailpit - email testing for developers
+<h1 align="center">
+  Mailpit - email testing for developers
+</h1>
 
-![Tests](https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg)
-![Build status](https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg)
-![Docker builds](https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg)
-![CodeQL](https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg)
-[![Go Report Card](https://goreportcard.com/badge/github.com/axllent/mailpit)](https://goreportcard.com/report/github.com/axllent/mailpit)
+<div align="center">
+    <a href="https://github.com/axllent/mailpit/actions/workflows/tests.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg" alt="CI Tests status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/release-build.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg" alt="CI build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg" alt="CI Docker build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg" alt="Code quality"></a>
+    <a href="https://goreportcard.com/report/github.com/axllent/mailpit"><img src="https://goreportcard.com/badge/github.com/axllent/mailpit" alt="Go Report Card"></a>
+    <br>
+    <a href="https://github.com/axllent/mailpit/releases/latest"><img src="https://img.shields.io/github/v/release/axllent/mailpit.svg" alt="Latest release"></a>
+    <a href="https://hub.docker.com/r/axllent/mailpit"><img src="https://img.shields.io/docker/pulls/axllent/mailpit.svg" alt="Docker pulls"></a>
+</div>
+<br>
+<p align="center">
+  <a href="https://mailpit.axllent.org">Website</a>  •
+  <a href="https://mailpit.axllent.org/docs/">Documentation</a>  •
+  <a href="https://mailpit.axllent.org/docs/api-v1/">API</a>
+</p>
 
-Mailpit is a multi-platform email testing tool & API for developers.
+<hr>
 
-It acts as both an SMTP server, and provides a web interface to view all captured emails.
+**Mailpit** is a small, fast, low memory, zero-dependency, multi-platform email testing tool & API for developers.
 
-Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster.
+It acts as an SMTP server, provides a modern web interface to view & test captured emails, and includes an API for automated integration testing.
 
-![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/docs/screenshot.png)
+Mailpit was originally **inspired** by MailHog which is [no longer maintained](https://github.com/mailhog/MailHog/issues/442#issuecomment-1493415258) and hasn't seen active development or security updates for a few years now.
+
+![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/server/ui-src/screenshot.png)
 
 
 ## Features
 
-- Runs entirely from a single binary, no installation required
-- SMTP server (default `0.0.0.0:1025`)
-- Web UI to view emails (formatted HTML, highlighted HTML source, text, headers, raw source and MIME attachments including image thumbnails)
+- Runs entirely from a single [static binary](https://mailpit.axllent.org/docs/install/) or multi-architecture [Docker images](https://mailpit.axllent.org/docs/install/docker/)
+- Modern web UI with advanced [mail search](https://mailpit.axllent.org/docs/usage/search-filters/) to view emails (formatted HTML, highlighted HTML source, text, headers, raw source, and MIME attachments
+including image thumbnails), including optional [HTTPS](https://mailpit.axllent.org/docs/configuration/http/) & [authentication](https://mailpit.axllent.org/docs/configuration/http/)
+- [SMTP server](https://mailpit.axllent.org/docs/configuration/smtp/) with optional STARTTLS or SSL/TLS, authentication (including an "accept any" mode)
+- A [REST API](https://mailpit.axllent.org/docs/api-v1/) for integration testing
+- Real-time web UI updates using web sockets for new mail & optional [browser notifications](https://mailpit.axllent.org/docs/usage/notifications/) when new mail is received
+- Optional [POP3 server](https://mailpit.axllent.org/docs/configuration/pop3/) to download captured message directly into your email client
+- [HTML check](https://mailpit.axllent.org/docs/usage/html-check/) to test & score mail client compatibility with HTML emails
+- [Link check](https://mailpit.axllent.org/docs/usage/link-check/) to test message links (HTML & text) & linked images
+- [Spam check](https://mailpit.axllent.org/docs/usage/spamassassin/) to test message "spamminess" using a running SpamAssassin server
+- [Create screenshots](https://mailpit.axllent.org/docs/usage/html-screenshots/) of HTML messages via web UI
 - Mobile and tablet HTML preview toggle in desktop mode
-- Advanced mail search ([see wiki](https://github.com/axllent/mailpit/wiki/Mail-search))
-- Message tagging ([see wiki](https://github.com/axllent/mailpit/wiki/Tagging))
-- Real-time web UI updates using web sockets for new mail
-- Optional browser notifications for new mail (HTTPS and `localhost` only)
-- Configurable automatic email pruning (default keeps the most recent 500 emails)
-- Email storage either in a temporary or persistent database ([see wiki](https://github.com/axllent/mailpit/wiki/Email-storage))
-- Fast SMTP processing & storing - approximately 70-100 emails per second depending on CPU, network speed & email size, easily handling tens of thousands of emails
-- SMTP relaying / message release - relay messages via a different SMTP server including an optional allowlist of accepted recipients ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-relay))
-- Optional SMTP with STARTTLS & SMTP authentication, including an "accept anything" mode ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-with-STARTTLS-and-authentication))
-- Optional HTTPS for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/HTTPS))
-- Optional basic authentication for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
-- A simple REST API ([see docs](docs/apiv1/README.md))
-- Multi-architecture [Docker images](https://github.com/axllent/mailpit/wiki/Docker-images)
+- [Message tagging](https://mailpit.axllent.org/docs/usage/tagging/) including manual tagging or automated tagging using filtering and "plus addressing"
+- [SMTP relaying](https://mailpit.axllent.org/docs/configuration/smtp-relay/) (message release) - relay messages via a different SMTP server including an optional allowlist of accepted recipients
+- [SMTP forwarding](https://mailpit.axllent.org/docs/configuration/smtp-forward/) - automatically forward messages via a different SMTP server to predefined email addresses
+- Fast message [storing & processing](https://mailpit.axllent.org/docs/configuration/email-storage/) - ingesting 100-200 emails per second over SMTP depending on CPU, network speed & email size,
+easily handling tens of thousands of emails, with automatic email pruning (by default keeping the most recent 500 emails)
+- [Chaos](https://mailpit.axllent.org/docs/integration/chaos/) feature to enable configurable SMTP errors to test application resilience
+- `List-Unsubscribe` syntax validation
+- Optional [webhook](https://mailpit.axllent.org/docs/integration/webhook/) for received messages
 
 
 ## Installation
 
-The Mailpit web UI listens by default on `http://0.0.0.0:8025`, and the SMTP port on `0.0.0.0:1025`.
+The Mailpit web UI listens by default on `http://0.0.0.0:8025` and the SMTP port on `0.0.0.0:1025`.
 
 Mailpit runs as a single binary and can be installed in different ways:
 
 
-### Install via Brew (Mac)
+### Install via package managers
 
-Add the repository to your taps with `brew tap axllent/apps`, and then install Mailpit with `brew install mailpit`.
+- **Mac**: `brew install mailpit` (to run automatically in the background: `brew services start mailpit`)
+- **Arch Linux**: available in the AUR as `mailpit`
+- **FreeBSD**: `pkg install mailpit`
 
 
 ### Install via bash script (Linux & Mac)
@@ -59,33 +79,33 @@ sudo bash < <(curl -sL https://raw.githubusercontent.com/axllent/mailpit/develop
 
 ### Download static binary (Windows, Linux and Mac)
 
-Static binaries can always be found on the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` binary can extracted and copied to your `$PATH`, or simply run as `./mailpit`.
+Static binaries can always be found on the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` binary can be extracted and copied to your `$PATH`, or simply run as `./mailpit`.
 
 
 ### Docker
 
-See [Docker instructions](https://github.com/axllent/mailpit/wiki/Docker-images).
+See [Docker instructions](https://mailpit.axllent.org/docs/install/docker/) for 386, amd64 & arm64 images.
 
 
 ### Compile from source
 
-To build Mailpit from source see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source).
+To build Mailpit from source, see [Building from source](https://mailpit.axllent.org/docs/install/source/).
+
+
+## Usage
+
+Run `mailpit -h` to see options. More information can be seen in [the docs](https://mailpit.axllent.org/docs/configuration/runtime-options/).
+
+If installed using homebrew, you may run `brew services start mailpit` to always run mailpit automatically.
 
 
 ### Testing Mailpit
 
-Please refer to [the documentation](https://github.com/axllent/mailpit/wiki/Testing-Mailpit) of how to easily test email delivery to Mailpit.
+Please refer to [the documentation](https://mailpit.axllent.org/docs/install/testing/) on how to easily test email delivery to Mailpit.
 
 
 ### Configuring sendmail
 
-Mailpit's SMTP server (by default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. A common MTA (Mail Transfer Agent) that delivers system emails to a SMTP server is `sendmail`, used by many applications including PHP. Mailpit can also act as substitute for sendmail. For instructions of how to set this up, please refer to the [sendmail documentation](https://github.com/axllent/mailpit/wiki/Configuring-sendmail).
-
-
-## Why rewrite MailHog?
-
-I had been using MailHog for a few years to intercept and test emails generated from several projects. MailHog has a number of performance issues, many of the frontend and Go modules are horribly out of date, and it is not actively developed.
-
-Initially I tried to upgrade a fork of MailHog (both the UI as well as the HTTP server & API), but soon discovered that it is (with all due respect to its authors) poorly designed. It is in my opinion over-engineered (split over 9 separate projects), and performs very poorly when dealing with large amounts of emails or processing emails with an attachments (a single email with a 3MB attachment can take over a minute to ingest). Finally, the API transmits a lot of duplicate and unnecessary data on every browser request, and there is no HTTP compression.
-
-In order to improve it I felt it needed to be completely rewritten, and so Mailpit was born.
+Mailpit's SMTP server (default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. 
+A common MTA (Mail Transfer Agent) that delivers system emails to an SMTP server is `sendmail`, used by many applications, including PHP. 
+Mailpit can also act as substitute for sendmail. For instructions on how to set this up, please refer to the [sendmail documentation](https://mailpit.axllent.org/docs/install/sendmail/).

SECURITY.md

@@ -0,0 +1,19 @@
+# Reporting security vulnerabilities
+
+Your efforts to responsibly disclose your findings are appreciated.
+
+** **Please do _not_ report security vulnerabilities through public GitHub issues.** **
+
+If you believe you have found a **security vulnerability**, then please report it to security@axllent.org so 
+your findings can be investigated, and if confirmed, fixed and released in a timely manner.
+
+Your report should include:
+
+- Mailpit version
+- A vulnerability description
+- Reproduction steps (if applicable)
+- Any other details you think are likely to be important
+
+You should receive an initial acknowledgement within 24 hours in most cases, and will kept updated throughout the process. 
+
+With your consent, your contributions will be publicly acknowledged.

cmd/dump.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/dump"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/spf13/cobra"
+)
+
+// dumpCmd represents the dump command
+var dumpCmd = &cobra.Command{
+	Use:   "dump <database> <output-dir>",
+	Short: "Dump all messages from a database to a directory",
+	Long: `Dump all messages stored in Mailpit into a local directory as individual files.
+
+The database can either be the database file (eg: --database /var/lib/mailpit/mailpit.db) or a
+URL of a running Mailpit instance (eg: --http http://127.0.0.1/). If dumping over HTTP, the URL
+should be the base URL of your running Mailpit instance, not the link to the API itself.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := dump.Sync(args[0]); err != nil {
+			logger.Log().Fatal(err)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(dumpCmd)
+
+	dumpCmd.Flags().SortFlags = false
+
+	dumpCmd.Flags().StringVar(&config.Database, "database", config.Database, "Dump messages directly from a database file")
+	dumpCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data (optional)")
+	dumpCmd.Flags().StringVar(&dump.URL, "http", dump.URL, "Dump messages via HTTP API (base URL of running Mailpit instance)")
+	dumpCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
+}

cmd/ingest.go

@@ -0,0 +1,173 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/logger"
+	sendmail "github.com/axllent/mailpit/sendmail/cmd"
+	"github.com/spf13/cobra"
+)
+
+var (
+	ingestRecent int
+)
+
+// ingestCmd represents the ingest command
+var ingestCmd = &cobra.Command{
+	Use:   "ingest <file|folder> ...[file|folder]",
+	Short: "Ingest a file or folder of emails for testing",
+	Long: `Ingest a file or folder of emails for testing.
+
+This command will scan the folder for emails and deliver them via SMTP to a running 
+Mailpit server. Each email must be a separate file (eg: Maildir format, not mbox).
+The --recent flag will only consider files with a modification date within the last X days.`,
+	// Hidden: true,
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var count int
+		var total int
+		var per100start = time.Now()
+
+		for _, a := range args {
+			err := filepath.Walk(a,
+				func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						logger.Log().Error(err)
+						return nil
+					}
+					if !isFile(path) {
+						return nil
+					}
+
+					if ingestRecent > 0 && time.Since(info.ModTime()) > time.Duration(ingestRecent)*24*time.Hour {
+						return nil
+					}
+
+					f, err := os.Open(filepath.Clean(path))
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+					defer f.Close() // #nosec
+
+					body, err := io.ReadAll(f)
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+
+					msg, err := mail.ReadMessage(bytes.NewReader(body))
+					if err != nil {
+						logger.Log().Errorf("error parsing message body: %s", err.Error())
+						return nil
+					}
+
+					recipients := []string{}
+					// get all recipients in To, Cc and Bcc
+					if to, err := msg.Header.AddressList("To"); err == nil {
+						for _, a := range to {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if cc, err := msg.Header.AddressList("Cc"); err == nil {
+						for _, a := range cc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if bcc, err := msg.Header.AddressList("Bcc"); err == nil {
+						for _, a := range bcc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+
+					if sendmail.FromAddr == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							sendmail.FromAddr = fromAddresses[0].Address
+						}
+					}
+
+					if len(recipients) == 0 {
+						// Bcc
+						recipients = []string{sendmail.FromAddr}
+					}
+
+					returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
+					if returnPath == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							returnPath = fromAddresses[0].Address
+						}
+					}
+
+					err = sendmail.Send(sendmail.SMTPAddr, returnPath, recipients, body)
+					if err != nil {
+						logger.Log().Errorf("error sending mail: %s (%s)", err.Error(), path)
+						return nil
+					}
+
+					count++
+					total++
+					if count%100 == 0 {
+						logger.Log().Infof("[%s] 100 messages in %s", format(total), time.Since(per100start))
+
+						per100start = time.Now()
+					}
+
+					return nil
+				})
+			if err != nil {
+				logger.Log().Error(err)
+			}
+		}
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(ingestCmd)
+
+	ingestCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
+	ingestCmd.Flags().IntVarP(&ingestRecent, "recent", "r", 0, "Only ingest messages from the last X days (default all)")
+}
+
+// IsFile returns if a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// Format a an integer 10000 => 10,000
+func format(n int) string {
+	in := fmt.Sprintf("%d", n)
+	numOfDigits := len(in)
+	if n < 0 {
+		numOfDigits-- // First character is the - sign (not a digit)
+	}
+	numOfCommas := (numOfDigits - 1) / 3
+
+	out := make([]byte, len(in)+numOfCommas)
+	if n < 0 {
+		in, out[0] = in[1:], '-'
+	}
+
+	for i, j, k := len(in)-1, len(out)-1, 0; ; i, j = i-1, j-1 {
+		out[j] = in[i]
+		if i == 0 {
+			return string(out)
+		}
+		if k++; k == 3 {
+			j, k = j-1, 0
+			out[j] = ','
+		}
+	}
+}

cmd/readyz.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/spf13/cobra"
+)
+
+var (
+	useHTTPS bool
+)
+
+// readyzCmd represents the healthcheck command
+var readyzCmd = &cobra.Command{
+	Use:   "readyz",
+	Short: "Run a healthcheck to test if Mailpit is running",
+	Long: `This command connects to the /readyz endpoint of a running Mailpit server
+and exits with a status of 0 if the connection is successful, else with a 
+status 1 if unhealthy.
+
+If running within Docker, it should automatically detect environment
+settings to determine the HTTP bind interface & port.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		webroot := strings.TrimRight(path.Join("/", config.Webroot, "/"), "/") + "/"
+		proto := "http"
+		if useHTTPS {
+			proto = "https"
+		}
+
+		uri := fmt.Sprintf("%s://%s%sreadyz", proto, config.HTTPListen, webroot)
+
+		conf := &http.Transport{
+			IdleConnTimeout:       time.Second * 5,
+			ExpectContinueTimeout: time.Second * 5,
+			TLSHandshakeTimeout:   time.Second * 5,
+			// do not verify TLS in case this instance is using HTTPS
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec
+		}
+		client := &http.Client{Transport: conf}
+
+		res, err := client.Get(uri)
+		if err != nil || res.StatusCode != 200 {
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(readyzCmd)
+
+	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
+		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+	}
+
+	if len(os.Getenv("MP_WEBROOT")) > 0 {
+		config.Webroot = os.Getenv("MP_WEBROOT")
+	}
+
+	config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
+
+	if config.UITLSCert != "" {
+		useHTTPS = true
+	}
+
+	readyzCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "Set the HTTP bind interface & port")
+	readyzCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
+	readyzCmd.Flags().BoolVar(&useHTTPS, "https", useHTTPS, "Connect via HTTPS (ignores HTTPS validation)")
+}

cmd/reindex.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/spf13/cobra"
+)
+
+// reindexCmd represents the reindex command
+var reindexCmd = &cobra.Command{
+	Use:   "reindex <database>",
+	Short: "Reindex the database",
+	Long: `This will reindex all messages in the entire database.
+
+If you have several thousand messages in your mailbox, then it is advised to shut down
+Mailpit while you reindex as this process will likely result in database locking issues.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		config.Database = args[0]
+		config.MaxMessages = 0
+
+		if err := storage.InitDB(); err != nil {
+			logger.Log().Error(err)
+			os.Exit(1)
+		}
+
+		storage.ReindexAll()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(reindexCmd)
+}

cmd/root.go

@@ -2,21 +2,22 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"strconv"
 	"strings"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/axllent/mailpit/server"
-	"github.com/axllent/mailpit/server/smtpd"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/server/webhook"
 	"github.com/spf13/cobra"
 )
 
-var cfgFile string
-
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "mailpit",
@@ -27,21 +28,22 @@ It acts as an SMTP server, and provides a web interface to view all captured ema
 
 Documentation:
   https://github.com/axllent/mailpit
-  https://github.com/axllent/mailpit/wiki`,
+  https://mailpit.axllent.org/docs/`,
 	Run: func(_ *cobra.Command, _ []string) {
 		if err := config.VerifyConfig(); err != nil {
 			logger.Log().Error(err.Error())
 			os.Exit(1)
 		}
 		if err := storage.InitDB(); err != nil {
-			logger.Log().Error(err.Error())
+			logger.Log().Fatal(err.Error())
 			os.Exit(1)
 		}
 
 		go server.Listen()
 
 		if err := smtpd.Listen(); err != nil {
-			logger.Log().Error(err.Error())
+			storage.Close()
+			logger.Log().Fatal(err.Error())
 			os.Exit(1)
 		}
 	},
@@ -77,52 +79,81 @@ func init() {
 	// load and warn deprecated ENV vars
 	initDeprecatedConfigFromEnv()
 
-	// load ENV vars
+	// load environment variables
 	initConfigFromEnv()
 
-	rootCmd.Flags().StringVarP(&config.DataFile, "db-file", "d", config.DataFile, "Database file to store persistent data")
-	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
-	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
+	rootCmd.Flags().StringVarP(&config.Database, "database", "d", config.Database, "Database to store persistent data")
+	rootCmd.Flags().BoolVar(&config.DisableWAL, "disable-wal", config.DisableWAL, "Disable WAL for local database (allows NFS mounted DBs)")
+	rootCmd.Flags().IntVar(&config.Compression, "compression", config.Compression, "Compression level to store raw messages (0-3)")
+	rootCmd.Flags().StringVar(&config.Label, "label", config.Label, "Optional label identify this Mailpit instance")
+	rootCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data")
 	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
-	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
-	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
+	rootCmd.Flags().StringVar(&config.MaxAge, "max-age", config.MaxAge, "Max age of messages in either (h)ours or (d)ays (eg: 3d)")
 	rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates")
-	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)")
+	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-ID)")
+	rootCmd.Flags().StringVar(&logger.LogFile, "log-file", logger.LogFile, "Log output to file instead of stdout")
+	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
+	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
 
-	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI authentication")
+	// Web UI / API
+	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface & port for UI")
+	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
+	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI & API authentication")
 	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-tls-cert", config.UITLSCert, "TLS certificate for web UI (HTTPS) - requires ui-tls-key")
 	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-tls-key", config.UITLSKey, "TLS key for web UI (HTTPS) - requires ui-tls-cert")
+	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
+	rootCmd.Flags().BoolVar(&config.BlockRemoteCSSAndFonts, "block-remote-css-and-fonts", config.BlockRemoteCSSAndFonts, "Block access to remote CSS & fonts")
+	rootCmd.Flags().StringVar(&config.EnableSpamAssassin, "enable-spamassassin", config.EnableSpamAssassin, "Enable integration with SpamAssassin")
+	rootCmd.Flags().BoolVar(&config.AllowUntrustedTLS, "allow-untrusted-tls", config.AllowUntrustedTLS, "Do not verify HTTPS certificates (link checker & screenshots)")
+	rootCmd.Flags().BoolVar(&config.DisableHTTPCompression, "disable-http-compression", config.DisableHTTPCompression, "Disable HTTP compression support (web UI & API)")
 
+	// SMTP server
+	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
 	rootCmd.Flags().StringVar(&config.SMTPAuthFile, "smtp-auth-file", config.SMTPAuthFile, "A password file for SMTP authentication")
 	rootCmd.Flags().BoolVar(&config.SMTPAuthAcceptAny, "smtp-auth-accept-any", config.SMTPAuthAcceptAny, "Accept any SMTP username and password, including none")
 	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key")
 	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert")
-	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPRequireSTARTTLS, "smtp-require-starttls", config.SMTPRequireSTARTTLS, "Require SMTP client use STARTTLS")
+	rootCmd.Flags().BoolVar(&config.SMTPRequireTLS, "smtp-require-tls", config.SMTPRequireTLS, "Require client use SSL/TLS")
+	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Allow insecure PLAIN & LOGIN SMTP authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain <CR><CR><LF>")
+	rootCmd.Flags().IntVar(&config.SMTPMaxRecipients, "smtp-max-recipients", config.SMTPMaxRecipients, "Maximum SMTP recipients allowed")
+	rootCmd.Flags().StringVar(&config.SMTPAllowedRecipients, "smtp-allowed-recipients", config.SMTPAllowedRecipients, "Only allow SMTP recipients matching a regular expression (default allow all)")
+	rootCmd.Flags().BoolVar(&smtpd.DisableReverseDNS, "smtp-disable-rdns", smtpd.DisableReverseDNS, "Disable SMTP reverse DNS lookups")
 
-	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
-	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
+	// SMTP relay
+	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP relay configuration file to allow releasing messages")
+	rootCmd.Flags().BoolVar(&config.SMTPRelayAll, "smtp-relay-all", config.SMTPRelayAll, "Auto-relay all new messages via external SMTP server (caution!)")
+	rootCmd.Flags().StringVar(&config.SMTPRelayMatching, "smtp-relay-matching", config.SMTPRelayMatching, "Auto-relay new messages to only matching recipients (regular expression)")
 
-	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
-	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
-	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
+	// SMTP forwarding
+	rootCmd.Flags().StringVar(&config.SMTPForwardConfigFile, "smtp-forward-config", config.SMTPForwardConfigFile, "SMTP forwarding configuration file for all messages")
 
-	// deprecated flags 2022/08/06
-	rootCmd.Flags().StringVarP(&config.UIAuthFile, "auth-file", "a", config.UIAuthFile, "A password file for web UI authentication")
-	rootCmd.Flags().StringVar(&config.UITLSCert, "ssl-cert", config.UITLSCert, "SSL certificate - requires ssl-key")
-	rootCmd.Flags().StringVar(&config.UITLSKey, "ssl-key", config.UITLSKey, "SSL key - requires ssl-cert")
-	rootCmd.Flags().Lookup("auth-file").Hidden = true
-	rootCmd.Flags().Lookup("auth-file").Deprecated = "use --ui-auth-file"
-	rootCmd.Flags().Lookup("ssl-cert").Hidden = true
-	rootCmd.Flags().Lookup("ssl-cert").Deprecated = "use --ui-tls-cert"
-	rootCmd.Flags().Lookup("ssl-key").Hidden = true
-	rootCmd.Flags().Lookup("ssl-key").Deprecated = "use --ui-tls-key"
+	// Chaos
+	rootCmd.Flags().BoolVar(&chaos.Enabled, "enable-chaos", chaos.Enabled, "Enable Chaos functionality (API / web UI)")
+	rootCmd.Flags().StringVar(&config.ChaosTriggers, "chaos-triggers", config.ChaosTriggers, "Enable Chaos & set the triggers for SMTP server")
 
-	// deprecated flags 2022/08/30
-	rootCmd.Flags().StringVar(&config.DataFile, "data", config.DataFile, "Database file to store persistent data")
-	rootCmd.Flags().Lookup("data").Hidden = true
-	rootCmd.Flags().Lookup("data").Deprecated = "use --db-file"
+	// POP3 server
+	rootCmd.Flags().StringVar(&config.POP3Listen, "pop3", config.POP3Listen, "POP3 server bind interface and port")
+	rootCmd.Flags().StringVar(&config.POP3AuthFile, "pop3-auth-file", config.POP3AuthFile, "A password file for POP3 server authentication (enables POP3 server)")
+	rootCmd.Flags().StringVar(&config.POP3TLSCert, "pop3-tls-cert", config.POP3TLSCert, "Optional TLS certificate for POP3 server - requires pop3-tls-key")
+	rootCmd.Flags().StringVar(&config.POP3TLSKey, "pop3-tls-key", config.POP3TLSKey, "Optional TLS key for POP3 server - requires pop3-tls-cert")
 
-	// deprecated flags 2023/03/12
+	// Tagging
+	rootCmd.Flags().StringVarP(&config.CLITagsArg, "tag", "t", config.CLITagsArg, "Tag new messages matching filters")
+	rootCmd.Flags().StringVar(&config.TagsConfig, "tags-config", config.TagsConfig, "Load tags filters from yaml configuration file")
+	rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "TitleCase new tags generated from plus-addresses and X-Tags")
+	rootCmd.Flags().StringVar(&config.TagsDisable, "tags-disable", config.TagsDisable, "Disable auto-tagging, comma separated (eg: plus-addresses,x-tags)")
+
+	// Webhook
+	rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages")
+	rootCmd.Flags().IntVar(&webhook.RateLimit, "webhook-limit", webhook.RateLimit, "Limit webhook requests per second")
+
+	// DEPRECATED FLAG 2024/04/12 - but will not be removed to maintain backwards compatibility
+	rootCmd.Flags().StringVar(&config.Database, "db-file", config.Database, "Database file to store persistent data")
+	rootCmd.Flags().Lookup("db-file").Hidden = true
+
+	// DEPRECATED FLAGS 2023/03/12
 	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-ssl-cert", config.UITLSCert, "SSL certificate for web UI - requires ui-ssl-key")
 	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-ssl-key", config.UITLSKey, "SSL key for web UI - requires ui-ssl-cert")
 	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-ssl-cert", config.SMTPTLSCert, "SSL certificate for SMTP - requires smtp-ssl-key")
@@ -135,69 +166,39 @@ func init() {
 	rootCmd.Flags().Lookup("smtp-ssl-cert").Deprecated = "use --smtp-tls-cert"
 	rootCmd.Flags().Lookup("smtp-ssl-key").Hidden = true
 	rootCmd.Flags().Lookup("smtp-ssl-key").Deprecated = "use --smtp-tls-key"
+
+	// DEPRECATED FLAGS 2024/03/16
+	rootCmd.Flags().BoolVar(&config.SMTPRequireSTARTTLS, "smtp-tls-required", config.SMTPRequireSTARTTLS, "smtp-require-starttls")
+	rootCmd.Flags().Lookup("smtp-tls-required").Hidden = true
+	rootCmd.Flags().Lookup("smtp-tls-required").Deprecated = "use --smtp-require-starttls"
+
+	// DEPRECATED FLAG 2024/04/13 - no longer used
+	rootCmd.Flags().BoolVar(&config.DisableHTMLCheck, "disable-html-check", config.DisableHTMLCheck, "Disable the HTML check functionality (web UI & API)")
+	rootCmd.Flags().Lookup("disable-html-check").Hidden = true
 }
 
 // Load settings from environment
 func initConfigFromEnv() {
-	// defaults from envars if provided
-	if len(os.Getenv("MP_DATA_FILE")) > 0 {
-		config.DataFile = os.Getenv("MP_DATA_FILE")
+	// General
+	if len(os.Getenv("MP_DATABASE")) > 0 {
+		config.Database = os.Getenv("MP_DATABASE")
 	}
-	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
-		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
-	}
-	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
-		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+
+	config.DisableWAL = getEnabledFromEnv("MP_DISABLE_WAL")
+
+	if len(os.Getenv("MP_COMPRESSION")) > 0 {
+		config.Compression, _ = strconv.Atoi(os.Getenv("MP_COMPRESSION"))
 	}
+
+	config.TenantID = os.Getenv("MP_TENANT_ID")
+
+	config.Label = os.Getenv("MP_LABEL")
+
 	if len(os.Getenv("MP_MAX_MESSAGES")) > 0 {
 		config.MaxMessages, _ = strconv.Atoi(os.Getenv("MP_MAX_MESSAGES"))
 	}
-	if len(os.Getenv("MP_TAG")) > 0 {
-		config.SMTPCLITags = os.Getenv("MP_TAG")
-	}
-
-	// UI
-	if len(os.Getenv("MP_UI_AUTH_FILE")) > 0 {
-		config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
-	}
-	if len(os.Getenv("MP_UI_TLS_CERT")) > 0 {
-		config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
-	}
-	if len(os.Getenv("MP_UI_TLS_KEY")) > 0 {
-		config.UITLSKey = os.Getenv("MP_UI_TLS_KEY")
-	}
-
-	// SMTP
-	if len(os.Getenv("MP_SMTP_AUTH_FILE")) > 0 {
-		config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
-	}
-	if len(os.Getenv("MP_SMTP_TLS_CERT")) > 0 {
-		config.SMTPTLSCert = os.Getenv("MP_SMTP_TLS_CERT")
-	}
-	if len(os.Getenv("MP_SMTP_TLS_KEY")) > 0 {
-		config.SMTPTLSKey = os.Getenv("MP_SMTP_TLS_KEY")
-	}
-	if getEnabledFromEnv("MP_SMTP_AUTH_ACCEPT_ANY") {
-		config.SMTPAuthAcceptAny = true
-	}
-	if getEnabledFromEnv("MP_SMTP_AUTH_ALLOW_INSECURE") {
-		config.SMTPAuthAllowInsecure = true
-	}
-
-	// Relay server config
-	if len(os.Getenv("MP_SMTP_RELAY_CONFIG")) > 0 {
-		config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")
-	}
-	if getEnabledFromEnv("MP_SMTP_RELAY_ALL") {
-		config.SMTPRelayAllIncoming = true
-	}
-
-	// Misc options
-	if len(os.Getenv("MP_WEBROOT")) > 0 {
-		config.Webroot = os.Getenv("MP_WEBROOT")
-	}
-	if len(os.Getenv("MP_API_CORS")) > 0 {
-		server.AccessControlAllowOrigin = os.Getenv("MP_API_CORS")
+	if len(os.Getenv("MP_MAX_AGE")) > 0 {
+		config.MaxAge = os.Getenv("MP_MAX_AGE")
 	}
 	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
 		config.UseMessageDates = true
@@ -205,53 +206,193 @@ func initConfigFromEnv() {
 	if getEnabledFromEnv("MP_IGNORE_DUPLICATE_IDS") {
 		config.IgnoreDuplicateIDs = true
 	}
+	if len(os.Getenv("MP_LOG_FILE")) > 0 {
+		logger.LogFile = os.Getenv("MP_LOG_FILE")
+	}
 	if getEnabledFromEnv("MP_QUIET") {
 		logger.QuietLogging = true
 	}
 	if getEnabledFromEnv("MP_VERBOSE") {
 		logger.VerboseLogging = true
 	}
+
+	// Web UI & API
+	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
+		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+	}
+	if len(os.Getenv("MP_WEBROOT")) > 0 {
+		config.Webroot = os.Getenv("MP_WEBROOT")
+	}
+	config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
+	if err := auth.SetUIAuth(os.Getenv("MP_UI_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
+	config.UITLSKey = os.Getenv("MP_UI_TLS_KEY")
+	if len(os.Getenv("MP_API_CORS")) > 0 {
+		server.AccessControlAllowOrigin = os.Getenv("MP_API_CORS")
+	}
+	if getEnabledFromEnv("MP_BLOCK_REMOTE_CSS_AND_FONTS") {
+		config.BlockRemoteCSSAndFonts = true
+	}
+	if len(os.Getenv("MP_ENABLE_SPAMASSASSIN")) > 0 {
+		config.EnableSpamAssassin = os.Getenv("MP_ENABLE_SPAMASSASSIN")
+	}
+	if getEnabledFromEnv("MP_ALLOW_UNTRUSTED_TLS") {
+		config.AllowUntrustedTLS = true
+	}
+	if getEnabledFromEnv("MP_DISABLE_HTTP_COMPRESSION") {
+		config.DisableHTTPCompression = true
+	}
+
+	// SMTP server
+	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
+		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
+	}
+	config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
+	if err := auth.SetSMTPAuth(os.Getenv("MP_SMTP_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ACCEPT_ANY") {
+		config.SMTPAuthAcceptAny = true
+	}
+	config.SMTPTLSCert = os.Getenv("MP_SMTP_TLS_CERT")
+	config.SMTPTLSKey = os.Getenv("MP_SMTP_TLS_KEY")
+	if getEnabledFromEnv("MP_SMTP_REQUIRE_STARTTLS") {
+		config.SMTPRequireSTARTTLS = true
+	}
+	if getEnabledFromEnv("MP_SMTP_REQUIRE_TLS") {
+		config.SMTPRequireTLS = true
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ALLOW_INSECURE") {
+		config.SMTPAuthAllowInsecure = true
+	}
+	if getEnabledFromEnv("MP_SMTP_STRICT_RFC_HEADERS") {
+		config.SMTPStrictRFCHeaders = true
+	}
+	if len(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) > 0 {
+		config.SMTPMaxRecipients, _ = strconv.Atoi(os.Getenv("MP_SMTP_MAX_RECIPIENTS"))
+	}
+	if len(os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")) > 0 {
+		config.SMTPAllowedRecipients = os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")
+	}
+	if getEnabledFromEnv("MP_SMTP_DISABLE_RDNS") {
+		smtpd.DisableReverseDNS = true
+	}
+
+	// SMTP relay
+	config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")
+	if getEnabledFromEnv("MP_SMTP_RELAY_ALL") {
+		config.SMTPRelayAll = true
+	}
+	config.SMTPRelayMatching = os.Getenv("MP_SMTP_RELAY_MATCHING")
+	config.SMTPRelayConfig = config.SMTPRelayConfigStruct{}
+	config.SMTPRelayConfig.Host = os.Getenv("MP_SMTP_RELAY_HOST")
+	if len(os.Getenv("MP_SMTP_RELAY_PORT")) > 0 {
+		config.SMTPRelayConfig.Port, _ = strconv.Atoi(os.Getenv("MP_SMTP_RELAY_PORT"))
+	}
+	config.SMTPRelayConfig.STARTTLS = getEnabledFromEnv("MP_SMTP_RELAY_STARTTLS")
+	config.SMTPRelayConfig.AllowInsecure = getEnabledFromEnv("MP_SMTP_RELAY_ALLOW_INSECURE")
+	config.SMTPRelayConfig.Auth = os.Getenv("MP_SMTP_RELAY_AUTH")
+	config.SMTPRelayConfig.Username = os.Getenv("MP_SMTP_RELAY_USERNAME")
+	config.SMTPRelayConfig.Password = os.Getenv("MP_SMTP_RELAY_PASSWORD")
+	config.SMTPRelayConfig.Secret = os.Getenv("MP_SMTP_RELAY_SECRET")
+	config.SMTPRelayConfig.ReturnPath = os.Getenv("MP_SMTP_RELAY_RETURN_PATH")
+	config.SMTPRelayConfig.OverrideFrom = os.Getenv("MP_SMTP_RELAY_OVERRIDE_FROM")
+	config.SMTPRelayConfig.AllowedRecipients = os.Getenv("MP_SMTP_RELAY_ALLOWED_RECIPIENTS")
+	config.SMTPRelayConfig.BlockedRecipients = os.Getenv("MP_SMTP_RELAY_BLOCKED_RECIPIENTS")
+
+	// SMTP forwarding
+	config.SMTPForwardConfigFile = os.Getenv("MP_SMTP_FORWARD_CONFIG")
+	config.SMTPForwardConfig = config.SMTPForwardConfigStruct{}
+	config.SMTPForwardConfig.Host = os.Getenv("MP_SMTP_FORWARD_HOST")
+	if len(os.Getenv("MP_SMTP_FORWARD_PORT")) > 0 {
+		config.SMTPForwardConfig.Port, _ = strconv.Atoi(os.Getenv("MP_SMTP_FORWARD_PORT"))
+	}
+	config.SMTPForwardConfig.STARTTLS = getEnabledFromEnv("MP_SMTP_FORWARD_STARTTLS")
+	config.SMTPForwardConfig.AllowInsecure = getEnabledFromEnv("MP_SMTP_FORWARD_ALLOW_INSECURE")
+	config.SMTPForwardConfig.Auth = os.Getenv("MP_SMTP_FORWARD_AUTH")
+	config.SMTPForwardConfig.Username = os.Getenv("MP_SMTP_FORWARD_USERNAME")
+	config.SMTPForwardConfig.Password = os.Getenv("MP_SMTP_FORWARD_PASSWORD")
+	config.SMTPForwardConfig.Secret = os.Getenv("MP_SMTP_FORWARD_SECRET")
+	config.SMTPForwardConfig.ReturnPath = os.Getenv("MP_SMTP_FORWARD_RETURN_PATH")
+	config.SMTPForwardConfig.OverrideFrom = os.Getenv("MP_SMTP_FORWARD_OVERRIDE_FROM")
+	config.SMTPForwardConfig.To = os.Getenv("MP_SMTP_FORWARD_TO")
+
+	// Chaos
+	chaos.Enabled = getEnabledFromEnv("MP_ENABLE_CHAOS")
+	config.ChaosTriggers = os.Getenv("MP_CHAOS_TRIGGERS")
+
+	// POP3 server
+	if len(os.Getenv("MP_POP3_BIND_ADDR")) > 0 {
+		config.POP3Listen = os.Getenv("MP_POP3_BIND_ADDR")
+	}
+	config.POP3AuthFile = os.Getenv("MP_POP3_AUTH_FILE")
+	if err := auth.SetPOP3Auth(os.Getenv("MP_POP3_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	config.POP3TLSCert = os.Getenv("MP_POP3_TLS_CERT")
+	config.POP3TLSKey = os.Getenv("MP_POP3_TLS_KEY")
+
+	// Tagging
+	config.CLITagsArg = os.Getenv("MP_TAG")
+	config.TagsConfig = os.Getenv("MP_TAGS_CONFIG")
+	tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE")
+	config.TagsDisable = os.Getenv("MP_TAGS_DISABLE")
+
+	// Webhook
+	if len(os.Getenv("MP_WEBHOOK_URL")) > 0 {
+		config.WebhookURL = os.Getenv("MP_WEBHOOK_URL")
+	}
+	if len(os.Getenv("MP_WEBHOOK_LIMIT")) > 0 {
+		webhook.RateLimit, _ = strconv.Atoi(os.Getenv("MP_WEBHOOK_LIMIT"))
+	}
+
+	// Demo mode
+	config.DemoMode = getEnabledFromEnv("MP_DEMO_MODE")
 }
 
 // load deprecated settings from environment and warn
 func initDeprecatedConfigFromEnv() {
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_AUTH_FILE")) > 0 {
-		fmt.Println("ENV MP_AUTH_FILE has been deprecated, use MP_UI_AUTH_FILE")
-		config.UIAuthFile = os.Getenv("MP_AUTH_FILE")
-	}
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_SSL_CERT")) > 0 {
-		fmt.Println("ENV MP_SSL_CERT has been deprecated, use MP_UI_TLS_CERT")
-		config.UITLSCert = os.Getenv("MP_SSL_CERT")
-	}
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_SSL_KEY")) > 0 {
-		fmt.Println("ENV MP_SSL_KEY has been deprecated, use MP_UI_TLS_KEY")
-		config.UITLSKey = os.Getenv("MP_TLS_KEY")
-	}
-	// deprecated 2022/08/28
-	if len(os.Getenv("MP_DATA_DIR")) > 0 {
-		fmt.Println("ENV MP_DATA_DIR has been deprecated, use MP_DATA_FILE")
-		config.DataFile = os.Getenv("MP_DATA_DIR")
+	// deprecated 2024/04/12 - but will not be removed to maintain backwards compatibility
+	if len(os.Getenv("MP_DATA_FILE")) > 0 {
+		config.Database = os.Getenv("MP_DATA_FILE")
 	}
+
 	// deprecated 2023/03/12
 	if len(os.Getenv("MP_UI_SSL_CERT")) > 0 {
-		fmt.Println("ENV MP_UI_SSL_CERT has been deprecated, use MP_UI_TLS_CERT")
+		logger.Log().Warn("ENV MP_UI_SSL_CERT has been deprecated, use MP_UI_TLS_CERT")
 		config.UITLSCert = os.Getenv("MP_UI_SSL_CERT")
 	}
+	// deprecated 2023/03/12
 	if len(os.Getenv("MP_UI_SSL_KEY")) > 0 {
-		fmt.Println("ENV MP_UI_SSL_KEY has been deprecated, use MP_UI_TLS_KEY")
+		logger.Log().Warn("ENV MP_UI_SSL_KEY has been deprecated, use MP_UI_TLS_KEY")
 		config.UITLSKey = os.Getenv("MP_UI_SSL_KEY")
 	}
+	// deprecated 2023/03/12
 	if len(os.Getenv("MP_SMTP_SSL_CERT")) > 0 {
-		fmt.Println("ENV MP_SMTP_CERT has been deprecated, use MP_SMTP_TLS_CERT")
+		logger.Log().Warn("ENV MP_SMTP_CERT has been deprecated, use MP_SMTP_TLS_CERT")
 		config.SMTPTLSCert = os.Getenv("MP_SMTP_SSL_CERT")
 	}
+	// deprecated 2023/03/12
 	if len(os.Getenv("MP_SMTP_SSL_KEY")) > 0 {
-		fmt.Println("ENV MP_SMTP_KEY has been deprecated, use MP_SMTP_TLS_KEY")
+		logger.Log().Warn("ENV MP_SMTP_KEY has been deprecated, use MP_SMTP_TLS_KEY")
 		config.SMTPTLSKey = os.Getenv("MP_SMTP_SMTP_KEY")
 	}
+	// deprecated 2023/12/10
+	if getEnabledFromEnv("MP_STRICT_RFC_HEADERS") {
+		logger.Log().Warn("ENV MP_STRICT_RFC_HEADERS has been deprecated, use MP_SMTP_STRICT_RFC_HEADERS")
+		config.SMTPStrictRFCHeaders = true
+	}
+	// deprecated 2024/03.16
+	if getEnabledFromEnv("MP_SMTP_TLS_REQUIRED") {
+		logger.Log().Warn("ENV MP_SMTP_TLS_REQUIRED has been deprecated, use MP_SMTP_REQUIRE_STARTTLS")
+		config.SMTPRequireSTARTTLS = true
+	}
+	if getEnabledFromEnv("MP_DISABLE_HTML_CHECK") {
+		logger.Log().Warn("ENV MP_DISABLE_HTML_CHECK has been deprecated and is no longer used")
+		config.DisableHTMLCheck = true
+	}
 }
 
 // Wrapper to get a boolean from an environment variable

cmd/sendmail.go

@@ -12,13 +12,13 @@ var sendmailCmd = &cobra.Command{
 	Use:   "sendmail [flags] [recipients]",
 	Short: "A sendmail command replacement for Mailpit",
 	Run: func(_ *cobra.Command, _ []string) {
-
 		sendmail.Run()
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(sendmailCmd)
+	var ignored string
 
 	// print out manual help screen
 	sendmailCmd.SetHelpTemplate(sendmail.HelpTemplate([]string{os.Args[0], "sendmail"}))
@@ -27,10 +27,13 @@ func init() {
 	// multi-letter single-dash variables (-bs)
 	sendmailCmd.Flags().StringVarP(&sendmail.FromAddr, "from", "f", sendmail.FromAddr, "SMTP sender")
 	sendmailCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
-	sendmailCmd.Flags().BoolVarP(&sendmail.UseB, "long-b", "b", false, "Handle SMTP commands on standard input (use as -bs)")
-	sendmailCmd.Flags().BoolVarP(&sendmail.UseS, "long-s", "s", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseB, "ignored-b", "b", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseS, "ignored-s", "s", false, "Handle SMTP commands on standard input (use as -bs)")
 	sendmailCmd.Flags().BoolP("verbose", "v", false, "Verbose mode (sends debug output to stderr)")
-	sendmailCmd.Flags().BoolP("long-i", "i", false, "Ignored")
-	sendmailCmd.Flags().BoolP("long-o", "o", false, "Ignored")
-	sendmailCmd.Flags().BoolP("long-t", "t", false, "Ignored")
+	sendmailCmd.Flags().BoolP("ignored-i", "i", false, "Ignored")
+	sendmailCmd.Flags().BoolP("ignored-o", "o", false, "Ignored")
+	sendmailCmd.Flags().BoolP("ignored-t", "t", false, "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-name", "F", "", "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-bits", "B", "", "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-errors", "e", "", "Ignored")
 }

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/updater"
+	"github.com/axllent/mailpit/internal/updater"
 	"github.com/spf13/cobra"
 )
 

config/config.go

@@ -4,17 +4,18 @@ package config
 import (
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
-	"github.com/mattn/go-shellwords"
-	"github.com/tg123/go-htpasswd"
-	"gopkg.in/yaml.v3"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"github.com/axllent/mailpit/internal/spamassassin"
+	"github.com/axllent/mailpit/internal/tools"
 )
 
 var (
@@ -24,12 +25,35 @@ var (
 	// HTTPListen to listen on <interface>:<port>
 	HTTPListen = "[::]:8025"
 
-	// DataFile for mail (optional)
-	DataFile string
+	// Database for mail (optional)
+	Database string
+
+	// DisableWAL will disable Write-Ahead Logging in SQLite
+	// @see https://sqlite.org/wal.html
+	DisableWAL bool
+
+	// Compression is the compression level used to store raw messages in the database:
+	// 0 = off, 1 = fastest (default), 2 = standard, 3 = best compression
+	Compression = 1
+
+	// TenantID is an optional prefix to be applied to all database tables,
+	// allowing multiple isolated instances of Mailpit to share a database.
+	TenantID string
+
+	// Label to identify this Mailpit instance (optional).
+	// This gets applied to web UI, SMTP and optional POP3 server.
+	Label string
 
 	// MaxMessages is the maximum number of messages a mailbox can have (auto-pruned every minute)
 	MaxMessages = 500
 
+	// MaxAge is the maximum age of messages (auto-pruned every hour).
+	// Value can be either <int>h for hours or <int>d for days
+	MaxAge string
+
+	// MaxAgeInHours is the maximum age of messages in hours, set with parseMaxAge() using MaxAge value
+	MaxAgeInHours int
+
 	// UseMessageDates sets the Created date using the message date, not the delivered date
 	UseMessageDates bool
 
@@ -39,60 +63,124 @@ var (
 	// UITLSKey file
 	UITLSKey string
 
-	// UIAuthFile for basic authentication
+	// UIAuthFile for UI & API authentication
 	UIAuthFile string
 
-	// UIAuth used for authentication
-	UIAuth *htpasswd.File
-
 	// Webroot to define the base path for the UI and API
 	Webroot = "/"
 
+	// DisableHTTPCompression will explicitly disable HTTP compression in the web UI and API
+	DisableHTTPCompression bool
+
 	// SMTPTLSCert file
 	SMTPTLSCert string
 
 	// SMTPTLSKey file
 	SMTPTLSKey string
 
+	// SMTPRequireSTARTTLS to enforce the use of STARTTLS
+	// The only allowed commands are NOOP, EHLO, STARTTLS and QUIT (as specified in RFC 3207) until
+	// the connection is upgraded to TLS i.e. until STARTTLS is issued.
+	SMTPRequireSTARTTLS bool
+
+	// SMTPRequireTLS to allow only SSL/TLS connections for all connections
+	//
+	SMTPRequireTLS bool
+
 	// SMTPAuthFile for SMTP authentication
 	SMTPAuthFile string
 
-	// SMTPAuthConfig used for authentication auto-generated from SMTPAuthFile
-	SMTPAuthConfig *htpasswd.File
-
 	// SMTPAuthAllowInsecure allows PLAIN & LOGIN unencrypted authentication
 	SMTPAuthAllowInsecure bool
 
 	// SMTPAuthAcceptAny accepts any username/password including none
 	SMTPAuthAcceptAny bool
 
+	// SMTPMaxRecipients is the maximum number of recipients a message may have.
+	// The SMTP RFC states that an server must handle a minimum of 100 recipients
+	// however some servers accept more.
+	SMTPMaxRecipients = 100
+
 	// IgnoreDuplicateIDs will skip messages with the same ID
 	IgnoreDuplicateIDs bool
 
-	// SMTPCLITags is used to map the CLI args
-	SMTPCLITags string
+	// BlockRemoteCSSAndFonts used to disable remote CSS & fonts
+	BlockRemoteCSSAndFonts = false
+
+	// CLITagsArg is used to map the CLI args
+	CLITagsArg string
 
 	// ValidTagRegexp represents a valid tag
-	ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_]){3,}$`)
+	ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_\.]){1,}$`)
 
-	// SMTPTags are expressions to apply tags to new mail
-	SMTPTags []AutoTag
+	// TagsConfig is a yaml file to pre-load tags
+	TagsConfig string
 
-	// SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server
+	// TagFilters are used to apply tags to new mail
+	TagFilters []autoTag
+
+	// TagsDisable accepts a comma-separated list of tag types to disable
+	// including x-tags & plus-addresses
+	TagsDisable string
+
+	// SMTPRelayConfigFile to parse a yaml file and store config of the relay SMTP server
 	SMTPRelayConfigFile string
 
-	// SMTPRelayConfig to parse a yaml file and store config of relay SMTP server
-	SMTPRelayConfig smtpRelayConfigStruct
+	// SMTPRelayConfig to parse a yaml file and store config of the the relay SMTP server
+	SMTPRelayConfig SMTPRelayConfigStruct
 
 	// ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile
 	ReleaseEnabled = false
 
-	// SMTPRelayAllIncoming is whether to relay all incoming messages via pre-configured SMTP server.
+	// SMTPRelayAll is whether to relay all incoming messages via pre-configured SMTP server.
 	// Use with extreme caution!
-	SMTPRelayAllIncoming = false
+	SMTPRelayAll = false
 
-	// ContentSecurityPolicy for HTTP server
-	ContentSecurityPolicy = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src 'self' data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';"
+	// SMTPRelayMatching if set, will auto-release to recipients matching this regular expression
+	SMTPRelayMatching string
+
+	// SMTPRelayMatchingRegexp is the compiled version of SMTPRelayMatching
+	SMTPRelayMatchingRegexp *regexp.Regexp
+
+	// SMTPForwardConfigFile to parse a yaml file and store config of the forwarding SMTP server
+	SMTPForwardConfigFile string
+
+	// SMTPForwardConfig to parse a yaml file and store config of the forwarding SMTP server
+	SMTPForwardConfig SMTPForwardConfigStruct
+
+	// SMTPStrictRFCHeaders will return an error if the email headers contain <CR><CR><LF> (\r\r\n)
+	// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
+	SMTPStrictRFCHeaders bool
+
+	// SMTPAllowedRecipients if set, will only accept recipients matching this regular expression
+	SMTPAllowedRecipients string
+
+	// SMTPAllowedRecipientsRegexp is the compiled version of SMTPAllowedRecipients
+	SMTPAllowedRecipientsRegexp *regexp.Regexp
+
+	// POP3Listen address - if set then Mailpit will start the POP3 server and listen on this address
+	POP3Listen = "[::]:1110"
+
+	// POP3AuthFile for POP3 authentication
+	POP3AuthFile string
+
+	// POP3TLSCert TLS certificate
+	POP3TLSCert string
+
+	// POP3TLSKey TLS certificate key
+	POP3TLSKey string
+
+	// EnableSpamAssassin must be either <host>:<port> or "postmark"
+	EnableSpamAssassin string
+
+	// WebhookURL for calling
+	WebhookURL string
+
+	// ContentSecurityPolicy for HTTP server - set via VerifyConfig()
+	ContentSecurityPolicy string
+
+	// AllowUntrustedTLS allows untrusted HTTPS connections link checking & screenshot generation
+	AllowUntrustedTLS bool
 
 	// Version is the default application version, updated on release
 	Version = "dev"
@@ -102,239 +190,341 @@ var (
 
 	// RepoBinaryName on Github for updater
 	RepoBinaryName = "mailpit"
+
+	// ChaosTriggers are parsed and set in the chaos module
+	ChaosTriggers string
+
+	// DisableHTMLCheck DEPRECATED 2024/04/13 - kept here to display console warning only
+	DisableHTMLCheck = false
+
+	// DemoMode disables SMTP relay, link checking & HTTP send functionality
+	DemoMode = false
 )
 
 // AutoTag struct for auto-tagging
-type AutoTag struct {
-	Tag   string
+type autoTag struct {
 	Match string
+	Tags  []string
 }
 
 // SMTPRelayConfigStruct struct for parsing yaml & storing variables
-type smtpRelayConfigStruct struct {
-	Host                     string `yaml:"host"`
-	Port                     int    `yaml:"port"`
-	STARTTLS                 bool   `yaml:"starttls"`
-	AllowInsecure            bool   `yaml:"allow-insecure"`
-	Auth                     string `yaml:"auth"`                // none, plain, login, cram-md5
-	Username                 string `yaml:"username"`            // plain & cram-md5
-	Password                 string `yaml:"password"`            // plain
-	Secret                   string `yaml:"secret"`              // cram-md5
-	ReturnPath               string `yaml:"return-path"`         // allow overriding the bounce address
-	RecipientAllowlist       string `yaml:"recipient-allowlist"` // regex, if set needs to match for mails to be relayed
-	RecipientAllowlistRegexp *regexp.Regexp
+type SMTPRelayConfigStruct struct {
+	Host                    string         `yaml:"host"`
+	Port                    int            `yaml:"port"`
+	STARTTLS                bool           `yaml:"starttls"`
+	AllowInsecure           bool           `yaml:"allow-insecure"`
+	Auth                    string         `yaml:"auth"`               // none, plain, login, cram-md5
+	Username                string         `yaml:"username"`           // plain & cram-md5
+	Password                string         `yaml:"password"`           // plain
+	Secret                  string         `yaml:"secret"`             // cram-md5
+	ReturnPath              string         `yaml:"return-path"`        // allow overriding the bounce address
+	OverrideFrom            string         `yaml:"override-from"`      // allow overriding of the from address
+	AllowedRecipients       string         `yaml:"allowed-recipients"` // regex, if set needs to match for mails to be relayed
+	AllowedRecipientsRegexp *regexp.Regexp // compiled regexp using AllowedRecipients
+	BlockedRecipients       string         `yaml:"blocked-recipients"` // regex, if set prevents relating to these addresses
+	BlockedRecipientsRegexp *regexp.Regexp // compiled regexp using BlockedRecipients
+
+	// DEPRECATED 2024/03/12
+	RecipientAllowlist string `yaml:"recipient-allowlist"`
+}
+
+// SMTPForwardConfigStruct struct for parsing yaml & storing variables
+type SMTPForwardConfigStruct struct {
+	To            string `yaml:"to"`             // comma-separated list of email addresses
+	Host          string `yaml:"host"`           // SMTP host
+	Port          int    `yaml:"port"`           // SMTP port
+	STARTTLS      bool   `yaml:"starttls"`       // whether to use STARTTLS
+	AllowInsecure bool   `yaml:"allow-insecure"` // allow insecure authentication
+	Auth          string `yaml:"auth"`           // none, plain, login, cram-md5
+	Username      string `yaml:"username"`       // plain & cram-md5
+	Password      string `yaml:"password"`       // plain
+	Secret        string `yaml:"secret"`         // cram-md5
+	ReturnPath    string `yaml:"return-path"`    // allow overriding the bounce address
+	OverrideFrom  string `yaml:"override-from"`  // allow overriding of the from address
 }
 
 // VerifyConfig wil do some basic checking
 func VerifyConfig() error {
-	if DataFile != "" && isDir(DataFile) {
-		DataFile = filepath.Join(DataFile, "mailpit.db")
+	cssFontRestriction := "*"
+	if BlockRemoteCSSAndFonts {
+		cssFontRestriction = "'self'"
 	}
 
-	re := regexp.MustCompile(`^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(\[([\da-fA-F:])+\])):\d+$`)
-	if !re.MatchString(SMTPListen) {
-		return errors.New("SMTP bind should be in the format of <ip>:<port>")
+	// The default Content Security Policy is updates on every application page load to replace script-src 'self'
+	// with a random nonce ID to prevent XSS. This applies to the Mailpit app & API.
+	// See server.middleWareFunc()
+	ContentSecurityPolicy = fmt.Sprintf("default-src 'self'; script-src 'self'; style-src %s 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src %s data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';",
+		cssFontRestriction, cssFontRestriction,
+	)
+
+	if Database != "" && isDir(Database) {
+		Database = filepath.Join(Database, "mailpit.db")
 	}
-	if !re.MatchString(HTTPListen) {
-		return errors.New("HTTP bind should be in the format of <ip>:<port>")
+
+	if Compression < 0 || Compression > 3 {
+		return errors.New("[db] compression level must be between 0 and 3")
+	}
+
+	Label = tools.Normalize(Label)
+
+	if err := parseMaxAge(); err != nil {
+		return err
+	}
+
+	TenantID = DBTenantID(TenantID)
+	if TenantID != "" {
+		logger.Log().Infof("[db] using tenant \"%s\"", TenantID)
+	}
+
+	re := regexp.MustCompile(`.*:\d+$`)
+	if _, _, isSocket := tools.UnixSocket(SMTPListen); !isSocket && !re.MatchString(SMTPListen) {
+		return errors.New("[smtp] bind should be in the format of <ip>:<port>")
+	}
+	if _, _, isSocket := tools.UnixSocket(HTTPListen); !isSocket && !re.MatchString(HTTPListen) {
+		return errors.New("[ui] HTTP bind should be in the format of <ip>:<port>")
 	}
 
 	if UIAuthFile != "" {
+		UIAuthFile = filepath.Clean(UIAuthFile)
+
 		if !isFile(UIAuthFile) {
-			return fmt.Errorf("HTTP password file not found: %s", UIAuthFile)
+			return fmt.Errorf("[ui] HTTP password file not found or readable: %s", UIAuthFile)
 		}
 
-		a, err := htpasswd.New(UIAuthFile, htpasswd.DefaultSystems, nil)
+		b, err := os.ReadFile(UIAuthFile)
 		if err != nil {
 			return err
 		}
-		UIAuth = a
+
+		if err := auth.SetUIAuth(string(b)); err != nil {
+			return err
+		}
 	}
 
 	if UITLSCert != "" && UITLSKey == "" || UITLSCert == "" && UITLSKey != "" {
-		return errors.New("You must provide both a UI TLS certificate and a key")
+		return errors.New("[ui] you must provide both a UI TLS certificate and a key")
 	}
 
 	if UITLSCert != "" {
+		UITLSCert = filepath.Clean(UITLSCert)
+		UITLSKey = filepath.Clean(UITLSKey)
+
 		if !isFile(UITLSCert) {
-			return fmt.Errorf("TLS certificate not found: %s", UITLSCert)
+			return fmt.Errorf("[ui] TLS certificate not found or readable: %s", UITLSCert)
 		}
 
 		if !isFile(UITLSKey) {
-			return fmt.Errorf("TLS key not found: %s", UITLSKey)
+			return fmt.Errorf("[ui] TLS key not found or readable: %s", UITLSKey)
 		}
 	}
 
 	if SMTPTLSCert != "" && SMTPTLSKey == "" || SMTPTLSCert == "" && SMTPTLSKey != "" {
-		return errors.New("You must provide both an SMTP TLS certificate and a key")
+		return errors.New("[smtp] You must provide both an SMTP TLS certificate and a key")
 	}
 
 	if SMTPTLSCert != "" {
+		SMTPTLSCert = filepath.Clean(SMTPTLSCert)
+		SMTPTLSKey = filepath.Clean(SMTPTLSKey)
+
 		if !isFile(SMTPTLSCert) {
-			return fmt.Errorf("SMTP TLS certificate not found: %s", SMTPTLSCert)
+			return fmt.Errorf("[smtp] TLS certificate not found or readable: %s", SMTPTLSCert)
 		}
 
 		if !isFile(SMTPTLSKey) {
-			return fmt.Errorf("SMTP TLS key not found: %s", SMTPTLSKey)
+			return fmt.Errorf("[smtp] TLS key not found or readable: %s", SMTPTLSKey)
 		}
+	} else if SMTPRequireTLS {
+		return errors.New("[smtp] TLS cannot be required without an SMTP TLS certificate and key")
+	} else if SMTPRequireSTARTTLS {
+		return errors.New("[smtp] STARTTLS cannot be required without an SMTP TLS certificate and key")
+	}
+	if SMTPRequireSTARTTLS && SMTPAuthAllowInsecure || SMTPRequireTLS && SMTPAuthAllowInsecure {
+		return errors.New("[smtp] TLS cannot be required with --smtp-auth-allow-insecure")
+	}
+	if SMTPRequireSTARTTLS && SMTPRequireTLS {
+		return errors.New("[smtp] TLS & STARTTLS cannot be required together")
 	}
 
 	if SMTPAuthFile != "" {
+		SMTPAuthFile = filepath.Clean(SMTPAuthFile)
+
 		if !isFile(SMTPAuthFile) {
-			return fmt.Errorf("SMTP password file not found: %s", SMTPAuthFile)
+			return fmt.Errorf("[smtp] password file not found or readable: %s", SMTPAuthFile)
 		}
 
-		if SMTPAuthAcceptAny {
-			return errors.New("SMTP authentication can either use --smtp-auth-file or --smtp-auth-accept-any")
-		}
-
-		a, err := htpasswd.New(SMTPAuthFile, htpasswd.DefaultSystems, nil)
+		b, err := os.ReadFile(SMTPAuthFile)
 		if err != nil {
 			return err
 		}
-		SMTPAuthConfig = a
+
+		if err := auth.SetSMTPAuth(string(b)); err != nil {
+			return err
+		}
+
+		if !SMTPAuthAllowInsecure {
+			// https://www.rfc-editor.org/rfc/rfc4954
+			// A server implementation MUST implement a configuration in which
+			// it does NOT permit any plaintext password mechanisms, unless either
+			// the STARTTLS [SMTP-TLS] command has been negotiated or some other
+			// mechanism that protects the session from password snooping has been
+			// provided.  Server sites SHOULD NOT use any configuration which
+			// permits a plaintext password mechanism without such a protection
+			// mechanism against password snooping.
+			SMTPRequireSTARTTLS = true
+		}
 	}
 
-	if SMTPTLSCert == "" && (SMTPAuthFile != "" || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure {
-		return errors.New("SMTP authentication requires TLS encryption, run with `--smtp-auth-allow-insecure` to allow insecure authentication")
+	if auth.SMTPCredentials != nil && SMTPAuthAcceptAny {
+		return errors.New("[smtp] authentication cannot use both credentials and --smtp-auth-accept-any")
 	}
 
-	validWebrootRe := regexp.MustCompile(`[^0-9a-zA-Z\/\-\_\.]`)
+	if SMTPTLSCert == "" && (auth.SMTPCredentials != nil || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure {
+		return errors.New("[smtp] authentication requires STARTTLS or TLS encryption, run with `--smtp-auth-allow-insecure` to allow insecure authentication")
+	}
+
+	if err := parseChaosTriggers(); err != nil {
+		return fmt.Errorf("[chaos] %s", err.Error())
+	}
+
+	if chaos.Enabled {
+		logger.Log().Info("[chaos] is enabled")
+	}
+
+	// POP3 server
+	if POP3TLSCert != "" {
+		POP3TLSCert = filepath.Clean(POP3TLSCert)
+		POP3TLSKey = filepath.Clean(POP3TLSKey)
+
+		if !isFile(POP3TLSCert) {
+			return fmt.Errorf("[pop3] TLS certificate not found or readable: %s", POP3TLSCert)
+		}
+
+		if !isFile(POP3TLSKey) {
+			return fmt.Errorf("[pop3] TLS key not found or readable: %s", POP3TLSKey)
+		}
+	}
+	if POP3TLSCert != "" && POP3TLSKey == "" || POP3TLSCert == "" && POP3TLSKey != "" {
+		return errors.New("[pop3] You must provide both a POP3 TLS certificate and a key")
+	}
+	if POP3Listen != "" {
+		_, err := net.ResolveTCPAddr("tcp", POP3Listen)
+		if err != nil {
+			return fmt.Errorf("[pop3] %s", err.Error())
+		}
+	}
+	if POP3AuthFile != "" {
+		POP3AuthFile = filepath.Clean(POP3AuthFile)
+
+		if !isFile(POP3AuthFile) {
+			return fmt.Errorf("[pop3] password file not found or readable: %s", POP3AuthFile)
+		}
+
+		b, err := os.ReadFile(POP3AuthFile)
+		if err != nil {
+			return err
+		}
+
+		if err := auth.SetPOP3Auth(string(b)); err != nil {
+			return err
+		}
+	}
+
+	// Web root
+	validWebrootRe := regexp.MustCompile(`[^0-9a-zA-Z\/\-\_\.@]`)
 	if validWebrootRe.MatchString(Webroot) {
-		return fmt.Errorf("Invalid characters in Webroot (%s). Valid chars include: [a-z A-Z 0-9 _ . - /]", Webroot)
+		return fmt.Errorf("invalid characters in Webroot (%s). Valid chars include: [a-z A-Z 0-9 _ . - / @]", Webroot)
 	}
 
 	s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/"
 	Webroot = s
 
-	SMTPTags = []AutoTag{}
+	if WebhookURL != "" && !isValidURL(WebhookURL) {
+		return fmt.Errorf("webhook URL does not appear to be a valid URL (%s)", WebhookURL)
+	}
 
-	p := shellwords.NewParser()
+	// DEPRECATED 2024/04/13
+	if DisableHTMLCheck {
+		logger.Log().Warn("--disable-html-check has been deprecated and is no longer used")
+	}
 
-	if SMTPCLITags != "" {
-		args, err := p.Parse(SMTPCLITags)
+	if EnableSpamAssassin != "" {
+		spamassassin.SetService(EnableSpamAssassin)
+		logger.Log().Infof("[spamassassin] enabled via %s", EnableSpamAssassin)
+
+		if err := spamassassin.Ping(); err != nil {
+			logger.Log().Warnf("[spamassassin] ping: %s", err.Error())
+		}
+	}
+
+	// load tag filters & options
+	TagFilters = []autoTag{}
+	if err := loadTagsFromArgs(CLITagsArg); err != nil {
+		return err
+	}
+	if err := loadTagsFromConfig(TagsConfig); err != nil {
+		return err
+	}
+	if err := parseTagsDisable(TagsDisable); err != nil {
+		return err
+	}
+
+	if SMTPAllowedRecipients != "" {
+		restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients)
 		if err != nil {
-			return fmt.Errorf("Error parsing tags (%s)", err)
+			return fmt.Errorf("[smtp] failed to compile smtp-allowed-recipients regexp: %s", err.Error())
 		}
 
-		for _, a := range args {
-			t := strings.Split(a, "=")
-			if len(t) > 1 {
-				tag := tools.CleanTag(t[0])
-				if !ValidTagRegexp.MatchString(tag) || len(tag) == 0 {
-					return fmt.Errorf("Invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tag)
-				}
-				match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
-				if len(match) == 0 {
-					return fmt.Errorf("Invalid tag match (%s) - no search detected", tag)
-				}
-				SMTPTags = append(SMTPTags, AutoTag{Tag: tag, Match: match})
-			} else {
-				return fmt.Errorf("Error parsing tags (%s)", a)
-			}
-		}
+		SMTPAllowedRecipientsRegexp = restrictRegexp
+		logger.Log().Infof("[smtp] only allowing recipients matching regexp: %s", SMTPAllowedRecipients)
 	}
 
 	if err := parseRelayConfig(SMTPRelayConfigFile); err != nil {
 		return err
 	}
 
-	if !ReleaseEnabled && SMTPRelayAllIncoming {
-		return errors.New("SMTP relay config must be set to relay all messages")
+	// separate relay config validation to account for environment variables
+	if err := validateRelayConfig(); err != nil {
+		return err
 	}
 
-	if SMTPRelayAllIncoming {
+	if !ReleaseEnabled && SMTPRelayAll || !ReleaseEnabled && SMTPRelayMatching != "" {
+		return errors.New("[relay] a relay configuration must be set to auto-relay any messages")
+	}
+
+	if SMTPRelayMatching != "" {
+		if SMTPRelayAll {
+			logger.Log().Warnf("[relay] ignoring smtp-relay-matching when smtp-relay-all is enabled")
+		} else {
+			re, err := regexp.Compile(SMTPRelayMatching)
+			if err != nil {
+				return fmt.Errorf("[relay] failed to compile smtp-relay-matching regexp: %s", err.Error())
+			}
+
+			SMTPRelayMatchingRegexp = re
+			logger.Log().Infof("[relay] auto-relaying new messages to recipients matching \"%s\" via %s:%d",
+				SMTPRelayMatching, SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+		}
+	}
+
+	if SMTPRelayAll {
 		// this deserves a warning
-		logger.Log().Warnf("[smtp] enabling automatic relay of all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+		logger.Log().Warnf("[relay] auto-relaying all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+	}
+
+	if err := parseForwardConfig(SMTPForwardConfigFile); err != nil {
+		return err
+	}
+
+	// separate forwarding config validation to account for environment variables
+	if err := validateForwardConfig(); err != nil {
+		return err
+	}
+
+	if DemoMode {
+		MaxMessages = 1000
+		// this deserves a warning
+		logger.Log().Info("demo mode enabled")
 	}
 
 	return nil
 }
-
-// Parse & validate the SMTPRelayConfigFile (if set)
-func parseRelayConfig(c string) error {
-	if c == "" {
-		return nil
-	}
-
-	if !isFile(c) {
-		return fmt.Errorf("SMTP relay configuration not found: %s", SMTPRelayConfigFile)
-	}
-
-	data, err := os.ReadFile(c)
-	if err != nil {
-		return err
-	}
-
-	if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil {
-		return err
-	}
-
-	if SMTPRelayConfig.Host == "" {
-		return errors.New("SMTP relay host not set")
-	}
-
-	if SMTPRelayConfig.Port == 0 {
-		SMTPRelayConfig.Port = 25 // default
-	}
-
-	SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth)
-
-	if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" {
-		SMTPRelayConfig.Auth = "none"
-	} else if SMTPRelayConfig.Auth == "plain" {
-		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
-			return fmt.Errorf("SMTP relay host username or password not set for PLAIN authentication (%s)", c)
-		}
-	} else if SMTPRelayConfig.Auth == "login" {
-		SMTPRelayConfig.Auth = "login"
-		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
-			return fmt.Errorf("SMTP relay host username or password not set for LOGIN authentication (%s)", c)
-		}
-	} else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") {
-		SMTPRelayConfig.Auth = "cram-md5"
-		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" {
-			return fmt.Errorf("SMTP relay host username or secret not set for CRAM-MD5 authentication (%s)", c)
-		}
-	} else {
-		return fmt.Errorf("SMTP relay authentication method not supported: %s", SMTPRelayConfig.Auth)
-	}
-
-	ReleaseEnabled = true
-
-	logger.Log().Infof("[smtp] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
-
-	allowlistRegexp, err := regexp.Compile(SMTPRelayConfig.RecipientAllowlist)
-
-	if SMTPRelayConfig.RecipientAllowlist != "" {
-		if err != nil {
-			return fmt.Errorf("failed to compile recipient allowlist regexp: %e", err)
-		}
-
-		SMTPRelayConfig.RecipientAllowlistRegexp = allowlistRegexp
-		logger.Log().Infof("[smtp] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist)
-
-	}
-
-	return nil
-}
-
-// IsFile returns if a path is a file
-func isFile(path string) bool {
-	info, err := os.Stat(path)
-	if os.IsNotExist(err) || !info.Mode().IsRegular() {
-		return false
-	}
-
-	return true
-}
-
-// IsDir returns whether a path is a directory
-func isDir(path string) bool {
-	info, err := os.Stat(path)
-	if os.IsNotExist(err) || !info.IsDir() {
-		return false
-	}
-
-	return true
-}

config/tags.go

@@ -0,0 +1,111 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	// TagsDisablePlus disables message tagging using plus-addresses (user+tag@example.com) - set via verifyConfig()
+	TagsDisablePlus bool
+
+	// TagsDisableXTags disables message tagging via the X-Tags header - set via verifyConfig()
+	TagsDisableXTags bool
+)
+
+type yamlTags struct {
+	Filters []yamlTag `yaml:"filters"`
+}
+
+type yamlTag struct {
+	Match string `yaml:"match"`
+	Tags  string `yaml:"tags"`
+}
+
+// Load tags from a configuration from a file, if set
+func loadTagsFromConfig(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[tags] configuration file not found or unreadable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return fmt.Errorf("[tags] %s", err.Error())
+	}
+
+	conf := yamlTags{}
+
+	if err := yaml.Unmarshal(data, &conf); err != nil {
+		return err
+	}
+
+	if conf.Filters == nil {
+		return fmt.Errorf("[tags] missing tag: array in %s", c)
+	}
+
+	for _, t := range conf.Filters {
+		tags := strings.Split(t.Tags, ",")
+		TagFilters = append(TagFilters, autoTag{Match: t.Match, Tags: tags})
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from config %s", tools.Plural(len(conf.Filters), "tag filter", "tag filters"), c)
+
+	return nil
+}
+
+func loadTagsFromArgs(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	args := tools.ArgsParser(c)
+
+	for _, a := range args {
+		t := strings.Split(a, "=")
+		if len(t) > 1 {
+			match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
+			tags := strings.Split(t[0], ",")
+			TagFilters = append(TagFilters, autoTag{Match: match, Tags: tags})
+		} else {
+			return fmt.Errorf("[tag] error parsing tags (%s)", a)
+		}
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from CLI args", tools.Plural(len(args), "tag filter", "tag filters"))
+
+	return nil
+}
+
+func parseTagsDisable(s string) error {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil
+	}
+
+	parts := strings.Split(strings.ToLower(s), ",")
+
+	for _, p := range parts {
+		switch strings.TrimSpace(p) {
+		case "x-tags", "xtags":
+			TagsDisableXTags = true
+		case "plus-addresses", "plus-addressing":
+			TagsDisablePlus = true
+		default:
+			return fmt.Errorf("[tags] invalid --tags-disable option: %s", p)
+		}
+	}
+
+	return nil
+}

config/utils.go

@@ -0,0 +1,51 @@
+package config
+
+import (
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// IsFile returns whether a file exists and is readable
+func isFile(path string) bool {
+	f, err := os.Open(filepath.Clean(path))
+	defer f.Close()
+	return err == nil
+}
+
+// IsDir returns whether a path is a directory
+func isDir(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || os.IsNotExist(err) || !info.IsDir() {
+		return false
+	}
+
+	return true
+}
+
+func isValidURL(s string) bool {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(u.Scheme, "http")
+}
+
+// DBTenantID converts a tenant ID to a DB-friendly value if set
+func DBTenantID(s string) string {
+	s = tools.Normalize(s)
+	if s != "" {
+		re := regexp.MustCompile(`[^a-zA-Z0-9\_]`)
+		s = re.ReplaceAllString(s, "_")
+		if !strings.HasSuffix(s, "_") {
+			s = s + "_"
+		}
+	}
+
+	return s
+}

config/validators.go

@@ -0,0 +1,282 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"gopkg.in/yaml.v3"
+)
+
+// Parse the --max-age value (if set)
+func parseMaxAge() error {
+	if MaxAge == "" {
+		return nil
+	}
+
+	re := regexp.MustCompile(`^\d+(h|d)$`)
+	if !re.MatchString(MaxAge) {
+		return fmt.Errorf("max-age must be either <int>h for hours or <int>d for days: %s", MaxAge)
+	}
+
+	if strings.HasSuffix(MaxAge, "h") {
+		hours, err := strconv.Atoi(strings.TrimSuffix(MaxAge, "h"))
+		if err != nil {
+			return err
+		}
+
+		MaxAgeInHours = hours
+
+		return nil
+	}
+
+	days, err := strconv.Atoi(strings.TrimSuffix(MaxAge, "d"))
+	if err != nil {
+		return err
+	}
+
+	logger.Log().Debugf("[db] auto-deleting messages older than %s", MaxAge)
+
+	MaxAgeInHours = days * 24
+	return nil
+}
+
+// Parse the SMTPRelayConfigFile (if set)
+func parseRelayConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[relay] configuration not found or readable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil {
+		return err
+	}
+
+	if SMTPRelayConfig.Host == "" {
+		return errors.New("[relay] host not set")
+	}
+
+	// DEPRECATED 2024/03/12
+	if SMTPRelayConfig.RecipientAllowlist != "" {
+		logger.Log().Warn("[relay] 'recipient-allowlist' is deprecated, use 'allowed-recipients' instead")
+		if SMTPRelayConfig.AllowedRecipients == "" {
+			SMTPRelayConfig.AllowedRecipients = SMTPRelayConfig.RecipientAllowlist
+		}
+	}
+
+	return nil
+}
+
+// Validate the SMTPRelayConfig (if Host is set)
+func validateRelayConfig() error {
+	if SMTPRelayConfig.Host == "" {
+		return nil
+	}
+
+	if SMTPRelayConfig.Port == 0 {
+		SMTPRelayConfig.Port = 25 // default
+	}
+
+	SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth)
+
+	if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" {
+		SMTPRelayConfig.Auth = "none"
+	} else if SMTPRelayConfig.Auth == "plain" {
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[relay] host username or password not set for PLAIN authentication")
+		}
+	} else if SMTPRelayConfig.Auth == "login" {
+		SMTPRelayConfig.Auth = "login"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[relay] host username or password not set for LOGIN authentication")
+		}
+	} else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") {
+		SMTPRelayConfig.Auth = "cram-md5"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" {
+			return fmt.Errorf("[relay] host username or secret not set for CRAM-MD5 authentication")
+		}
+	} else {
+		return fmt.Errorf("[relay] authentication method not supported: %s", SMTPRelayConfig.Auth)
+	}
+
+	if SMTPRelayConfig.AllowedRecipients != "" {
+		re, err := regexp.Compile(SMTPRelayConfig.AllowedRecipients)
+		if err != nil {
+			return fmt.Errorf("[relay] failed to compile recipient allowlist regexp: %s", err.Error())
+		}
+
+		SMTPRelayConfig.AllowedRecipientsRegexp = re
+		logger.Log().Infof("[relay] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.AllowedRecipients)
+	}
+
+	if SMTPRelayConfig.BlockedRecipients != "" {
+		re, err := regexp.Compile(SMTPRelayConfig.BlockedRecipients)
+		if err != nil {
+			return fmt.Errorf("[relay] failed to compile recipient blocklist regexp: %s", err.Error())
+		}
+
+		SMTPRelayConfig.BlockedRecipientsRegexp = re
+		logger.Log().Infof("[relay] recipient blocklist is active with the following regexp: %s", SMTPRelayConfig.BlockedRecipients)
+	}
+
+	if SMTPRelayConfig.OverrideFrom != "" {
+		m, err := mail.ParseAddress(SMTPRelayConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("[relay] override-from is not a valid email address: %s", SMTPRelayConfig.OverrideFrom)
+		}
+
+		SMTPRelayConfig.OverrideFrom = m.Address
+	}
+
+	ReleaseEnabled = true
+
+	logger.Log().Infof("[relay] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+
+	return nil
+}
+
+// Parse the SMTPForwardConfigFile (if set)
+func parseForwardConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[forward] configuration not found or readable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPForwardConfig); err != nil {
+		return err
+	}
+
+	if SMTPForwardConfig.Host == "" {
+		return errors.New("[forward] host not set")
+	}
+
+	return nil
+}
+
+// Validate the SMTPForwardConfig (if Host is set)
+func validateForwardConfig() error {
+	if SMTPForwardConfig.Host == "" {
+		return nil
+	}
+
+	if SMTPForwardConfig.Port == 0 {
+		SMTPForwardConfig.Port = 25 // default
+	}
+
+	SMTPForwardConfig.Auth = strings.ToLower(SMTPForwardConfig.Auth)
+
+	if SMTPForwardConfig.Auth == "" || SMTPForwardConfig.Auth == "none" || SMTPForwardConfig.Auth == "false" {
+		SMTPForwardConfig.Auth = "none"
+	} else if SMTPForwardConfig.Auth == "plain" {
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Password == "" {
+			return fmt.Errorf("[forward] host username or password not set for PLAIN authentication")
+		}
+	} else if SMTPForwardConfig.Auth == "login" {
+		SMTPForwardConfig.Auth = "login"
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Password == "" {
+			return fmt.Errorf("[forward] host username or password not set for LOGIN authentication")
+		}
+	} else if strings.HasPrefix(SMTPForwardConfig.Auth, "cram") {
+		SMTPForwardConfig.Auth = "cram-md5"
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Secret == "" {
+			return fmt.Errorf("[forward] host username or secret not set for CRAM-MD5 authentication")
+		}
+	} else {
+		return fmt.Errorf("[forward] authentication method not supported: %s", SMTPForwardConfig.Auth)
+	}
+
+	if SMTPForwardConfig.To == "" {
+		return errors.New("[forward] To addresses missing")
+	}
+
+	to := []string{}
+	addresses := strings.Split(SMTPForwardConfig.To, ",")
+	for _, a := range addresses {
+		a = strings.TrimSpace(a)
+		m, err := mail.ParseAddress(a)
+		if err != nil {
+			return fmt.Errorf("[forward] To address is not a valid email address: %s", a)
+		}
+		to = append(to, m.Address)
+	}
+
+	if len(to) == 0 {
+		return errors.New("[forward] no valid To addresses found")
+	}
+
+	// overwrite the To field with the cleaned up list
+	SMTPForwardConfig.To = strings.Join(to, ",")
+
+	if SMTPForwardConfig.OverrideFrom != "" {
+		m, err := mail.ParseAddress(SMTPForwardConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("[forward] override-from is not a valid email address: %s", SMTPForwardConfig.OverrideFrom)
+		}
+
+		SMTPForwardConfig.OverrideFrom = m.Address
+	}
+
+	logger.Log().Infof("[forward] enabling message forwarding to %s via %s:%d", SMTPForwardConfig.To, SMTPForwardConfig.Host, SMTPForwardConfig.Port)
+
+	return nil
+}
+
+func parseChaosTriggers() error {
+	if ChaosTriggers == "" {
+		return nil
+	}
+
+	re := regexp.MustCompile(`^([a-zA-Z0-0]+):(\d\d\d):(\d+(\.\d)?)$`)
+
+	parts := strings.Split(ChaosTriggers, ",")
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if !re.MatchString(p) {
+			return fmt.Errorf("invalid argument: %s", p)
+		}
+
+		matches := re.FindAllStringSubmatch(p, 1)
+		key := matches[0][1]
+		errorCode, err := strconv.Atoi(matches[0][2])
+		if err != nil {
+			return err
+		}
+		probability, err := strconv.Atoi(matches[0][3])
+		if err != nil {
+			return err
+		}
+
+		if err := chaos.Set(key, errorCode, probability); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

docs/apiv1/Message.md

@@ -1,118 +0,0 @@
-# Message
-
-## Message summary
-
-Returns a JSON summary of the message and attachments.
-
-**URL** : `api/v1/message/<ID>`
-
-**Method** : `GET`
-
-## Response
-
-**Status** : `200`
-
-```json
-{
-  "ID": "d7a5543b-96dd-478b-9b60-2b465c9884de",
-  "MessageID": "12345.67890@localhost",
-  "Read": true,
-  "From": {
-    "Name": "John Doe",
-    "Address": "john@example.com"
-  },
-  "To": [
-    {
-      "Name": "Jane Smith",
-      "Address": "jane@example.com"
-    }
-  ],
-  "Cc": [],
-  "Bcc": [],
-  "ReplyTo": [],
-  "Subject": "Message subject",
-  "Date": "2016-09-07T16:46:00+13:00",
-  "Tags": ["test"],
-  "Text": "Plain text MIME part of the email",
-  "HTML": "HTML MIME part (if exists)",
-  "Size": 79499,
-  "Inline": [
-    {
-      "PartID": "1.2",
-      "FileName": "filename.gif",
-      "ContentType": "image/gif",
-      "ContentID": "919564503@07092006-1525",
-      "Size": 7760
-    }
-  ],
-  "Attachments": [
-    {
-      "PartID": "2",
-      "FileName": "filename.doc",
-      "ContentType": "application/msword",
-      "ContentID": "",
-      "Size": 43520
-    }
-  ]
-}
-```
-### Notes
-
-- `Read` - always true (message marked read on open)
-- `From` - Name & Address, or null
-- `To`, `CC`, `BCC`, `ReplyTo` - Array of Names & Address
-- `Date` - Parsed email local date & time from headers
-- `Size` - Total size of raw email
-- `Inline`, `Attachments` - Array of attachments and inline images.
-
-
----
-## Attachments
-
-**URL** : `api/v1/message/<ID>/part/<PartID>`
-
-**Method** : `GET`
-
-Returns the attachment using the MIME type provided by the attachment `ContentType`.
-
----
-## Headers
-
-**URL** : `api/v1/message/<ID>/headers`
-
-**Method** : `GET`
-
-Returns all message headers as a JSON array.
-Each unique header key contains an array of one or more values (email headers can be listed multiple times.)
-
-```json
-{
-  "Content-Type": [
-    "multipart/related; type=\"multipart/alternative\"; boundary=\"----=_NextPart_000_0013_01C6A60C.47EEAB80\""
-  ],
-  "Date": [
-    "Wed, 12 Jul 2006 23:38:30 +1200"
-  ],
-  "Delivered-To": [
-    "user@example.com",
-    "user-alias@example.com"
-  ],
-  "From": [
-    "\"User Name\" \\u003remote@example.com\\u003e"
-  ],
-  "Message-Id": [
-    "\\u003c001701c6a5a7$b3205580$0201010a@HomeOfficeSM\\u003e"
-  ],
-....
-}
-```
-
-
----
-## Raw (source) email
-
-**URL** : `api/v1/message/<ID>/raw`
-
-**Method** : `GET`
-
-Returns the original email source including headers and attachments.

docs/apiv1/Messages.md

@@ -1,169 +0,0 @@
-# Messages
-
-List & delete messages.
-
-
----
-## List
-
-List messages in the mailbox. Messages are returned in the order of latest received to oldest.
-
-**URL** : `api/v1/messages`
-
-**Method** : `GET`
-
-
-### Query parameters
-
-| Parameter | Type    | Required | Description                |
-|-----------|---------|----------|----------------------------|
-| limit     | integer | false    | Limit results (default 50) |
-| start     | integer | false    | Pagination offset          |
-
-
-### Response
-
-**Status** : `200`
-
-```json
-{
-  "total": 500,
-  "unread": 500,
-  "count": 50,
-  "start": 0,
-  "tags": ["test"],
-  "messages": [
-    {
-      "ID": "1c575821-70ba-466f-8cee-2e1cf0fcdd0f",
-      "MessageID": "12345.67890@localhost",
-      "Read": false,
-      "From": {
-        "Name": "John Doe",
-        "Address": "john@example.com"
-      },
-      "To": [
-        {
-          "Name": "Jane Smith",
-          "Address": "jane@example.com"
-        }
-      ],
-      "Cc": [
-        {
-          "Name": "Accounts",
-          "Address": "accounts@example.com"
-        }
-      ],
-      "Bcc": [],
-      "Subject": "Message subject",
-      "Created": "2022-10-03T21:35:32.228605299+13:00",
-      "Tags": ["test"],
-      "Size": 6144,
-      "Attachments": 0
-    },
-    ...
-  ]
-}
-```
-
-### Notes
-
-- `total` - Total messages in mailbox
-- `unread` - Total unread messages in mailbox
-- `count` - Number of messages returned in request
-- `start` - The offset (default `0`) for pagination
-- `Read` - The read/unread status of the message
-- `From` - Name & Address, or null if none
-- `To`, `CC`, `BCC` - Array of Names & Address
-- `Created` - Local date & time the message was received
-- `Size` - Total size of raw email in bytes
-
-
----
-## Delete individual messages
-
-Delete one or more messages by ID.
-
-**URL** : `api/v1/messages`
-
-**Method** : `DELETE`
-
-### Request
-
-```json
-{
-  "ids": ["<ID>","<ID>"...]
-}
-```
-
-### Response
-
-**Status** : `200`
-
-
----
-## Delete all messages
-
-Delete all messages (same as deleting individual messages, but with the "ids" either empty or omitted entirely).
-
-**URL** : `api/v1/messages`
-
-**Method** : `DELETE`
-
-### Request
-
-```json
-{
-  "ids": []
-}
-```
-
-### Response
-
-**Status** : `200`
-
-
----
-## Update individual read statuses
-
-Set the read status of one or more messages. 
-The `read` status can be `true` or `false`.
-
-**URL** : `api/v1/messages`
-
-**Method** : `PUT`
-
-### Request
-
-```json
-{
-  "ids": ["<ID>","<ID>"...],
-  "read": false
-}
-```
-
-### Response
-
-**Status** : `200`
-
----
-## Update all messages read status
-
-Set the read status of all messages. 
-The `read` status can be `true` or `false`.
-
-**URL** : `api/v1/messages`
-
-**Method** : `PUT`
-
-### Request
-
-```json
-{
-  "ids": [],
-  "read": false
-}
-```
-
-### Response
-
-**Status** : `200`

docs/apiv1/README.md

@@ -1,14 +0,0 @@
-# API v1
-
-Mailpit provides a simple REST API to access and delete stored messages.
-
-If the Mailpit server is set to use Basic Authentication, then API requests must use Basic Authentication too.
-
-You can view the Swagger API documentation directly within Mailpit by going to `http://0.0.0.0:8025/api/v1/`.
-
-The API is split into three main parts:
-
-- [Messages](Messages.md) - Listing, deleting & marking messages as read/unread.
-- [Message](Message.md) - Return message data & attachments
-- [Tags](Tags.md) - Set message tags
-- [Search](Search.md) - Searching messages

docs/apiv1/Search.md

@@ -1,70 +0,0 @@
-# Search
-
-**URL** : `api/v1/search?query=<string>`
-
-**Method** : `GET`
-
-The search returns the most recent matches (default 50).
-Matching messages are returned in the order of latest received to oldest.
-
-
-## Query parameters
-
-| Parameter | Type    | Required | Description                |
-|-----------|---------|----------|----------------------------|
-| query     | string  | true     | Search query               |
-| limit     | integer | false    | Limit results (default 50) |
-| start     | integer | false    | Pagination offset          |
-
-
-## Response
-
-**Status** : `200`
-
-```json
-{
-  "total": 500,
-  "unread": 500,
-  "count": 25,
-  "start": 0,
-  "messages": [
-    {
-      "ID": "1c575821-70ba-466f-8cee-2e1cf0fcdd0f",
-      "MessageID": "12345.67890@localhost",
-      "Read": false,
-      "From": {
-        "Name": "John Doe",
-        "Address": "john@example.com"
-      },
-      "To": [
-        {
-          "Name": "Jane Smith",
-          "Address": "jane@example.com"
-        }
-      ],
-      "Cc": [
-        {
-          "Name": "Accounts",
-          "Address": "accounts@example.com"
-        }
-      ],
-      "Bcc": [],
-      "Subject": "Test email",
-      "Created": "2022-10-03T21:35:32.228605299+13:00",
-      "Size": 6144,
-      "Attachments": 0
-    },
-    ...
-  ]
-}
-```
-
-### Notes
-
-- `total` - Total messages in mailbox (all messages, not search)
-- `unread` - Total unread messages in mailbox (all messages, not search)
-- `count` - Number of messages returned in request
-- `start` - The offset (default `0`) for pagination
-- `From` - Singular Name & Address, or null if none
-- `To`, `CC`, `BCC` - Array of Name & Address
-- `Size` - Total size of raw email in bytes

docs/apiv1/Tags.md

@@ -1,27 +0,0 @@
-# Tags
-
-Set message tags.
-
-
----
-## Update message tags
-
-Set the tags for one or more messages.
-If the tags array is empty then all tags are removed from the messages.
-
-**URL** : `api/v1/tags`
-
-**Method** : `PUT`
-
-### Request
-
-```json
-{
-  "ids": ["<ID>","<ID>"...],
-  "tags": ["<tag>","<tag>"]
-}
-```
-
-### Response
-
-**Status** : `200`

esbuild.config.mjs

@@ -14,8 +14,19 @@ const ctx = await esbuild.context(
         bundle: true,
         minify: doMinify,
         sourcemap: false,
+        define: {
+            '__VUE_OPTIONS_API__': 'true',
+            '__VUE_PROD_DEVTOOLS__': 'false',
+            '__VUE_PROD_HYDRATION_MISMATCH_DETAILS__': 'false',
+        },
         outdir: "server/ui/dist/",
-        plugins: [pluginVue(), sassPlugin()],
+        plugins: [
+            pluginVue(),
+            sassPlugin({
+                silenceDeprecations: ['import'],
+                quietDeps: true,
+            })
+        ],
         loader: {
             ".svg": "file",
             ".woff": "file",

go.mod

@@ -1,65 +1,63 @@
 module github.com/axllent/mailpit
 
-go 1.18
+go 1.23.0
+
+toolchain go1.23.2
 
 require (
-	github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244
+	github.com/PuerkitoBio/goquery v1.10.2
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/axllent/semver v0.0.1
-	github.com/disintegration/imaging v1.6.2
-	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/websocket v1.5.0
-	github.com/jhillyerd/enmime v0.11.1
-	github.com/k3a/html2text v1.2.1
-	github.com/klauspost/compress v1.16.5
+	github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b
+	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/websocket v1.5.3
+	github.com/jhillyerd/enmime v1.3.0
+	github.com/klauspost/compress v1.18.0
+	github.com/kovidgoyal/imaging v1.6.4
 	github.com/leporo/sqlf v1.4.0
-	github.com/mattn/go-shellwords v1.0.12
-	github.com/mhale/smtpd v0.8.0
-	github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
-	github.com/satori/go.uuid v1.2.0
-	github.com/sirupsen/logrus v1.9.2
-	github.com/spf13/cobra v1.7.0
-	github.com/spf13/pflag v1.0.5
-	github.com/tg123/go-htpasswd v1.2.1
-	golang.org/x/text v0.9.0
+	github.com/lithammer/shortuuid/v4 v4.2.0
+	github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62
+	github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.9.1
+	github.com/spf13/pflag v1.0.6
+	github.com/tg123/go-htpasswd v1.2.3
+	github.com/vanng822/go-premailer v1.23.0
+	golang.org/x/net v0.37.0
+	golang.org/x/text v0.23.0
+	golang.org/x/time v0.11.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.22.1
+	modernc.org/sqlite v1.36.1
 )
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect
 	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
-	github.com/cznic/ql v1.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/reiver/go-oi v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/image v0.7.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/tools v0.9.3 // indirect
+	github.com/vanng822/css v1.0.1 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
+	golang.org/x/image v0.25.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	lukechampine.com/uint128 v1.3.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.6 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.1.0 // indirect
+	modernc.org/libc v1.61.13 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.8.2 // indirect
 )

go.sum

@@ -1,72 +1,52 @@
-github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI=
 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
-github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 h1:dqzm54OhCqY8RinR/cx+Ppb0y56Ds5I3wwWhx4XybDg=
-github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244/go.mod h1:3sqgkckuISJ5rs1EpOp6vCvwOUKe/z9vPmyuIlq8Q/A=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/PuerkitoBio/goquery v1.10.2 h1:7fh2BdHcG6VFZsK7toXBT/Bh1z5Wmy8Q9MV9HqT2AM8=
+github.com/PuerkitoBio/goquery v1.10.2/go.mod h1:0guWGjcLu9AYC7C1GHnpysHy056u9aEkUHwhdnePMCU=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E=
 github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk=
-github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
-github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas=
-github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
-github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4 h1:CVAqftqbj+exlab+8KJQrE+kNIVlQfJt58j4GxCMF1s=
-github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
-github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00 h1:FHpbUtp2K8X53/b4aFNj4my5n+i3x+CQCZWNuHWH/+E=
-github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4=
-github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs=
-github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A=
-github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk=
-github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
-github.com/cznic/ql v1.2.0 h1:lcKp95ZtdF0XkWhGnVIXGF8dVD2X+ClS08tglKtf+ak=
-github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE=
-github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
-github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ=
-github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA=
-github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
-github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc h1:YKKpTb2BrXN2GYyGaygIdis1vXbE7SSAG9axGWIMClg=
-github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
-github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
-github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk=
+github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
 github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jhillyerd/enmime v0.11.1 h1:U6ToGVxfxNQQhKrAaGxtwOf7Zqksb8AQ3j1CyAWOk5k=
-github.com/jhillyerd/enmime v0.11.1/go.mod h1:EktNOa/V6ka9yCrfoB2uxgefp1lno6OVdszW0iQ5LnM=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY=
-github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/jhillyerd/enmime v1.3.0 h1:LV5kzfLidiOr8qRGIpYYmUZCnhrPbcFAnAFUnWn99rw=
+github.com/jhillyerd/enmime v1.3.0/go.mod h1:6c6jg5HdRRV2FtvVL69LjiX1M8oE0xDX9VEhV3oy4gs=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kovidgoyal/imaging v1.6.4 h1:K0idhRPXnRrJBKnBYcTfI1HTWSNDeAn7hYDvf9I0dCk=
+github.com/kovidgoyal/imaging v1.6.4/go.mod h1:bEIgsaZmXlvFfkv/CUxr9rJook6AQkJnpB5EPosRfRY=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -76,17 +56,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leporo/sqlf v1.4.0 h1:SyWnX/8GSGOzVmanG0Ub1c04mR9nNl6Tq3IeFKX2/4c=
 github.com/leporo/sqlf v1.4.0/go.mod h1:pgN9yKsAnQ+2ewhbZogr98RcasUjPsHF3oXwPPhHvBw=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c=
+github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
-github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=
-github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
+github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62 h1:XMG5DklHoioVYysfYglOB7vRBg/LOUJZy2mq2QyedLg=
+github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62/go.mod h1:niAM5cni0I/47IFA995xQfeK58Mkbb7FHJjacY4OGQg=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -95,92 +78,145 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/reiver/go-oi v1.0.0 h1:nvECWD7LF+vOs8leNGV/ww+F2iZKf3EYjYZ527turzM=
 github.com/reiver/go-oi v1.0.0/go.mod h1:RrDBct90BAhoDTxB1fenZwfykqeGvhI6LsNfStJoEkI=
-github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e h1:quuzZLi72kkJjl+f5AQ93FMcadG19WkS7MO6TXFOSas=
-github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e/go.mod h1:+5vNVvEWwEIx86DB9Ke/+a5wBI464eDRo3eF0LcfpWg=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a h1:9O8zgGrMBuTsnA3yyFd+JWhFSflQwzSUEB4AMnFHKhU=
+github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
-github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU=
-github.com/tg123/go-htpasswd v1.2.1/go.mod h1:erHp1B86KXdwQf1X5ZrLb7erXZnWueEQezb2dql4q58=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tg123/go-htpasswd v1.2.3 h1:ALR6ZBIc2m9u70m+eAWUFt5p43ISbIvAvRFYzZPTOY8=
+github.com/tg123/go-htpasswd v1.2.3/go.mod h1:FcIrK0J+6zptgVwK1JDlqyajW/1B4PtuJ/FLWl7nx8A=
+github.com/unrolled/render v1.7.0/go.mod h1:LwQSeDhjml8NLjIO9GJO1/1qpFJxtfVIpzxXKjfVkoI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/vanng822/css v1.0.1 h1:10yiXc4e8NI8ldU6mSrWmSWMuyWgPr9DZ63RSlsgDw8=
+github.com/vanng822/css v1.0.1/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w=
+github.com/vanng822/go-premailer v1.23.0 h1:vZp2wuz1jb4q/DurUV18VGjXWtTFYZHwTCw2EAWKO74=
+github.com/vanng822/go-premailer v1.23.0/go.mod h1:0+z0UJ6ZGQatzkWlaQNl50M7fLz5f6FcP8V2p0oie88=
+github.com/vanng822/r2router v0.0.0-20150523112421-1023140a4f30/go.mod h1:1BVq8p2jVr55Ost2PkZWDrG86PiJ/0lxqcXoAcGxvWU=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
-golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
+golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
+golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
-golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
+golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -190,27 +226,27 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
-lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
-modernc.org/libc v1.22.6 h1:cbXU8R+A6aOjRuhsFh3nbDWXO/Hs4ClJRXYB11KmPDo=
-modernc.org/libc v1.22.6/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
-modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE=
-modernc.org/sqlite v1.22.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
+modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
+modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.23.16 h1:Z2N+kk38b7SfySC1ZkpGLN2vthNJP1+ZzGZIlH7uBxo=
+modernc.org/ccgo/v4 v4.23.16/go.mod h1:nNma8goMTY7aQZQNTyN9AIoJfxav4nvTnvKThAeMDdo=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.6.3 h1:aJVhcqAte49LF+mGveZ5KPlsp4tdGdAOT4sipJXADjw=
+modernc.org/gc/v2 v2.6.3/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8=
+modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI=
+modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.36.1 h1:bDa8BJUH4lg6EGkLbahKe/8QqoF8p9gArSc6fTqYhyQ=
+modernc.org/sqlite v1.36.1/go.mod h1:7MPwH7Z6bREicF9ZVUR78P1IKuxfZ8mRIDHD0iD+8TU=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
 modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=

internal/auth/auth.go

@@ -0,0 +1,90 @@
+// Package auth handles the web UI and SMTP authentication
+package auth
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/tg123/go-htpasswd"
+)
+
+var (
+	// UICredentials passwords
+	UICredentials *htpasswd.File
+	// SMTPCredentials passwords
+	SMTPCredentials *htpasswd.File
+	// POP3Credentials passwords
+	POP3Credentials *htpasswd.File
+)
+
+// SetUIAuth will set Basic Auth credentials required for the UI & API
+func SetUIAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	UICredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetSMTPAuth will set SMTP credentials
+func SetSMTPAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	SMTPCredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetPOP3Auth will set POP3 server credentials
+func SetPOP3Auth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	POP3Credentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func credentialsFromString(s string) []string {
+	// split string by any whitespace character
+	re := regexp.MustCompile(`\s+`)
+
+	words := re.Split(s, -1)
+	credentials := []string{}
+	for _, w := range words {
+		if w != "" {
+			credentials = append(credentials, w)
+		}
+	}
+
+	return credentials
+}

internal/dump/dump.go

@@ -0,0 +1,163 @@
+// Package dump is used to export all messages from mailpit into a directory
+package dump
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/apiv1"
+)
+
+var (
+	linkRe = regexp.MustCompile(`(?i)^https?:\/\/`)
+
+	outDir string
+
+	// Base URL of mailpit instance
+	base string
+
+	// URL is the base URL of a remove Mailpit instance
+	URL string
+
+	summary = []storage.MessageSummary{}
+)
+
+// Sync will sync all messages from the specified database or API to the specified output directory
+func Sync(d string) error {
+
+	outDir = path.Clean(d)
+
+	if URL != "" {
+		if !linkRe.MatchString(URL) {
+			return errors.New("Invalid URL")
+		}
+
+		base = strings.TrimRight(URL, "/") + "/"
+	}
+
+	if base == "" && config.Database == "" {
+		return errors.New("No database or API URL specified")
+	}
+
+	if !tools.IsDir(outDir) {
+		if err := os.MkdirAll(outDir, 0755); /* #nosec */ err != nil {
+			return err
+		}
+	}
+
+	if err := loadIDs(); err != nil {
+		return err
+	}
+
+	if err := saveMessages(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// LoadIDs will load all message IDs from the specified database or API
+func loadIDs() error {
+	if base != "" {
+		// remote
+		logger.Log().Debugf("Fetching messages summary from %s", base)
+		res, err := http.Get(base + "api/v1/messages?limit=0")
+
+		if err != nil {
+			return err
+		}
+
+		body, err := io.ReadAll(res.Body)
+
+		if err != nil {
+			return err
+		}
+
+		var data apiv1.MessagesSummary
+		if err := json.Unmarshal(body, &data); err != nil {
+			return err
+		}
+
+		summary = data.Messages
+
+	} else {
+		// make sure the database isn't pruned while open
+		config.MaxMessages = 0
+
+		var err error
+		// local database
+		if err = storage.InitDB(); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("Fetching messages summary from %s", config.Database)
+
+		summary, err = storage.List(0, 0, 0)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(summary) == 0 {
+		return errors.New("No messages found")
+	}
+
+	return nil
+}
+
+func saveMessages() error {
+	for _, m := range summary {
+		out := path.Join(outDir, m.ID+".eml")
+
+		// skip if message exists
+		if tools.IsFile(out) {
+			continue
+		}
+
+		var b []byte
+
+		if base != "" {
+			res, err := http.Get(base + "api/v1/message/" + m.ID + "/raw")
+
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+
+			b, err = io.ReadAll(res.Body)
+
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+		} else {
+			var err error
+			b, err = storage.GetMessageRaw(m.ID)
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+		}
+
+		if err := os.WriteFile(out, b, 0644); /* #nosec */ err != nil {
+			logger.Log().Errorf("Error writing message %s: %s", m.ID, err.Error())
+			continue
+		}
+
+		_ = os.Chtimes(out, m.Created, m.Created)
+
+		logger.Log().Debugf("Saved message %s to %s", m.ID, out)
+	}
+
+	return nil
+}

internal/html2text/html2text.go

@@ -0,0 +1,83 @@
+// Package html2text is a simple library to convert HTML to plain text
+package html2text
+
+import (
+	"bytes"
+	"log"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"golang.org/x/net/html"
+)
+
+var (
+	re      = regexp.MustCompile(`\s+`)
+	spaceRe = regexp.MustCompile(`(?mi)<\/(div|p|td|th|h[1-6]|ul|ol|li|address|article|aside|blockquote|dl|dt|footer|header|hr|main|nav|pre|table|thead|tfoot|video)><`)
+	brRe    = regexp.MustCompile(`(?mi)<(br /|br)>`)
+	imgRe   = regexp.MustCompile(`(?mi)<(img)`)
+	skip    = make(map[string]bool)
+)
+
+func init() {
+	skip["script"] = true
+	skip["title"] = true
+	skip["head"] = true
+	skip["link"] = true
+	skip["meta"] = true
+	skip["style"] = true
+	skip["noscript"] = true
+}
+
+// Strip will convert a HTML string to plain text
+func Strip(h string, includeLinks bool) string {
+	h = spaceRe.ReplaceAllString(h, "</$1> <")
+	h = brRe.ReplaceAllString(h, " ")
+	h = imgRe.ReplaceAllString(h, " <$1")
+	var buffer bytes.Buffer
+	doc, err := html.Parse(strings.NewReader(h))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	extract(doc, &buffer, includeLinks)
+	return clean(buffer.String())
+}
+
+func extract(node *html.Node, buff *bytes.Buffer, includeLinks bool) {
+	if node.Type == html.TextNode {
+		data := node.Data
+		if data != "" {
+			buff.WriteString(data)
+		}
+	}
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		if _, skip := skip[c.Data]; !skip {
+			if includeLinks && c.Data == "a" {
+				for _, a := range c.Attr {
+					if a.Key == "href" && strings.HasPrefix(strings.ToLower(a.Val), "http") {
+						buff.WriteString(" " + a.Val + " ")
+					}
+				}
+			}
+			extract(c, buff, includeLinks)
+		}
+	}
+}
+
+func clean(text string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	text = strings.ReplaceAll(text, string('\uFEFF'), " ")
+
+	// remove non-printable characters
+	text = strings.Map(func(r rune) rune {
+		if unicode.IsPrint(r) {
+			return r
+		}
+		return []rune(" ")[0]
+	}, text)
+
+	text = re.ReplaceAllString(text, " ")
+
+	return strings.TrimSpace(text)
+}

internal/html2text/html2text_test.go

@@ -0,0 +1,250 @@
+package html2text
+
+import "testing"
+
+func TestPlain(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, false)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestWithLinks(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading https://github.com linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading https://github.com linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, true)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func BenchmarkPlain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, false)
+	}
+}
+
+func BenchmarkLinks(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, true)
+	}
+}
+
+var htmlTestData = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en" style="font-family: sans-serif; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; box-sizing: border-box;" xml:lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>[axllent/mailpit] Run failed: .github/workflows/tests.yml - feature/swagger (284335a)</title>
+    
+  </head>
+  <body style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;; font-size: 14px; line-height: 1.5; color: #24292e; background-color: #fff; margin: 0;" bgcolor="#fff">
+    <table align="center" class="container-sm width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 544px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <td class="center p-3" align="center" valign="top" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 16px;">
+          <center style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full container-md" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 768px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                  <td class="text-left" style="box-sizing: border-box; text-align: left !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;" align="left">
+                    <img src="https://github.githubassets.com/images/email/global/octocat-logo.png" alt="GitHub" width="32" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+                    <h2 class="lh-condensed mt-2 text-normal" style="box-sizing: border-box; margin-top: 8px !important; margin-bottom: 0; font-size: 24px; font-weight: 400 !important; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                        [axllent/mailpit] .github/workflows/tests.yml workflow run
+
+                    </h2>
+                  </td>
+                </tr>
+              </table>
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+</td>
+  </tr>
+</table>
+            <table width="100%" class="width-full" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+              <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <td class="border rounded-2 d-block" style="box-sizing: border-box; border-radius: 6px !important; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0; border: 1px solid #e1e4e8;">
+                  <table align="center" class="width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                      <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                        <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                          
+<table align="center" class="border-bottom width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; border-bottom-width: 1px !important; border-bottom-color: #e1e4e8 !important; border-bottom-style: solid !important; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td class="d-block px-3 pt-3 p-sm-4" style="box-sizing: border-box; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 24px;">
+      <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+        
+    <img src="https://github.githubassets.com/images/email/icons/actions.png" width="56" height="56" alt="" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="12" style="font-size: 12px; line-height: 12px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+<h3 class="lh-condensed" style="box-sizing: border-box; margin-top: 0; margin-bottom: 0; font-size: 20px; font-weight: 600; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">.github/workflows/tests.yml: No jobs were run</h3>
+<table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+
+  <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+    <table width="100%" border="0" cellspacing="0" cellpadding="0" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+      <table border="0" cellspacing="0" cellpadding="0" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+          <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <!--[if mso]> <table><tr><td align="center" bgcolor="#28a745"> <![endif]-->
+                <a href="https://github.com/axllent/mailpit/actions/runs/6522820865" target="_blank" rel="noopener noreferrer" class="btn btn-large btn-primary" style="background-color: #1f883d !important; box-sizing: border-box; color: #fff; text-decoration: none; position: relative; display: inline-block; font-size: inherit; font-weight: 500; line-height: 1.5; white-space: nowrap; vertical-align: middle; cursor: pointer; -webkit-user-select: none; user-select: none; border-radius: .5em; -webkit-appearance: none; appearance: none; box-shadow: 0 1px 0 rgba(27,31,35,.1),inset 0 1px 0 rgba(255,255,255,.03); transition: background-color .2s cubic-bezier(0.3, 0, 0.5, 1); font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: .75em 1.5em; border: 1px solid #1f883d;">View workflow run</a>
+              <!--[if mso]> </td></tr></table> <![endif]-->
+          </td>
+        </tr>
+      </table>
+    </td>
+  </tr>
+</table>
+
+</td>
+  </tr>
+</table>
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="32" style="font-size: 32px; line-height: 32px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+</td>
+  </tr>
+</table>
+    </td>
+  </tr>
+</table>
+
+
+
+
+</td>
+  </tr>
+</table>
+                      </td>
+                    </tr>
+                  </table>
+                </td>
+              </tr>
+            </table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <p class="f5 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 14px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">  </p><p style="font-size: small; -webkit-text-size-adjust: none; color: #666; box-sizing: border-box; margin-top: 0; margin-bottom: 10px; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">&#8212;<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" />You are receiving this because you are subscribed to this thread.<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" /><a href="https://github.com/settings/notifications" style="background-color: transparent; box-sizing: border-box; color: #0366d6; text-decoration: none; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">Manage your GitHub Actions notifications</a></p>
+
+</td>
+  </tr>
+</table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+  <p class="f6 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 12px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">GitHub, Inc. &#12539;88 Colin P Kelly Jr Street &#12539;San Francisco, CA 94107</p>
+</td>
+  </tr>
+</table>
+
+          </center>
+        </td>
+      </tr>
+    </table>
+    <!-- prevent Gmail on iOS font size manipulation -->
+   <div style="display: none; white-space: nowrap; box-sizing: border-box; font: 15px/0 apple-system, BlinkMacSystemFont, &quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;"> &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; </div>
+  </body>
+</html>`

internal/htmlcheck/README.md

@@ -0,0 +1,5 @@
+# HTML check
+
+The database used for HTML support tests is based on [can I email](https://www.caniemail.com/).
+
+The `caniemail-data.json` file used to determine client support is copied from the [API](https://www.caniemail.com/api/data.json)

internal/htmlcheck/caniemail.go

@@ -0,0 +1,74 @@
+// Package htmlcheck is used for parsing HTML and returning
+// HTML compatibility errors and warnings
+package htmlcheck
+
+import (
+	"embed"
+	"encoding/json"
+	"regexp"
+)
+
+//go:embed caniemail-data.json
+var embeddedFS embed.FS
+
+var (
+	cie = CanIEmail{}
+
+	noteMatch = regexp.MustCompile(` #(\d)+$`)
+
+	// LimitFamilies will limit results to families if set
+	LimitFamilies = []string{}
+
+	// LimitPlatforms will limit results to platforms if set
+	LimitPlatforms = []string{}
+
+	// LimitClients will limit results to clients if set
+	LimitClients = []string{}
+)
+
+// CanIEmail struct for JSON data
+type CanIEmail struct {
+	APIVersion     string `json:"api_version"`
+	LastUpdateDate string `json:"last_update_date"`
+	// NiceNames map[string]string `json:"last_update_date"`
+	NiceNames struct {
+		Family   map[string]string `json:"family"`
+		Platform map[string]string `json:"platform"`
+		Support  map[string]string `json:"support"`
+		Category map[string]string `json:"category"`
+	} `json:"nicenames"`
+	Data []JSONResult `json:"data"`
+}
+
+// JSONResult struct for CanIEmail Data
+type JSONResult struct {
+	Slug           string                 `json:"slug"`
+	Title          string                 `json:"title"`
+	Description    string                 `json:"description"`
+	URL            string                 `json:"url"`
+	Category       string                 `json:"category"`
+	Tags           []string               `json:"tags"`
+	Keywords       string                 `json:"keywords"`
+	LastTestDate   string                 `json:"last_test_date"`
+	TestURL        string                 `json:"test_url"`
+	TestResultsURL string                 `json:"test_results_url"`
+	Stats          map[string]interface{} `json:"stats"`
+	Notes          string                 `json:"notes"`
+	NotesByNumber  map[string]string      `json:"notes_by_num"`
+}
+
+// Load the JSON data
+func loadJSONData() error {
+	if cie.APIVersion != "" {
+		return nil
+	}
+
+	b, err := embeddedFS.ReadFile("caniemail-data.json")
+	if err != nil {
+		return err
+	}
+
+	cie = CanIEmail{}
+
+	return json.Unmarshal(b, &cie)
+}

internal/htmlcheck/config.go

@@ -0,0 +1,215 @@
+package htmlcheck
+
+import "regexp"
+
+// HTML tests
+var htmlTests = map[string]string{
+	// body check is manually done because it always exists in *goquery.Document
+	"html-body": "body",
+	// HTML tests
+	"html-object":         "object, embed, image, pdf",
+	"html-link":           "link",
+	"html-hr":             "hr",
+	"html-dialog":         "dialog",
+	"html-srcset":         "[srcset]",
+	"html-picture":        "picture",
+	"html-svg":            "svg",
+	"html-progress":       "progress",
+	"html-required":       "[required]",
+	"html-meter":          "meter",
+	"html-audio":          "audio",
+	"html-form":           "form",
+	"html-input-submit":   "submit",
+	"html-button-reset":   "button[type=\"reset\"]",
+	"html-button-submit":  "submit, button[type=\"submit\"]",
+	"html-base":           "base",
+	"html-input-checkbox": "checkbox",
+	"html-input-hidden":   "[type=\"hidden\"]",
+	"html-input-radio":    "radio",
+	"html-input-text":     "input[type=\"text\"]",
+	"html-video":          "video",
+	"html-semantics":      "article, aside, details, figcaption, figure, footer, header, main, mark, nav, section, summary, time",
+	"html-select":         "select",
+	"html-textarea":       "textarea",
+	"html-anchor-links":   "a[href^=\"#\"]",
+	"html-style":          "style",
+	"html-image-maps":     "map, img[usemap]",
+}
+
+// Image tests using regex to match against img[src]
+var imageRegexpTests = map[string]*regexp.Regexp{
+	"image-apng":   regexp.MustCompile(`(?i)\.apng$`),
+	"image-avif":   regexp.MustCompile(`(?i)\.avif$`),
+	"image-base64": regexp.MustCompile(`^(?i)data:image\/`),
+	"image-bmp":    regexp.MustCompile(`(?i)\.bmp$`),
+	"image-gif":    regexp.MustCompile(`(?i)\.gif$`),
+	"image-hdr":    regexp.MustCompile(`(?i)\.hdr$`),
+	"image-heif":   regexp.MustCompile(`(?i)\.heif$`),
+	"image-ico":    regexp.MustCompile(`(?i)\.ico$`),
+	"image-mp4":    regexp.MustCompile(`(?i)\.mp4$`),
+	"image-ppm":    regexp.MustCompile(`(?i)\.ppm$`),
+	"image-svg":    regexp.MustCompile(`(?i)\.svg$`),
+	"image-tiff":   regexp.MustCompile(`(?i)\.tiff?$`),
+	"image-webp":   regexp.MustCompile(`(?i)\.webp$`),
+}
+
+// inline attribute <match>=""
+var styleInlineAttributes = map[string]string{
+	"css-background-color": "[bgcolor]",
+	"css-background":       "[background]",
+	"css-border":           "[border]",
+	"css-height":           "[height]",
+	"css-padding":          "[padding]",
+	"css-width":            "[width]",
+}
+
+// inline style="<match>"
+var cssInlineRegexTests = map[string]*regexp.Regexp{
+	"css-accent-color":                   regexp.MustCompile(`(?i)(^|\s|;)accent-color(\s+)?:`),
+	"css-align-items":                    regexp.MustCompile(`(?i)(^|\s|;)align-items(\s+)?:`),
+	"css-aspect-ratio":                   regexp.MustCompile(`(?i)(^|\s|;)aspect-ratio(\s+)?:`),
+	"css-background-blend-mode":          regexp.MustCompile(`(?i)(^|\s|;)background-blend-mode(\s+)?:`),
+	"css-background-clip":                regexp.MustCompile(`(?i)(^|\s|;)background-clip(\s+)?:`),
+	"css-background-color":               regexp.MustCompile(`(?i)(^|\s|;)background-color(\s+)?:`),
+	"css-background-image":               regexp.MustCompile(`(?i)(^|\s|;)background-image(\s+)?:`),
+	"css-background-origin":              regexp.MustCompile(`(?i)(^|\s|;)background-origin(\s+)?:`),
+	"css-background-position":            regexp.MustCompile(`(?i)(^|\s|;)background-position(\s+)?:`),
+	"css-background-repeat":              regexp.MustCompile(`(?i)(^|\s|;)background-repeat(\s+)?:`),
+	"css-background-size":                regexp.MustCompile(`(?i)(^|\s|;)background-size(\s+)?:`),
+	"css-background":                     regexp.MustCompile(`(?i)(^|\s|;)background(\s+)?:`),
+	"css-block-inline-size":              regexp.MustCompile(`(?i)(^|\s|;)block-inline-size(\s+)?:`),
+	"css-border-image":                   regexp.MustCompile(`(?i)(^|\s|;)border-image(\s+)?:`),
+	"css-border-inline-block-individual": regexp.MustCompile(`(?i)(^|\s|;)border-inline(\s+)?:`),
+	"css-border-radius":                  regexp.MustCompile(`(?i)(^|\s|;)border-radius(\s+)?:`),
+	"css-border":                         regexp.MustCompile(`(?i)(^|\s|;)border(\s+)?:`),
+	"css-box-shadow":                     regexp.MustCompile(`(?i)(^|\s|;)box-shadow(\s+)?:`),
+	"css-box-sizing":                     regexp.MustCompile(`(?i)(^|\s|;)box-sizing(\s+)?:`),
+	"css-caption-side":                   regexp.MustCompile(`(?i)(^|\s|;)caption-side(\s+)?:`),
+	"css-clip-path":                      regexp.MustCompile(`(?i)(^|\s|;)clip-path(\s+)?:`),
+	"css-column-count":                   regexp.MustCompile(`(?i)(^|\s|;)column-count(\s+)?:`),
+	"css-column-layout-properties":       regexp.MustCompile(`(?i)(^|\s|;)column-layout-properties(\s+)?:`),
+	"css-conic-gradient":                 regexp.MustCompile(`(?i)(^|\s|;)conic-gradient(\s+)?:`),
+	"css-direction":                      regexp.MustCompile(`(?i)(^|\s|;)direction(\s+)?:`),
+	"css-display-flex":                   regexp.MustCompile(`(?i)(^|\s|;)display(\s+)?:(\s+)?flex($|\s|;)`),
+	"css-display-grid":                   regexp.MustCompile(`(?i)(^|\s|;)display:grid`),
+	"css-display-none":                   regexp.MustCompile(`(?i)(^|\s|;)display:none`),
+	"css-display":                        regexp.MustCompile(`(?i)(^|\s|;)display(\s+)?:`),
+	"css-filter":                         regexp.MustCompile(`(?i)(^|\s|;)filter(\s+)?:`),
+	"css-flex-direction":                 regexp.MustCompile(`(?i)(^|\s|;)flex-direction(\s+)?:`),
+	"css-flex-wrap":                      regexp.MustCompile(`(?i)(^|\s|;)flex-wrap(\s+)?:`),
+	"css-float":                          regexp.MustCompile(`(?i)(^|\s|;)float(\s+)?:`),
+	"css-font-kerning":                   regexp.MustCompile(`(?i)(^|\s|;)font-kerning(\s+)?:`),
+	"css-font-weight":                    regexp.MustCompile(`(?i)(^|\s|;)font-weight(\s+)?:`),
+	"css-font":                           regexp.MustCompile(`(?i)(^|\s|;)font(\s+)?:`),
+	"css-gap":                            regexp.MustCompile(`(?i)(^|\s|;)gap(\s+)?:`),
+	"css-grid-template":                  regexp.MustCompile(`(?i)(^|\s|;)grid-template(\s+)?:`),
+	"css-height":                         regexp.MustCompile(`(?i)(^|\s|;)height(\s+)?:`),
+	"css-hyphens":                        regexp.MustCompile(`(?i)(^|\s|;)hyphens(\s+)?:`),
+	"css-important":                      regexp.MustCompile(`(?i)!important($|\s|;)`),
+	"css-inline-size":                    regexp.MustCompile(`(?i)(^|\s|;)inline-size(\s+)?:`),
+	"css-intrinsic-size":                 regexp.MustCompile(`(?i)(^|\s|;)intrinsic-size(\s+)?:`),
+	"css-justify-content":                regexp.MustCompile(`(?i)(^|\s|;)justify-content(\s+)?:`),
+	"css-letter-spacing":                 regexp.MustCompile(`(?i)(^|\s|;)letter-spacing(\s+)?:`),
+	"css-line-height":                    regexp.MustCompile(`(?i)(^|\s|;)line-height(\s+)?:`),
+	"css-list-style-image":               regexp.MustCompile(`(?i)(^|\s|;)list-style-image(\s+)?:`),
+	"css-list-style-position":            regexp.MustCompile(`(?i)(^|\s|;)list-style-position(\s+)?:`),
+	"css-list-style":                     regexp.MustCompile(`(?i)(^|\s|;)list-style(\s+)?:`),
+	"css-margin-block-start-end":         regexp.MustCompile(`(?i)(^|\s|;)margin-block-(start|end)(\s+)?:`),
+	"css-margin-inline-block":            regexp.MustCompile(`(?i)(^|\s|;)margin-inline-block(\s+)?:`),
+	"css-margin-inline-start-end":        regexp.MustCompile(`(?i)(^|\s|;)margin-inline-(start|end)(\s+)?:`),
+	"css-margin-inline":                  regexp.MustCompile(`(?i)(^|\s|;)margin-inline(\s+)?:`),
+	"css-margin":                         regexp.MustCompile(`(?i)(^|\s|;)margin(\s+)?:`),
+	"css-max-block-size":                 regexp.MustCompile(`(?i)(^|\s|;)max-block-size(\s+)?:`),
+	"css-max-height":                     regexp.MustCompile(`(?i)(^|\s|;)max-height(\s+)?:`),
+	"css-max-width":                      regexp.MustCompile(`(?i)(^|\s|;)max-width(\s+)?:`),
+	"css-min-height":                     regexp.MustCompile(`(?i)(^|\s|;)min-height(\s+)?:`),
+	"css-min-inline-size":                regexp.MustCompile(`(?i)(^|\s|;)min-inline-size(\s+)?:`),
+	"css-min-width":                      regexp.MustCompile(`(?i)(^|\s|;)min-width(\s+)?:`),
+	"css-mix-blend-mode":                 regexp.MustCompile(`(?i)(^|\s|;)mix-blend-mode(\s+)?:`),
+	"css-modern-color":                   regexp.MustCompile(`(?i)(^|\s|;)modern-color(\s+)?:`),
+	"css-object-fit":                     regexp.MustCompile(`(?i)(^|\s|;)object-fit(\s+)?:`),
+	"css-object-position":                regexp.MustCompile(`(?i)(^|\s|;)object-position(\s+)?:`),
+	"css-opacity":                        regexp.MustCompile(`(?i)(^|\s|;)opacity(\s+)?:`),
+	"css-outline-offset":                 regexp.MustCompile(`(?i)(^|\s|;)outline-offset(\s+)?:`),
+	"css-outline":                        regexp.MustCompile(`(?i)(^|\s|;)outline(\s+)?:`),
+	"css-overflow-wrap":                  regexp.MustCompile(`(?i)(^|\s|;)overflow-wrap(\s+)?:`),
+	"css-overflow":                       regexp.MustCompile(`(?i)(^|\s|;)overflow(\s+)?:`),
+	"css-padding-block-start-end":        regexp.MustCompile(`(?i)(^|\s|;)padding-block-(start|end)(\s+)?:`),
+	"css-padding-inline-block":           regexp.MustCompile(`(?i)(^|\s|;)padding-inline-block(\s+)?:`),
+	"css-padding-inline-start-end":       regexp.MustCompile(`(?i)(^|\s|;)padding-inline-(start|end)(\s+)?:`),
+	"css-padding":                        regexp.MustCompile(`(?i)(^|\s|;)padding(\s+)?:`),
+	"css-position":                       regexp.MustCompile(`(?i)(^|\s|;)position(\s+)?:`),
+	"css-radial-gradient":                regexp.MustCompile(`(?i)(^|\s|;)radial-gradient(\s+)?:`),
+	"css-rgb":                            regexp.MustCompile(`(?i)(\s|:)rgb\(`),
+	"css-rgba":                           regexp.MustCompile(`(?i)(\s|:)rgba\(`),
+	"css-scroll-snap":                    regexp.MustCompile(`(?i)(^|\s|;)roll-snap(\s+)?:`),
+	"css-tab-size":                       regexp.MustCompile(`(?i)(^|\s|;)tab-size(\s+)?:`),
+	"css-table-layout":                   regexp.MustCompile(`(?i)(^|\s|;)table-layout(\s+)?:`),
+	"css-text-align-last":                regexp.MustCompile(`(?i)(^|\s|;)text-align-last(\s+)?:`),
+	"css-text-align":                     regexp.MustCompile(`(?i)(^|\s|;)text-align(\s+)?:`),
+	"css-text-decoration-color":          regexp.MustCompile(`(?i)(^|\s|;)text-decoration-color(\s+)?:`),
+	"css-text-decoration-thickness":      regexp.MustCompile(`(?i)(^|\s|;)text-decoration-thickness(\s+)?:`),
+	"css-text-decoration":                regexp.MustCompile(`(?i)(^|\s|;)text-decoration(\s+)?:`),
+	"css-text-emphasis-position":         regexp.MustCompile(`(?i)(^|\s|;)text-emphasis-position(\s+)?:`),
+	"css-text-emphasis":                  regexp.MustCompile(`(?i)(^|\s|;)text-emphasis(\s+)?:`),
+	"css-text-indent":                    regexp.MustCompile(`(?i)(^|\s|;)text-indent(\s+)?:`),
+	"css-text-overflow":                  regexp.MustCompile(`(?i)(^|\s|;)text-overflow(\s+)?:`),
+	"css-text-shadow":                    regexp.MustCompile(`(?i)(^|\s|;)text-shadow(\s+)?:`),
+	"css-text-transform":                 regexp.MustCompile(`(?i)(^|\s|;)text-transform(\s+)?:`),
+	"css-text-underline-offset":          regexp.MustCompile(`(?i)(^|\s|;)text-underline-offset(\s+)?:`),
+	"css-transform":                      regexp.MustCompile(`(?i)(^|\s|;)transform(\s+)?:`),
+	"css-unit-calc":                      regexp.MustCompile(`(?i)(\s|:)calc\(`),
+	"css-variables":                      regexp.MustCompile(`(?i)(^|\s|;)variables(\s+)?:`),
+	"css-visibility":                     regexp.MustCompile(`(?i)(^|\s|;)visibility(\s+)?:`),
+	"css-white-space":                    regexp.MustCompile(`(?i)(^|\s|;)white-space(\s+)?:`),
+	"css-width":                          regexp.MustCompile(`(?i)(^|\s|;)width(\s+)?:`),
+	"css-word-break":                     regexp.MustCompile(`(?i)(^|\s|;)word-break(\s+)?:`),
+	"css-writing-mode":                   regexp.MustCompile(`(?i)(^|\s|;)writing-mode(\s+)?:`),
+	"css-z-index":                        regexp.MustCompile(`(?i)(^|\s|;)z-index(\s+)?:`),
+}
+
+// some CSS tests using regex for things that can't be merged inline
+var cssRegexpTests = map[string]*regexp.Regexp{
+	"css-at-font-face":                  regexp.MustCompile(`(?mi)@font\-face\s+?{`),
+	"css-at-import":                     regexp.MustCompile(`(?mi)@import\s`),
+	"css-at-keyframes":                  regexp.MustCompile(`(?mi)@keyframes\s`),
+	"css-at-media":                      regexp.MustCompile(`(?mi)@media\s?\(`),
+	"css-at-supports":                   regexp.MustCompile(`(?mi)@supports\s?\(`),
+	"css-pseudo-class-active":           regexp.MustCompile(`:active`),
+	"css-pseudo-class-checked":          regexp.MustCompile(`:checked`),
+	"css-pseudo-class-first-child":      regexp.MustCompile(`:first\-child`),
+	"css-pseudo-class-first-of-type":    regexp.MustCompile(`:first\-of\-type`),
+	"css-pseudo-class-focus":            regexp.MustCompile(`:focus`),
+	"css-pseudo-class-has":              regexp.MustCompile(`:has`),
+	"css-pseudo-class-hover":            regexp.MustCompile(`:hover`),
+	"css-pseudo-class-lang":             regexp.MustCompile(`:lang\s?\(`),
+	"css-pseudo-class-last-child":       regexp.MustCompile(`:last\-child`),
+	"css-pseudo-class-last-of-type":     regexp.MustCompile(`:last\-of\-type`),
+	"css-pseudo-class-link":             regexp.MustCompile(`:link`),
+	"css-pseudo-class-not":              regexp.MustCompile(`:not(\s+)?\(`),
+	"css-pseudo-class-nth-child":        regexp.MustCompile(`:nth\-child(\s+)?\(`),
+	"css-pseudo-class-nth-last-child":   regexp.MustCompile(`:nth\-last\-child(\s+)?\(`),
+	"css-pseudo-class-nth-last-of-type": regexp.MustCompile(`:nth\-last\-of\-type(\s+)?\(`),
+	"css-pseudo-class-nth-of-type":      regexp.MustCompile(`:nth\-of\-type(\s+)?\(`),
+	"css-pseudo-class-only-child":       regexp.MustCompile(`:only\-child(\s+)?\(`),
+	"css-pseudo-class-only-of-type":     regexp.MustCompile(`:only\-of\-type(\s+)?\(`),
+	"css-pseudo-class-target":           regexp.MustCompile(`:target`),
+	"css-pseudo-class-visited":          regexp.MustCompile(`:visited`),
+	"css-pseudo-element-after":          regexp.MustCompile(`:after`),
+	"css-pseudo-element-before":         regexp.MustCompile(`:before`),
+	"css-pseudo-element-first-letter":   regexp.MustCompile(`::first\-letter`),
+	"css-pseudo-element-first-line":     regexp.MustCompile(`::first\-line`),
+	"css-pseudo-element-marker":         regexp.MustCompile(`::marker`),
+	"css-pseudo-element-placeholder":    regexp.MustCompile(`::placeholder`),
+}
+
+// some CSS tests using regex for units
+var cssRegexpUnitTests = map[string]*regexp.Regexp{
+	"css-unit-ch":      regexp.MustCompile(`\b\d+ch\b`),
+	"css-unit-initial": regexp.MustCompile(`:\s?initial\b`),
+	"css-unit-rem":     regexp.MustCompile(`\b\d+rem\b`),
+	"css-unit-vh":      regexp.MustCompile(`\b\d+vh\b`),
+	"css-unit-vmax":    regexp.MustCompile(`\b\d+vmax\b`),
+	"css-unit-vmin":    regexp.MustCompile(`\b\d+vmin\b`),
+	"css-unit-vw":      regexp.MustCompile(`\b\d+vw\b`),
+}

internal/htmlcheck/css.go

@@ -0,0 +1,251 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/vanng822/go-premailer/premailer"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Go cannot calculate any rendered CSS attributes, so we merge all styles
+// into the HTML and detect elements with styles containing the keywords.
+func runCSSTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	inlined, err := inlineRemoteCSS(html)
+	if err != nil {
+		inlined = html
+	}
+
+	// merge all CSS inline
+	merged, err := mergeInlineCSS(inlined)
+	if err != nil {
+		merged = inlined
+	}
+
+	reader := strings.NewReader(merged)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	inlineStyleResults := testInlineStyles(doc)
+	totalTests = totalTests + len(cssInlineRegexTests) + len(styleInlineAttributes)
+	for key, count := range inlineStyleResults {
+		result, err := cie.getTest(key)
+		if err == nil {
+			result.Score.Found = count
+			results = append(results, result)
+		}
+
+	}
+
+	// get a list of all generated styles from all nodes
+	allNodeStyles := []string{}
+	for _, n := range doc.Find("*[style]").Nodes {
+		style, err := tools.GetHTMLAttributeVal(n, "style")
+		if err == nil {
+			allNodeStyles = append(allNodeStyles, style)
+		}
+	}
+
+	for key, re := range cssRegexpUnitTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := 0
+		// loop through all styles to count total
+		for _, styles := range allNodeStyles {
+			found = found + len(re.FindAllString(styles, -1))
+		}
+
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	// get all inline CSS block data
+	reader = strings.NewReader(inlined)
+
+	// Load the HTML document
+	doc, _ = goquery.NewDocumentFromReader(reader)
+
+	cssCode := ""
+	for _, n := range doc.Find("style").Nodes {
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			cssCode = cssCode + c.Data
+		}
+	}
+
+	for key, re := range cssRegexpTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := len(re.FindAllString(cssCode, -1))
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	return results, totalTests, nil
+}
+
+// MergeInlineCSS merges header CSS into element attributes
+func mergeInlineCSS(html string) (string, error) {
+	options := premailer.NewOptions()
+	// options.RemoveClasses = true
+	// options.CssToAttributes = false
+	options.KeepBangImportant = true
+	pre, err := premailer.NewPremailerFromString(html, options)
+	if err != nil {
+		return "", err
+	}
+
+	return pre.Transform()
+}
+
+// InlineRemoteCSS searches the HTML for linked stylesheets, downloads the content, and
+// inserts new <style> blocks into the head, unless BlockRemoteCSSAndFonts is set
+func inlineRemoteCSS(h string) (string, error) {
+	reader := strings.NewReader(h)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return h, err
+	}
+
+	remoteCSS := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range remoteCSS {
+		attributes := link.Attr
+		for _, a := range attributes {
+			if a.Key == "href" {
+				if !isURL(a.Val) {
+					// skip invalid URL
+					continue
+				}
+
+				if config.BlockRemoteCSSAndFonts {
+					logger.Log().Debugf("[html-check] skip testing remote CSS content: %s (--block-remote-css-and-fonts)", a.Val)
+					return h, nil
+				}
+
+				resp, err := downloadToBytes(a.Val)
+				if err != nil {
+					logger.Log().Warnf("[html-check] failed to download %s", a.Val)
+					continue
+				}
+
+				// create new <style> block and insert downloaded CSS
+				styleBlock := &html.Node{
+					Type:     html.ElementNode,
+					Data:     "style",
+					DataAtom: atom.Style,
+				}
+				styleBlock.AppendChild(&html.Node{
+					Type: html.TextNode,
+					Data: string(resp),
+				})
+
+				link.Parent.AppendChild(styleBlock)
+			}
+		}
+	}
+
+	newDoc, err := doc.Html()
+	if err != nil {
+		logger.Log().Warnf("[html-check] failed to download %s", err.Error())
+		return h, err
+	}
+
+	return newDoc, nil
+}
+
+// DownloadToBytes returns a []byte slice from a URL
+func downloadToBytes(url string) ([]byte, error) {
+	client := http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	// Get the link response data
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		err := fmt.Errorf("Error downloading %s", url)
+		return nil, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+// Test if str is a URL
+func isURL(str string) bool {
+	u, err := url.Parse(str)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}
+
+// Test the HTML for inline CSS styles and styling attributes
+func testInlineStyles(doc *goquery.Document) map[string]int {
+	matches := make(map[string]int)
+
+	// find all elements containing a style attribute
+	styles := doc.Find("[style]").Nodes
+	for _, s := range styles {
+		style, err := tools.GetHTMLAttributeVal(s, "style")
+		if err != nil {
+			continue
+		}
+
+		for id, test := range cssInlineRegexTests {
+			if test.MatchString(style) {
+				if _, ok := matches[id]; !ok {
+					matches[id] = 0
+				}
+				matches[id]++
+			}
+		}
+	}
+
+	// find all elements containing styleInlineAttributes
+	for id, test := range styleInlineAttributes {
+		a := doc.Find(test).Nodes
+		if len(a) > 0 {
+			if _, ok := matches[id]; !ok {
+				matches[id] = 0
+			}
+			matches[id]++
+		}
+	}
+
+	return matches
+}

internal/htmlcheck/html.go

@@ -0,0 +1,102 @@
+package htmlcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// HTML tests
+func runHTMLTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	reader := strings.NewReader(html)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	// Almost all <script> is bad
+	scripts := len(doc.Find("script:not([type=\"application/ld+json\"]):not([type=\"application/json\"])").Nodes)
+	if scripts > 0 {
+		var result = Warning{}
+		result.Title = "<script> element"
+		result.Slug = "html-script"
+		result.Category = "html"
+		result.Description = "JavaScript is not supported in any email client."
+		result.Tags = []string{}
+		result.Results = []Result{}
+		result.NotesByNumber = map[string]string{}
+		result.Score.Found = scripts
+		result.Score.Supported = 0
+		result.Score.Partial = 0
+		result.Score.Unsupported = 100
+		results = append(results, result)
+		totalTests++
+	}
+
+	for key, test := range htmlTests {
+		totalTests++
+		if test == "body" {
+			re := regexp.MustCompile(`(?im)</body>`)
+			if re.MatchString(html) {
+				result, err := cie.getTest(key)
+				if err != nil {
+					return results, totalTests, err
+				}
+
+				result.Score.Found = 1
+				results = append(results, result)
+			}
+		} else if len(doc.Find(test).Nodes) > 0 {
+			result, err := cie.getTest(key)
+			if err != nil {
+				return results, totalTests, err
+			}
+			totalTests++
+
+			result.Score.Found = len(doc.Find(test).Nodes)
+
+			results = append(results, result)
+		}
+	}
+
+	// find all images
+	images := doc.Find("img[src]").Nodes
+	imageResults := make(map[string]int)
+	totalTests = totalTests + len(imageRegexpTests)
+
+	for _, image := range images {
+		src, err := tools.GetHTMLAttributeVal(image, "src")
+		if err != nil {
+			continue
+		}
+		for key, test := range imageRegexpTests {
+			if test.MatchString(src) {
+				matches, exists := imageResults[key]
+				if exists {
+					imageResults[key] = matches + 1
+				} else {
+					imageResults[key] = 1
+				}
+
+			}
+		}
+	}
+
+	for key, found := range imageResults {
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+		result.Score.Found = found
+		results = append(results, result)
+	}
+
+	return results, totalTests, nil
+}

internal/htmlcheck/inline_test.go

@@ -0,0 +1,81 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+func TestInlineStyleDetection(t *testing.T) {
+	/// tests should contain the HTML test, and expected test results in alphabetical order
+	tests := map[string]string{}
+	tests[`<h1 style="transform: rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="color: green; transform:rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="color:green; transform :rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="transform:rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="TRANSFORM:rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="transform:	rotate(20deg)">Heading</h1>`] = "css-transform"
+	tests[`<h1 style="ignore-transform: something">Heading</h1>`] = "" // no match
+	tests[`<h1 style="text-transform: uppercase">Heading</h1>`] = "css-text-transform"
+	tests[`<h1 style="text-transform: uppercase; text-transform: uppercase">Heading</h1>`] = "css-text-transform"
+	tests[`<h1 style="test-transform: uppercase">Heading</h1>`] = "" // no match
+	tests[`<h1 style="padding-inline-start: 5rem">Heading</h1>`] = "css-padding-inline-start-end"
+	tests[`<h1 style="margin-inline-end: 5rem">Heading</h1>`] = "css-margin-inline-start-end"
+	tests[`<h1 style="margin-inline-middle: 5rem">Heading</h1>`] = "" // no match
+	tests[`<h1 style="color:green!important">Heading</h1>`] = "css-important"
+	tests[`<h1 style="color: green !important">Heading</h1>`] = "css-important"
+	tests[`<h1 style="color: green!important;">Heading</h1>`] = "css-important"
+	tests[`<h1 style="color:green!important-stuff;">Heading</h1>`] = "" // no match
+	tests[`<h1 style="background-image:url('img.jpg')">Heading</h1>`] = "css-background-image"
+	tests[`<h1 style="background-image:url('img.jpg'); color: green">Heading</h1>`] = "css-background-image"
+	tests[`<h1 style=" color:green; background-image:url('img.jpg');">Heading</h1>`] = "css-background-image"
+	tests[`<h1 style="display  : flex ;">Heading</h1>`] = "css-display,css-display-flex"
+	tests[`<h1 style="DISPLAY:FLEX;">Heading</h1>`] = "css-display,css-display-flex"
+	tests[`<h1 style="display: flexing;">Heading</h1>`] = "css-display" // should not match css-display-flex rule
+	tests[`<h1 style="line-height: 1rem;opacity: 0.5; width: calc(10px + 100px)">Heading</h1>`] = "css-line-height,css-opacity,css-unit-calc,css-width"
+	tests[`<h1 style="color: rgb(255,255,255);">Heading</h1>`] = "css-rgb"
+	tests[`<h1 style="color:rgb(255,255,255);">Heading</h1>`] = "css-rgb"
+	tests[`<h1 style="color:rgb(255,255,255);">Heading</h1>`] = "css-rgb"
+	tests[`<h1 style="color:rgba(255,255,255, 0);">Heading</h1>`] = "css-rgba"
+	tests[`<h1 style="border: solid rgb(255,255,255) 1px; color:rgba(255,255,255, 0);">Heading</h1>`] = "css-border,css-rgb,css-rgba"
+	tests[`<h1 border="2">Heading</h1>`] = "css-border"
+	tests[`<h1 border="2" background="green">Heading</h1>`] = "css-background,css-border"
+	tests[`<h1 BORDER="2" BACKGROUND="GREEN">Heading</h1>`] = "css-background,css-border"
+	tests[`<h1 border-something="2" background-something="green">Heading</h1>`] = "" // no match
+	tests[`<h1 border="2" style="border: solid green 1px!important">Heading</h1>`] = "css-border,css-important"
+
+	for html, expected := range tests {
+		reader := strings.NewReader(html)
+		doc, err := goquery.NewDocumentFromReader(reader)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		results := testInlineStyles(doc)
+
+		matches := []string{}
+		uniqMap := make(map[string]bool)
+		for key := range results {
+			if _, exists := uniqMap[key]; !exists {
+				matches = append(matches, key)
+			}
+		}
+
+		// ensure results are sorted to ensure consistent results
+		sort.Strings(matches)
+
+		assertEqual(t, expected, strings.Join(matches, ","), fmt.Sprintf("inline style detection \"%s\"", html))
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}

internal/htmlcheck/main.go

@@ -0,0 +1,201 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+)
+
+// RunTests will run all tests on an HTML string
+func RunTests(html string) (Response, error) {
+	s := Response{}
+	s.Warnings = []Warning{}
+	if platforms, err := Platforms(); err == nil {
+		s.Platforms = platforms
+	}
+
+	s.Total = Total{}
+
+	// crude way to determine whether the HTML contains a <body> structure
+	// or whether it's just plain HTML content
+	re := regexp.MustCompile(`(?im)</body>`)
+	nodeMatch := "body *, script"
+	if re.MatchString(html) {
+		nodeMatch = "*:not(html):not(head):not(meta), script"
+	}
+	reader := strings.NewReader(html)
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return s, err
+	}
+	// calculate the number of nodes in HTML
+	s.Total.Nodes = len(doc.Find(nodeMatch).Nodes)
+
+	if err := loadJSONData(); err != nil {
+		return s, err
+	}
+
+	// HTML tests
+	htmlResults, totalTests, err := runHTMLTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add html test totals
+	s.Warnings = append(s.Warnings, htmlResults...)
+
+	// CSS tests
+	cssResults, totalTests, err := runCSSTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add css test totals
+	s.Warnings = append(s.Warnings, cssResults...)
+
+	// calculate total score
+	var partial, unsupported float32
+	partial = 0
+	unsupported = 0
+
+	for _, w := range s.Warnings {
+		if w.Score.Found == 0 {
+			continue
+		}
+
+		// supported is calculated by subtracting partial and unsupported from 100%
+		if w.Score.Partial > 0 {
+			weighted := w.Score.Partial * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > partial {
+				partial = weighted
+			}
+		}
+		if w.Score.Unsupported > 0 {
+			weighted := w.Score.Unsupported * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > unsupported {
+				unsupported = weighted
+			}
+		}
+	}
+
+	s.Total.Supported = 100 - partial - unsupported
+	s.Total.Partial = partial
+	s.Total.Unsupported = unsupported
+
+	// sort slice to get lowest scores first
+	sort.Slice(s.Warnings, func(i, j int) bool {
+		return (s.Warnings[i].Score.Unsupported+s.Warnings[i].Score.Partial)*float32(s.Warnings[i].Score.Found)/float32(s.Total.Nodes) >
+			(s.Warnings[j].Score.Unsupported+s.Warnings[j].Score.Partial)*float32(s.Warnings[j].Score.Found)/float32(s.Total.Nodes)
+	})
+
+	return s, nil
+}
+
+// Test returns a test
+func (c CanIEmail) getTest(k string) (Warning, error) {
+	warning := Warning{}
+	exists := false
+	found := JSONResult{}
+	for _, r := range cie.Data {
+		if r.Slug == k {
+			found = r
+			exists = true
+			break
+		}
+	}
+
+	if !exists {
+		return warning, fmt.Errorf("%s does not exist", k)
+	}
+
+	warning.Slug = found.Slug
+	warning.Title = found.Title
+	warning.Description = mdToHTML(found.Description)
+	warning.Category = found.Category
+	warning.URL = found.URL
+	warning.Tags = found.Tags
+	// warning.Keywords = found.Keywords
+	// warning.Notes = found.Notes
+	warning.NotesByNumber = make(map[string]string, len(found.NotesByNumber))
+	for nr, note := range found.NotesByNumber {
+		warning.NotesByNumber[nr] = mdToHTML(note)
+	}
+	warning.Results = []Result{}
+
+	var y, n, p float32
+
+	for family, stats := range found.Stats {
+		if len(LimitFamilies) != 0 && !tools.InArray(family, LimitFamilies) {
+			continue
+		}
+
+		for platform, clients := range stats.(map[string]interface{}) {
+			if len(LimitPlatforms) != 0 && !tools.InArray(platform, LimitPlatforms) {
+				continue
+			}
+			for version, support := range clients.(map[string]interface{}) {
+				s := Result{}
+				s.Name = fmt.Sprintf("%s %s (%s)", c.NiceNames.Family[family], c.NiceNames.Platform[platform], version)
+				s.Family = family
+				s.Platform = platform
+				s.Version = version
+
+				if support == "y" {
+					y++
+					s.Support = "yes"
+				} else if support == "n" {
+					n++
+					s.Support = "no"
+				} else {
+					p++
+					s.Support = "partial"
+
+					noteIDS := noteMatch.FindStringSubmatch(fmt.Sprintf("%s", support))
+
+					for _, id := range noteIDS {
+						s.NoteNumber = id
+					}
+				}
+
+				warning.Results = append(warning.Results, s)
+			}
+		}
+
+	}
+
+	total := y + n + p
+	warning.Score.Supported = y / total * 100
+	warning.Score.Unsupported = n / total * 100
+	warning.Score.Partial = p / total * 100
+
+	return warning, nil
+}
+
+// Convert markdown to HTML, stripping <p> & </p>
+func mdToHTML(str string) string {
+	md := []byte(str)
+	// create markdown parser with extensions
+	extensions := parser.CommonExtensions | parser.AutoHeadingIDs | parser.NoEmptyLineBeforeBlock
+	// extensions := parser.NoExtensions
+	p := parser.NewWithExtensions(extensions)
+	doc := p.Parse(md)
+
+	// create HTML renderer with extensions
+	htmlFlags := html.CommonFlags | html.HrefTargetBlank
+	opts := html.RendererOptions{Flags: htmlFlags}
+	renderer := html.NewRenderer(opts)
+
+	return strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(string(markdown.Render(doc, renderer))), "<p>"), "</p>")
+}

internal/htmlcheck/platforms.go

@@ -0,0 +1,42 @@
+package htmlcheck
+
+import (
+	"sort"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Platforms returns all platforms with their respective email clients
+func Platforms() (map[string][]string, error) {
+	// [platform]clients
+	data := make(map[string][]string)
+
+	if err := loadJSONData(); err != nil {
+		return data, err
+	}
+
+	for _, t := range cie.Data {
+		for family, stats := range t.Stats {
+			niceFamily := cie.NiceNames.Family[family]
+			for platform := range stats.(map[string]interface{}) {
+				c, found := data[platform]
+				if !found {
+					data[platform] = []string{}
+				}
+				if !tools.InArray(niceFamily, c) {
+					c = append(c, niceFamily)
+					data[platform] = c
+				}
+			}
+		}
+	}
+
+	for group, clients := range data {
+		sort.Slice(clients, func(i, j int) bool {
+			return clients[i] < clients[j]
+		})
+		data[group] = clients
+	}
+
+	return data, nil
+}

internal/htmlcheck/structs.go

@@ -0,0 +1,87 @@
+package htmlcheck
+
+// Response represents the HTML check response struct
+//
+// swagger:model HTMLCheckResponse
+type Response struct {
+	// List of warnings from tests
+	Warnings []Warning `json:"Warnings"`
+	// All platforms tested, mainly for the web UI
+	Platforms map[string][]string `json:"Platforms"`
+	// Total overall score
+	Total Total `json:"Total"`
+}
+
+// Warning represents a failed test
+//
+// swagger:model HTMLCheckWarning
+type Warning struct {
+	// Slug identifier
+	Slug string `json:"Slug"`
+	// Friendly title
+	Title string `json:"Title"`
+	// Description
+	Description string `json:"Description"`
+	// URL to caniemail.com
+	URL string `json:"URL"`
+	// Category [css, html]
+	Category string `json:"Category"`
+	// Tags
+	Tags []string `json:"Tags"`
+	// Keywords
+	Keywords string `json:"Keywords"`
+	// Test results
+	Results []Result `json:"Results"`
+	// Notes based on results
+	NotesByNumber map[string]string `json:"NotesByNumber"`
+	// Test score calculated from results
+	Score Score `json:"Score"`
+}
+
+// Result struct
+//
+// swagger:model HTMLCheckResult
+type Result struct {
+	// Friendly name of result, combining family, platform & version
+	Name string `json:"Name"`
+	// Platform eg: ios, android, windows
+	Platform string `json:"Platform"`
+	// Family eg: Outlook, Mozilla Thunderbird
+	Family string `json:"Family"`
+	// Family version eg: 4.7.1, 2019-10, 10.3
+	Version string `json:"Version"`
+	// Support [yes, no, partial]
+	Support string `json:"Support"`
+	// Note number for partially supported if applicable
+	NoteNumber string `json:"NoteNumber"` // where applicable
+}
+
+// Score struct
+//
+// swagger:model HTMLCheckScore
+type Score struct {
+	// Number of matches in the document
+	Found int `json:"Found"`
+	// Total percentage supported
+	Supported float32 `json:"Supported"`
+	// Total percentage partially supported
+	Partial float32 `json:"Partial"`
+	// Total percentage unsupported
+	Unsupported float32 `json:"Unsupported"`
+}
+
+// Total weighted result for all scores
+//
+// swagger:model HTMLCheckTotal
+type Total struct {
+	// Total number of tests done
+	Tests int `json:"Tests"`
+	// Total number of HTML nodes detected in message
+	Nodes int `json:"Nodes"`
+	// Overall percentage supported
+	Supported float32 `json:"Supported"`
+	// Overall percentage partially supported
+	Partial float32 `json:"Partial"` // total percentage
+	// Overall percentage unsupported
+	Unsupported float32 `json:"Unsupported"` // total percentage
+}

internal/linkcheck/linkcheck_test.go

@@ -0,0 +1,71 @@
+package linkcheck
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/axllent/mailpit/internal/storage"
+)
+
+var (
+	testHTML = `
+	<html>
+	<head>
+		<link rel=stylesheet href="http://remote-host/style.css"></link>
+		<script async src="https://www.googletagmanager.com/gtag/js?id=ignored"></script>
+	</head>
+	<body>
+		<div>
+			<p><a href="http://example.com">HTTP link</a></p>
+			<p><a href="https://example.com">HTTPS link</a></p>
+			<p><a href="HTTPS://EXAMPLE.COM">HTTPS link</a></p>
+			<p><a href="http://localhost">Localhost link</a> (ignored)</p>
+			<p><a href="https://localhost">Localhost link</a> (ignored)</p>
+			<p><a href='https://127.0.0.1'>Single quotes link</a> (ignored)</p>
+			<p><img src=https://example.com/image.jpg></p>
+			<p href="http://invalid-link.com">This should be ignored</p>
+			<p><a href="http://link with spaces">Link with spaces</a></p>
+			<p><a href="http://example.com/?blaah=yes&amp;test=true">URL-encoded characters</a></p>
+		</div>
+	</body>
+	</html>`
+
+	expectedHTMLLinks = []string{
+		"http://example.com", "https://example.com", "HTTPS://EXAMPLE.COM", "http://localhost", "https://localhost", "https://127.0.0.1", "http://link with spaces", "http://example.com/?blaah=yes&test=true",
+		"http://remote-host/style.css",  // css
+		"https://example.com/image.jpg", // images
+	}
+
+	testTextLinks = `This is a line with http://example.com https://example.com
+		HTTPS://EXAMPLE.COM
+		[http://localhost]
+		www.google.com < ignored
+		|||http://example.com/?some=query-string|||
+	`
+
+	expectedTextLinks = []string{
+		"http://example.com", "https://example.com", "HTTPS://EXAMPLE.COM", "http://localhost", "http://example.com/?some=query-string",
+	}
+)
+
+func TestLinkDetection(t *testing.T) {
+
+	t.Log("Testing HTML link detection")
+
+	m := storage.Message{}
+
+	m.Text = testTextLinks
+	m.HTML = testHTML
+
+	textLinks := extractTextLinks(&m)
+
+	if !reflect.DeepEqual(textLinks, expectedTextLinks) {
+		t.Fatalf("Failed to detect text links correctly")
+	}
+
+	htmlLinks := extractHTMLLinks(&m)
+
+	if !reflect.DeepEqual(htmlLinks, expectedHTMLLinks) {
+		t.Fatalf("Failed to detect HTML links correctly")
+	}
+}

internal/linkcheck/main.go

@@ -0,0 +1,90 @@
+// Package linkcheck handles message links checking
+package linkcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+var linkRe = regexp.MustCompile(`(?im)\b(http|https):\/\/([\-\w@:%_\+'!.~#?,&\/\/=;]+)`)
+
+// RunTests will run all tests on an HTML string
+func RunTests(msg *storage.Message, followRedirects bool) (Response, error) {
+	s := Response{}
+
+	allLinks := extractHTMLLinks(msg)
+	allLinks = strUnique(append(allLinks, extractTextLinks(msg)...))
+	s.Links = getHTTPStatuses(allLinks, followRedirects)
+
+	for _, l := range s.Links {
+		if l.StatusCode >= 400 || l.StatusCode == 0 {
+			s.Errors++
+		}
+	}
+
+	return s, nil
+}
+
+func extractTextLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	links = append(links, linkRe.FindAllString(msg.Text, -1)...)
+
+	return links
+}
+
+func extractHTMLLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	reader := strings.NewReader(msg.HTML)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return links
+	}
+
+	aLinks := doc.Find("a[href]").Nodes
+	for _, link := range aLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	cssLinks := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range cssLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	imgLinks := doc.Find("img[src]").Nodes
+	for _, link := range imgLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "src")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	return links
+}
+
+// strUnique return a slice of unique strings from a slice
+func strUnique(strSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range strSlice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	return list
+}

internal/linkcheck/status.go

@@ -0,0 +1,114 @@
+package linkcheck
+
+import (
+	"crypto/tls"
+	"net/http"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+func getHTTPStatuses(links []string, followRedirects bool) []Link {
+	// allow 5 threads
+	threads := make(chan int, 5)
+
+	results := make(map[string]Link, len(links))
+	resultsMutex := sync.RWMutex{}
+
+	output := []Link{}
+
+	var wg sync.WaitGroup
+
+	for _, l := range links {
+		wg.Add(1)
+		go func(link string, w *sync.WaitGroup) {
+			threads <- 1 // will block if MAX threads
+			defer w.Done()
+
+			code, err := doHead(link, followRedirects)
+			l := Link{}
+			l.URL = link
+			if err != nil {
+				l.StatusCode = 0
+				l.Status = httpErrorSummary(err)
+			} else {
+				l.StatusCode = code
+				l.Status = http.StatusText(code)
+			}
+			resultsMutex.Lock()
+			results[link] = l
+			resultsMutex.Unlock()
+
+			<-threads // remove from threads
+		}(l, &wg)
+	}
+
+	wg.Wait()
+
+	for _, l := range results {
+		output = append(output, l)
+	}
+
+	return output
+}
+
+// Do a HEAD request to return HTTP status code
+func doHead(link string, followRedirects bool) (int, error) {
+
+	timeout := time.Duration(10 * time.Second)
+
+	tr := &http.Transport{}
+
+	if config.AllowUntrustedTLS {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec
+	}
+
+	client := http.Client{
+		Timeout:   timeout,
+		Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if followRedirects {
+				return nil
+			}
+			return http.ErrUseLastResponse
+		},
+	}
+
+	req, err := http.NewRequest("HEAD", link, nil)
+	if err != nil {
+		logger.Log().Errorf("[link-check] %s", err.Error())
+		return 0, err
+	}
+
+	req.Header.Set("User-Agent", "Mailpit/"+config.Version)
+
+	res, err := client.Do(req)
+	if err != nil {
+		if res != nil {
+			return res.StatusCode, err
+		}
+
+		return 0, err
+
+	}
+
+	return res.StatusCode, nil
+}
+
+// HTTP errors include a lot more info that just the actual error, so this
+// tries to take the final part of it, eg: `no such host`
+func httpErrorSummary(err error) string {
+	var re = regexp.MustCompile(`.*: (.*)$`)
+
+	e := err.Error()
+	if !re.MatchString(e) {
+		return e
+	}
+
+	parts := re.FindAllStringSubmatch(e, -1)
+
+	return parts[0][len(parts[0])-1]
+}

internal/linkcheck/structs.go

@@ -0,0 +1,21 @@
+package linkcheck
+
+// Response represents the Link check response
+//
+// swagger:model LinkCheckResponse
+type Response struct {
+	// Total number of errors
+	Errors int `json:"Errors"`
+	// Tested links
+	Links []Link `json:"Links"`
+}
+
+// Link struct
+type Link struct {
+	// Link URL
+	URL string `json:"URL"`
+	// HTTP status code
+	StatusCode int `json:"StatusCode"`
+	// HTTP status definition
+	Status string `json:"Status"`
+}

internal/logger/logger.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 
 	"github.com/sirupsen/logrus"
@@ -18,6 +19,8 @@ var (
 	QuietLogging bool
 	// NoLogging shows only fatal errors
 	NoLogging bool
+	// LogFile sets a log file
+	LogFile string
 )
 
 // Log returns the logger instance
@@ -36,11 +39,21 @@ func Log() *logrus.Logger {
 			log.SetLevel(logrus.PanicLevel)
 		}
 
-		log.Out = os.Stdout
+		if LogFile != "" {
+			file, err := os.OpenFile(filepath.Clean(LogFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0664) // #nosec
+			if err == nil {
+				log.Out = file
+			} else {
+				log.Out = os.Stdout
+				log.Warn("Failed to log to file, using default stderr")
+			}
+		} else {
+			log.Out = os.Stdout
+		}
+
 		log.SetFormatter(&logrus.TextFormatter{
 			FullTimestamp:   true,
 			TimestampFormat: "2006/01/02 15:04:05",
-			ForceColors:     true,
 		})
 	}
 

internal/pop3/functions.go

@@ -0,0 +1,84 @@
+package pop3
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+func authUser(username, password string) bool {
+	return auth.POP3Credentials.Match(username, password)
+}
+
+// Send a response with debug logging
+func sendResponse(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+	logger.Log().Debugf("[pop3] response: %s", m)
+
+	if strings.HasPrefix(m, "-ERR ") {
+		sub, _ := strings.CutPrefix(m, "-ERR ")
+		websockets.BroadCastClientError("error", "pop3", c.RemoteAddr().String(), sub)
+	}
+}
+
+// Send a response without debug logging (for data)
+func sendData(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+}
+
+// Get the latest 100 messages
+func getMessages() ([]message, error) {
+	messages := []message{}
+	list, err := storage.List(0, 0, 100)
+	if err != nil {
+		return messages, err
+	}
+
+	for _, m := range list {
+		msg := message{}
+		msg.ID = m.ID
+		msg.Size = m.Size
+		messages = append(messages, msg)
+	}
+
+	return messages, nil
+}
+
+// POP3 TOP command returns the headers, followed by the next x lines
+func getTop(id string, nr int) (string, string, error) {
+	var header, body string
+	raw, err := storage.GetMessageRaw(id)
+	if err != nil {
+		return header, body, errors.New("-ERR no such message")
+	}
+
+	parts := strings.SplitN(string(raw), "\r\n\r\n", 2)
+	header = parts[0]
+	lines := []string{}
+	if nr > 0 && len(parts) == 2 {
+		lines = strings.SplitN(parts[1], "\r\n", nr)
+	}
+
+	return header, strings.Join(lines, "\r\n"), nil
+}
+
+// cuts the line into command and arguments
+func getCommand(line string) (string, []string) {
+	line = strings.Trim(line, "\r \n")
+	cmd := strings.Split(line, " ")
+	return cmd[0], cmd[1:]
+}
+
+func getSafeArg(args []string, nr int) (string, error) {
+	if nr < len(args) {
+		return args[nr], nil
+	}
+
+	return "", errors.New("-ERR out of range")
+}

internal/pop3/pop3_test.go

@@ -0,0 +1,365 @@
+package pop3
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/pop3client"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	testingPort int
+)
+
+func TestPOP3(t *testing.T) {
+	t.Log("Testing POP3 server")
+	setup()
+	defer storage.Close()
+
+	// connect with bad password
+	t.Log("Testing invalid login")
+	c, err := connectBadAuth()
+	if err == nil {
+		t.Error("invalid login gained access")
+		return
+	}
+
+	t.Log("Testing valid login")
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, size, err := c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 0, "incorrect message count")
+	assertEqual(t, size, 0, "incorrect size")
+
+	// quit else we get old data
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Inserting 50 messages")
+
+	insertEmailData(t) // insert 50 messages
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 50, "incorrect message count")
+
+	t.Log("Fetching 20 messages")
+
+	for i := 1; i <= 20; i++ {
+		_, err := c.Retr(i)
+		if err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	t.Log("Deleting 25 messages")
+
+	for i := 1; i <= 25; i++ {
+		if err := c.Dele(i); err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	// messages get deleted after a QUIT
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Fetching message count")
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 25, "incorrect message count")
+
+	// messages get deleted after a QUIT
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Deleting 25 messages")
+
+	for i := 1; i <= 25; i++ {
+		if err := c.Dele(i); err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	t.Log("Undeleting messages")
+
+	if err := c.Rset(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 25, "incorrect message count")
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+}
+
+func TestAuthentication(t *testing.T) {
+	// commands only allowed after authentication
+	authCommands := make(map[string]bool)
+	authCommands["STAT"] = false
+	authCommands["LIST"] = true
+	authCommands["NOOP"] = false
+	authCommands["RSET"] = false
+	authCommands["RETR 1"] = true
+
+	t.Log("Testing authenticated commands while not logged in")
+	setup()
+	defer storage.Close()
+
+	insertEmailData(t) // insert 50 messages
+
+	// non-authenticated connection
+	c, err := connect()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	for cmd, multi := range authCommands {
+		if _, err := c.Cmd(cmd, multi); err == nil {
+			t.Errorf("%s should require authentication", cmd)
+			return
+		}
+
+		if _, err := c.Cmd(strings.ToLower(cmd), multi); err == nil {
+			t.Errorf("%s should require authentication", cmd)
+			return
+		}
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Testing authenticated commands while logged in")
+
+	// authenticated connection
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	for cmd, multi := range authCommands {
+		if _, err := c.Cmd(cmd, multi); err != nil {
+			t.Errorf("%s should work when authenticated", cmd)
+			return
+		}
+
+		if _, err := c.Cmd(strings.ToLower(cmd), multi); err != nil {
+			t.Errorf("%s should work when authenticated", cmd)
+			return
+		}
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+}
+
+func setup() {
+	auth.SetPOP3Auth("username:password")
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.Database = os.Getenv("MP_DATABASE")
+	var foundPort bool
+	for !foundPort {
+		testingPort = randRange(1111, 2000)
+		if portFree(testingPort) {
+			foundPort = true
+		}
+	}
+
+	config.POP3Listen = fmt.Sprintf("localhost:%d", testingPort)
+
+	if err := storage.InitDB(); err != nil {
+		panic(err)
+	}
+
+	if err := storage.DeleteAllMessages(); err != nil {
+		panic(err)
+	}
+
+	go Run()
+
+	time.Sleep(time.Second)
+}
+
+// connect and authenticate
+func connectAuth() (*pop3client.Conn, error) {
+	c, err := connect()
+	if err != nil {
+		return c, err
+	}
+
+	err = c.Auth("username", "password")
+
+	return c, err
+}
+
+// connect and authenticate
+func connectBadAuth() (*pop3client.Conn, error) {
+	c, err := connect()
+	if err != nil {
+		return c, err
+	}
+
+	err = c.Auth("username", "notPassword")
+
+	return c, err
+}
+
+// connect but do not authenticate
+func connect() (*pop3client.Conn, error) {
+	p := pop3client.New(pop3client.Opt{
+		Host:       "localhost",
+		Port:       testingPort,
+		TLSEnabled: false,
+	})
+
+	c, err := p.NewConn()
+	if err != nil {
+		return c, err
+	}
+
+	return c, err
+}
+
+func portFree(port int) bool {
+	ln, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
+	if err != nil {
+		return false
+	}
+
+	if err := ln.Close(); err != nil {
+		panic(err)
+	}
+
+	return true
+}
+
+func randRange(min, max int) int {
+	return rand.IntN(max-min) + min
+}
+
+func insertEmailData(t *testing.T) {
+	for i := 0; i < 50; i++ {
+		msg := enmime.Builder().
+			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+			Subject(fmt.Sprintf("Subject line %d end", i)).
+			Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+			To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i))
+
+		env, err := msg.Build()
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := env.Encode(buf); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		bufBytes := buf.Bytes()
+
+		id, err := storage.Store(&bufBytes)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		if _, err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}

internal/pop3/server.go

@@ -0,0 +1,319 @@
+// Package pop3 is a simple POP3 server for Mailpit.
+// By default it is disabled unless password credentials have been loaded.
+//
+// References: https://github.com/r0stig/golang-pop3 | https://github.com/inbucket/inbucket
+// See RFC: https://datatracker.ietf.org/doc/html/rfc1939
+package pop3
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+const (
+	// AUTHORIZATION is the initial state
+	AUTHORIZATION = 1
+	// TRANSACTION is the state after login
+	TRANSACTION = 2
+	// UPDATE is the state before closing
+	UPDATE = 3
+)
+
+// Run will start the POP3 server if enabled
+func Run() {
+	if auth.POP3Credentials == nil || config.POP3Listen == "" {
+		// POP3 server is disabled without authentication
+		return
+	}
+
+	var listener net.Listener
+	var err error
+
+	if config.POP3TLSCert != "" {
+		cer, err2 := tls.LoadX509KeyPair(config.POP3TLSCert, config.POP3TLSKey)
+		if err2 != nil {
+			logger.Log().Errorf("[pop3] %s", err2.Error())
+			return
+		}
+
+		tlsConfig := &tls.Config{
+			Certificates: []tls.Certificate{cer},
+			MinVersion:   tls.VersionTLS12,
+		}
+
+		listener, err = tls.Listen("tcp", config.POP3Listen, tlsConfig)
+	} else {
+		// unencrypted
+		listener, err = net.Listen("tcp", config.POP3Listen)
+	}
+
+	if err != nil {
+		logger.Log().Errorf("[pop3] %s", err.Error())
+		return
+	}
+
+	logger.Log().Infof("[pop3] starting on %s", config.POP3Listen)
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			logger.Log().Errorf("[pop3] accept error: %s", err.Error())
+			continue
+		}
+
+		// run as goroutine
+		go handleClient(conn)
+	}
+}
+
+type message struct {
+	ID   string
+	Size float64
+}
+
+func handleClient(conn net.Conn) {
+	var (
+		user     = ""
+		state    = AUTHORIZATION // Start with AUTHORIZATION state
+		toDelete []string        // Track messages marked for deletion
+		messages []message
+	)
+
+	defer func() {
+		if state == UPDATE {
+			if len(toDelete) > 0 {
+				if err := storage.DeleteMessages(toDelete); err != nil {
+					logger.Log().Errorf("[pop3] error deleting: %s", err.Error())
+				}
+				// Update web UI to remove deleted messages
+				websockets.Broadcast("prune", nil)
+			}
+		}
+
+		if err := conn.Close(); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+		}
+	}()
+
+	reader := bufio.NewReader(conn)
+
+	logger.Log().Debugf("[pop3] connection opened by %s", conn.RemoteAddr().String())
+
+	// First welcome the new connection
+	serverName := "Mailpit"
+	if config.Label != "" {
+		serverName = fmt.Sprintf("Mailpit (%s)", config.Label)
+	}
+	sendResponse(conn, fmt.Sprintf("+OK %s POP3 server", serverName))
+
+	// Set 10 minutes timeout according to RFC1939
+	timeoutDuration := 600 * time.Second
+
+	for {
+		// Set read deadline
+		if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+			return
+		}
+
+		// Reads a line from the client
+		rawLine, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				logger.Log().Debugf("[pop3] client disconnected: %s", conn.RemoteAddr().String())
+			} else {
+				logger.Log().Errorf("[pop3] read error: %s", err.Error())
+			}
+			return
+		}
+
+		// Parses the command
+		cmd, args := getCommand(rawLine)
+		cmd = strings.ToUpper(cmd) // Commands in the POP3 are case-insensitive
+
+		logger.Log().Debugf("[pop3] received: %s (%s)", strings.TrimSpace(rawLine), conn.RemoteAddr().String())
+
+		switch cmd {
+		case "CAPA":
+			// List our capabilities per RFC2449
+			sendResponse(conn, "+OK capability list follows")
+			sendResponse(conn, "TOP")
+			sendResponse(conn, "USER")
+			sendResponse(conn, "UIDL")
+			sendResponse(conn, "IMPLEMENTATION Mailpit")
+			sendResponse(conn, ".")
+		case "USER":
+			if state == AUTHORIZATION {
+				if len(args) != 1 {
+					sendResponse(conn, "-ERR must supply a user")
+					return
+				}
+				sendResponse(conn, "+OK")
+				user = args[0]
+			} else {
+				sendResponse(conn, "-ERR user already specified")
+			}
+		case "PASS":
+			if state == AUTHORIZATION {
+				if user == "" {
+					sendResponse(conn, "-ERR must supply a user")
+					return
+				}
+				if len(args) != 1 {
+					sendResponse(conn, "-ERR must supply a password")
+					return
+				}
+
+				pass := args[0]
+				if authUser(user, pass) {
+					sendResponse(conn, "+OK signed in")
+					var err error
+					messages, err = getMessages()
+					if err != nil {
+						logger.Log().Errorf("[pop3] %s", err.Error())
+					}
+					state = TRANSACTION
+				} else {
+					sendResponse(conn, "-ERR invalid password")
+					logger.Log().Warnf("[pop3] failed login: %s", user)
+				}
+			} else {
+				sendResponse(conn, "-ERR user not specified")
+			}
+		case "STAT", "LIST", "UIDL", "RETR", "TOP", "NOOP", "DELE", "RSET":
+			if state == TRANSACTION {
+				handleTransactionCommand(conn, cmd, args, messages, &toDelete)
+			} else {
+				sendResponse(conn, "-ERR user not authenticated")
+			}
+		case "QUIT":
+			sendResponse(conn, "+OK goodbye")
+			state = UPDATE
+			return
+		default:
+			sendResponse(conn, "-ERR unknown command")
+		}
+	}
+}
+
+func handleTransactionCommand(conn net.Conn, cmd string, args []string, messages []message, toDelete *[]string) {
+	switch cmd {
+	case "STAT":
+		totalSize := float64(0)
+		for _, m := range messages {
+			totalSize += m.Size
+		}
+		sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), int64(totalSize)))
+	case "LIST":
+		totalSize := float64(0)
+		for _, m := range messages {
+			totalSize += m.Size
+		}
+		sendResponse(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), int64(totalSize)))
+
+		for row, m := range messages {
+			sendResponse(conn, fmt.Sprintf("%d %d", row+1, int64(m.Size))) // Convert Size to int64 when printing
+		}
+		sendResponse(conn, ".")
+	case "UIDL":
+		sendResponse(conn, "+OK unique-id listing follows")
+		for row, m := range messages {
+			sendResponse(conn, fmt.Sprintf("%d %s", row+1, m.ID))
+		}
+		sendResponse(conn, ".")
+	case "RETR":
+		if len(args) != 1 {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		nr, err := strconv.Atoi(args[0])
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		m := messages[nr-1]
+		raw, err := storage.GetMessageRaw(m.ID)
+		if err != nil {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+		size := len(raw)
+		sendResponse(conn, fmt.Sprintf("+OK %d octets", size))
+
+		// When all lines of the response have been sent, a
+		// final line is sent, consisting of a termination octet (decimal code
+		// 046, ".") and a CRLF pair. If any line of the multi-line response
+		// begins with the termination octet, the line is "byte-stuffed" by
+		// pre-pending the termination octet to that line of the response.
+		// @see: https://www.ietf.org/rfc/rfc1939.txt
+		sendData(conn, strings.Replace(string(raw), "\n.", "\n..", -1))
+		sendResponse(conn, ".")
+	case "TOP":
+		arg, err := getSafeArg(args, 0)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+		nr, err := strconv.Atoi(arg)
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		arg2, err := getSafeArg(args, 1)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+
+		lines, err := strconv.Atoi(arg2)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+
+		m := messages[nr-1]
+		headers, body, err := getTop(m.ID, lines)
+		if err != nil {
+			sendResponse(conn, err.Error())
+			return
+		}
+
+		sendResponse(conn, "+OK top of message follows")
+		sendData(conn, headers+"\r\n")
+		sendData(conn, body)
+		sendResponse(conn, ".")
+	case "NOOP":
+		sendResponse(conn, "+OK")
+	case "DELE":
+		arg, _ := getSafeArg(args, 0)
+		nr, err := strconv.Atoi(arg)
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		m := messages[nr-1]
+		*toDelete = append(*toDelete, m.ID)
+		sendResponse(conn, "+OK message marked for deletion")
+	case "RSET":
+		*toDelete = []string{}
+		sendResponse(conn, "+OK")
+	default:
+		sendResponse(conn, "-ERR unknown command")
+	}
+}

internal/pop3client/client.go

@@ -0,0 +1,453 @@
+// Package pop3client is borrowed directly from https://github.com/knadh/go-pop3 to reduce dependencies.
+// This is used solely for testing the POP3 server
+package pop3client
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Client implements a Client e-mail client.
+type Client struct {
+	opt    Opt
+	dialer Dialer
+}
+
+// Conn is a stateful connection with the POP3 server/
+type Conn struct {
+	conn net.Conn
+	r    *bufio.Reader
+	w    *bufio.Writer
+}
+
+// Opt represents the client configuration.
+type Opt struct {
+	// Host name
+	Host string `json:"host"`
+	// Port number
+	Port int `json:"port"`
+	// DialTimeout default is 3 seconds.
+	DialTimeout time.Duration `json:"dial_timeout"`
+	// Dialer
+	Dialer Dialer `json:"-"`
+	// TLSEnabled sets whether SLS is enabled
+	TLSEnabled bool `json:"tls_enabled"`
+	// TLSSkipVerify skips TLS verification (ie: self-signed)
+	TLSSkipVerify bool `json:"tls_skip_verify"`
+}
+
+// Dialer interface
+type Dialer interface {
+	Dial(network, address string) (net.Conn, error)
+}
+
+// MessageID contains the ID and size of an individual message.
+type MessageID struct {
+	// ID is the numerical index (non-unique) of the message.
+	ID int
+	// Size in bytes
+	Size int
+	// UID is only present if the response is to the UIDL command.
+	UID string
+}
+
+var (
+	lineBreak   = []byte("\r\n")
+	respOK      = []byte("+OK")   // `+OK` without additional info
+	respOKInfo  = []byte("+OK ")  // `+OK <info>`
+	respErr     = []byte("-ERR")  // `-ERR` without additional info
+	respErrInfo = []byte("-ERR ") // `-ERR <info>`
+)
+
+// New returns a new client object using an existing connection.
+func New(opt Opt) *Client {
+	if opt.DialTimeout < time.Millisecond {
+		opt.DialTimeout = time.Second * 3
+	}
+
+	c := &Client{
+		opt:    opt,
+		dialer: opt.Dialer,
+	}
+
+	if c.dialer == nil {
+		c.dialer = &net.Dialer{Timeout: opt.DialTimeout}
+	}
+
+	return c
+}
+
+// NewConn creates and returns live POP3 server connection.
+func (c *Client) NewConn() (*Conn, error) {
+	var (
+		addr = fmt.Sprintf("%s:%d", c.opt.Host, c.opt.Port)
+	)
+
+	conn, err := c.dialer.Dial("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+
+	// No TLS.
+	if c.opt.TLSEnabled {
+		// Skip TLS host verification.
+		tlsCfg := tls.Config{} // #nosec
+		if c.opt.TLSSkipVerify {
+			tlsCfg.InsecureSkipVerify = c.opt.TLSSkipVerify // #nosec
+		} else {
+			tlsCfg.ServerName = c.opt.Host
+		}
+
+		conn = tls.Client(conn, &tlsCfg)
+	}
+
+	pCon := &Conn{
+		conn: conn,
+		r:    bufio.NewReader(conn),
+		w:    bufio.NewWriter(conn),
+	}
+
+	// Verify the connection by reading the welcome +OK greeting.
+	if _, err := pCon.ReadOne(); err != nil {
+		return nil, err
+	}
+
+	return pCon, nil
+}
+
+// Send sends a POP3 command to the server. The given comand is suffixed with "\r\n".
+func (c *Conn) Send(b string) error {
+	if _, err := c.w.WriteString(b + "\r\n"); err != nil {
+		return err
+	}
+
+	return c.w.Flush()
+}
+
+// Cmd sends a command to the server. POP3 responses are either single line or multi-line.
+// The first line always with -ERR in case of an error or +OK in case of a successful operation.
+// OK+ is always followed by a response on the same line which is either the actual response data
+// in case of single line responses, or a help message followed by multiple lines of actual response
+// data in case of multiline responses.
+// See https://www.shellhacks.com/retrieve-email-pop3-server-command-line/ for examples.
+func (c *Conn) Cmd(cmd string, isMulti bool, args ...interface{}) (*bytes.Buffer, error) {
+	var cmdLine string
+
+	// Repeat a %v to format each arg.
+	if len(args) > 0 {
+		format := " " + strings.TrimRight(strings.Repeat("%v ", len(args)), " ")
+
+		// CMD arg1 argn ...\r\n
+		cmdLine = fmt.Sprintf(cmd+format, args...)
+	} else {
+		cmdLine = cmd
+	}
+
+	if err := c.Send(cmdLine); err != nil {
+		return nil, err
+	}
+
+	// Read the first line of response to get the +OK/-ERR status.
+	b, err := c.ReadOne()
+	if err != nil {
+		return nil, err
+	}
+
+	// Single line response.
+	if !isMulti {
+		return bytes.NewBuffer(b), err
+	}
+
+	buf, err := c.ReadAll()
+	return buf, err
+}
+
+// ReadOne reads a single line response from the conn.
+func (c *Conn) ReadOne() ([]byte, error) {
+	b, _, err := c.r.ReadLine()
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := parseResp(b)
+	return r, err
+}
+
+// ReadAll reads all lines from the connection until the POP3 multiline terminator "." is encountered
+// and returns a bytes.Buffer of all the read lines.
+func (c *Conn) ReadAll() (*bytes.Buffer, error) {
+	buf := &bytes.Buffer{}
+
+	for {
+		b, _, err := c.r.ReadLine()
+		if err != nil {
+			return nil, err
+		}
+
+		// "." indicates the end of a multi-line response.
+		if bytes.Equal(b, []byte(".")) {
+			break
+		}
+
+		if _, err := buf.Write(b); err != nil {
+			return nil, err
+		}
+		if _, err := buf.Write(lineBreak); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf, nil
+}
+
+// Auth authenticates the given credentials with the server.
+func (c *Conn) Auth(user, password string) error {
+	if err := c.User(user); err != nil {
+		return err
+	}
+
+	if err := c.Pass(password); err != nil {
+		return err
+	}
+
+	// Issue a NOOP to force the server to respond to the auth.
+	// Courtesy: github.com/TheCreeper/go-pop3
+	return c.Noop()
+}
+
+// User sends the username to the server.
+func (c *Conn) User(s string) error {
+	_, err := c.Cmd("USER", false, s)
+
+	return err
+}
+
+// Pass sends the password to the server.
+func (c *Conn) Pass(s string) error {
+	_, err := c.Cmd("PASS", false, s)
+
+	return err
+}
+
+// Stat returns the number of messages and their total size in bytes in the inbox.
+func (c *Conn) Stat() (int, int, error) {
+	b, err := c.Cmd("STAT", false)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// count size
+	f := bytes.Fields(b.Bytes())
+
+	// Total number of messages.
+	count, err := strconv.Atoi(string(f[0]))
+	if err != nil {
+		return 0, 0, err
+	}
+	if count == 0 {
+		return 0, 0, nil
+	}
+
+	// Total size of all messages in bytes.
+	size, err := strconv.Atoi(string(f[1]))
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return count, size, nil
+}
+
+// List returns a list of (message ID, message Size) pairs.
+// If the optional msgID > 0, then only that particular message is listed.
+// The message IDs are sequential, 1 to N.
+func (c *Conn) List(msgID int) ([]MessageID, error) {
+	var (
+		buf *bytes.Buffer
+		err error
+	)
+
+	if msgID <= 0 {
+		// Multiline response listing all messages.
+		buf, err = c.Cmd("LIST", true)
+	} else {
+		// Single line response listing one message.
+		buf, err = c.Cmd("LIST", false, msgID)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		out   []MessageID
+		lines = bytes.Split(buf.Bytes(), lineBreak)
+	)
+
+	for _, l := range lines {
+		// id size
+		f := bytes.Fields(l)
+		if len(f) == 0 {
+			break
+		}
+
+		id, err := strconv.Atoi(string(f[0]))
+		if err != nil {
+			return nil, err
+		}
+
+		size, err := strconv.Atoi(string(f[1]))
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, MessageID{ID: id, Size: size})
+	}
+
+	return out, nil
+}
+
+// Uidl returns a list of (message ID, message UID) pairs. If the optional msgID
+// is > 0, then only that particular message is listed. It works like Top() but only works on
+// servers that support the UIDL command. Messages size field is not available in the UIDL response.
+func (c *Conn) Uidl(msgID int) ([]MessageID, error) {
+	var (
+		buf *bytes.Buffer
+		err error
+	)
+
+	if msgID <= 0 {
+		// Multiline response listing all messages.
+		buf, err = c.Cmd("UIDL", true)
+	} else {
+		// Single line response listing one message.
+		buf, err = c.Cmd("UIDL", false, msgID)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		out   []MessageID
+		lines = bytes.Split(buf.Bytes(), lineBreak)
+	)
+
+	for _, l := range lines {
+		// id size
+		f := bytes.Fields(l)
+		if len(f) == 0 {
+			break
+		}
+
+		id, err := strconv.Atoi(string(f[0]))
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, MessageID{ID: id, UID: string(f[1])})
+	}
+
+	return out, nil
+}
+
+// Retr downloads a message by the given msgID, parses it and returns it as a *mail.Message.
+func (c *Conn) Retr(msgID int) (*mail.Message, error) {
+	b, err := c.Cmd("RETR", true, msgID)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := mail.ReadMessage(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// RetrRaw downloads a message by the given msgID and returns the raw []byte
+// of the entire message.
+func (c *Conn) RetrRaw(msgID int) (*bytes.Buffer, error) {
+	b, err := c.Cmd("RETR", true, msgID)
+	return b, err
+}
+
+// Top retrieves a message by its ID with full headers and numLines lines of the body.
+func (c *Conn) Top(msgID int, numLines int) (*mail.Message, error) {
+	b, err := c.Cmd("TOP", true, msgID, numLines)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := mail.ReadMessage(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// Dele deletes one or more messages. The server only executes the
+// deletions after a successful Quit().
+func (c *Conn) Dele(msgID ...int) error {
+	for _, id := range msgID {
+		_, err := c.Cmd("DELE", false, id)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Rset clears the messages marked for deletion in the current session.
+func (c *Conn) Rset() error {
+	_, err := c.Cmd("RSET", false)
+	return err
+}
+
+// Noop issues a do-nothing NOOP command to the server. This is useful for
+// prolonging open connections.
+func (c *Conn) Noop() error {
+	_, err := c.Cmd("NOOP", false)
+	return err
+}
+
+// Quit sends the QUIT command to server and gracefully closes the connection.
+// Message deletions (DELE command) are only executed by the server on a graceful
+// quit and close.
+func (c *Conn) Quit() error {
+	defer c.conn.Close()
+
+	if _, err := c.Cmd("QUIT", false); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// parseResp checks if the response is an error that starts with `-ERR`
+// and returns an error with the message that succeeds the error indicator.
+// For success `+OK` messages, it returns the remaining response bytes.
+func parseResp(b []byte) ([]byte, error) {
+	if len(b) == 0 {
+		return nil, nil
+	}
+
+	if bytes.Equal(b, respOK) {
+		return nil, nil
+	} else if bytes.HasPrefix(b, respOKInfo) {
+		return bytes.TrimPrefix(b, respOKInfo), nil
+	} else if bytes.Equal(b, respErr) {
+		return nil, errors.New("unknown error (no info specified in response)")
+	} else if bytes.HasPrefix(b, respErrInfo) {
+		return nil, errors.New(string(bytes.TrimPrefix(b, respErrInfo)))
+	}
+
+	return nil, fmt.Errorf("unknown response: %s. Neither -ERR, nor +OK", string(b))
+}

internal/smtpd/chaos/chaos.go

@@ -0,0 +1,121 @@
+// Package chaos is used to simulate Chaos engineering (random failures) in the SMTPD server.
+// See https://en.wikipedia.org/wiki/Chaos_engineering
+// See https://mailpit.axllent.org/docs/integration/chaos/
+package chaos
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var (
+	// Enabled is a flag to enable or disable support for chaos
+	Enabled = false
+
+	// Config is the global Chaos configuration
+	Config = Triggers{
+		Sender:         Trigger{ErrorCode: 451, Probability: 0},
+		Recipient:      Trigger{ErrorCode: 451, Probability: 0},
+		Authentication: Trigger{ErrorCode: 535, Probability: 0},
+	}
+)
+
+// Triggers for the Chaos configuration
+// swagger:model Triggers
+type Triggers struct {
+	// Sender trigger to fail on From, Sender
+	Sender Trigger
+	// Recipient trigger to fail on To, Cc, Bcc
+	Recipient Trigger
+	// Authentication trigger to fail while authenticating (auth must be configured)
+	Authentication Trigger
+}
+
+// Trigger for Chaos
+// swagger:model Trigger
+type Trigger struct {
+	// SMTP error code to return. The value must range from 400 to 599.
+	// required: true
+	// example: 451
+	ErrorCode int
+
+	// Probability (chance) of triggering the error. The value must range from 0 to 100.
+	// required: true
+	// example: 5
+	Probability int
+}
+
+// SetFromStruct will set a whole map of chaos configurations (ie: API)
+func SetFromStruct(c Triggers) error {
+	if c.Sender.ErrorCode == 0 {
+		c.Sender.ErrorCode = 451 // default
+	}
+
+	if c.Recipient.ErrorCode == 0 {
+		c.Recipient.ErrorCode = 451 // default
+	}
+
+	if c.Authentication.ErrorCode == 0 {
+		c.Authentication.ErrorCode = 535 // default
+	}
+
+	if err := Set("Sender", c.Sender.ErrorCode, c.Sender.Probability); err != nil {
+		return err
+	}
+	if err := Set("Recipient", c.Recipient.ErrorCode, c.Recipient.Probability); err != nil {
+		return err
+	}
+	if err := Set("Authentication", c.Authentication.ErrorCode, c.Authentication.Probability); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Set will set the chaos configuration for the given key (CLI & setMap())
+func Set(key string, errorCode int, probability int) error {
+	Enabled = true
+	if errorCode < 400 || errorCode > 599 {
+		return fmt.Errorf("error code must be between 400 and 599")
+	}
+
+	if probability > 100 || probability < 0 {
+		return fmt.Errorf("probability must be between 0 and 100")
+	}
+
+	key = strings.ToLower(key)
+
+	switch key {
+	case "sender":
+		Config.Sender = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Sender to return %d error with %d%% probability", errorCode, probability)
+	case "recipient", "recipients":
+		Config.Recipient = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Recipient to return %d error with %d%% probability", errorCode, probability)
+	case "auth", "authentication":
+		Config.Authentication = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Authentication to return %d error with %d%% probability", errorCode, probability)
+	default:
+		return fmt.Errorf("unknown key %s", key)
+	}
+
+	return nil
+}
+
+// Trigger will return whether the Chaos rule is triggered based on the configuration
+// and a randomly-generated percentage value.
+func (c Trigger) Trigger() (bool, int) {
+	if !Enabled || c.Probability == 0 {
+		return false, 0
+	}
+
+	nBig, _ := rand.Int(rand.Reader, big.NewInt(100))
+
+	// rand.IntN(100) will return 0-99, whereas probability is 1-100,
+	// so value must be less than (not <=) to the probability to trigger
+	return int(nBig.Int64()) < c.Probability, c.ErrorCode
+}

internal/smtpd/forward.go

@@ -0,0 +1,111 @@
+package smtpd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/smtp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Wrapper to forward messages if configured
+func autoForwardMessage(from string, data *[]byte) {
+	if config.SMTPForwardConfig.Host == "" {
+		return
+	}
+
+	if err := forward(from, *data); err != nil {
+		logger.Log().Errorf("[forward] error: %s", err.Error())
+	} else {
+		logger.Log().Debugf("[forward] message from %s to %s via %s:%d",
+			from, config.SMTPForwardConfig.To, config.SMTPForwardConfig.Host, config.SMTPForwardConfig.Port)
+	}
+}
+
+// Forward will connect to a pre-configured SMTP server and send a message to one or more recipients.
+func forward(from string, msg []byte) error {
+	addr := fmt.Sprintf("%s:%d", config.SMTPForwardConfig.Host, config.SMTPForwardConfig.Port)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		return fmt.Errorf("error connecting to %s: %s", addr, err.Error())
+	}
+
+	defer c.Close()
+
+	if config.SMTPForwardConfig.STARTTLS {
+		conf := &tls.Config{ServerName: config.SMTPForwardConfig.Host} // #nosec
+
+		conf.InsecureSkipVerify = config.SMTPForwardConfig.AllowInsecure
+
+		if err = c.StartTLS(conf); err != nil {
+			return fmt.Errorf("error creating StartTLS config: %s", err.Error())
+		}
+	}
+
+	auth := forwardAuthFromConfig()
+
+	if auth != nil {
+		if err = c.Auth(auth); err != nil {
+			return fmt.Errorf("error response to AUTH command: %s", err.Error())
+		}
+	}
+
+	if config.SMTPForwardConfig.OverrideFrom != "" {
+		msg, err = tools.OverrideFromHeader(msg, config.SMTPForwardConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("error overriding From header: %s", err.Error())
+		}
+
+		from = config.SMTPForwardConfig.OverrideFrom
+	}
+
+	if err = c.Mail(from); err != nil {
+		return fmt.Errorf("error response to MAIL command: %s", err.Error())
+	}
+
+	to := strings.Split(config.SMTPForwardConfig.To, ",")
+
+	for _, addr := range to {
+		if err = c.Rcpt(addr); err != nil {
+			logger.Log().Warnf("error response to RCPT command for %s: %s", addr, err.Error())
+		}
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return fmt.Errorf("error response to DATA command: %s", err.Error())
+	}
+
+	if _, err := w.Write(msg); err != nil {
+		return fmt.Errorf("error sending message: %s", err.Error())
+	}
+
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("error closing connection: %s", err.Error())
+	}
+
+	return c.Quit()
+}
+
+// Return the SMTP forwarding authentication based on config
+func forwardAuthFromConfig() smtp.Auth {
+	var a smtp.Auth
+
+	if config.SMTPForwardConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Password, config.SMTPForwardConfig.Host)
+	}
+
+	if config.SMTPForwardConfig.Auth == "login" {
+		a = LoginAuth(config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Password)
+	}
+
+	if config.SMTPForwardConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Secret)
+	}
+
+	return a
+}

internal/smtpd/main.go

@@ -0,0 +1,317 @@
+// Package smtpd is the SMTP daemon
+package smtpd
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/mail"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/stats"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+var (
+	// DisableReverseDNS allows rDNS to be disabled
+	DisableReverseDNS bool
+
+	warningResponse = regexp.MustCompile(`^4\d\d `)
+	errorResponse   = regexp.MustCompile(`^5\d\d `)
+)
+
+// MailHandler handles the incoming message to store in the database
+func mailHandler(origin net.Addr, from string, to []string, data []byte) (string, error) {
+	return SaveToDatabase(origin, from, to, data)
+}
+
+// SaveToDatabase will attempt to save a message to the database
+func SaveToDatabase(origin net.Addr, from string, to []string, data []byte) (string, error) {
+	if !config.SMTPStrictRFCHeaders && bytes.Contains(data, []byte("\r\r\n")) {
+		// replace all <CR><CR><LF> (\r\r\n) with <CR><LF> (\r\n)
+		// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
+		data = bytes.ReplaceAll(data, []byte("\r\r\n"), []byte("\r\n"))
+	}
+
+	msg, err := mail.ReadMessage(bytes.NewReader(data))
+	if err != nil {
+		logger.Log().Warnf("[smtpd] error parsing message: %s", err.Error())
+		stats.LogSMTPRejected()
+		return "", err
+	}
+
+	// check / set the Return-Path based on SMTP from
+	returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
+	if returnPath != from {
+		data, err = tools.SetMessageHeader(data, "Return-Path", "<"+from+">")
+		if err != nil {
+			return "", err
+		}
+	}
+
+	messageID := strings.Trim(msg.Header.Get("Message-ID"), "<>")
+
+	// add a message ID if not set
+	if messageID == "" {
+		// generate unique ID
+		messageID = shortuuid.New() + "@mailpit"
+		// add unique ID
+		data = append([]byte("Message-ID: <"+messageID+">\r\n"), data...)
+	} else if config.IgnoreDuplicateIDs {
+		if storage.MessageIDExists(messageID) {
+			logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID)
+			stats.LogSMTPIgnored()
+			return "", nil
+		}
+	}
+
+	// if enabled, this may conditionally relay the email through to the preconfigured smtp server
+	autoRelayMessage(from, to, &data)
+
+	// if enabled, this will forward a copy to preconfigured addresses
+	autoForwardMessage(from, &data)
+
+	// build array of all addresses in the header to compare to the []to array
+	emails, hasBccHeader := scanAddressesInHeader(msg.Header)
+
+	missingAddresses := []string{}
+	for _, a := range to {
+		// loop through passed email addresses to check if they are in the headers
+		if _, err := mail.ParseAddress(a); err == nil {
+			_, ok := emails[strings.ToLower(a)]
+			if !ok {
+				missingAddresses = append(missingAddresses, a)
+			}
+		} else {
+			logger.Log().Warnf("[smtpd] ignoring invalid email address: %s", a)
+		}
+	}
+
+	// add missing email addresses to Bcc (eg: Laravel doesn't include these in the headers)
+	if len(missingAddresses) > 0 {
+		bccVal := strings.Join(missingAddresses, ", ")
+		if hasBccHeader {
+			b := msg.Header.Get("Bcc")
+			bccVal = ", " + b
+		}
+
+		data, err = tools.SetMessageHeader(data, "Bcc", bccVal)
+		if err != nil {
+			return "", err
+		}
+
+		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
+	}
+
+	id, err := storage.Store(&data)
+	if err != nil {
+		logger.Log().Errorf("[db] error storing message: %s", err.Error())
+		return "", err
+	}
+
+	stats.LogSMTPAccepted(len(data))
+
+	data = nil // avoid memory leaks
+
+	subject := msg.Header.Get("Subject")
+	logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject)
+
+	return id, err
+}
+
+func authHandler(remoteAddr net.Addr, mechanism string, username []byte, password []byte, _ []byte) (bool, error) {
+	allow := auth.SMTPCredentials.Match(string(username), string(password))
+	if allow {
+		logger.Log().Debugf("[smtpd] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+	} else {
+		logger.Log().Warnf("[smtpd] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+	}
+
+	return allow, nil
+}
+
+// Allow any username and password
+func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, _ []byte, _ []byte) (bool, error) {
+	logger.Log().Debugf("[smtpd] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr))
+
+	return true, nil
+}
+
+// HandlerRcpt used to optionally restrict recipients based on `--smtp-allowed-recipients`
+func handlerRcpt(remoteAddr net.Addr, from string, to string) bool {
+	if config.SMTPAllowedRecipientsRegexp == nil {
+		return true
+	}
+
+	result := config.SMTPAllowedRecipientsRegexp.MatchString(to)
+
+	if !result {
+		logger.Log().Warnf("[smtpd] rejected message to %s from %s (%s)", to, from, cleanIP(remoteAddr))
+		stats.LogSMTPRejected()
+	}
+
+	return result
+}
+
+// Listen starts the SMTPD server
+func Listen() error {
+	if config.SMTPAuthAllowInsecure {
+		if auth.SMTPCredentials != nil {
+			logger.Log().Info("[smtpd] enabling login authentication (insecure)")
+		} else if config.SMTPAuthAcceptAny {
+			logger.Log().Info("[smtpd] enabling any authentication (insecure)")
+		}
+	} else {
+		if auth.SMTPCredentials != nil {
+			logger.Log().Info("[smtpd] enabling login authentication")
+		} else if config.SMTPAuthAcceptAny {
+			logger.Log().Info("[smtpd] enabling any authentication")
+		}
+	}
+
+	return listenAndServe(config.SMTPListen, mailHandler, authHandler)
+}
+
+// Translate the smtpd verb from READ/WRITE
+func verbLogTranslator(verb string) string {
+	if verb == "READ" {
+		return "received"
+	}
+
+	return "response"
+}
+
+func listenAndServe(addr string, handler MsgIDHandler, authHandler AuthHandler) error {
+
+	socketAddr, perm, isSocket := tools.UnixSocket(addr)
+
+	Debug = true // to enable Mailpit logging
+	srv := &Server{
+		Addr:              addr,
+		MsgIDHandler:      handler,
+		HandlerRcpt:       handlerRcpt,
+		AppName:           "Mailpit",
+		Hostname:          "",
+		AuthHandler:       nil,
+		AuthRequired:      false,
+		MaxRecipients:     config.SMTPMaxRecipients,
+		DisableReverseDNS: DisableReverseDNS,
+		LogRead: func(remoteIP, verb, line string) {
+			logger.Log().Debugf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+		},
+		LogWrite: func(remoteIP, verb, line string) {
+			if warningResponse.MatchString(line) {
+				logger.Log().Warnf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+				websockets.BroadCastClientError("warning", "smtpd", remoteIP, line)
+			} else if errorResponse.MatchString(line) {
+				logger.Log().Errorf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+				websockets.BroadCastClientError("error", "smtpd", remoteIP, line)
+			} else {
+				logger.Log().Debugf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+			}
+		},
+	}
+
+	if config.Label != "" {
+		srv.AppName = fmt.Sprintf("Mailpit (%s)", config.Label)
+	}
+
+	if config.SMTPAuthAllowInsecure {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+	}
+
+	if auth.SMTPCredentials != nil {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+		srv.AuthHandler = authHandler
+		srv.AuthRequired = true
+	} else if config.SMTPAuthAcceptAny {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+		srv.AuthHandler = authHandlerAny
+	}
+
+	if config.SMTPTLSCert != "" {
+		srv.TLSRequired = config.SMTPRequireSTARTTLS
+		srv.TLSListener = config.SMTPRequireTLS // if true overrules srv.TLSRequired
+		if err := srv.ConfigureTLS(config.SMTPTLSCert, config.SMTPTLSKey); err != nil {
+			return err
+		}
+	}
+
+	if isSocket {
+		srv.Addr = socketAddr
+		srv.Protocol = "unix"
+		srv.SocketPerm = perm
+
+		if err := tools.PrepareSocket(srv.Addr); err != nil {
+			storage.Close()
+			return err
+		}
+
+		// delete the Unix socket file on exit
+		storage.AddTempFile(srv.Addr)
+
+		logger.Log().Infof("[smtpd] starting on %s", config.SMTPListen)
+	} else {
+		smtpType := "no encryption"
+
+		if config.SMTPTLSCert != "" {
+			if config.SMTPRequireTLS {
+				smtpType = "SSL/TLS required"
+			} else if config.SMTPRequireSTARTTLS {
+				smtpType = "STARTTLS required"
+			} else {
+				smtpType = "STARTTLS optional"
+				if !config.SMTPAuthAllowInsecure && auth.SMTPCredentials != nil {
+					smtpType = "STARTTLS required"
+				}
+			}
+		}
+
+		logger.Log().Infof("[smtpd] starting on %s (%s)", config.SMTPListen, smtpType)
+	}
+
+	return srv.ListenAndServe()
+}
+
+func cleanIP(i net.Addr) string {
+	parts := strings.Split(i.String(), ":")
+
+	return parts[0]
+}
+
+// Returns a list of all lowercased emails found in To, Cc and Bcc,
+// as well as whether there is a Bcc field
+func scanAddressesInHeader(h mail.Header) (map[string]bool, bool) {
+	emails := make(map[string]bool)
+	hasBccHeader := false
+
+	if recipients, err := h.AddressList("To"); err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+	}
+
+	if recipients, err := h.AddressList("Cc"); err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+	}
+
+	recipients, err := h.AddressList("Bcc")
+	if err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+
+		hasBccHeader = true
+	}
+
+	return emails, hasBccHeader
+}

internal/smtpd/relay.go

@@ -0,0 +1,173 @@
+package smtpd
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/smtp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Wrapper to auto relay messages if configured
+func autoRelayMessage(from string, to []string, data *[]byte) {
+	if config.SMTPRelayConfig.BlockedRecipientsRegexp != nil {
+		filteredTo := []string{}
+		for _, address := range to {
+			if config.SMTPRelayConfig.BlockedRecipientsRegexp.MatchString(address) {
+				logger.Log().Debugf("[relay] ignoring auto-relay to %s: found in blocklist", address)
+				continue
+			}
+
+			filteredTo = append(filteredTo, address)
+		}
+		to = filteredTo
+	}
+
+	if len(to) == 0 {
+		return
+	}
+
+	if config.SMTPRelayAll {
+		if err := Relay(from, to, *data); err != nil {
+			logger.Log().Errorf("[relay] error: %s", err.Error())
+		} else {
+			logger.Log().Debugf("[relay] sent message to %s from %s via %s:%d",
+				strings.Join(to, ", "), from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		}
+	} else if config.SMTPRelayMatchingRegexp != nil {
+		filtered := []string{}
+		for _, t := range to {
+			if config.SMTPRelayMatchingRegexp.MatchString(t) {
+				filtered = append(filtered, t)
+			}
+		}
+
+		if len(filtered) == 0 {
+			return
+		}
+
+		if err := Relay(from, filtered, *data); err != nil {
+			logger.Log().Errorf("[relay] error: %s", err.Error())
+		} else {
+			logger.Log().Debugf("[relay] auto-relay message to %s from %s via %s:%d",
+				strings.Join(filtered, ", "), from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		}
+	}
+}
+
+// Relay will connect to a pre-configured SMTP server and send a message to one or more recipients.
+func Relay(from string, to []string, msg []byte) error {
+	addr := fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		return fmt.Errorf("error connecting to %s: %s", addr, err.Error())
+	}
+
+	defer c.Close()
+
+	if config.SMTPRelayConfig.STARTTLS {
+		conf := &tls.Config{ServerName: config.SMTPRelayConfig.Host} // #nosec
+
+		conf.InsecureSkipVerify = config.SMTPRelayConfig.AllowInsecure
+
+		if err = c.StartTLS(conf); err != nil {
+			return fmt.Errorf("error creating StartTLS config: %s", err.Error())
+		}
+	}
+
+	auth := relayAuthFromConfig()
+
+	if auth != nil {
+		if err = c.Auth(auth); err != nil {
+			return fmt.Errorf("error response to AUTH command: %s", err.Error())
+		}
+	}
+
+	if config.SMTPRelayConfig.OverrideFrom != "" {
+		msg, err = tools.OverrideFromHeader(msg, config.SMTPRelayConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("error overriding From header: %s", err.Error())
+		}
+
+		from = config.SMTPRelayConfig.OverrideFrom
+	}
+
+	if err = c.Mail(from); err != nil {
+		return fmt.Errorf("error response to MAIL command: %s", err.Error())
+	}
+
+	for _, addr := range to {
+		if err = c.Rcpt(addr); err != nil {
+			logger.Log().Warnf("error response to RCPT command for %s: %s", addr, err.Error())
+		}
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return fmt.Errorf("error response to DATA command: %s", err.Error())
+	}
+
+	if _, err := w.Write(msg); err != nil {
+		return fmt.Errorf("error sending message: %s", err.Error())
+	}
+
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("error closing connection: %s", err.Error())
+	}
+
+	return c.Quit()
+}
+
+// Return the SMTP relay authentication based on config
+func relayAuthFromConfig() smtp.Auth {
+	var a smtp.Auth
+
+	if config.SMTPRelayConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host)
+	}
+
+	if config.SMTPRelayConfig.Auth == "login" {
+		a = LoginAuth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password)
+	}
+
+	if config.SMTPRelayConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret)
+	}
+
+	return a
+}
+
+// Custom implementation of LOGIN SMTP authentication
+// @see https://gist.github.com/andelf/5118732
+type loginAuth struct {
+	username, password string
+}
+
+// LoginAuth authentication
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(_ *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch string(fromServer) {
+		case "Username:":
+			return []byte(a.username), nil
+		case "Password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.New("Unknown fromServer")
+		}
+	}
+
+	return nil, nil
+}

internal/smtpd/smtpd.go

@@ -0,0 +1,979 @@
+// Package smtpd implements a basic SMTP server.
+//
+// This is a modified version of https://github.com/mhale/smtpd to
+// add support for unix sockets and Mailpit Chaos.
+package smtpd
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"net"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+)
+
+var (
+	// Debug `true` enables verbose logging.
+	Debug      = false
+	rcptToRE   = regexp.MustCompile(`(?i)TO: ?<([^<>\v]+)>( |$)(.*)?`)
+	mailFromRE = regexp.MustCompile(`(?i)FROM: ?<(|[^<>\v]+)>( |$)(.*)?`) // Delivery Status Notifications are sent with "MAIL FROM:<>"
+
+	// extract mail size from 'MAIL FROM' parameter
+	mailFromSizeRE = regexp.MustCompile(`(?U)(^| |,)[Ss][Ii][Zz][Ee]=(.*)($|,| )`)
+)
+
+// Handler function called upon successful receipt of an email.
+// Results in a "250 2.0.0 Ok: queued" response.
+type Handler func(remoteAddr net.Addr, from string, to []string, data []byte) error
+
+// MsgIDHandler function called upon successful receipt of an email. Returns a message ID.
+// Results in a "250 2.0.0 Ok: queued as <message-id>" response.
+type MsgIDHandler func(remoteAddr net.Addr, from string, to []string, data []byte) (string, error)
+
+// HandlerRcpt function called on RCPT. Return accept status.
+type HandlerRcpt func(remoteAddr net.Addr, from string, to string) bool
+
+// AuthHandler function called when a login attempt is performed. Returns true if credentials are correct.
+type AuthHandler func(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error)
+
+// ErrServerClosed is the default message when a server closes a connection
+var ErrServerClosed = errors.New("Server has been closed")
+
+// ListenAndServe listens on the TCP network address addr
+// and then calls Serve with handler to handle requests
+// on incoming connections.
+func ListenAndServe(addr string, handler Handler, appName string, hostname string) error {
+	srv := &Server{Addr: addr, Handler: handler, AppName: appName, Hostname: hostname}
+	return srv.ListenAndServe()
+}
+
+// ListenAndServeTLS listens on the TCP network address addr
+// and then calls Serve with handler to handle requests
+// on incoming connections. Connections may be upgraded to TLS if the client requests it.
+func ListenAndServeTLS(addr string, certFile string, keyFile string, handler Handler, appName string, hostname string) error {
+	srv := &Server{Addr: addr, Handler: handler, AppName: appName, Hostname: hostname}
+	err := srv.ConfigureTLS(certFile, keyFile)
+	if err != nil {
+		return err
+	}
+	return srv.ListenAndServe()
+}
+
+type maxSizeExceededError struct {
+	limit int
+}
+
+func maxSizeExceeded(limit int) maxSizeExceededError {
+	return maxSizeExceededError{limit}
+}
+
+// Error uses the RFC 5321 response message in preference to RFC 1870.
+// RFC 3463 defines enhanced status code x.3.4 as "Message too big for system".
+func (err maxSizeExceededError) Error() string {
+	return fmt.Sprintf("552 5.3.4 Requested mail action aborted: exceeded storage allocation (%d)", err.limit)
+}
+
+// LogFunc is a function capable of logging the client-server communication.
+type LogFunc func(remoteIP, verb, line string)
+
+// Server is an SMTP server.
+type Server struct {
+	Addr              string // TCP address to listen on, defaults to ":25" (all addresses, port 25) if empty
+	AppName           string
+	AuthHandler       AuthHandler
+	AuthMechs         map[string]bool // Override list of allowed authentication mechanisms. Currently supported: LOGIN, PLAIN, CRAM-MD5. Enabling LOGIN and PLAIN will reduce RFC 4954 compliance.
+	AuthRequired      bool            // Require authentication for every command except AUTH, EHLO, HELO, NOOP, RSET or QUIT as per RFC 4954. Ignored if AuthHandler is not configured.
+	DisableReverseDNS bool            // Disable reverse DNS lookups, enforces "unknown" hostname
+	Handler           Handler
+	HandlerRcpt       HandlerRcpt
+	Hostname          string
+	LogRead           LogFunc
+	LogWrite          LogFunc
+	MaxSize           int // Maximum message size allowed, in bytes
+	MaxRecipients     int // Maximum number of recipients, defaults to 100.
+	MsgIDHandler      MsgIDHandler
+	Timeout           time.Duration
+	TLSConfig         *tls.Config
+	TLSListener       bool        // Listen for incoming TLS connections only (not recommended as it may reduce compatibility). Ignored if TLS is not configured.
+	TLSRequired       bool        // Require TLS for every command except NOOP, EHLO, STARTTLS, or QUIT as per RFC 3207. Ignored if TLS is not configured.
+	Protocol          string      // Default tcp, supports unix
+	SocketPerm        fs.FileMode // if using Unix socket, socket permissions
+
+	inShutdown   int32 // server was closed or shutdown
+	openSessions int32 // count of open sessions
+	mu           sync.Mutex
+	shutdownChan chan struct{} // let the sessions know we are shutting down
+
+	XClientAllowed []string // List of XCLIENT allowed IP addresses
+}
+
+// ConfigureTLS creates a TLS configuration from certificate and key files.
+func (srv *Server) ConfigureTLS(certFile string, keyFile string) error {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return err
+	}
+	srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cert}} // #nosec
+	return nil
+}
+
+// // ConfigureTLSWithPassphrase creates a TLS configuration from a certificate,
+// // an encrypted key file and the associated passphrase:
+// func (srv *Server) ConfigureTLSWithPassphrase(
+// 	certFile string,
+// 	keyFile string,
+// 	passphrase string,
+// ) error {
+// 	certPEMBlock, err := os.ReadFile(certFile)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	keyPEMBlock, err := os.ReadFile(keyFile)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	keyDERBlock, _ := pem.Decode(keyPEMBlock)
+// 	keyPEMDecrypted, err := x509.DecryptPEMBlock(keyDERBlock, []byte(passphrase))
+// 	if err != nil {
+// 		return err
+// 	}
+// 	var pemBlock pem.Block
+// 	pemBlock.Type = keyDERBlock.Type
+// 	pemBlock.Bytes = keyPEMDecrypted
+// 	keyPEMBlock = pem.EncodeToMemory(&pemBlock)
+// 	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cert}}
+// 	return nil
+// }
+
+// ListenAndServe listens on the either a TCP network address srv.Addr or
+// alternatively a Unix socket. and then calls Serve to handle requests on
+// incoming connections. If srv.Addr is blank, ":25" is used.
+func (srv *Server) ListenAndServe() error {
+	if atomic.LoadInt32(&srv.inShutdown) != 0 {
+		return ErrServerClosed
+	}
+
+	if srv.Addr == "" {
+		srv.Addr = ":25"
+	}
+	if srv.AppName == "" {
+		srv.AppName = "smtpd"
+	}
+	if srv.Hostname == "" {
+		srv.Hostname, _ = os.Hostname()
+	}
+	if srv.Timeout == 0 {
+		srv.Timeout = 5 * time.Minute
+	}
+	if srv.Protocol == "" {
+		srv.Protocol = "tcp"
+	}
+
+	var ln net.Listener
+	var err error
+
+	// If TLSListener is enabled, listen for TLS connections only.
+	if srv.TLSConfig != nil && srv.TLSListener {
+		ln, err = tls.Listen(srv.Protocol, srv.Addr, srv.TLSConfig)
+	} else {
+		ln, err = net.Listen(srv.Protocol, srv.Addr)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if srv.Protocol == "unix" {
+		// set permissions
+		if err := os.Chmod(srv.Addr, srv.SocketPerm); err != nil {
+			return err
+		}
+	}
+
+	return srv.Serve(ln)
+}
+
+// Serve creates a new SMTP session after a network connection is established.
+func (srv *Server) Serve(ln net.Listener) error {
+	if atomic.LoadInt32(&srv.inShutdown) != 0 {
+		return ErrServerClosed
+	}
+
+	defer ln.Close()
+
+	for {
+		// if we are shutting down, don't accept new connections
+		select {
+		case <-srv.getShutdownChan():
+			return ErrServerClosed
+		default:
+		}
+
+		conn, err := ln.Accept()
+		if err != nil {
+			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
+				continue
+			}
+			return err
+		}
+
+		session := srv.newSession(conn)
+		atomic.AddInt32(&srv.openSessions, 1)
+		go session.serve()
+	}
+}
+
+type session struct {
+	srv           *Server
+	conn          net.Conn
+	br            *bufio.Reader
+	bw            *bufio.Writer
+	remoteIP      string // Remote IP address
+	remoteHost    string // Remote hostname according to reverse DNS lookup
+	remoteName    string // Remote hostname as supplied with EHLO
+	xClient       string // Information string as supplied with XCLIENT
+	xClientADDR   string // Information string as supplied with XCLIENT ADDR
+	xClientNAME   string // Information string as supplied with XCLIENT NAME
+	xClientTrust  bool   // Trust XCLIENT from current IP address
+	tls           bool
+	authenticated bool
+}
+
+// Create new session from connection.
+func (srv *Server) newSession(conn net.Conn) (s *session) {
+	s = &session{
+		srv:  srv,
+		conn: conn,
+		br:   bufio.NewReader(conn),
+		bw:   bufio.NewWriter(conn),
+	}
+
+	// Get remote end info for the Received header.
+	s.remoteIP, _, _ = net.SplitHostPort(s.conn.RemoteAddr().String())
+	if s.remoteIP == "" {
+		s.remoteIP = "127.0.0.1"
+	}
+	if !s.srv.DisableReverseDNS {
+		names, err := net.LookupAddr(s.remoteIP)
+		if err == nil && len(names) > 0 {
+			s.remoteHost = names[0]
+		} else {
+			s.remoteHost = "unknown"
+		}
+	} else {
+		s.remoteHost = "unknown"
+	}
+
+	// Set tls = true if TLS is already in use.
+	_, s.tls = s.conn.(*tls.Conn)
+
+	for _, checkIP := range srv.XClientAllowed {
+		if s.remoteIP == checkIP {
+			s.xClientTrust = true
+		}
+	}
+	return
+}
+
+func (srv *Server) getShutdownChan() <-chan struct{} {
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+	if srv.shutdownChan == nil {
+		srv.shutdownChan = make(chan struct{})
+	}
+
+	return srv.shutdownChan
+}
+
+func (srv *Server) closeShutdownChan() {
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+	if srv.shutdownChan == nil {
+		srv.shutdownChan = make(chan struct{})
+	}
+
+	select {
+	case <-srv.shutdownChan:
+	default:
+		close(srv.shutdownChan)
+	}
+}
+
+// Close - closes the connection without waiting
+func (srv *Server) Close() error {
+	atomic.StoreInt32(&srv.inShutdown, 1)
+	srv.closeShutdownChan()
+	return nil
+}
+
+// Shutdown - waits for current sessions to complete before closing
+func (srv *Server) Shutdown(ctx context.Context) error {
+	atomic.StoreInt32(&srv.inShutdown, 1)
+	srv.closeShutdownChan()
+
+	// wait for up to 30 seconds to allow the current sessions to
+	// end
+	timer := time.NewTimer(100 * time.Millisecond)
+	defer timer.Stop()
+
+	for i := 0; i < 300; i++ {
+		// wait for open sessions to close
+		if atomic.LoadInt32(&srv.openSessions) == 0 {
+			break
+		}
+
+		select {
+		case <-timer.C:
+			timer.Reset(100 * time.Millisecond)
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+	}
+
+	return nil
+}
+
+// Function called to handle connection requests.
+func (s *session) serve() {
+	defer atomic.AddInt32(&s.srv.openSessions, -1)
+	defer s.conn.Close()
+
+	var from string
+	var gotFrom bool
+	var to []string
+	var buffer bytes.Buffer
+
+	// RFC 5321 specifies support for minimum of 100 recipients is required.
+	if s.srv.MaxRecipients == 0 {
+		s.srv.MaxRecipients = 100
+	}
+
+	// Send banner.
+	s.writef("220 %s %s ESMTP Service ready", s.srv.Hostname, s.srv.AppName)
+
+loop:
+	for {
+		// Attempt to read a line from the socket.
+		// On timeout, send a timeout message and return from serve().
+		// On error, assume the client has gone away i.e. return from serve().
+		line, err := s.readLine()
+		if err != nil {
+			if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+				s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+			}
+			break
+		}
+
+		verb, args := s.parseLine(line)
+
+		switch verb {
+		case "HELO":
+			s.remoteName = args
+			s.writef("250 %s greets %s", s.srv.Hostname, s.remoteName)
+
+			// RFC 2821 section 4.1.4 specifies that EHLO has the same effect as RSET, so reset for HELO too.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "EHLO":
+			s.remoteName = args
+			s.writef("%s", s.makeEHLOResponse())
+
+			// RFC 2821 section 4.1.4 specifies that EHLO has the same effect as RSET.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "MAIL":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+
+			match := mailFromRE.FindStringSubmatch(args)
+			if match == nil {
+				s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid FROM parameter)")
+			} else {
+				// Mailpit Chaos
+				if fail, code := chaos.Config.Sender.Trigger(); fail {
+					s.writef("%d Chaos sender error", code)
+					break
+				}
+
+				// Validate the SIZE parameter if one was sent.
+				if len(match[2]) > 0 { // A parameter is present
+					sizeMatch := mailFromSizeRE.FindStringSubmatch(match[3])
+					if sizeMatch == nil {
+						// ignore other parameter
+						from = match[1]
+						gotFrom = true
+						s.writef("250 2.1.0 Ok")
+					} else {
+						// Enforce the maximum message size if one is set.
+						size, err := strconv.Atoi(sizeMatch[2])
+						if err != nil { // Bad SIZE parameter
+							s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid SIZE parameter)")
+						} else if s.srv.MaxSize > 0 && size > s.srv.MaxSize { // SIZE above maximum size, if set
+							err = maxSizeExceeded(s.srv.MaxSize)
+							s.writef("%s", err.Error())
+						} else { // SIZE ok
+							from = match[1]
+							gotFrom = true
+							s.writef("250 2.1.0 Ok")
+						}
+					}
+				} else { // No parameters after FROM
+					from = match[1]
+					gotFrom = true
+					s.writef("250 2.1.0 Ok")
+				}
+			}
+
+			to = nil
+			buffer.Reset()
+		case "RCPT":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+			if !gotFrom {
+				s.writef("503 5.5.1 Bad sequence of commands (MAIL required before RCPT)")
+				break
+			}
+
+			match := rcptToRE.FindStringSubmatch(args)
+			if match == nil {
+				s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid TO parameter)")
+			} else {
+				// Mailpit Chaos
+				if fail, code := chaos.Config.Recipient.Trigger(); fail {
+					s.writef("%d Chaos recipient error", code)
+					break
+				}
+
+				if len(to) >= s.srv.MaxRecipients {
+					s.writef("452 4.5.3 Too many recipients")
+				} else {
+					accept := true
+					if s.srv.HandlerRcpt != nil {
+						accept = s.srv.HandlerRcpt(s.conn.RemoteAddr(), from, match[1])
+					}
+					if accept {
+						to = append(to, match[1])
+						s.writef("250 2.1.5 Ok")
+					} else {
+						s.writef("550 5.1.0 Requested action not taken: mailbox unavailable")
+					}
+				}
+			}
+		case "DATA":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+			if !gotFrom || len(to) == 0 {
+				s.writef("503 5.5.1 Bad sequence of commands (MAIL & RCPT required before DATA)")
+				break
+			}
+
+			s.writef("354 Start mail input; end with <CR><LF>.<CR><LF>")
+
+			// Attempt to read message body from the socket.
+			// On timeout, send a timeout message and return from serve().
+			// On net.Error, assume the client has gone away i.e. return from serve().
+			// On other errors, allow the client to try again.
+			data, err := s.readData()
+			if err != nil {
+				switch err.(type) {
+				case net.Error:
+					if err.(net.Error).Timeout() {
+						s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+					}
+					break loop
+				case maxSizeExceededError:
+					s.writef("%s", err.Error())
+					continue
+				default:
+					s.writef("451 4.3.0 Requested action aborted: local error in processing")
+					continue
+				}
+			}
+
+			// Create Received header & write message body into buffer.
+			buffer.Reset()
+			buffer.Write(s.makeHeaders(to))
+			buffer.Write(data)
+
+			// Pass mail on to handler.
+			if s.srv.Handler != nil {
+				err := s.srv.Handler(s.conn.RemoteAddr(), from, to, buffer.Bytes())
+				if err != nil {
+					checkErrFormat := regexp.MustCompile(`^([2-5][0-9]{2})[\s\-](.+)$`)
+					if checkErrFormat.MatchString(err.Error()) {
+						s.writef("%s", err.Error())
+					} else {
+						s.writef("451 4.3.5 Unable to process mail")
+					}
+					break
+				}
+				s.writef("250 2.0.0 Ok: queued")
+			} else if s.srv.MsgIDHandler != nil {
+				msgID, err := s.srv.MsgIDHandler(s.conn.RemoteAddr(), from, to, buffer.Bytes())
+				if err != nil {
+					checkErrFormat := regexp.MustCompile(`^([2-5][0-9]{2})[\s\-](.+)$`)
+					if checkErrFormat.MatchString(err.Error()) {
+						s.writef("%s", err.Error())
+					} else {
+						s.writef("451 4.3.5 Unable to process mail")
+					}
+					break
+				}
+
+				if msgID != "" {
+					s.writef("250 2.0.0 Ok: queued as %s", msgID)
+				} else {
+					s.writef("250 2.0.0 Ok: queued")
+				}
+			} else {
+				s.writef("250 2.0.0 Ok: queued")
+			}
+
+			// Reset for next mail.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "QUIT":
+			s.writef("221 2.0.0 %s %s ESMTP Service closing transmission channel", s.srv.Hostname, s.srv.AppName)
+			break loop
+		case "RSET":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			s.writef("250 2.0.0 Ok")
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "NOOP":
+			s.writef("250 2.0.0 Ok")
+		case "XCLIENT":
+			s.xClient = args
+			if s.xClientTrust {
+				xCArgs := strings.Split(args, " ")
+				for _, xCArg := range xCArgs {
+					xCParse := strings.Split(strings.TrimSpace(xCArg), "=")
+					if strings.ToUpper(xCParse[0]) == "ADDR" && (net.ParseIP(xCParse[1]) != nil) {
+						s.xClientADDR = xCParse[1]
+					}
+					if strings.ToUpper(xCParse[0]) == "NAME" && len(xCParse[1]) > 0 {
+						if xCParse[1] != "[UNAVAILABLE]" {
+							s.xClientNAME = xCParse[1]
+						}
+					}
+				}
+				if len(s.xClientADDR) > 7 {
+					s.remoteIP = s.xClientADDR
+					if len(s.xClientNAME) > 4 {
+						s.remoteHost = s.xClientNAME
+					} else {
+						names, err := net.LookupAddr(s.remoteIP)
+						if err == nil && len(names) > 0 {
+							s.remoteHost = names[0]
+						} else {
+							s.remoteHost = "unknown"
+						}
+					}
+				}
+			}
+			s.writef("250 2.0.0 Ok")
+		case "HELP", "VRFY", "EXPN":
+			// See RFC 5321 section 4.2.4 for usage of 500 & 502 response codes.
+			s.writef("502 5.5.1 Command not implemented")
+		case "STARTTLS":
+			// Parameters are not allowed (RFC 3207 section 4).
+			if args != "" {
+				s.writef("501 5.5.2 Syntax error (no parameters allowed)")
+				break
+			}
+
+			// Handle case where TLS is requested but not configured (and therefore not listed as a service extension).
+			if s.srv.TLSConfig == nil {
+				s.writef("502 5.5.1 Command not implemented")
+				break
+			}
+
+			// Handle case where STARTTLS is received when TLS is already in use.
+			if s.tls {
+				s.writef("503 5.5.1 Bad sequence of commands (TLS already in use)")
+				break
+			}
+
+			s.writef("220 2.0.0 Ready to start TLS")
+
+			// Establish a TLS connection with the client.
+			tlsConn := tls.Server(s.conn, s.srv.TLSConfig)
+			err := tlsConn.Handshake()
+			if err != nil {
+				s.writef("403 4.7.0 TLS handshake failed")
+				break
+			}
+
+			// TLS handshake succeeded, switch to using the TLS connection.
+			s.conn = tlsConn
+			s.br = bufio.NewReader(s.conn)
+			s.bw = bufio.NewWriter(s.conn)
+			s.tls = true
+
+			// RFC 3207 specifies that the server must discard any prior knowledge obtained from the client.
+			s.remoteName = ""
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "AUTH":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			// Handle case where AUTH is requested but not configured (and therefore not listed as a service extension).
+			if s.srv.AuthHandler == nil {
+				s.writef("502 5.5.1 Command not implemented")
+				break
+			}
+
+			// Handle case where AUTH is received when already authenticated.
+			if s.authenticated {
+				s.writef("503 5.5.1 Bad sequence of commands (already authenticated for this session)")
+				break
+			}
+
+			// RFC 4954 specifies that AUTH is not permitted during mail transactions.
+			if gotFrom || len(to) > 0 {
+				s.writef("503 5.5.1 Bad sequence of commands (AUTH not permitted during mail transaction)")
+				break
+			}
+
+			// RFC 4954 requires a mechanism parameter.
+			authType, authArgs := s.parseLine(args)
+			if authType == "" {
+				s.writef("501 5.5.4 Malformed AUTH input (argument required)")
+				break
+			}
+
+			// RFC 4954 requires rejecting unsupported authentication mechanisms with a 504 response.
+			allowedAuth := s.authMechs()
+			if allowed, found := allowedAuth[authType]; !found || !allowed {
+				s.writef("504 5.5.4 Unrecognized authentication type")
+				break
+			}
+
+			// Mailpit Chaos
+			if fail, code := chaos.Config.Authentication.Trigger(); fail {
+				s.writef("%d Chaos authentication error", code)
+				break
+			}
+
+			// RFC 4954 also specifies that ESMTP code 5.5.4 ("Invalid command arguments") should be returned
+			// when attempting to use an unsupported authentication type.
+			// Many servers return 5.7.4 ("Security features not supported") instead.
+			switch authType {
+			case "PLAIN":
+				s.authenticated, err = s.handleAuthPlain(authArgs)
+			case "LOGIN":
+				s.authenticated, err = s.handleAuthLogin(authArgs)
+			case "CRAM-MD5":
+				s.authenticated, err = s.handleAuthCramMD5()
+			}
+
+			if err != nil {
+				if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+					s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+					break loop
+				}
+
+				s.writef("%s", err.Error())
+				break
+			}
+
+			if s.authenticated {
+				s.writef("235 2.7.0 Authentication successful")
+			} else {
+				s.writef("535 5.7.8 Authentication credentials invalid")
+			}
+		default:
+			// See RFC 5321 section 4.2.4 for usage of 500 & 502 response codes.
+			s.writef("500 5.5.2 Syntax error, command unrecognized")
+		}
+	}
+}
+
+// Wrapper function for writing a complete line to the socket.
+func (s *session) writef(format string, args ...interface{}) {
+	if s.srv.Timeout > 0 {
+		_ = s.conn.SetWriteDeadline(time.Now().Add(s.srv.Timeout))
+	}
+
+	line := fmt.Sprintf(format, args...)
+	fmt.Fprintf(s.bw, "%s\r\n", line)
+	_ = s.bw.Flush()
+
+	if Debug {
+		verb := "WROTE"
+		if s.srv.LogWrite != nil {
+			s.srv.LogWrite(s.remoteIP, verb, line)
+		} else {
+			log.Println(s.remoteIP, verb, line)
+		}
+	}
+}
+
+// Read a complete line from the socket.
+func (s *session) readLine() (string, error) {
+	if s.srv.Timeout > 0 {
+		_ = s.conn.SetReadDeadline(time.Now().Add(s.srv.Timeout))
+	}
+
+	line, err := s.br.ReadString('\n')
+	if err != nil {
+		return "", err
+	}
+	line = strings.TrimSpace(line) // Strip trailing \r\n
+
+	if Debug {
+		verb := "READ"
+		if s.srv.LogRead != nil {
+			s.srv.LogRead(s.remoteIP, verb, line)
+		} else {
+			log.Println(s.remoteIP, verb, line)
+		}
+	}
+
+	return line, err
+}
+
+// Parse a line read from the socket.
+func (s *session) parseLine(line string) (verb string, args string) {
+	if idx := strings.Index(line, " "); idx != -1 {
+		verb = strings.ToUpper(line[:idx])
+		args = strings.TrimSpace(line[idx+1:])
+	} else {
+		verb = strings.ToUpper(line)
+		args = ""
+	}
+	return verb, args
+}
+
+// Read the message data following a DATA command.
+func (s *session) readData() ([]byte, error) {
+	var data []byte
+	for {
+		if s.srv.Timeout > 0 {
+			_ = s.conn.SetReadDeadline(time.Now().Add(s.srv.Timeout))
+		}
+
+		line, err := s.br.ReadBytes('\n')
+		if err != nil {
+			return nil, err
+		}
+		// Handle end of data denoted by lone period (\r\n.\r\n)
+		if bytes.Equal(line, []byte(".\r\n")) {
+			break
+		}
+		// Remove leading period (RFC 5321 section 4.5.2)
+		if line[0] == '.' {
+			line = line[1:]
+		}
+
+		// Enforce the maximum message size limit.
+		if s.srv.MaxSize > 0 {
+			if len(data)+len(line) > s.srv.MaxSize {
+				_, _ = s.br.Discard(s.br.Buffered()) // Discard the buffer remnants.
+				return nil, maxSizeExceeded(s.srv.MaxSize)
+			}
+		}
+
+		data = append(data, line...)
+	}
+	return data, nil
+}
+
+// Create the Received header to comply with RFC 2821 section 3.8.2.
+// TODO: Work out what to do with multiple to addresses.
+func (s *session) makeHeaders(to []string) []byte {
+	var buffer bytes.Buffer
+	now := time.Now().Format("Mon, 2 Jan 2006 15:04:05 -0700 (MST)")
+	buffer.WriteString(fmt.Sprintf("Received: from %s (%s [%s])\r\n", s.remoteName, s.remoteHost, s.remoteIP))
+	buffer.WriteString(fmt.Sprintf("        by %s (%s) with SMTP\r\n", s.srv.Hostname, s.srv.AppName))
+	buffer.WriteString(fmt.Sprintf("        for <%s>; %s\r\n", to[0], now))
+	return buffer.Bytes()
+}
+
+// Determine allowed authentication mechanisms.
+// RFC 4954 specifies that plaintext authentication mechanisms such as LOGIN and PLAIN require a TLS connection.
+// This can be explicitly overridden e.g. setting s.srv.AuthMechs["LOGIN"] = true.
+func (s *session) authMechs() (mechs map[string]bool) {
+	mechs = map[string]bool{"LOGIN": s.tls, "PLAIN": s.tls, "CRAM-MD5": true}
+
+	for mech := range mechs {
+		allowed, found := s.srv.AuthMechs[mech]
+		if found {
+			mechs[mech] = allowed
+		}
+	}
+
+	return
+}
+
+// Create the greeting string sent in response to an EHLO command.
+func (s *session) makeEHLOResponse() (response string) {
+	response = fmt.Sprintf("250-%s greets %s\r\n", s.srv.Hostname, s.remoteName)
+
+	// RFC 1870 specifies that "SIZE 0" indicates no maximum size is in force.
+	response += fmt.Sprintf("250-SIZE %d\r\n", s.srv.MaxSize)
+
+	// Only list STARTTLS if TLS is configured, but not currently in use.
+	if s.srv.TLSConfig != nil && !s.tls {
+		response += "250-STARTTLS\r\n"
+	}
+
+	// Only list AUTH if an AuthHandler is configured and at least one mechanism is allowed.
+	if s.srv.AuthHandler != nil {
+		var mechs []string
+		for mech, allowed := range s.authMechs() {
+			if allowed {
+				mechs = append(mechs, mech)
+			}
+		}
+		if len(mechs) > 0 {
+			response += "250-AUTH " + strings.Join(mechs, " ") + "\r\n"
+		}
+	}
+
+	response += "250 ENHANCEDSTATUSCODES"
+	return
+}
+
+func (s *session) handleAuthLogin(arg string) (bool, error) {
+	var err error
+
+	if arg == "" {
+		s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte("Username:")))
+		arg, err = s.readLine()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	username, err := base64.StdEncoding.DecodeString(arg)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte("Password:")))
+	line, err := s.readLine()
+	if err != nil {
+		return false, err
+	}
+
+	password, err := base64.StdEncoding.DecodeString(line)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "LOGIN", username, password, nil)
+
+	return authenticated, err
+}
+
+func (s *session) handleAuthPlain(arg string) (bool, error) {
+	var err error
+
+	// If fast mode (AUTH PLAIN [arg]) is not used, prompt for credentials.
+	if arg == "" {
+		s.writef("334 ")
+		arg, err = s.readLine()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	data, err := base64.StdEncoding.DecodeString(arg)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	parts := bytes.Split(data, []byte{0})
+	if len(parts) != 3 {
+		return false, errors.New("501 5.5.2 Syntax error (unable to parse)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "PLAIN", parts[1], parts[2], nil)
+
+	return authenticated, err
+}
+
+func (s *session) handleAuthCramMD5() (bool, error) {
+	shared := "<" + strconv.Itoa(os.Getpid()) + "." + strconv.Itoa(time.Now().Nanosecond()) + "@" + s.srv.Hostname + ">"
+
+	s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte(shared)))
+
+	data, err := s.readLine()
+	if err != nil {
+		return false, err
+	}
+
+	if data == "*" {
+		return false, errors.New("501 5.7.0 Authentication cancelled")
+	}
+
+	buf, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	fields := strings.Split(string(buf), " ")
+	if len(fields) < 2 {
+		return false, errors.New("501 5.5.2 Syntax error (unable to parse)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "CRAM-MD5", []byte(fields[0]), []byte(fields[1]), []byte(shared))
+
+	return authenticated, err
+}

internal/spamassassin/postmark/postmark.go

@@ -0,0 +1,100 @@
+// Package postmark uses the free https://spamcheck.postmarkapp.com/
+// See https://spamcheck.postmarkapp.com/doc/ for more details.
+package postmark
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Response struct
+type Response struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"` // for errors only
+	Score   string `json:"score"`
+	Rules   []Rule `json:"rules"`
+	Report  string `json:"report"` // ignored
+}
+
+// Rule struct
+type Rule struct {
+	Score string `json:"score"`
+	// Name not returned by postmark but rather extracted from description
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// Check will post the email data to Postmark
+func Check(email []byte, timeout int) (Response, error) {
+	r := Response{}
+	// '{"email":"raw dump of email", "options":"short"}'
+	var d struct {
+		// The raw dump of the email to be filtered, including all headers.
+		Email string `json:"email"`
+		// Default "long". Must either be "long" for a full report of processing rules, or "short" for a score request.
+		Options string `json:"options"`
+	}
+
+	d.Email = string(email)
+	d.Options = "long"
+
+	data, err := json.Marshal(d)
+	if err != nil {
+		return r, err
+	}
+
+	client := http.Client{
+		Timeout: time.Duration(timeout) * time.Second,
+	}
+
+	resp, err := client.Post("https://spamcheck.postmarkapp.com/filter", "application/json",
+		bytes.NewBuffer(data))
+
+	if err != nil {
+		return r, err
+	}
+
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&r)
+
+	// remove trailing line spaces for all lines in report
+	re := regexp.MustCompile("\r?\n")
+	lines := re.Split(r.Report, -1)
+	reportLines := []string{}
+	for _, l := range lines {
+		line := strings.TrimRight(l, " ")
+		reportLines = append(reportLines, line)
+	}
+	reportRaw := strings.Join(reportLines, "\n")
+
+	// join description lines to make a single line per rule
+	re2 := regexp.MustCompile("\n                                ")
+	report := re2.ReplaceAllString(reportRaw, "")
+	for i, rule := range r.Rules {
+		// populate rule name
+		r.Rules[i].Name = nameFromReport(rule.Score, rule.Description, report)
+	}
+
+	return r, err
+}
+
+// Extract the name of the test from the report as Postmark does not include this in the JSON reports
+func nameFromReport(score, description, report string) string {
+	score = regexp.QuoteMeta(score)
+	description = regexp.QuoteMeta(description)
+	str := fmt.Sprintf("%s\\s+([A-Z0-9\\_]+)\\s+%s", score, description)
+	re := regexp.MustCompile(str)
+
+	matches := re.FindAllStringSubmatch(report, 1)
+	if len(matches) > 0 && len(matches[0]) == 2 {
+		return strings.TrimSpace(matches[0][1])
+	}
+
+	return ""
+}

internal/spamassassin/spamassassin.go

@@ -0,0 +1,147 @@
+// Package spamassassin will return results from either a SpamAssassin server or
+// Postmark's public API depending on configuration
+package spamassassin
+
+import (
+	"errors"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/spamassassin/postmark"
+	"github.com/axllent/mailpit/internal/spamassassin/spamc"
+)
+
+var (
+	// Service to use, either "<host>:<ip>" for self-hosted SpamAssassin or "postmark"
+	service string
+
+	// SpamScore is the score at which a message is determined to be spam
+	spamScore = 5.0
+
+	// Timeout in seconds
+	timeout = 8
+)
+
+// Result is a SpamAssassin result
+//
+// swagger:model SpamAssassinResponse
+type Result struct {
+	// Whether the message is spam or not
+	IsSpam bool
+	// If populated will return an error string
+	Error string
+	// Total spam score based on triggered rules
+	Score float64
+	// Spam rules triggered
+	Rules []Rule
+}
+
+// Rule struct
+type Rule struct {
+	// Spam rule score
+	Score float64
+	// SpamAssassin rule name
+	Name string
+	// SpamAssassin rule description
+	Description string
+}
+
+// SetService defines which service should be used.
+func SetService(s string) {
+	switch s {
+	case "postmark":
+		service = "postmark"
+	default:
+		service = s
+	}
+}
+
+// SetTimeout defines the timeout
+func SetTimeout(t int) {
+	if t > 0 {
+		timeout = t
+	}
+}
+
+// Ping returns whether a service is active or not
+func Ping() error {
+	if service == "postmark" {
+		return nil
+	}
+
+	var client *spamc.Client
+	if strings.HasPrefix(service, "unix:") {
+		client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+	} else {
+		client = spamc.NewTCP(service, timeout)
+	}
+
+	return client.Ping()
+}
+
+// Check will return a Result
+func Check(msg []byte) (Result, error) {
+	r := Result{Score: 0}
+
+	if service == "" {
+		return r, errors.New("no SpamAssassin service defined")
+	}
+
+	if service == "postmark" {
+		res, err := postmark.Check(msg, timeout)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		resFloat, err := strconv.ParseFloat(res.Score, 32)
+		if err == nil {
+			r.Score = round1dm(resFloat)
+			r.IsSpam = resFloat >= spamScore
+		}
+		r.Error = res.Message
+		for _, pr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(pr.Score, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = pr.Name
+			rule.Description = pr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	} else {
+		var client *spamc.Client
+		if strings.HasPrefix(service, "unix:") {
+			client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+		} else {
+			client = spamc.NewTCP(service, timeout)
+		}
+
+		res, err := client.Report(msg)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		r.IsSpam = res.Score >= spamScore
+		r.Score = round1dm(res.Score)
+		r.Rules = []Rule{}
+		for _, sr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(sr.Points, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = sr.Name
+			rule.Description = sr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	}
+
+	return r, nil
+}
+
+// Round to one decimal place
+func round1dm(n float64) float64 {
+	return math.Floor(n*10) / 10
+}

internal/spamassassin/spamc/spamc.go

@@ -0,0 +1,250 @@
+// Package spamc provides a client for the SpamAssassin spamd protocol.
+// http://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+//
+// Modified to add timeouts from https://github.com/cgt/spamc
+package spamc
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// ProtoVersion is the protocol version
+const ProtoVersion = "1.5"
+
+var (
+	spamInfoRe    = regexp.MustCompile(`(.+)\/(.+) (\d+) (.+)`)
+	spamMainRe    = regexp.MustCompile(`^Spam: (.+) ; (.+) . (.+)$`)
+	spamDetailsRe = regexp.MustCompile(`^\s?(-?[0-9\.]+)\s([a-zA-Z0-9_]*)(\W*)(.*)`)
+)
+
+// connection is like net.Conn except that it also has a CloseWrite method.
+// CloseWrite is implemented by net.TCPConn and net.UnixConn, but for some
+// reason it is not present in the net.Conn interface.
+type connection interface {
+	net.Conn
+	CloseWrite() error
+}
+
+// Client is a spamd client.
+type Client struct {
+	net     string
+	addr    string
+	timeout int
+}
+
+// NewTCP returns a *Client that connects to spamd via the given TCP address.
+func NewTCP(addr string, timeout int) *Client {
+	return &Client{"tcp", addr, timeout}
+}
+
+// NewUnix returns a *Client that connects to spamd via the given Unix socket.
+func NewUnix(addr string) *Client {
+	return &Client{"unix", addr, 0}
+}
+
+// Rule represents a matched SpamAssassin rule.
+type Rule struct {
+	Points      string
+	Name        string
+	Description string
+}
+
+// Result struct
+type Result struct {
+	ResponseCode int
+	Message      string
+	Spam         bool
+	Score        float64
+	Threshold    float64
+	Rules        []Rule
+}
+
+// dial connects to spamd through TCP or a Unix socket.
+func (c *Client) dial() (connection, error) {
+	if c.net == "tcp" {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialTCP("tcp", nil, tcpAddr)
+	} else if c.net == "unix" {
+		unixAddr, err := net.ResolveUnixAddr("unix", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialUnix("unix", nil, unixAddr)
+	}
+
+	panic("Client.net must be either \"tcp\" or \"unix\"")
+}
+
+// Report checks if message is spam or not, and returns score plus report
+func (c *Client) Report(email []byte) (Result, error) {
+	output, err := c.report(email)
+	if err != nil {
+		return Result{}, err
+	}
+
+	return c.parseOutput(output), nil
+}
+
+func (c *Client) report(email []byte) ([]string, error) {
+	conn, err := c.dial()
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return nil, err
+	}
+
+	bw := bufio.NewWriter(conn)
+	if _, err := bw.WriteString("REPORT SPAMC/" + ProtoVersion + "\r\n"); err != nil {
+		return nil, err
+	}
+
+	if _, err := bw.WriteString("Content-length: " + strconv.Itoa(len(email)) + "\r\n\r\n"); err != nil {
+		return nil, err
+	}
+
+	if _, err := bw.Write(email); err != nil {
+		return nil, err
+	}
+
+	if err := bw.Flush(); err != nil {
+		return nil, err
+	}
+
+	// Client is supposed to close its writing side of the connection
+	// after sending its request.
+	if err := conn.CloseWrite(); err != nil {
+		return nil, err
+	}
+
+	var (
+		lines []string
+		br    = bufio.NewReader(conn)
+	)
+
+	for {
+		line, err := br.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		line = strings.TrimRight(line, " \t\r\n")
+		lines = append(lines, line)
+	}
+
+	// join lines, and replace multi-line descriptions with single line for each
+	tmp := strings.Join(lines, "\n")
+	re := regexp.MustCompile("\n                            ")
+	n := re.ReplaceAllString(tmp, " ")
+
+	//split lines again
+	return strings.Split(n, "\n"), nil
+}
+
+func (c *Client) parseOutput(output []string) Result {
+	var result Result
+	var reachedRules bool
+	for _, row := range output {
+		// header
+		if spamInfoRe.MatchString(row) {
+			res := spamInfoRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				resCode, err := strconv.Atoi(res[3])
+				if err == nil {
+					result.ResponseCode = resCode
+				}
+				result.Message = res[4]
+				continue
+			}
+		}
+
+		// summary
+		if spamMainRe.MatchString(row) {
+			res := spamMainRe.FindStringSubmatch(row)
+			if len(res) == 4 {
+				if tools.InArray(res[1], []string{"true", "yes"}) {
+					result.Spam = true
+				} else {
+					result.Spam = false
+				}
+				resFloat, err := strconv.ParseFloat(res[2], 32)
+				if err == nil {
+					result.Score = resFloat
+					continue
+				}
+				resFloat, err = strconv.ParseFloat(res[3], 32)
+				if err == nil {
+					result.Threshold = resFloat
+					continue
+				}
+			}
+		}
+
+		if strings.HasPrefix(row, "Content analysis details") {
+			reachedRules = true
+			continue
+		}
+
+		// details
+		if reachedRules && spamDetailsRe.MatchString(row) {
+			res := spamDetailsRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				rule := Rule{Points: res[1], Name: res[2], Description: res[4]}
+				result.Rules = append(result.Rules, rule)
+			}
+		}
+	}
+
+	return result
+}
+
+// Ping the spamd
+func (c *Client) Ping() error {
+	conn, err := c.dial()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return err
+	}
+
+	if _, err := io.WriteString(conn, fmt.Sprintf("PING SPAMC/%s\r\n\r\n", ProtoVersion)); err != nil {
+		return err
+	}
+
+	if err := conn.CloseWrite(); err != nil {
+		return err
+	}
+
+	br := bufio.NewReader(conn)
+	for {
+		_, err = br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/stats/stats.go

@@ -0,0 +1,131 @@
+// Package stats stores and returns Mailpit statistics
+package stats
+
+import (
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/updater"
+)
+
+var (
+	// to prevent hammering Github for latest version
+	latestVersionCache string
+
+	// StartedAt is set to the current ime when Mailpit starts
+	startedAt time.Time
+
+	mu sync.RWMutex
+
+	smtpAccepted     float64
+	smtpAcceptedSize float64
+	smtpRejected     float64
+	smtpIgnored      float64
+)
+
+// AppInformation struct
+// swagger:model AppInformation
+type AppInformation struct {
+	// Current Mailpit version
+	Version string
+	// Latest Mailpit version
+	LatestVersion string
+	// Database path
+	Database string
+	// Database size in bytes
+	DatabaseSize float64
+	// Total number of messages in the database
+	Messages float64
+	// Total number of messages in the database
+	Unread float64
+	// Tags and message totals per tag
+	Tags map[string]int64
+	// Runtime statistics
+	RuntimeStats struct {
+		// Mailpit server uptime in seconds
+		Uptime float64
+		// Current memory usage in bytes
+		Memory uint64
+		// Database runtime messages deleted
+		MessagesDeleted float64
+		// Accepted runtime SMTP messages
+		SMTPAccepted float64
+		// Total runtime accepted messages size in bytes
+		SMTPAcceptedSize float64
+		// Rejected runtime SMTP messages
+		SMTPRejected float64
+		// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
+		SMTPIgnored float64
+	}
+}
+
+// Load the current statistics
+func Load() AppInformation {
+	info := AppInformation{}
+	info.Version = config.Version
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	info.RuntimeStats.Memory = m.Sys - m.HeapReleased
+	info.RuntimeStats.Uptime = time.Since(startedAt).Seconds()
+	info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
+	info.RuntimeStats.SMTPAccepted = smtpAccepted
+	info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
+	info.RuntimeStats.SMTPRejected = smtpRejected
+	info.RuntimeStats.SMTPIgnored = smtpIgnored
+
+	if latestVersionCache != "" {
+		info.LatestVersion = latestVersionCache
+	} else {
+		latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
+		if err == nil {
+			info.LatestVersion = latest
+			latestVersionCache = latest
+
+			// clear latest version cache after 5 minutes
+			go func() {
+				time.Sleep(5 * time.Minute)
+				latestVersionCache = ""
+			}()
+		}
+	}
+
+	info.Database = config.Database
+	info.DatabaseSize = storage.DbSize()
+	info.Messages = storage.CountTotal()
+	info.Unread = storage.CountUnread()
+	info.Tags = storage.GetAllTagsCount()
+
+	return info
+}
+
+// Track will start the statistics logging in memory
+func Track() {
+	startedAt = time.Now()
+}
+
+// LogSMTPAccepted logs a successful SMTP transaction
+func LogSMTPAccepted(size int) {
+	mu.Lock()
+	smtpAccepted = smtpAccepted + 1
+	smtpAcceptedSize = smtpAcceptedSize + float64(size)
+	mu.Unlock()
+}
+
+// LogSMTPRejected logs a rejected SMTP transaction
+func LogSMTPRejected() {
+	mu.Lock()
+	smtpRejected = smtpRejected + 1
+	mu.Unlock()
+}
+
+// LogSMTPIgnored logs an ignored SMTP transaction
+func LogSMTPIgnored() {
+	mu.Lock()
+	smtpIgnored = smtpIgnored + 1
+	mu.Unlock()
+}

internal/storage/cron.go

@@ -0,0 +1,216 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+// Database cron runs every minute
+func dbCron() {
+	for {
+		time.Sleep(60 * time.Second)
+
+		currentTime := time.Now()
+		sinceLastDbAction := currentTime.Sub(dbLastAction)
+
+		// only run the database has been idle for 5 minutes
+		if math.Floor(sinceLastDbAction.Minutes()) == 5 {
+			deletedSize := getDeletedSize()
+
+			if deletedSize > 0 {
+				total := totalMessagesSize()
+				var deletedPercent float64
+				if total == 0 {
+					deletedPercent = 100
+				} else {
+					deletedPercent = deletedSize * 100 / total
+				}
+				// only vacuum the DB if at least 1% of mail storage size has been deleted
+				if deletedPercent >= 1 {
+					logger.Log().Debugf("[db] deleted messages is %f%% of total size, reclaim space", deletedPercent)
+					vacuumDb()
+				}
+			}
+		}
+
+		pruneMessages()
+	}
+}
+
+// PruneMessages will auto-delete the oldest messages if messages > config.MaxMessages.
+// Set config.MaxMessages to 0 to disable.
+func pruneMessages() {
+	if config.MaxMessages < 1 && config.MaxAgeInHours == 0 {
+		return
+	}
+
+	start := time.Now()
+
+	ids := []string{}
+	var prunedSize int64
+	var size float64
+
+	// prune using `--max` if set
+	if config.MaxMessages > 0 {
+		total := CountTotal()
+		if total > float64(config.MaxAgeInHours) {
+			offset := config.MaxMessages
+			if config.DemoMode {
+				offset = 500
+			}
+			q := sqlf.Select("ID, Size").
+				From(tenant("mailbox")).
+				OrderBy("Created DESC").
+				Limit(5000).
+				Offset(offset)
+
+			if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+				var id string
+
+				if err := row.Scan(&id, &size); err != nil {
+					logger.Log().Errorf("[db] %s", err.Error())
+					return
+				}
+				ids = append(ids, id)
+				prunedSize = prunedSize + int64(size)
+
+			}); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+		}
+	}
+
+	// prune using `--max-age` if set
+	if config.MaxAgeInHours > 0 {
+		// now() minus the number of hours
+		ts := time.Now().Add(time.Duration(-config.MaxAgeInHours) * time.Hour).UnixMilli()
+
+		q := sqlf.Select("ID, Size").
+			From(tenant("mailbox")).
+			Where("Created < ?", ts).
+			Limit(5000)
+
+		if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			var id string
+
+			if err := row.Scan(&id, &size); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+
+			if !tools.InArray(id, ids) {
+				ids = append(ids, id)
+				prunedSize = prunedSize + int64(size)
+			}
+
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+	}
+
+	if len(ids) == 0 {
+		return
+	}
+
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("mailbox_data")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("message_tags")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("mailbox")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	err = tx.Commit()
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		if err := tx.Rollback(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	if err := pruneUnusedTags(); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	addDeletedSize(prunedSize)
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed)
+
+	logMessagesDeleted(len(ids))
+
+	if config.DemoMode {
+		vacuumDb()
+	}
+
+	websockets.Broadcast("prune", nil)
+}
+
+// Vacuum the database to reclaim space from deleted messages
+func vacuumDb() {
+	if sqlDriver == "rqlite" {
+		// let rqlite handle vacuuming
+		return
+	}
+
+	start := time.Now()
+
+	// set WAL file checkpoint
+	if _, err := db.Exec("PRAGMA wal_checkpoint"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	// vacuum database
+	if _, err := db.Exec("VACUUM"); err != nil {
+		logger.Log().Errorf("[db] VACUUM: %s", err.Error())
+		return
+	}
+
+	// truncate WAL file
+	if _, err := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] vacuum completed in %s", elapsed)
+}

internal/storage/database.go

@@ -0,0 +1,284 @@
+// Package storage handles all database actions
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"os/signal"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/klauspost/compress/zstd"
+	"github.com/leporo/sqlf"
+
+	// sqlite - https://gitlab.com/cznic/sqlite
+	_ "modernc.org/sqlite"
+
+	// rqlite - https://github.com/rqlite/gorqlite | https://rqlite.io/
+	_ "github.com/rqlite/gorqlite/stdlib"
+)
+
+var (
+	db           *sql.DB
+	dbFile       string
+	sqlDriver    string
+	dbLastAction time.Time
+
+	// zstd compression encoder & decoder
+	dbEncoder    *zstd.Encoder
+	dbDecoder, _ = zstd.NewReader(nil)
+
+	temporaryFiles = []string{}
+)
+
+// InitDB will initialise the database
+func InitDB() error {
+	// dbEncoder
+	var (
+		dsn string
+		err error
+	)
+
+	if config.Compression > 0 {
+		var compression zstd.EncoderLevel
+		switch config.Compression {
+		case 1:
+			compression = zstd.SpeedFastest
+		case 2:
+			compression = zstd.SpeedDefault
+		case 3:
+			compression = zstd.SpeedBestCompression
+		}
+		dbEncoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(compression))
+		if err != nil {
+			return err
+		}
+		logger.Log().Debugf("[db] storing messages with compression: %s", compression.String())
+	} else {
+		logger.Log().Debug("[db] storing messages with no compression")
+	}
+
+	p := config.Database
+
+	if p == "" {
+		// when no path is provided then we create a temporary file
+		// which will get deleted on Close(), SIGINT or SIGTERM
+		p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano())
+		// delete the Unix socket file on exit
+		AddTempFile(p)
+		sqlDriver = "sqlite"
+		dsn = p
+		logger.Log().Debugf("[db] using temporary database: %s", p)
+	} else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
+		sqlDriver = "rqlite"
+		dsn = p
+		logger.Log().Debugf("[db] opening rqlite database %s", p)
+	} else {
+		p = filepath.Clean(p)
+		sqlDriver = "sqlite"
+		dsn = fmt.Sprintf("file:%s?cache=shared", p)
+		logger.Log().Debugf("[db] opening database %s", p)
+	}
+
+	config.Database = p
+
+	if sqlDriver == "sqlite" {
+		if !isFile(p) {
+			// try create a file to ensure permissions
+			f, err := os.Create(p)
+			if err != nil {
+				return fmt.Errorf("[db] %s", err.Error())
+			}
+			_ = f.Close()
+		}
+	}
+
+	db, err = sql.Open(sqlDriver, dsn)
+	if err != nil {
+		return err
+	}
+
+	for i := 1; i < 6; i++ {
+		if err := Ping(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			logger.Log().Infof("[db] reconnecting in 5 seconds (attempt %d/5)", i)
+			time.Sleep(5 * time.Second)
+		} else {
+			continue
+		}
+	}
+
+	// prevent "database locked" errors
+	// @see https://github.com/mattn/go-sqlite3#faq
+	db.SetMaxOpenConns(1)
+
+	if sqlDriver == "sqlite" {
+		if config.DisableWAL {
+			// disable WAL mode for SQLite, allows NFS mounted DBs
+			_, err = db.Exec("PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL;")
+		} else {
+			// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
+			_, err = db.Exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	// create tables if necessary & apply migrations
+	if err := dbApplySchemas(); err != nil {
+		return err
+	}
+
+	LoadTagFilters()
+
+	dbFile = p
+	dbLastAction = time.Now()
+
+	sigs := make(chan os.Signal, 1)
+	// catch all signals since not explicitly listing
+	// Program that will listen to the SIGINT and SIGTERM
+	// SIGINT will listen to CTRL-C.
+	// SIGTERM will be caught if kill command executed
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
+	// method invoked upon seeing signal
+	go func() {
+		s := <-sigs
+		fmt.Printf("[db] got %s signal, shutting down\n", s)
+		Close()
+		os.Exit(0)
+	}()
+
+	// auto-prune & delete
+	go dbCron()
+
+	go dataMigrations()
+
+	return nil
+}
+
+// Tenant applies an optional prefix to the table name
+func tenant(table string) string {
+	return fmt.Sprintf("%s%s", config.TenantID, table)
+}
+
+// Close will close the database, and delete if temporary
+func Close() {
+	// on a fatal exit (eg: ports blocked), allow Mailpit to run migration tasks before closing the DB
+	time.Sleep(200 * time.Millisecond)
+
+	if db != nil {
+		if err := db.Close(); err != nil {
+			logger.Log().Warn("[db] error closing database, ignoring")
+		}
+	}
+
+	// allow SQLite to finish closing DB & write WAL logs if local
+	time.Sleep(100 * time.Millisecond)
+
+	// delete all temporary files
+	deleteTempFiles()
+}
+
+// Ping the database connection and return an error if unsuccessful
+func Ping() error {
+	return db.Ping()
+}
+
+// StatsGet returns the total/unread statistics for a mailbox
+func StatsGet() MailboxStats {
+	var (
+		total  = CountTotal()
+		unread = CountUnread()
+		tags   = GetAllTags()
+	)
+
+	dbLastAction = time.Now()
+
+	return MailboxStats{
+		Total:  total,
+		Unread: unread,
+		Tags:   tags,
+	}
+}
+
+// CountTotal returns the number of emails in the database
+func CountTotal() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// CountUnread returns the number of emails in the database that are unread.
+func CountUnread() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 0).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// CountRead returns the number of emails in the database that are read.
+func CountRead() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 1).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// DbSize returns the size of the SQLite database.
+func DbSize() float64 {
+	var total sql.NullFloat64
+
+	err := db.QueryRow("SELECT page_count * page_size AS size FROM pragma_page_count(), pragma_page_size()").Scan(&total)
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return total.Float64
+	}
+
+	return total.Float64
+}
+
+// IsUnread returns whether a message is unread or not.
+func IsUnread(id string) bool {
+	var unread int
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&unread).
+		Where("Read = ?", 0).
+		Where("ID = ?", id).
+		QueryRowAndClose(context.TODO(), db)
+
+	return unread == 1
+}
+
+// MessageIDExists checks whether a Message-ID exists in the DB
+func MessageIDExists(id string) bool {
+	var total int
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("MessageID = ?", id).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total != 0
+}

internal/storage/messages.go

@@ -0,0 +1,751 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/mail"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/webhook"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+// Store will save an email to the database tables.
+// Returns the database ID of the saved message.
+func Store(body *[]byte) (string, error) {
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	// Parse message body with enmime
+	env, err := parser.ReadEnvelope(bytes.NewReader(*body))
+	if err != nil {
+		logger.Log().Warnf("[message] %s", err.Error())
+		return "", nil
+	}
+
+	from := &mail.Address{}
+	fromJSON := addressToSlice(env, "From")
+	if len(fromJSON) > 0 {
+		from = fromJSON[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	obj := DBMailSummary{
+		From:    from,
+		To:      addressToSlice(env, "To"),
+		Cc:      addressToSlice(env, "Cc"),
+		Bcc:     addressToSlice(env, "Bcc"),
+		ReplyTo: addressToSlice(env, "Reply-To"),
+	}
+
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+	created := time.Now()
+
+	// use message date instead of created date
+	if config.UseMessageDates {
+		if mDate, err := env.Date(); err == nil {
+			created = mDate
+		}
+	}
+
+	// generate the search text
+	searchText := createSearchText(env)
+
+	// generate unique ID
+	id := shortuuid.New()
+
+	summaryJSON, err := json.Marshal(obj)
+	if err != nil {
+		return "", err
+	}
+
+	// begin a transaction to ensure both the message
+	// and data are stored successfully
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return "", err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	subject := env.GetHeader("Subject")
+	size := float64(len(*body))
+	inline := len(env.Inlines)
+	attachments := len(env.Attachments)
+	snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+	sql := fmt.Sprintf(`INSERT INTO %s 
+		(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) 
+		VALUES(?,?,?,?,?,?,?,?,?,0,?)`,
+		tenant("mailbox"),
+	) // #nosec
+
+	// insert mail summary data
+	_, err = tx.Exec(sql, created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet)
+	if err != nil {
+		return "", err
+	}
+
+	if config.Compression > 0 {
+		// insert compressed raw message
+		compressed := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size)))
+
+		if sqlDriver == "rqlite" {
+			// rqlite does not support binary data in query, so we need to encode the compressed message into hexadecimal
+			// string and then generate the SQL query, which is more memory intensive, especially with large messages
+			hexStr := hex.EncodeToString(compressed)
+			_, err = tx.Exec(fmt.Sprintf(`INSERT INTO %s (ID, Email, Compressed) VALUES(?, x'%s', 1)`, tenant("mailbox_data"), hexStr), id) // #nosec
+		} else {
+			_, err = tx.Exec(fmt.Sprintf(`INSERT INTO %s (ID, Email, Compressed) VALUES(?, ?, 1)`, tenant("mailbox_data")), id, compressed) // #nosec
+		}
+
+		compressed = nil
+	} else {
+		// insert uncompressed raw message
+		_, err = tx.Exec(fmt.Sprintf(`INSERT INTO %s (ID, Email, Compressed) VALUES(?, ?, 0)`, tenant("mailbox_data")), id, string(*body)) // #nosec
+	}
+
+	if err != nil {
+		return "", err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return "", err
+	}
+
+	// extract tags using pre-set tag filters, empty slice if not set
+	tags := findTagsInRawMessage(body)
+
+	if !config.TagsDisableXTags {
+		xTagsHdr := env.GetHeader("X-Tags")
+		if xTagsHdr != "" {
+			// extract tags from X-Tags header
+			tags = append(tags, tools.SetTagCasing(strings.Split(strings.TrimSpace(xTagsHdr), ","))...)
+		}
+	}
+
+	if !config.TagsDisablePlus {
+		// get tags from plus-addresses
+		tags = append(tags, obj.tagsFromPlusAddresses()...)
+	}
+
+	// extract tags from search matches, and sort and extract unique tags
+	tags = sortedUniqueTags(append(tags, tagFilterMatches(id)...))
+
+	setTags := []string{}
+	if len(tags) > 0 {
+		setTags, err = SetMessageTags(id, tags)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	c := &MessageSummary{}
+	if err := json.Unmarshal(summaryJSON, c); err != nil {
+		return "", err
+	}
+
+	c.Created = created
+	c.ID = id
+	c.MessageID = messageID
+	c.Attachments = attachments
+	c.Subject = subject
+	c.Size = size
+	c.Tags = setTags
+	c.Snippet = snippet
+
+	websockets.Broadcast("new", c)
+	webhook.Send(c)
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	logger.Log().Debugf("[db] saved message %s (%d bytes)", id, int64(size))
+
+	return id, nil
+}
+
+// List returns a subset of messages from the mailbox,
+// sorted latest to oldest
+func List(start int, beforeTS int64, limit int) ([]MessageSummary, error) {
+	results := []MessageSummary{}
+	tsStart := time.Now()
+
+	q := sqlf.From(tenant("mailbox") + " m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet`).
+		OrderBy("m.Created DESC")
+
+	if limit > 0 {
+		q = q.Limit(limit).Offset(start)
+	}
+
+	if beforeTS > 0 {
+		q = q.Where("Created < ?", beforeTS)
+	}
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var read int
+		var snippet string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[json] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(int64(created))
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+		// artificially generate ReplyTo if legacy data is missing Reply-To field
+		if em.ReplyTo == nil {
+			em.ReplyTo = []*mail.Address{}
+		}
+
+		results = append(results, em)
+	}); err != nil {
+		return results, err
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] list INBOX in %s", elapsed)
+
+	return results, nil
+}
+
+// GetMessage returns a Message generated from the mailbox_data collection.
+// If the message lacks a date header, then the received datetime is used.
+func GetMessage(id string) (*Message, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	env, err := parser.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var from *mail.Address
+	fromData := addressToSlice(env, "From")
+	if len(fromData) > 0 {
+		from = fromData[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
+	returnPath := strings.Trim(env.GetHeader("Return-Path"), "<>")
+	if returnPath == "" && from != nil {
+		returnPath = from.Address
+	}
+
+	date, err := env.Date()
+	if err != nil {
+		// return received datetime when message does not contain a date header
+		q := sqlf.From(tenant("mailbox")).
+			Select(`Created`).
+			Where(`ID = ?`, id)
+
+		if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			var created float64
+
+			if err := row.Scan(&created); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+
+			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
+
+			date = time.UnixMilli(int64(created))
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	obj := Message{
+		ID:         id,
+		MessageID:  messageID,
+		From:       from,
+		Date:       date,
+		To:         addressToSlice(env, "To"),
+		Cc:         addressToSlice(env, "Cc"),
+		Bcc:        addressToSlice(env, "Bcc"),
+		ReplyTo:    addressToSlice(env, "Reply-To"),
+		ReturnPath: returnPath,
+		Subject:    env.GetHeader("Subject"),
+		Tags:       getMessageTags(id),
+		Size:       float64(len(raw)),
+		Text:       env.Text,
+	}
+
+	obj.HTML = env.HTML
+	obj.Inline = []Attachment{}
+	obj.Attachments = []Attachment{}
+
+	for _, i := range env.Inlines {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, i := range env.OtherParts {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.FileName != "" || a.ContentID != "" {
+			obj.Attachments = append(obj.Attachments, AttachmentSummary(a))
+		}
+	}
+
+	// get List-Unsubscribe links if set
+	obj.ListUnsubscribe = ListUnsubscribe{}
+	obj.ListUnsubscribe.Links = []string{}
+	if env.GetHeader("List-Unsubscribe") != "" {
+		l := env.GetHeader("List-Unsubscribe")
+		links, err := tools.ListUnsubscribeParser(l)
+		obj.ListUnsubscribe.Header = l
+		obj.ListUnsubscribe.Links = links
+		if err != nil {
+			obj.ListUnsubscribe.Errors = err.Error()
+		}
+		obj.ListUnsubscribe.HeaderPost = env.GetHeader("List-Unsubscribe-Post")
+	}
+
+	// mark message as read
+	if err := MarkRead(id); err != nil {
+		return &obj, err
+	}
+
+	dbLastAction = time.Now()
+
+	return &obj, nil
+}
+
+// GetMessageRaw returns an []byte of the full message
+func GetMessageRaw(id string) ([]byte, error) {
+	var i, msg string
+	var compressed int
+	q := sqlf.From(tenant("mailbox_data")).
+		Select(`ID`).To(&i).
+		Select(`Email`).To(&msg).
+		Select(`Compressed`).To(&compressed).
+		Where(`ID = ?`, id)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return nil, err
+	}
+
+	if i == "" {
+		return nil, errors.New("message not found")
+	}
+
+	var data []byte
+	if sqlDriver == "rqlite" && compressed == 1 {
+		data, err = base64.StdEncoding.DecodeString(msg)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding base64 message: %w", err)
+		}
+	} else {
+		data = []byte(msg)
+	}
+
+	dbLastAction = time.Now()
+
+	if compressed == 1 {
+		raw, err := dbDecoder.DecodeAll(data, nil)
+		if err != nil {
+			return nil, fmt.Errorf("error decompressing message: %s", err.Error())
+		}
+
+		return raw, err
+	}
+
+	return data, nil
+}
+
+// GetAttachmentPart returns an *enmime.Part (attachment or inline) from a message
+func GetAttachmentPart(id, partID string) (*enmime.Part, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	env, err := parser.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range env.Inlines {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.OtherParts {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	dbLastAction = time.Now()
+
+	return nil, errors.New("attachment not found")
+}
+
+// AttachmentSummary returns a summary of the attachment without any binary data
+func AttachmentSummary(a *enmime.Part) Attachment {
+	o := Attachment{}
+	o.PartID = a.PartID
+	o.FileName = a.FileName
+	if o.FileName == "" {
+		o.FileName = a.ContentID
+	}
+	o.ContentType = a.ContentType
+	o.ContentID = a.ContentID
+	o.Size = float64(len(a.Content))
+
+	return o
+}
+
+// LatestID returns the latest message ID
+//
+// If a query argument is set in the request the function will return the
+// latest message matching the search
+func LatestID(r *http.Request) (string, error) {
+	var messages []MessageSummary
+	var err error
+
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search != "" {
+		messages, _, err = Search(search, r.URL.Query().Get("tz"), 0, 0, 1)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		messages, err = List(0, 0, 1)
+		if err != nil {
+			return "", err
+		}
+	}
+	if len(messages) == 0 {
+		return "", errors.New("Message not found")
+	}
+
+	return messages[0].ID, nil
+}
+
+// MarkRead will mark a message as read
+func MarkRead(id string) error {
+	if !IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 1).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as read", id)
+	}
+
+	BroadcastMailboxStats()
+
+	d := struct {
+		ID   string
+		Read bool
+	}{ID: id, Read: true}
+
+	websockets.Broadcast("update", d)
+
+	return err
+}
+
+// MarkAllRead will mark all messages as read
+func MarkAllRead() error {
+	var (
+		start = time.Now()
+		total = CountUnread()
+	)
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 1).
+		Where("Read = ?", 0).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %v messages as read in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkAllUnread will mark all messages as unread
+func MarkAllUnread() error {
+	var (
+		start = time.Now()
+		total = CountRead()
+	)
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 0).
+		Where("Read = ?", 1).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %v messages as unread in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkUnread will mark a message as unread
+func MarkUnread(id string) error {
+	if IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 0).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as unread", id)
+	}
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	d := struct {
+		ID   string
+		Read bool
+	}{ID: id, Read: false}
+
+	websockets.Broadcast("update", d)
+
+	return err
+}
+
+// DeleteMessages deletes one or more messages in bulk
+func DeleteMessages(ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	start := time.Now()
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	sql := fmt.Sprintf(`SELECT ID, Size FROM %s WHERE  ID IN (?%s)`, tenant("mailbox"), strings.Repeat(",?", len(args)-1)) // #nosec
+	rows, err := db.Query(sql, args...)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	toDelete := []string{}
+	var totalSize float64
+
+	for rows.Next() {
+		var id string
+		var size float64
+		if err := rows.Scan(&id, &size); err != nil {
+			return err
+		}
+		toDelete = append(toDelete, id)
+		totalSize = totalSize + size
+	}
+
+	if err = rows.Err(); err != nil {
+		return err
+	}
+
+	if len(toDelete) == 0 {
+		return nil // nothing to delete
+	}
+
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	args = make([]interface{}, len(toDelete))
+	for i, id := range toDelete {
+		args[i] = id
+	}
+
+	tables := []string{"mailbox", "mailbox_data", "message_tags"}
+
+	for _, t := range tables {
+		sql = fmt.Sprintf(`DELETE FROM %s WHERE ID IN (?%s)`, tenant(t), strings.Repeat(",?", len(ids)-1))
+
+		_, err = tx.Exec(sql, args...) // #nosec
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	dbLastAction = time.Now()
+	addDeletedSize(int64(totalSize))
+
+	logMessagesDeleted(len(toDelete))
+
+	_ = pruneUnusedTags()
+
+	elapsed := time.Since(start)
+
+	messages := "messages"
+	if len(toDelete) == 1 {
+		messages = "message"
+	}
+
+	logger.Log().Debugf("[db] deleted %d %s in %s", len(toDelete), messages, elapsed)
+
+	BroadcastMailboxStats()
+
+	// broadcast individual message deletions
+	for _, id := range toDelete {
+		d := struct {
+			ID string
+		}{ID: id}
+
+		websockets.Broadcast("delete", d)
+	}
+
+	return nil
+}
+
+// DeleteAllMessages will delete all messages from a mailbox
+func DeleteAllMessages() error {
+	var (
+		start = time.Now()
+		total int
+	)
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(context.TODO(), db)
+
+	// begin a transaction to ensure both the message
+	// summaries and data are deleted successfully
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	tables := []string{"mailbox", "mailbox_data", "tags", "message_tags"}
+
+	for _, t := range tables {
+		sql := fmt.Sprintf(`DELETE FROM %s`, tenant(t)) // #nosec
+		_, err := tx.Exec(sql)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed)
+
+	vacuumDb()
+
+	dbLastAction = time.Now()
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Warnf("[db] %s", err.Error())
+	}
+
+	logMessagesDeleted(total)
+
+	BroadcastMailboxStats()
+
+	websockets.Broadcast("truncate", nil)
+
+	return err
+}

internal/storage/messages_test.go

@@ -0,0 +1,206 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+)
+
+func TestTextEmailInserts(t *testing.T) {
+	setup("")
+	defer Close()
+
+	t.Log("Testing text email storage")
+
+	start := time.Now()
+
+	for i := 0; i < testRuns; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of text emails stored")
+
+	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+	delStart := time.Now()
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, CountTotal(), float64(0), "incorrect number of text emails deleted")
+
+	t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
+
+	assertEqualStats(t, 0, 0)
+}
+
+func TestMimeEmailInserts(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing mime email storage")
+		} else {
+			t.Logf("Testing mime email storage (tenant %s)", tenantID)
+		}
+
+		start := time.Now()
+
+		for i := 0; i < testRuns; i++ {
+			if _, err := Store(&testMimeEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of mime emails stored")
+
+		t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+		delStart := time.Now()
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, CountTotal(), float64(0), "incorrect number of mime emails deleted")
+
+		t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart))
+
+		Close()
+	}
+}
+
+func TestRetrieveMimeEmail(t *testing.T) {
+	compressionLevels := []int{0, 1, 2, 3}
+
+	for _, compressionLevel := range compressionLevels {
+		t.Logf("Testing compression level: %d", compressionLevel)
+		for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+			tenantID = config.DBTenantID(tenantID)
+			config.Compression = compressionLevel
+			setup(tenantID)
+
+			if tenantID == "" {
+				t.Log("Testing mime email retrieval")
+			} else {
+				t.Logf("Testing mime email retrieval (tenant %s)", tenantID)
+			}
+
+			id, err := Store(&testMimeEmail)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			msg, err := GetMessage(id)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+			assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+			assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+			assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+			assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+			assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+			assertEqual(t, len(msg.Attachments), 1, "incorrect number of attachments")
+			assertEqual(t, msg.Attachments[0].FileName, "Sample PDF.pdf", "attachment filename does not match")
+			assertEqual(t, len(msg.Inline), 1, "incorrect number of inline attachments")
+			assertEqual(t, msg.Inline[0].FileName, "inline-image.jpg", "inline attachment filename does not match")
+
+			attachmentData, err := GetAttachmentPart(id, msg.Attachments[0].PartID)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			assertEqual(t, float64(len(attachmentData.Content)), msg.Attachments[0].Size, "attachment size does not match")
+
+			inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			assertEqual(t, float64(len(inlineData.Content)), msg.Inline[0].Size, "inline attachment size does not match")
+
+			Close()
+		}
+	}
+
+	// reset compression
+	config.Compression = 1
+}
+
+func TestMessageSummary(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing message summary")
+		} else {
+			t.Logf("Testing message summary (tenant %s)", tenantID)
+		}
+
+		if _, err := Store(&testMimeEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		summaries, err := List(0, 0, 1)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, len(summaries), 1, "Expected 1 result")
+
+		msg := summaries[0]
+
+		assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+		assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+		assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+		assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+		assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+		assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+		assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match")
+		assertEqual(t, msg.Attachments, 1, "Expected 1 attachment")
+		assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match")
+
+		Close()
+	}
+}
+
+func BenchmarkImportText(b *testing.B) {
+	setup("")
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+}
+
+func BenchmarkImportMime(b *testing.B) {
+	setup("")
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testMimeEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+
+}

internal/storage/notifications.go

@@ -0,0 +1,38 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+var bcStatsDelay = false
+
+// BroadcastMailboxStats broadcasts the total number of messages
+// displayed to the web UI, as well as the total unread messages.
+// The lookup is very fast (< 10ms / 100k messages under load).
+// Rate limited to 4x per second.
+func BroadcastMailboxStats() {
+	if bcStatsDelay {
+		return
+	}
+
+	bcStatsDelay = true
+
+	go func() {
+		time.Sleep(250 * time.Millisecond)
+		bcStatsDelay = false
+		b := struct {
+			Total   float64
+			Unread  float64
+			Version string
+		}{
+			Total:   CountTotal(),
+			Unread:  CountUnread(),
+			Version: config.Version,
+		}
+
+		websockets.Broadcast("stats", b)
+	}()
+}

internal/storage/reindex.go

@@ -0,0 +1,146 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/mail"
+	"os"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+)
+
+// ReindexAll will regenerate the search text and snippet for a message
+// and update the database.
+func ReindexAll() {
+	ids := []string{}
+	var i string
+	chunkSize := 1000
+
+	finished := 0
+
+	err := sqlf.Select("ID").To(&i).
+		From(tenant("mailbox")).
+		OrderBy("Created DESC").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			ids = append(ids, i)
+		})
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		os.Exit(1)
+	}
+
+	total := len(ids)
+
+	chunks := chunkBy(ids, chunkSize)
+
+	logger.Log().Infof("reindexing %d messages", total)
+
+	type updateStruct struct {
+		// ID in database
+		ID string
+		// SearchText for searching
+		SearchText string
+		// Snippet for UI
+		Snippet string
+		// Metadata info
+		Metadata string
+	}
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	for _, ids := range chunks {
+		updates := []updateStruct{}
+
+		for _, id := range ids {
+			raw, err := GetMessageRaw(id)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+
+			r := bytes.NewReader(raw)
+
+			env, err := parser.ReadEnvelope(r)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			from := &mail.Address{}
+			fromJSON := addressToSlice(env, "From")
+			if len(fromJSON) > 0 {
+				from = fromJSON[0]
+			} else if env.GetHeader("From") != "" {
+				from = &mail.Address{Name: env.GetHeader("From")}
+			}
+
+			obj := DBMailSummary{
+				From:    from,
+				To:      addressToSlice(env, "To"),
+				Cc:      addressToSlice(env, "Cc"),
+				Bcc:     addressToSlice(env, "Bcc"),
+				ReplyTo: addressToSlice(env, "Reply-To"),
+			}
+
+			MetadataJSON, err := json.Marshal(obj)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			searchText := createSearchText(env)
+			snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+			u := updateStruct{}
+			u.ID = id
+			u.SearchText = searchText
+			u.Snippet = snippet
+			u.Metadata = string(MetadataJSON)
+
+			updates = append(updates, u)
+		}
+
+		ctx := context.Background()
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		// insert mail summary data
+		for _, u := range updates {
+			_, err = tx.Exec(fmt.Sprintf(`UPDATE %s SET SearchText = ?, Snippet = ?, Metadata = ? WHERE ID = ?`, tenant("mailbox")), u.SearchText, u.Snippet, u.Metadata, u.ID)
+			if err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				continue
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		finished += len(updates)
+
+		logger.Log().Printf("reindexed: %d / %d (%d%%)", finished, total, finished*100/total)
+	}
+}
+
+func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) {
+	for chunkSize < len(items) {
+		items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
+	}
+
+	return append(chunks, items)
+}

internal/storage/schemas.go

@@ -0,0 +1,152 @@
+package storage
+
+import (
+	"bytes"
+	"embed"
+	"log"
+	"path"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/semver"
+)
+
+//go:embed schemas/*
+var schemaScripts embed.FS
+
+// Create tables and apply schemas if required
+func dbApplySchemas() error {
+	if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS ` + tenant("schemas") + ` (Version TEXT PRIMARY KEY NOT NULL)`); err != nil {
+		return err
+	}
+
+	var legacyMigrationTable int
+	err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)`, tenant("darwin_migrations")).Scan(&legacyMigrationTable)
+	if err != nil {
+		return err
+	}
+
+	if legacyMigrationTable == 1 {
+		rows, err := db.Query(`SELECT version FROM ` + tenant("darwin_migrations"))
+		if err != nil {
+			return err
+		}
+
+		legacySchemas := []string{}
+
+		for rows.Next() {
+			var oldID string
+			if err := rows.Scan(&oldID); err == nil {
+				legacySchemas = append(legacySchemas, semver.MajorMinor(oldID)+"."+semver.Patch(oldID))
+			}
+		}
+
+		legacySchemas = semver.SortMin(legacySchemas)
+
+		for _, v := range legacySchemas {
+			var migrated int
+			err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, v).Scan(&migrated)
+			if err != nil {
+				return err
+			}
+			if migrated == 0 {
+				// copy to tenant("schemas")
+				if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, v); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	schemaFiles, err := schemaScripts.ReadDir("schemas")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	temp := template.New("")
+	temp.Funcs(
+		template.FuncMap{
+			"tenant": tenant,
+		},
+	)
+
+	type schema struct {
+		Name   string
+		Semver string
+	}
+
+	scripts := []schema{}
+
+	for _, s := range schemaFiles {
+		if !s.Type().IsRegular() || !strings.HasSuffix(s.Name(), ".sql") {
+			continue
+		}
+
+		schemaID := strings.TrimRight(s.Name(), ".sql")
+
+		if !semver.IsValid(schemaID) {
+			logger.Log().Warnf("[db] invalid schema name: %s", s.Name())
+			continue
+		}
+
+		script := schema{s.Name(), semver.MajorMinor(schemaID) + "." + semver.Patch(schemaID)}
+		scripts = append(scripts, script)
+	}
+
+	// sort schemas by semver, low to high
+	sort.Slice(scripts, func(i, j int) bool {
+		return semver.Compare(scripts[j].Semver, scripts[i].Semver) == 1
+	})
+
+	for _, s := range scripts {
+		var complete int
+		err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, s.Semver).Scan(&complete)
+		if err != nil {
+			return err
+		}
+
+		if complete == 1 {
+			// already completed, ignore
+			continue
+		}
+		// use path.Join for Windows compatibility, see https://github.com/golang/go/issues/44305
+		b, err := schemaScripts.ReadFile(path.Join("schemas", s.Name))
+		if err != nil {
+			return err
+		}
+
+		// parse import script
+		t1, err := temp.Parse(string(b))
+		if err != nil {
+			return err
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := t1.Execute(buf, nil); err != nil {
+			return err
+		}
+
+		if _, err := db.Exec(buf.String()); err != nil {
+			return err
+		}
+
+		if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, s.Semver); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("[db] applied schema: %s", s.Name)
+	}
+
+	return nil
+}
+
+// These functions are used to migrate data formats/structure on startup.
+func dataMigrations() {
+	// ensure DeletedSize has a value if empty
+	if SettingGet("DeletedSize") == "" {
+		_ = SettingPut("DeletedSize", "0")
+	}
+}

internal/storage/schemas/1.0.0.sql

@@ -0,0 +1,19 @@
+-- CREATE TABLES
+CREATE TABLE IF NOT EXISTS {{ tenant "mailbox" }} (
+	Sort INTEGER PRIMARY KEY AUTOINCREMENT,
+	ID TEXT NOT NULL,
+	Data BLOB,
+	Search TEXT,
+	Read INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_sort" }} ON {{ tenant "mailbox" }} (Sort);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read);
+
+CREATE TABLE IF NOT EXISTS {{ tenant "mailbox_data" }} (
+	ID TEXT KEY NOT NULL,
+	Email BLOB
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_data_id" }} ON {{ tenant "mailbox_data" }} (ID);

internal/storage/schemas/1.1.0.sql

@@ -0,0 +1,3 @@
+-- CREATE TAGS COLUMN
+ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Tags Text NOT NULL DEFAULT '[]';
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags);

internal/storage/schemas/1.2.0.sql

@@ -0,0 +1,36 @@
+-- CREATING NEW MAILBOX FORMAT
+CREATE TABLE IF NOT EXISTS {{ tenant "mailboxtmp" }} (
+	Created INTEGER NOT NULL,
+	ID TEXT NOT NULL,
+	MessageID TEXT NOT NULL,
+	Subject TEXT NOT NULL,
+	Metadata TEXT,
+	Size INTEGER NOT NULL,
+	Inline INTEGER NOT NULL,
+	Attachments INTEGER NOT NULL,
+	Read INTEGER,
+	Tags TEXT,
+	SearchText TEXT
+);
+
+INSERT INTO {{ tenant "mailboxtmp" }}
+	(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+	SELECT 
+		Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+		json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+		Search, Read, Tags
+	FROM {{ tenant "mailbox" }};
+
+DROP TABLE IF EXISTS {{ tenant "mailbox" }};
+
+ALTER TABLE {{ tenant "mailboxtmp" }} RENAME TO {{ tenant "mailbox" }};
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_created" }} ON {{ tenant "mailbox" }} (Created);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_id" }} ON {{ tenant "mailbox" }} (MessageID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_subject" }} ON {{ tenant "mailbox" }} (Subject);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_size" }} ON {{ tenant "mailbox" }} (Size);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_inline" }} ON {{ tenant "mailbox" }} (Inline);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_attachments" }} ON {{ tenant "mailbox" }} (Attachments);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags);

internal/storage/schemas/1.21.2.sql

@@ -0,0 +1,6 @@
+-- DROP LEGACY MIGRATION TABLE
+DROP TABLE IF EXISTS {{ tenant "darwin_migrations" }};
+
+-- DROP LEGACY TAGS COLUMN
+DROP INDEX IF EXISTS {{ tenant "idx_tags" }};
+ALTER TABLE {{ tenant "mailbox" }} DROP COLUMN Tags;

internal/storage/schemas/1.21.8.sql

@@ -0,0 +1,22 @@
+-- Rebuild message_tags to remove FOREIGN KEY REFERENCES
+PRAGMA foreign_keys=OFF;
+
+DROP INDEX IF EXISTS {{ tenant "idx_message_tag_id" }};
+DROP INDEX IF EXISTS {{ tenant "idx_message_tag_tagid" }};
+
+ALTER TABLE {{ tenant "message_tags" }} RENAME TO _{{ tenant "message_tags" }}_old;
+
+CREATE TABLE IF NOT EXISTS {{ tenant "message_tags" }} (
+	Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	ID TEXT NOT NULL,
+	TagID INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tags_id" }} ON {{ tenant "message_tags" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tags_tagid" }} ON {{ tenant "message_tags" }} (TagID);
+
+INSERT INTO {{ tenant "message_tags" }} SELECT * FROM _{{ tenant "message_tags" }}_old;
+
+DROP TABLE IF EXISTS _{{ tenant "message_tags" }}_old;
+
+PRAGMA foreign_keys=ON;

internal/storage/schemas/1.23.0.sql

@@ -0,0 +1,5 @@
+-- CREATE Compressed COLUMN IN mailbox_data
+ALTER TABLE {{ tenant "mailbox_data" }} ADD COLUMN Compressed INTEGER NOT NULL DEFAULT '0';
+
+-- SET Compressed = 1 for all existing data
+UPDATE {{ tenant "mailbox_data" }} SET Compressed = 1;

internal/storage/schemas/1.3.0.sql

@@ -0,0 +1,2 @@
+-- CREATE SNIPPET COLUMN
+ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Snippet TEXT NOT NULL DEFAULT '';

internal/storage/schemas/1.4.0.sql

@@ -0,0 +1,16 @@
+-- CREATE TAG TABLES
+CREATE TABLE IF NOT EXISTS {{ tenant "tags" }} (
+	ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	Name TEXT COLLATE NOCASE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_tag_name" }} ON {{ tenant "tags" }} (Name);
+
+CREATE TABLE IF NOT EXISTS {{ tenant "message_tags" }} (
+	Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	ID TEXT REFERENCES {{ tenant "mailbox" }} (ID),
+	TagID INT REFERENCES {{ tenant "tags" }} (ID)
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_id" }} ON {{ tenant "message_tags" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_tagid" }} ON {{ tenant "message_tags" }} (TagID);

internal/storage/schemas/1.5.0.sql

@@ -0,0 +1,7 @@
+-- CREATE SETTINGS TABLE
+CREATE TABLE IF NOT EXISTS {{ tenant "settings" }} (
+	Key TEXT,
+	Value TEXT
+);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_settings_key" }} ON {{ tenant "settings" }} (Key);
+INSERT INTO {{ tenant "settings" }} (Key, Value) VALUES ("DeletedSize", (SELECT SUM(Size)/2 FROM {{ tenant "mailbox" }}));

internal/storage/schemas/README.md

@@ -0,0 +1,5 @@
+# Migration scripts
+
+- Scripts should be named using semver and have the `.sql` extension.
+- Inline comments should be prefixed with a `--`
+- All references to tables and indexes should be wrapped with a `{{ tenant "<name>" }}`

internal/storage/search.go

@@ -0,0 +1,478 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/araddon/dateparse"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+// Search will search a mailbox for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func Search(search, timezone string, start int, beforeTS int64, limit int) ([]MessageSummary, int, error) {
+	results := []MessageSummary{}
+	allResults := []MessageSummary{}
+	tsStart := time.Now()
+	nrResults := 0
+	if limit < 0 {
+		limit = 50
+	}
+
+	q := searchQueryBuilder(search, timezone)
+
+	if beforeTS > 0 {
+		q = q.Where(`Created < ?`, beforeTS)
+	}
+
+	var err error
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var snippet string
+		var read int
+		var ignore string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(int64(created))
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+
+		allResults = append(allResults, em)
+	}); err != nil {
+		return results, nrResults, err
+	}
+
+	dbLastAction = time.Now()
+
+	nrResults = len(allResults)
+
+	if nrResults > start {
+		end := nrResults
+		if nrResults >= start+limit {
+			end = start + limit
+		}
+
+		results = allResults[start:end]
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
+
+	return results, nrResults, err
+}
+
+// DeleteSearch will delete all messages for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func DeleteSearch(search, timezone string) error {
+	q := searchQueryBuilder(search, timezone)
+
+	ids := []string{}
+	deleteSize := float64(0)
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var read int
+		var snippet string
+		var ignore string
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		ids = append(ids, id)
+		deleteSize = deleteSize + size
+	}); err != nil {
+		return err
+	}
+
+	if len(ids) > 0 {
+		total := len(ids)
+
+		// split ids into chunks of 1000 ids
+		var chunks [][]string
+		if total > 1000 {
+			chunkSize := 1000
+			chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize)
+			for chunkSize < len(ids) {
+				ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize])
+			}
+			if len(ids) > 0 {
+				// add remaining ids <= 1000
+				chunks = append(chunks, ids)
+			}
+		} else {
+			chunks = append(chunks, ids)
+		}
+
+		// begin a transaction to ensure both the message
+		// and data are deleted successfully
+		tx, err := db.BeginTx(context.Background(), nil)
+		if err != nil {
+			return err
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		for _, ids := range chunks {
+			delIDs := make([]interface{}, len(ids))
+			for i, id := range ids {
+				delIDs[i] = id
+			}
+
+			sqlDelete1 := `DELETE FROM ` + tenant("mailbox") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete1, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete2 := `DELETE FROM ` + tenant("mailbox_data") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete2, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete3 := `DELETE FROM ` + tenant("message_tags") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete3, delIDs...)
+			if err != nil {
+				return err
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			return err
+		}
+
+		if err := pruneUnusedTags(); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("[db] deleted %d messages matching %s", total, search)
+
+		dbLastAction = time.Now()
+
+		// broadcast changes
+		if len(ids) > 200 {
+			websockets.Broadcast("prune", nil)
+		} else {
+			for _, id := range ids {
+				d := struct {
+					ID string
+				}{ID: id}
+				websockets.Broadcast("delete", d)
+			}
+		}
+
+		addDeletedSize(int64(deleteSize))
+
+		logMessagesDeleted(total)
+
+		BroadcastMailboxStats()
+	}
+
+	return nil
+}
+
+// SearchParser returns the SQL syntax for the database search based on the search arguments
+func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt {
+	// group strings with quotes as a single argument and remove quotes
+	args := tools.ArgsParser(searchString)
+
+	if timezone != "" {
+		loc, err := time.LoadLocation(timezone)
+		if err != nil {
+			logger.Log().Warnf("ignoring invalid timezone:\"%s\"", timezone)
+		} else {
+			time.Local = loc
+		}
+	}
+
+	q := sqlf.From(tenant("mailbox") + " m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read,
+			m.Snippet,
+			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
+			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
+			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
+			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON,
+			IFNULL(json_extract(Metadata, '$.ReplyTo'), '{}') as ReplyToJSON
+		`).
+		OrderBy("m.Created DESC")
+
+	for _, w := range args {
+		if cleanString(w) == "" {
+			continue
+		}
+
+		// lowercase search to try match search prefixes
+		lw := strings.ToLower(w)
+
+		exclude := false
+		// search terms starting with a `-` or `!` imply an exclude
+		if len(w) > 1 && (strings.HasPrefix(w, "-") || strings.HasPrefix(w, "!")) {
+			exclude = true
+			w = w[1:]
+			lw = lw[1:]
+		}
+
+		// ignore blank searches
+		if len(w) == 0 {
+			continue
+		}
+
+		if strings.HasPrefix(lw, "to:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("ToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "from:") {
+			w = cleanString(w[5:])
+			if w != "" {
+				if exclude {
+					q.Where("FromJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("FromJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "cc:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("CcJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("CcJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "bcc:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where("BccJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("BccJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "reply-to:") {
+			w = cleanString(w[9:])
+			if w != "" {
+				if exclude {
+					q.Where("ReplyToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ReplyToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "addressed:") {
+			w = cleanString(w[10:])
+			arg := "%" + escPercentChar(w) + "%"
+			if w != "" {
+				if exclude {
+					q.Where("(ToJSON NOT LIKE ? AND FromJSON NOT LIKE ? AND CcJSON NOT LIKE ? AND BccJSON NOT LIKE ? AND ReplyToJSON NOT LIKE ?)", arg, arg, arg, arg, arg)
+				} else {
+					q.Where("(ToJSON LIKE ? OR FromJSON LIKE ? OR CcJSON LIKE ? OR BccJSON LIKE ? OR ReplyToJSON LIKE ?)", arg, arg, arg, arg, arg)
+				}
+			}
+		} else if strings.HasPrefix(lw, "subject:") {
+			w = w[8:]
+			if w != "" {
+				if exclude {
+					q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "message-id:") {
+			w = cleanString(w[11:])
+			if w != "" {
+				if exclude {
+					q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "tag:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where(`m.ID NOT IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				} else {
+					q.Where(`m.ID IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				}
+			}
+		} else if lw == "is:read" {
+			if exclude {
+				q.Where("Read = 0")
+			} else {
+				q.Where("Read = 1")
+			}
+		} else if lw == "is:unread" {
+			if exclude {
+				q.Where("Read = 1")
+			} else {
+				q.Where("Read = 0")
+			}
+		} else if lw == "is:tagged" {
+			if exclude {
+				q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`)
+			} else {
+				q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`)
+			}
+		} else if lw == "has:inline" || lw == "has:inlines" {
+			if exclude {
+				q.Where("Inline = 0")
+			} else {
+				q.Where("Inline > 0")
+			}
+		} else if lw == "has:attachment" || lw == "has:attachments" {
+			if exclude {
+				q.Where("Attachments = 0")
+			} else {
+				q.Where("Attachments > 0")
+			}
+		} else if strings.HasPrefix(lw, "after:") {
+			w = cleanString(w[6:])
+			if w != "" {
+				t, err := dateparse.ParseLocal(w)
+				if err != nil {
+					logger.Log().Warnf("ignoring invalid after: date \"%s\"", w)
+				} else {
+					timestamp := t.UnixMilli()
+					if exclude {
+						q.Where(`m.Created <= ?`, timestamp)
+					} else {
+						q.Where(`m.Created >= ?`, timestamp)
+					}
+				}
+			}
+		} else if strings.HasPrefix(lw, "before:") {
+			w = cleanString(w[7:])
+			if w != "" {
+				t, err := dateparse.ParseLocal(w)
+				if err != nil {
+					logger.Log().Warnf("ignoring invalid before: date \"%s\"", w)
+				} else {
+					timestamp := t.UnixMilli()
+					if exclude {
+						q.Where(`m.Created >= ?`, timestamp)
+					} else {
+						q.Where(`m.Created <= ?`, timestamp)
+					}
+				}
+			}
+		} else if strings.HasPrefix(lw, "larger:") && sizeToBytes(cleanString(w[7:])) > 0 {
+			w = cleanString(w[7:])
+			size := sizeToBytes(w)
+			if exclude {
+				q.Where("Size < ?", size)
+			} else {
+				q.Where("Size > ?", size)
+			}
+		} else if strings.HasPrefix(lw, "smaller:") && sizeToBytes(cleanString(w[8:])) > 0 {
+			w = cleanString(w[8:])
+			size := sizeToBytes(w)
+			if exclude {
+				q.Where("Size > ?", size)
+			} else {
+				q.Where("Size < ?", size)
+			}
+		} else {
+			// search text
+			if exclude {
+				q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			} else {
+				q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			}
+		}
+	}
+
+	return q
+}
+
+// Simple function to return a size in bytes, eg 2kb, 4MB or 1.5m.
+//
+// K, k, Kb, KB, kB and kb are treated as Kilobytes.
+// M, m, Mb, MB and mb are treated as Megabytes.
+func sizeToBytes(v string) int64 {
+	v = strings.ToLower(v)
+	re := regexp.MustCompile(`^(\d+)(\.\d+)?\s?([a-z]{1,2})?$`)
+
+	m := re.FindAllStringSubmatch(v, -1)
+	if len(m) == 0 {
+		return 0
+	}
+
+	val := fmt.Sprintf("%s%s", m[0][1], m[0][2])
+	unit := m[0][3]
+
+	i, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
+	if err != nil {
+		return 0
+	}
+
+	if unit == "" {
+		return int64(i)
+	}
+
+	if unit == "k" || unit == "kb" {
+		return int64(i * 1024)
+	}
+
+	if unit == "m" || unit == "mb" {
+		return int64(i * 1024 * 1024)
+	}
+
+	return 0
+}

internal/storage/search_test.go

@@ -0,0 +1,225 @@
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/jhillyerd/enmime"
+)
+
+func TestSearch(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing search")
+		} else {
+			t.Logf("Testing search (tenant %s)", tenantID)
+		}
+
+		for i := 0; i < testRuns; i++ {
+			msg := enmime.Builder().
+				From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+				CC(fmt.Sprintf("CC %d", i), fmt.Sprintf("cc-%d@example.com", i)).
+				CC(fmt.Sprintf("CC2 %d", i), fmt.Sprintf("cc2-%d@example.com", i)).
+				Subject(fmt.Sprintf("Subject line %d end", i)).
+				Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+				To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i)).
+				To(fmt.Sprintf("To2 %d", i), fmt.Sprintf("to2-%d@example.com", i)).
+				ReplyTo(fmt.Sprintf("Reply To %d", i), fmt.Sprintf("reply-to-%d@example.com", i))
+
+			env, err := msg.Build()
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			buf := new(bytes.Buffer)
+
+			if err := env.Encode(buf); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			bufBytes := buf.Bytes()
+
+			if _, err := Store(&bufBytes); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		for i := 1; i < 51; i++ {
+			// search a random something that will return a single result
+			uniqueSearches := []string{
+				fmt.Sprintf("from-%d@example.com", i),
+				fmt.Sprintf("from:from-%d@example.com", i),
+				fmt.Sprintf("to-%d@example.com", i),
+				fmt.Sprintf("to:to-%d@example.com", i),
+				fmt.Sprintf("to2-%d@example.com", i),
+				fmt.Sprintf("to:to2-%d@example.com", i),
+				fmt.Sprintf("cc-%d@example.com", i),
+				fmt.Sprintf("cc:cc-%d@example.com", i),
+				fmt.Sprintf("cc2-%d@example.com", i),
+				fmt.Sprintf("cc:cc2-%d@example.com", i),
+				fmt.Sprintf("reply-to-%d@example.com", i),
+				fmt.Sprintf("reply-to:\"reply-to-%d@example.com\"", i),
+				fmt.Sprintf("\"Subject line %d end\"", i),
+				fmt.Sprintf("subject:\"Subject line %d end\"", i),
+				fmt.Sprintf("\"the email body %d jdsauk dwqmdqw\"", i),
+			}
+			searchIdx := rand.Intn(len(uniqueSearches))
+
+			search := uniqueSearches[searchIdx]
+
+			summaries, _, err := Search(search, "", 0, 0, 100)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			assertEqual(t, len(summaries), 1, "search result expected")
+
+			assertEqual(t, summaries[0].From.Name, fmt.Sprintf("From %d", i), "\"From\" name does not match")
+			assertEqual(t, summaries[0].From.Address, fmt.Sprintf("from-%d@example.com", i), "\"From\" address does not match")
+			assertEqual(t, summaries[0].To[0].Name, fmt.Sprintf("To %d", i), "\"To\" name does not match")
+			assertEqual(t, summaries[0].To[0].Address, fmt.Sprintf("to-%d@example.com", i), "\"To\" address does not match")
+			assertEqual(t, summaries[0].Subject, fmt.Sprintf("Subject line %d end", i), "\"Subject\" does not match")
+		}
+
+		// search something that will return 200 results
+		summaries, _, err := Search("This is the email body", "", 0, 0, testRuns)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		assertEqual(t, len(summaries), testRuns, "search results expected")
+
+		Close()
+	}
+}
+
+func TestSearchDelete100(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing search delete of 100 messages")
+		} else {
+			t.Logf("Testing search delete of 100 messages (tenant %s)", tenantID)
+		}
+
+		for i := 0; i < 100; i++ {
+			if _, err := Store(&testTextEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			if _, err := Store(&testMimeEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		_, total, err := Search("from:sender@example.com", "", 0, 0, 100)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, total, 100, "100 search results expected")
+
+		if err := DeleteSearch("from:sender@example.com", ""); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		_, total, err = Search("from:sender@example.com", "", 0, 0, 100)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, total, 0, "0 search results expected")
+
+		Close()
+	}
+}
+
+func TestSearchDelete1100(t *testing.T) {
+	setup("")
+	defer Close()
+
+	t.Log("Testing search delete of 1100 messages")
+	for i := 0; i < 1100; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	_, total, err := Search("from:sender@example.com", "", 0, 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 1100, "100 search results expected")
+
+	if err := DeleteSearch("from:sender@example.com", ""); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	_, total, err = Search("from:sender@example.com", "", 0, 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 0, "0 search results expected")
+}
+
+func TestEscPercentChar(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a test"] = "this is a test"
+	tests["this is% a test"] = "this is%% a test"
+	tests["this is%% a test"] = "this is%%%% a test"
+	tests["this is%%% a test"] = "this is%%%%%% a test"
+	tests["%this is% a test"] = "%%this is%% a test"
+	tests["Ä"] = "Ä"
+	tests["Ä%"] = "Ä%%"
+
+	for search, expected := range tests {
+		res := escPercentChar(search)
+		assertEqual(t, res, expected, "no match")
+	}
+}
+
+func TestSizeToBytes(t *testing.T) {
+	tests := map[string]int64{}
+	tests["1m"] = 1048576
+	tests["1mb"] = 1048576
+	tests["1 M"] = 1048576
+	tests["1 MB"] = 1048576
+	tests["1k"] = 1024
+	tests["1kb"] = 1024
+	tests["1 K"] = 1024
+	tests["1 kB"] = 1024
+	tests["1.5M"] = 1572864
+	tests["1234567890"] = 1234567890
+	tests["invalid"] = 0
+	tests["1.2.3"] = 0
+	tests["1.2.3M"] = 0
+
+	for search, expected := range tests {
+		res := sizeToBytes(search)
+		assertEqual(t, res, expected, "size does not match")
+	}
+}

internal/storage/settings.go

@@ -0,0 +1,76 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/leporo/sqlf"
+)
+
+// SettingGet returns a setting string value, blank is it does not exist
+func SettingGet(k string) string {
+	var result sql.NullString
+	err := sqlf.From(tenant("settings")).
+		Select("Value").To(&result).
+		Where("Key = ?", k).
+		Limit(1).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return ""
+	}
+
+	return result.String
+}
+
+// SettingPut sets a setting string value, inserting if new
+func SettingPut(k, v string) error {
+	_, err := db.Exec(`INSERT INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?) ON CONFLICT(Key) DO UPDATE SET Value = ?`, k, v, v)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return err
+}
+
+// The total deleted message size as an int64 value
+func getDeletedSize() float64 {
+	var result sql.NullFloat64
+	err := sqlf.From(tenant("settings")).
+		Select("Value").To(&result).
+		Where("Key = ?", "DeletedSize").
+		Limit(1).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Float64
+}
+
+// The total raw non-compressed messages size in bytes of all messages in the database
+func totalMessagesSize() float64 {
+	var result sql.NullFloat64
+	err := sqlf.From(tenant("mailbox")).
+		Select("SUM(Size)").To(&result).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Float64
+}
+
+// AddDeletedSize will add the value to the DeletedSize setting
+func addDeletedSize(v int64) {
+	if _, err := db.Exec(`INSERT OR IGNORE INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?)`, "DeletedSize", 0); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	if _, err := db.Exec(`UPDATE `+tenant("settings")+` SET Value = Value + ? WHERE Key = ?`, v, "DeletedSize"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+}

internal/storage/structs.go

@@ -3,8 +3,6 @@ package storage
 import (
 	"net/mail"
 	"time"
-
-	"github.com/jhillyerd/enmime"
 )
 
 // Message data excluding physical attachments
@@ -15,8 +13,6 @@ type Message struct {
 	ID string
 	// Message ID
 	MessageID string
-	// Read status
-	Read bool
 	// From address
 	From *mail.Address
 	// To addresses
@@ -31,6 +27,9 @@ type Message struct {
 	ReturnPath string
 	// Message subject
 	Subject string
+	// List-Unsubscribe header information
+	// swagger:ignore
+	ListUnsubscribe ListUnsubscribe
 	// Message date if set, else date received
 	Date time.Time
 	// Message tags
@@ -40,7 +39,7 @@ type Message struct {
 	// Message body HTML
 	HTML string
 	// Message size in bytes
-	Size int
+	Size float64
 	// Inline message attachments
 	Inline []Attachment
 	// Message attachments
@@ -60,7 +59,7 @@ type Attachment struct {
 	// Content ID
 	ContentID string
 	// Size in bytes
-	Size int
+	Size float64
 }
 
 // MessageSummary struct for frontend messages
@@ -81,6 +80,8 @@ type MessageSummary struct {
 	Cc []*mail.Address
 	// Bcc addresses
 	Bcc []*mail.Address
+	// Reply-To address
+	ReplyTo []*mail.Address
 	// Email subject
 	Subject string
 	// Created time
@@ -88,29 +89,38 @@ type MessageSummary struct {
 	// Message tags
 	Tags []string
 	// Message size in bytes (total)
-	Size int
+	Size float64
 	// Whether the message has any attachments
 	Attachments int
+	// Message snippet includes up to 250 characters
+	Snippet string
 }
 
 // MailboxStats struct for quick mailbox total/read lookups
 type MailboxStats struct {
-	Total  int
-	Unread int
+	Total  float64
+	Unread float64
 	Tags   []string
 }
 
-// AttachmentSummary returns a summary of the attachment without any binary data
-func AttachmentSummary(a *enmime.Part) Attachment {
-	o := Attachment{}
-	o.PartID = a.PartID
-	o.FileName = a.FileName
-	if o.FileName == "" {
-		o.FileName = a.ContentID
-	}
-	o.ContentType = a.ContentType
-	o.ContentID = a.ContentID
-	o.Size = len(a.Content)
-
-	return o
+// DBMailSummary struct for storing mail summary
+type DBMailSummary struct {
+	From    *mail.Address
+	To      []*mail.Address
+	Cc      []*mail.Address
+	Bcc     []*mail.Address
+	ReplyTo []*mail.Address
+}
+
+// ListUnsubscribe contains a summary of List-Unsubscribe & List-Unsubscribe-Post headers
+// including validation of the link structure
+type ListUnsubscribe struct {
+	// List-Unsubscribe header value
+	Header string
+	// Detected links, maximum one email and one HTTP(S)
+	Links []string
+	// Validation errors if any
+	Errors string
+	// List-Unsubscribe-Post value if set
+	HeaderPost string
 }

internal/storage/tagfilters.go

@@ -0,0 +1,87 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/leporo/sqlf"
+)
+
+// TagFilter struct
+type TagFilter struct {
+	// Match is the user-defined match
+	Match string
+	// SQL represents the SQL equivalent of Match
+	SQL *sqlf.Stmt
+	// Tags to add on match
+	Tags []string
+}
+
+var tagFilters = []TagFilter{}
+
+// LoadTagFilters loads tag filters from the config and pre-generates the SQL query
+func LoadTagFilters() {
+	tagFilters = []TagFilter{}
+
+	for _, t := range config.TagFilters {
+		match := strings.TrimSpace(t.Match)
+		if match == "" {
+			logger.Log().Warnf("[tags] ignoring tag item with missing 'match'")
+			continue
+		}
+		if t.Tags == nil || len(t.Tags) == 0 {
+			logger.Log().Warnf("[tags] ignoring tag items with missing 'tags' array")
+			continue
+		}
+
+		validTags := []string{}
+		for _, tag := range t.Tags {
+			tagName := tools.CleanTag(tag)
+			if !config.ValidTagRegexp.MatchString(tagName) || len(tagName) == 0 {
+				logger.Log().Warnf("[tags] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tagName)
+				continue
+			}
+			validTags = append(validTags, tagName)
+		}
+
+		if len(validTags) == 0 {
+			continue
+		}
+
+		tagFilters = append(tagFilters, TagFilter{Match: match, Tags: validTags, SQL: searchQueryBuilder(match, "")})
+	}
+}
+
+// TagFilterMatches returns a slice of matching tags from a message
+func tagFilterMatches(id string) []string {
+	tags := []string{}
+
+	if len(tagFilters) == 0 {
+		return tags
+	}
+
+	for _, f := range tagFilters {
+		var matchID string
+		q := f.SQL.Clone().Where("ID = ?", id)
+		if err := q.QueryAndClose(context.Background(), db, func(row *sql.Rows) {
+			var ignore sql.NullString
+
+			if err := row.Scan(&ignore, &matchID, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return tags
+		}
+		if matchID == id {
+			tags = append(tags, f.Tags...)
+		}
+	}
+
+	return tags
+}

internal/storage/tags.go

@@ -0,0 +1,405 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+var (
+	addressPlusRe = regexp.MustCompile(`(?U)^(.*){1,}\+(.*)@`)
+	addTagMutex   sync.RWMutex
+)
+
+// SetMessageTags will set the tags for a given database ID, removing any not in the array
+func SetMessageTags(id string, tags []string) ([]string, error) {
+	applyTags := []string{}
+	for _, t := range tags {
+		t = tools.CleanTag(t)
+		if t != "" && config.ValidTagRegexp.MatchString(t) && !tools.InArray(t, applyTags) {
+			applyTags = append(applyTags, t)
+		}
+	}
+
+	tagNames := []string{}
+	currentTags := getMessageTags(id)
+	origTagCount := len(currentTags)
+
+	for _, t := range applyTags {
+		if t == "" || !config.ValidTagRegexp.MatchString(t) || tools.InArray(t, currentTags) {
+			continue
+		}
+
+		name, err := addMessageTag(id, t)
+		if err != nil {
+			return []string{}, err
+		}
+
+		tagNames = append(tagNames, name)
+	}
+
+	if origTagCount > 0 {
+		currentTags = getMessageTags(id)
+
+		for _, t := range currentTags {
+			if !tools.InArray(t, applyTags) {
+				if err := deleteMessageTag(id, t); err != nil {
+					return []string{}, err
+				}
+			}
+		}
+	}
+
+	d := struct {
+		ID   string
+		Tags []string
+	}{ID: id, Tags: applyTags}
+
+	websockets.Broadcast("update", d)
+
+	return tagNames, nil
+}
+
+// AddMessageTag adds a tag to a message
+func addMessageTag(id, name string) (string, error) {
+	// prevent two identical tags being added at the same time
+	addTagMutex.Lock()
+
+	var tagID int
+	var foundName sql.NullString
+
+	q := sqlf.From(tenant("tags")).
+		Select("ID").To(&tagID).
+		Select("Name").To(&foundName).
+		Where("Name = ?", name)
+
+	// if tag exists - add tag to message
+	if err := q.QueryRowAndClose(context.TODO(), db); err == nil {
+		addTagMutex.Unlock()
+		// check message does not already have this tag
+		var exists int
+
+		if err := sqlf.From(tenant("message_tags")).
+			Select("COUNT(ID)").To(&exists).
+			Where("ID = ?", id).
+			Where("TagID = ?", tagID).
+			QueryRowAndClose(context.Background(), db); err != nil {
+			return "", err
+		}
+		if exists > 0 {
+			// already exists
+			return foundName.String, nil
+		}
+
+		logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
+
+		_, err := sqlf.InsertInto(tenant("message_tags")).
+			Set("ID", id).
+			Set("TagID", tagID).
+			ExecAndClose(context.TODO(), db)
+
+		return foundName.String, err
+	}
+
+	// new tag, add to the database
+	if _, err := sqlf.InsertInto(tenant("tags")).
+		Set("Name", name).
+		ExecAndClose(context.TODO(), db); err != nil {
+		addTagMutex.Unlock()
+		return name, err
+	}
+
+	addTagMutex.Unlock()
+
+	// add tag to the message
+	return addMessageTag(id, name)
+}
+
+// DeleteMessageTag deletes a tag from a message
+func deleteMessageTag(id, name string) error {
+	if _, err := sqlf.DeleteFrom(tenant("message_tags")).
+		Where(tenant("message_tags.ID")+" = ?", id).
+		Where(tenant("message_tags.Key")+` IN (SELECT Key FROM `+tenant("message_tags")+` LEFT JOIN `+tenant("tags")+` ON TagID=`+tenant("tags.ID")+` WHERE Name = ?)`, name).
+		ExecAndClose(context.TODO(), db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// DeleteAllMessageTags deleted all tags from a message
+func DeleteAllMessageTags(id string) error {
+	if _, err := sqlf.DeleteFrom(tenant("message_tags")).
+		Where(tenant("message_tags.ID")+" = ?", id).
+		ExecAndClose(context.TODO(), db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// GetAllTags returns all used tags
+func GetAllTags() []string {
+	var tags = []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`DISTINCT Name`).
+		From(tenant("tags")).To(&name).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// GetAllTagsCount returns all used tags with their total messages
+func GetAllTagsCount() map[string]int64 {
+	var tags = make(map[string]int64)
+	var name string
+	var total int64
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		Select(`COUNT(`+tenant("message_tags.TagID")+`) as total`).To(&total).
+		From(tenant("tags")).
+		LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")).
+		GroupBy(tenant("message_tags.TagID")).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags[name] = total
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// RenameTag renames a tag
+func RenameTag(from, to string) error {
+	to = tools.CleanTag(to)
+	if to == "" || !config.ValidTagRegexp.MatchString(to) {
+		return fmt.Errorf("invalid tag name: %s", to)
+	}
+
+	if from == to {
+		return nil // ignore
+	}
+
+	var id, existsID int
+
+	q := sqlf.From(tenant("tags")).
+		Select(`ID`).To(&id).
+		Where(`Name = ?`, from).
+		Limit(1)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("tag not found: %s", from)
+	}
+
+	// check if another tag by this name already exists
+	q = sqlf.From(tenant("tags")).
+		Select("ID").To(&existsID).
+		Where(`Name = ?`, to).
+		Where(`ID != ?`, id).
+		Limit(1)
+	err = q.QueryRowAndClose(context.Background(), db)
+	if err == nil || existsID != 0 {
+		return fmt.Errorf("tag already exists: %s", to)
+	}
+
+	q = sqlf.Update(tenant("tags")).
+		Set("Name", to).
+		Where("ID = ?", id)
+	_, err = q.ExecAndClose(context.Background(), db)
+
+	return err
+}
+
+// DeleteTag deleted a tag and removed all references to the tag
+func DeleteTag(tag string) error {
+	var id int
+
+	q := sqlf.From(tenant("tags")).
+		Select(`ID`).To(&id).
+		Where(`Name = ?`, tag).
+		Limit(1)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("tag not found: %s", tag)
+	}
+
+	// delete all references
+	q = sqlf.DeleteFrom(tenant("message_tags")).
+		Where(`TagID = ?`, id)
+	_, err = q.ExecAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("error deleting tag references: %s", err.Error())
+	}
+
+	// delete tag
+	q = sqlf.DeleteFrom(tenant("tags")).
+		Where(`ID = ?`, id)
+	_, err = q.ExecAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("error deleting tag: %s", err.Error())
+	}
+
+	return nil
+}
+
+// PruneUnusedTags will delete all unused tags from the database
+func pruneUnusedTags() error {
+	q := sqlf.From(tenant("tags")).
+		Select(tenant("tags.ID")+", "+tenant("tags.Name")+", COUNT("+tenant("message_tags.ID")+") as COUNT").
+		LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")).
+		GroupBy(tenant("tags.ID"))
+
+	toDel := []int{}
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var n string
+		var id int
+		var c int
+
+		if err := row.Scan(&id, &n, &c); err != nil {
+			logger.Log().Errorf("[tags] %s", err.Error())
+			return
+		}
+
+		if c == 0 {
+			logger.Log().Debugf("[tags] deleting unused tag \"%s\"", n)
+			toDel = append(toDel, id)
+		}
+	}); err != nil {
+		return err
+	}
+
+	if len(toDel) > 0 {
+		for _, id := range toDel {
+			if _, err := sqlf.DeleteFrom(tenant("tags")).
+				Where("ID = ?", id).
+				ExecAndClose(context.TODO(), db); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Find tags set via --tags in raw message, useful for matching all headers etc.
+// This function is largely superseded by the database searching, however this
+// includes literally everything and is kept for backwards compatibility.
+// Returns a comma-separated string.
+func findTagsInRawMessage(message *[]byte) []string {
+	tags := []string{}
+	if len(tagFilters) == 0 {
+		return tags
+	}
+
+	str := bytes.ToLower(*message)
+	for _, t := range tagFilters {
+		if bytes.Contains(str, []byte(t.Match)) {
+			tags = append(tags, t.Tags...)
+		}
+	}
+
+	return tags
+}
+
+// Returns tags found in email plus addresses (eg: test+tagname@example.com)
+func (d DBMailSummary) tagsFromPlusAddresses() []string {
+	tags := []string{}
+	for _, c := range d.To {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Cc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Bcc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	matches := addressPlusRe.FindAllStringSubmatch(d.From.Address, 1)
+	if len(matches) == 1 {
+		tags = append(tags, strings.Split(matches[0][2], "+")...)
+	}
+
+	return tools.SetTagCasing(tags)
+}
+
+// Get message tags from the database for a given database ID
+// Used when parsing a raw email.
+func getMessageTags(id string) []string {
+	tags := []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		From(tenant("Tags")).
+		LeftJoin(tenant("message_tags"), tenant("Tags.ID")+"="+tenant("message_tags.TagID")).
+		Where(tenant("message_tags.ID")+` = ?`, id).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[tags] %s", err.Error())
+		return tags
+	}
+
+	return tags
+}
+
+// SortedUniqueTags will return a unique slice of normalised tags
+func sortedUniqueTags(s []string) []string {
+	tags := []string{}
+	added := make(map[string]bool)
+
+	if len(s) == 0 {
+		return tags
+	}
+
+	for _, p := range s {
+		w := tools.CleanTag(p)
+		if w == "" {
+			continue
+		}
+		lc := strings.ToLower(w)
+		if _, exists := added[lc]; exists {
+			continue
+		}
+		if config.ValidTagRegexp.MatchString(w) {
+			added[lc] = true
+			tags = append(tags, w)
+		} else {
+			logger.Log().Debugf("[tags] ignoring invalid tag: %s", w)
+		}
+	}
+
+	sort.Strings(tags)
+
+	return tags
+}

internal/storage/tags_test.go

@@ -0,0 +1,143 @@
+package storage
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+)
+
+func TestTags(t *testing.T) {
+
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing tags")
+		} else {
+			t.Logf("Testing tags (tenant %s)", tenantID)
+		}
+
+		ids := []string{}
+
+		for i := 0; i < 10; i++ {
+			id, err := Store(&testMimeEmail)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			ids = append(ids, id)
+		}
+
+		for i := 0; i < 10; i++ {
+			if _, err := SetMessageTags(ids[i], []string{fmt.Sprintf("Tag-%d", i)}); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		for i := 0; i < 10; i++ {
+			message, err := GetMessage(ids[i])
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			if len(message.Tags) != 1 || message.Tags[0] != fmt.Sprintf("Tag-%d", i) {
+				t.Fatal("Message tags do not match")
+			}
+		}
+
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// test 20 tags
+		id, err := Store(&testMimeEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		newTags := []string{}
+		for i := 0; i < 20; i++ {
+			// pad number with 0 to ensure they are returned alphabetically
+			newTags = append(newTags, fmt.Sprintf("AnotherTag %02d", i))
+		}
+		if _, err := SetMessageTags(id, newTags); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags := getMessageTags(id)
+		assertEqual(t, strings.Join(newTags, "|"), strings.Join(returnedTags, "|"), "Message tags do not match")
+
+		// remove first tag
+		if err := deleteMessageTag(id, newTags[0]); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, strings.Join(newTags[1:], "|"), strings.Join(returnedTags, "|"), "Message tags do not match after deleting 1")
+
+		// remove all tags
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "", strings.Join(returnedTags, "|"), "Message tags should be empty")
+
+		// apply the same tag twice
+		if _, err := SetMessageTags(id, []string{"Duplicate Tag", "Duplicate Tag"}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "Duplicate Tag", strings.Join(returnedTags, "|"), "Message tags should be duplicated")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// apply tag with invalid characters
+		if _, err := SetMessageTags(id, []string{"Dirty! \"Tag\""}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "Dirty Tag", strings.Join(returnedTags, "|"), "Dirty message tag did not clean as expected")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// Check deleted message tags also prune the tags database
+		allTags := GetAllTags()
+		assertEqual(t, "", strings.Join(allTags, "|"), "Tags did not delete as expected")
+
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// test 20 tags
+		id, err = Store(&testTagEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "BccTag|CcTag|FromFag|ToTag|X-tag1|X-tag2", strings.Join(returnedTags, "|"), "Tags not detected correctly")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		Close()
+	}
+
+}

internal/storage/testdata/mime-attachment.eml

@@ -1,4 +1,4 @@
-Delivered-To: recipient@example.com
+Delivered-To: recipient2@example.com
 Received: by 2002:a0c:fe87:0:0:0:0:0 with SMTP id d7csp145570qvs;
         Tue, 26 Jul 2022 20:42:36 -0700 (PDT)
 X-Received: by 2002:a17:902:f788:b0:16c:f48b:905e with SMTP id q8-20020a170902f78800b0016cf48b905emr19885972pln.60.1658893355881;
@@ -23,18 +23,18 @@ ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc
          uSfA==
 ARC-Authentication-Results: i=1; mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])
         by mx.google.com with SMTPS id 11-20020aa7914b000000b0052ab192de4fsor8543241pfi.101.2022.07.26.20.42.35
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (Google Transport Security);
         Tue, 26 Jul 2022 20:42:35 -0700 (PDT)
-Received-SPF: pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
+Received-SPF: pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
 Authentication-Results: mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
         d=gmail.com; s=20210112;
@@ -63,10 +63,10 @@ X-Gm-Message-State: AJIora/WUqr3biShTHQBjSlCKazFbrLxeYpxmr1VF0TpBUbjnJrcLT77
 X-Google-Smtp-Source: AGRyM1tai6X1Bx130Y1yHG5w2e0r8wx6bbI+H+YppWmQoT28TV3dSoYCqmeQK5VViW8WuvdOpQzhPQ==
 X-Received: by 2002:a62:29c3:0:b0:52b:f774:7242 with SMTP id p186-20020a6229c3000000b0052bf7747242mr12504553pfp.67.1658893354675;
         Tue, 26 Jul 2022 20:42:34 -0700 (PDT)
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from [192.168.1.2] ([8.8.8.8])
         by smtp.gmail.com with ESMTPSA id oj16-20020a17090b4d9000b001f291c9d3bdsm387578pjb.48.2022.07.26.20.42.32
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);
         Tue, 26 Jul 2022 20:42:33 -0700 (PDT)
 Content-Type: multipart/mixed; boundary="------------ae0qIOkrNQLQHe1YyfTsUXrk"
@@ -76,8 +76,8 @@ MIME-Version: 1.0
 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101
  Thunderbird/91.11.0
 Content-Language: en-NZ
-To: "Recipient Ross" <recipient@example.com>
-From: Sender Smith <sender@example.com>
+To: "Recipient Ross" <recipient2@example.com>
+From: Sender Smith <sender2@example.com>
 Subject: inline + attachment
 
 This is a multi-part message in MIME format.
@@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit
     <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   </head>
   <body>
-    Message with inline image and attachment:<br>
+    <h1>Message with inline image and attachment:</h1>
     <br>
-    <img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"
-      moz-do-not-send="false"><br>
+    <p><img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"></p>
     <br>
     <br>
   </body>

internal/storage/testdata/tags.eml

@@ -0,0 +1,49 @@
+Date: Wed, 27 Jul 2022 15:44:41 +1200
+From: Sender Smith <sender+FromFag@example.com>
+To: Recipient Ross <recipient+ToTag@example.com>
+Cc: Recipient Ross <cc+CcTag@example.com>
+Bcc: <bcc+BccTag@example.com>
+Subject: Plain text message
+X-Tags: X-tag1, X-tag2
+Message-ID: <20220727034441.7za34h6ljuzfpmj3@localhost.localhost>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras non massa lacinia, 
+fringilla ex vel, ornare nulla. Suspendisse dapibus commodo sapien, non 
+hendrerit diam feugiat sit amet. Nulla lorem quam, laoreet vitae nisl volutpat, 
+mollis bibendum felis. In eget ultricies justo. Donec vitae hendrerit tortor, at 
+posuere libero. Fusce a gravida nibh. Nulla ac odio ex.
+
+Aliquam sem turpis, cursus vitae condimentum at, scelerisque pulvinar lectus. 
+Cras tempor nisl ut arcu interdum, et luctus arcu cursus. Maecenas mollis 
+sagittis commodo. Mauris ac lorem nec ex interdum consequat. Morbi congue 
+ultrices ullamcorper. Aenean ex tortor, dapibus quis dapibus iaculis, iaculis 
+eget felis. Vestibulum purus ante, efficitur in turpis ac, tristique laoreet 
+orci. Nulla facilisi. Praesent mollis orci posuere elementum laoreet. 
+Pellentesque enim nibh, varius at ante id, consequat posuere ante.
+
+Cras maximus venenatis nulla nec cursus. Morbi convallis, enim eget viverra 
+vulputate, ipsum arcu tincidunt tortor, ut cursus dui enim commodo quam. Donec 
+et vulputate quam. Vivamus non posuere erat. Nam commodo pellentesque 
+condimentum. Vivamus condimentum eros at odio dictum feugiat. Ut imperdiet 
+tempor luctus. Aenean varius libero ac faucibus dictum. Aliquam sed finibus 
+massa. Morbi dolor lorem, feugiat quis neque et, suscipit posuere ex. Sed auctor 
+et augue at finibus. Vestibulum interdum mi ac justo porta aliquam. Curabitur 
+nec enim sit amet enim aliquet accumsan. Etiam accumsan tellus tortor, interdum 
+sodales odio finibus eu. Integer eget ante eu nisi lobortis pulvinar et vel 
+ipsum. Cras condimentum posuere vulputate.
+
+Cras nulla felis, blandit vitae egestas quis, fringilla ut dolor. Phasellus est 
+augue, feugiat eu risus quis, posuere ultrices libero. Phasellus non nunc eget 
+justo sollicitudin tincidunt. Praesent pretium dui id felis bibendum sodales. 
+Phasellus eget dictum libero, auctor tempor nibh. Suspendisse posuere libero 
+venenatis elit imperdiet porttitor. In condimentum dictum luctus. Nullam in 
+nulla vitae augue blandit posuere. Vestibulum consectetur ultricies tincidunt. 
+Vivamus dolor quam, pharetra sed eros sed, hendrerit ultrices diam. Vestibulum 
+vulputate tellus eget tellus lacinia, a pulvinar velit vulputate. Suspendisse 
+mauris odio, scelerisque eget turpis sed, tincidunt ultrices magna. Nunc arcu 
+arcu, commodo et porttitor quis, accumsan viverra purus. Fusce id libero iaculis 
+lorem tristique commodo porttitor id ipsum. Vestibulum odio dui, tincidunt eget 
+lectus vel, tristique lacinia libero. Aliquam dapibus ac felis vitae cursus.

internal/storage/testing.go

@@ -0,0 +1,69 @@
+package storage
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var (
+	testTextEmail []byte
+	testTagEmail  []byte
+	testMimeEmail []byte
+	testRuns      = 100
+)
+
+func setup(tenantID string) {
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.Database = os.Getenv("MP_DATABASE")
+	config.TenantID = config.DBTenantID(tenantID)
+
+	if err := InitDB(); err != nil {
+		panic(err)
+	}
+
+	var err error
+
+	// ensure DB is empty
+	if err := DeleteAllMessages(); err != nil {
+		panic(err)
+	}
+
+	testTextEmail, err = os.ReadFile("testdata/plain-text.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testTagEmail, err = os.ReadFile("testdata/tags.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testMimeEmail, err = os.ReadFile("testdata/mime-attachment.eml")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}
+
+func assertEqualStats(t *testing.T, total int, unread int) {
+	s := StatsGet()
+	if float64(total) != s.Total {
+		t.Fatalf("Incorrect total mailbox stats: \"%v\" != \"%v\"", total, s.Total)
+	}
+
+	if float64(unread) != s.Unread {
+		t.Fatalf("Incorrect unread mailbox stats: \"%v\" != \"%v\"", unread, s.Unread)
+	}
+}

internal/storage/utils.go

@@ -0,0 +1,108 @@
+package storage
+
+import (
+	"net/mail"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/axllent/mailpit/internal/html2text"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	// for stats to prevent import cycle
+	mu sync.RWMutex
+	// StatsDeleted for counting the number of messages deleted
+	StatsDeleted float64
+)
+
+// AddTempFile adds a file to the slice of files to delete on exit
+func AddTempFile(s string) {
+	temporaryFiles = append(temporaryFiles, s)
+}
+
+// DeleteTempFiles will delete files added via AddTempFiles
+func deleteTempFiles() {
+	for _, f := range temporaryFiles {
+		if err := os.Remove(f); err == nil {
+			logger.Log().Debugf("removed temporary file: %s", f)
+		}
+	}
+}
+
+// Return a header field as a []*mail.Address, or "null" is not found/empty
+func addressToSlice(env *enmime.Envelope, key string) []*mail.Address {
+	data, err := env.AddressList(key)
+	if err != nil || data == nil {
+		return []*mail.Address{}
+	}
+
+	return data
+}
+
+// Generate the search text based on some header fields (to, from, subject etc)
+// and either the stripped HTML body (if exists) or text body
+func createSearchText(env *enmime.Envelope) string {
+	var b strings.Builder
+
+	b.WriteString(env.GetHeader("From") + " ")
+	b.WriteString(env.GetHeader("Subject") + " ")
+	b.WriteString(env.GetHeader("To") + " ")
+	b.WriteString(env.GetHeader("Cc") + " ")
+	b.WriteString(env.GetHeader("Bcc") + " ")
+	b.WriteString(env.GetHeader("Reply-To") + " ")
+	b.WriteString(env.GetHeader("Return-Path") + " ")
+
+	h := html2text.Strip(env.HTML, true)
+	if h != "" {
+		b.WriteString(h + " ")
+	} else {
+		b.WriteString(env.Text + " ")
+	}
+	// add attachment filenames
+	for _, a := range env.Attachments {
+		b.WriteString(a.FileName + " ")
+	}
+
+	d := cleanString(b.String())
+
+	return d
+}
+
+// CleanString removes unwanted characters from stored search text and search queries
+func cleanString(str string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	str = strings.ReplaceAll(str, string('\uFEFF'), " ")
+
+	// remove/replace new lines
+	re := regexp.MustCompile(`(\r?\n|\t|>|<|"|\,|;|\(|\))`)
+	str = re.ReplaceAllString(str, " ")
+
+	// remove duplicate whitespace and trim
+	return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " "))
+}
+
+// LogMessagesDeleted logs the number of messages deleted
+func logMessagesDeleted(n int) {
+	mu.Lock()
+	StatsDeleted = StatsDeleted + float64(n)
+	mu.Unlock()
+}
+
+// IsFile returns whether a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// Convert `%` to `%%` for SQL searches
+func escPercentChar(s string) string {
+	return strings.ReplaceAll(s, "%", "%%")
+}

internal/tools/argsparser.go

@@ -0,0 +1,32 @@
+package tools
+
+import "strings"
+
+// ArgsParser will split a string by new words and quotes phrases
+func ArgsParser(s string) []string {
+	args := []string{}
+	sb := &strings.Builder{}
+	quoted := false
+	for _, r := range s {
+		if r == '"' {
+			quoted = !quoted
+			sb.WriteRune(r) // keep '"' otherwise comment this line
+		} else if !quoted && r == ' ' {
+			v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+			if v != "" {
+				args = append(args, v)
+			}
+			sb.Reset()
+		} else {
+			sb.WriteRune(r)
+		}
+	}
+	if sb.Len() > 0 {
+		v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+		if v != "" {
+			args = append(args, v)
+		}
+	}
+
+	return args
+}

internal/tools/fs.go

@@ -0,0 +1,23 @@
+package tools
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// IsFile returns whether a file exists and is readable
+func IsFile(path string) bool {
+	f, err := os.Open(filepath.Clean(path))
+	defer f.Close()
+	return err == nil
+}
+
+// IsDir returns whether a path is a directory
+func IsDir(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || os.IsNotExist(err) || !info.IsDir() {
+		return false
+	}
+
+	return true
+}

internal/tools/headers.go

@@ -0,0 +1,160 @@
+// Package tools provides various methods for various things
+package tools
+
+import (
+	"bufio"
+	"bytes"
+	"net/mail"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+// RemoveMessageHeaders scans a message for headers, if found them removes them.
+// It will only remove a single instance of any given message header.
+func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	reBlank := regexp.MustCompile(`^\s+`)
+
+	for _, hdr := range headers {
+		// case-insensitive
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(hdr+":"))
+
+		// header := []byte(hdr + ":")
+		if m.Header.Get(hdr) != "" {
+			scanner := bufio.NewScanner(bytes.NewReader(msg))
+			found := false
+			hdr := []byte("")
+			for scanner.Scan() {
+				line := scanner.Bytes()
+				if !found && reHdr.Match(line) {
+					// add the first line starting with <header>:
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+					found = true
+				} else if found && reBlank.Match(line) {
+					// add any following lines starting with a whitespace (tab or space)
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+				} else if found {
+					// stop scanning, we have the full <header>
+					break
+				}
+			}
+
+			if len(hdr) > 0 {
+				logger.Log().Debugf("[relay] removed %s header", hdr)
+				msg = bytes.Replace(msg, hdr, []byte(""), 1)
+			}
+		}
+	}
+
+	return msg, nil
+}
+
+// SetMessageHeader scans a message for a header and updates its value if found.
+// It does not consider multiple instances of the same header.
+// If not found it will add the header to the beginning of the message.
+func SetMessageHeader(msg []byte, header, value string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get(header) != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(header+":"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		return bytes.Replace(msg, hdr, []byte(header+": "+value+"\r\n"), 1), nil
+	}
+
+	// no header, so add one to beginning
+	return append([]byte(header+": "+value+"\r\n"), msg...), nil
+}
+
+// OverrideFromHeader scans a message for the From header and replaces it with a different email address.
+func OverrideFromHeader(msg []byte, address string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get("From") != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta("From:"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		if len(hdr) > 0 {
+			originalFrom := strings.TrimRight(string(hdr[5:]), "\r\n")
+
+			from, err := mail.ParseAddress(originalFrom)
+			if err != nil {
+				// error parsing the from address, so just replace the whole line
+				msg = bytes.Replace(msg, hdr, []byte("From: "+address+"\r\n"), 1)
+			} else {
+				originalFrom = from.Address
+				// replace the from email, but keep the original name
+				from.Address = address
+				msg = bytes.Replace(msg, hdr, []byte("From: "+from.String()+"\r\n"), 1)
+			}
+
+			// insert the original From header as X-Original-From
+			msg = append([]byte("X-Original-From: "+originalFrom+"\r\n"), msg...)
+
+			logger.Log().Debugf("[relay] Replaced From email address with %s", address)
+		}
+	} else {
+		// no From header, so add one
+		msg = append([]byte("From: "+address+"\r\n"), msg...)
+		logger.Log().Debugf("[relay] Added From email: %s", address)
+	}
+
+	return msg, nil
+}

internal/tools/html.go

@@ -0,0 +1,50 @@
+package tools
+
+import (
+	"fmt"
+
+	"golang.org/x/net/html"
+)
+
+// GetHTMLAttributeVal returns the value of an HTML Attribute, else an error.
+// Returns a blank value if the attribute is set but empty.
+func GetHTMLAttributeVal(e *html.Node, key string) (string, error) {
+	for _, a := range e.Attr {
+		if a.Key == key {
+			return a.Val, nil
+		}
+	}
+
+	return "", fmt.Errorf("%s not found", key)
+}
+
+// SetHTMLAttributeVal sets an attribute on a node.
+func SetHTMLAttributeVal(n *html.Node, key, val string) {
+	for i := range n.Attr {
+		a := &n.Attr[i]
+		if a.Key == key {
+			a.Val = val
+			return
+		}
+	}
+	n.Attr = append(n.Attr, html.Attribute{
+		Key: key,
+		Val: val,
+	})
+}
+
+// WalkHTML traverses the entire HTML tree and calls fn on each node.
+func WalkHTML(n *html.Node, fn func(*html.Node)) {
+	if n == nil {
+		return
+	}
+
+	fn(n)
+
+	// Each node has a pointer to its first child and next sibling. To traverse
+	// all children of a node, we need to start from its first child and then
+	// traverse the next sibling until nil.
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		WalkHTML(c, fn)
+	}
+}

internal/tools/listunsubscribeparser.go

@@ -0,0 +1,99 @@
+package tools
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+// ListUnsubscribeParser will attempt to parse a `List-Unsubscribe` header and return
+// a slide of addresses (mail & URLs)
+func ListUnsubscribeParser(v string) ([]string, error) {
+	var results = []string{}
+	var re = regexp.MustCompile(`(?mU)<(.*)>`)
+	var reJoins = regexp.MustCompile(`(?imUs)>(.*)<`)
+	var reValidJoinChars = regexp.MustCompile(`(?imUs)^(\s+)?,(\s+)?$`)
+	var reWrapper = regexp.MustCompile(`(?imUs)^<(.*)>$`)
+	var reMailTo = regexp.MustCompile(`^mailto:[a-zA-Z0-9]`)
+	var reHTTP = regexp.MustCompile(`^(?i)https?://[a-zA-Z0-9]`)
+	var reSpaces = regexp.MustCompile(`\s`)
+	var reComments = regexp.MustCompile(`(?mUs)\(.*\)`)
+	var hasMailTo bool
+	var hasHTTP bool
+
+	v = strings.TrimSpace(v)
+
+	comments := reComments.FindAllStringSubmatch(v, -1)
+	for _, c := range comments {
+		// strip comments
+		v = strings.Replace(v, c[0], "", -1)
+		v = strings.TrimSpace(v)
+	}
+
+	if !re.MatchString(v) {
+		return results, fmt.Errorf("\"%s\" no valid unsubscribe links found", v)
+	}
+
+	errors := []string{}
+
+	if !reWrapper.MatchString(v) {
+		return results, fmt.Errorf("\"%s\" should be enclosed in <>", v)
+	}
+
+	matches := re.FindAllStringSubmatch(v, -1)
+
+	if len(matches) > 2 {
+		errors = append(errors, fmt.Sprintf("\"%s\" should include a maximum of one email and one HTTP link", v))
+	} else {
+		splits := reJoins.FindAllStringSubmatch(v, -1)
+		for _, g := range splits {
+			if !reValidJoinChars.MatchString(g[1]) {
+				return results, fmt.Errorf("\"%s\" <> should be split with a comma and optional spaces", v)
+			}
+		}
+
+		for _, m := range matches {
+			r := m[1]
+			if reSpaces.MatchString(r) {
+				errors = append(errors, fmt.Sprintf("\"%s\" should not contain spaces", r))
+				continue
+			}
+
+			if reMailTo.MatchString(r) {
+				if hasMailTo {
+					errors = append(errors, fmt.Sprintf("\"%s\" should only contain one mailto:", r))
+					continue
+				}
+
+				hasMailTo = true
+			} else if reHTTP.MatchString(r) {
+				if hasHTTP {
+					errors = append(errors, fmt.Sprintf("\"%s\" should only contain one HTTP link", r))
+					continue
+				}
+
+				hasHTTP = true
+
+			} else {
+				errors = append(errors, fmt.Sprintf("\"%s\" should start with either http(s):// or mailto:", r))
+				continue
+			}
+
+			_, err := url.ParseRequestURI(r)
+			if err != nil {
+				errors = append(errors, err.Error())
+				continue
+			}
+
+			results = append(results, r)
+		}
+	}
+
+	var err error
+	if len(errors) > 0 {
+		err = fmt.Errorf("%s", strings.Join(errors, ", "))
+	}
+
+	return results, err
+}

internal/tools/snippets.go

@@ -0,0 +1,69 @@
+package tools
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/html2text"
+)
+
+// CreateSnippet returns a message snippet. It will use the HTML version (if it exists)
+// otherwise the text version.
+func CreateSnippet(text, html string) string {
+	text = strings.TrimSpace(text)
+	html = strings.TrimSpace(html)
+	limit := 200
+	spaceRe := regexp.MustCompile(`\s+`)
+
+	if text == "" && html == "" {
+		return ""
+	}
+
+	if html != "" {
+		data := html2text.Strip(html, false)
+
+		if len(data) <= limit {
+			return data
+		}
+
+		return truncate(data, limit) + "..."
+	}
+
+	if text != "" {
+		// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+		text = strings.ReplaceAll(text, string('\uFEFF'), " ")
+		text = strings.TrimSpace(spaceRe.ReplaceAllString(text, " "))
+		if len(text) <= limit {
+			return text
+		}
+
+		return truncate(text, limit) + "..."
+	}
+
+	return ""
+}
+
+// Truncate a string allowing for multi-byte encoding.
+// Shamelessly borrowed from Tailscale.
+// See https://github.com/tailscale/tailscale/blob/main/util/truncate/truncate.go
+func truncate(s string, n int) string {
+	if n >= len(s) {
+		return s
+	}
+
+	// Back up until we find the beginning of a UTF-8 encoding.
+	for n > 0 && s[n-1]&0xc0 == 0x80 { // 0x10... is a continuation byte
+		n--
+	}
+
+	// If we're at the beginning of a multi-byte encoding, back up one more to
+	// skip it. It's possible the value was already complete, but it's simpler
+	// if we only have to check in one direction.
+	//
+	// Otherwise, we have a single-byte code (0x00... or 0x01...).
+	if n > 0 && s[n-1]&0xc0 == 0xc0 { // 0x11... starts a multibyte encoding
+		n--
+	}
+
+	return s[:n]
+}