Merge branch 'release/v1.22.3'

To reach healthy state faster at startup.

.chglog/CHANGELOG.tpl.md

@@ -19,7 +19,7 @@ Notable changes to Mailpit will be documented in this file.
 
 {{ range .Versions }} 
 {{- if .CommitGroups -}}
-## {{ .Tag.Name }}
+## [{{ .Tag.Name }}]
 
 {{ if .NoteGroups -}}
 {{ range .NoteGroups -}}

.chglog/config.yml

@@ -17,6 +17,24 @@ options:
       fix: Fix
     #   perf: Performance Improvements
     #   refactor: Code Refactoring
+    sort_by: Custom
+    title_order:
+      - Feature
+      - Chore
+      - UI
+      - API
+      - Libs
+      - Docker
+      - Security
+      - Fix
+      - Bugfix
+      - Docs
+      - Swagger
+      - Build
+      - Testing
+      - Test
+      - Tests
+      - Pull Requests
   header:
     pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$"
     pattern_maps:

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [axllent]

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "docker"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/build-docker-edge.yml

@@ -0,0 +1,45 @@
+on:
+  push:
+    branches: [ develop ]
+
+name: Build docker edge images
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log into Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+    - name: Log into GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+
+    - uses: benjlevesque/short-sha@v3.0
+      id: short-sha
+
+    - name: Build and push
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        platforms: linux/386,linux/amd64,linux/arm64
+        build-args: |
+          "VERSION=edge-${{ steps.short-sha.outputs.sha }}"
+        push: true
+        tags: |
+          axllent/mailpit:edge
+          ghcr.io/${{ github.repository }}:edge

.github/workflows/build-docker.yml

@@ -8,30 +8,46 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Get tag
-      id: tag
-      uses: dawidd6/action-get-tag@v1
+      uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v2
+    - name: Log into Docker Hub
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
+    - name: Log into GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+
+    - name: Parse semver
+      id: semver_parser 
+      uses: booxmedialtd/ws-action-parse-semver@v1.4.7
+      with:
+        input_string: '${{ github.ref_name }}'
+        version_extractor_regex: 'v(.*)$'
+
     - name: Build and push
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v6
       with:
         context: .
-        platforms: linux/386,linux/amd64,linux/arm,linux/arm64
+        platforms: linux/386,linux/amd64,linux/arm64
         build-args: |
-          "VERSION=${{ steps.tag.outputs.tag }}"
+          "VERSION=${{ github.ref_name }}"
         push: true
-        tags: axllent/mailpit:latest,axllent/mailpit:${{ steps.tag.outputs.tag }}
+        tags: |
+          axllent/mailpit:latest
+          axllent/mailpit:${{ github.ref_name }}
+          axllent/mailpit:v${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}
+          ghcr.io/${{ github.repository }}:latest
+          ghcr.io/${{ github.repository }}:${{ github.ref_name }}
+          ghcr.io/${{ github.repository }}:v${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}

.github/workflows/close-stale-issues.yml

@@ -0,0 +1,23 @@
+name: Close stale issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9.1.0
+        with:
+          days-before-issue-stale: 7
+          days-before-issue-close: 3
+          exempt-issue-labels: "enhancement,bug,awaiting feedback"
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue has been marked as stale because it has been open for 7 days with no activity."
+          close-issue-message: "This issue was closed because there has been no activity since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +69,4 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/release-build.yml

@@ -5,7 +5,7 @@ on:
 name: Build & release
 jobs:
   releases-matrix:
-    name: Release Go Binary
+    name: Build
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -21,22 +21,19 @@ jobs:
           - goarch: arm
             goos: windows
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Get tag
-      id: tag
-      uses: dawidd6/action-get-tag@v1
+    - uses: actions/checkout@v4
 
     # build the assets
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 18
         cache: 'npm'
+    - run: echo "Building assets for ${{ github.ref_name }}"
     - run: npm install
     - run: npm run package
 
     # build the binaries
-    - uses: wangyoucao577/go-release-action@v1.30
+    - uses: wangyoucao577/go-release-action@v1
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}
@@ -46,4 +43,6 @@ jobs:
         asset_name: mailpit-${{ matrix.goos }}-${{ matrix.goarch }}
         extra_files: LICENSE README.md
         md5sum: false
-        ldflags: -w -X "github.com/axllent/mailpit/config.Version=${{ steps.tag.outputs.tag }}"
+        overwrite: true
+        retry: 5
+        ldflags: -w -X "github.com/axllent/mailpit/config.Version=${{ github.ref_name }}"

.github/workflows/tests.yml

@@ -1,22 +1,24 @@
 name: Tests
 on:
   pull_request:
-    branches: [ develop ]
+    branches: [ develop, 'feature/**' ]
   push:
     branches: [ develop, 'feature/**' ]
 jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.18.x]
-        os: [ubuntu-latest]
+        go-version: ['1.23']
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+        cache: false
+    - uses: actions/checkout@v4
+    - name: Run Go tests
+      uses: actions/cache@v4
       with:
         path: |
           ~/.cache/go-build
@@ -24,13 +26,24 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test ./storage ./server -v
-    - run: go test ./storage -bench=.
+    - run: go test -p 1 ./internal/storage ./server ./internal/smtpd ./internal/pop3 ./internal/tools ./internal/html2text ./internal/linkcheck -v
+    - run: go test -p 1 ./internal/storage ./internal/html2text -bench=.
     
     # build the assets
-    - uses: actions/setup-node@v3
+    - name: Build web UI
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 18
         cache: 'npm'
-    - run: npm install
-    - run: npm run package
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm install
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm run package
+
+    # validate the swagger file
+    - name: Validate OpenAPI definition
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: swaggerexpert/swagger-editor-validate@v1
+      with:
+        definition-file: server/ui/api/v1/swagger.json

.gitignore

@@ -1,7 +1,9 @@
 /node_modules/
 /send
+/sendmail/sendmail
 /server/ui/dist
 /Makefile
 /mailpit*
+/.idea
 *.old
 *.db

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:alpine as builder
+FROM golang:alpine AS builder
 
 ARG VERSION=dev
 
@@ -6,15 +6,25 @@ COPY . /app
 
 WORKDIR /app
 
-RUN apk add --no-cache git npm && \
+RUN  apk upgrade && apk add git npm && \
 npm install && npm run package && \
 CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/axllent/mailpit/config.Version=${VERSION}" -o /mailpit
 
-
 FROM alpine:latest
 
+LABEL org.opencontainers.image.title="Mailpit" \
+  org.opencontainers.image.description="An email and SMTP testing tool with API for developers" \
+  org.opencontainers.image.source="https://github.com/axllent/mailpit" \
+  org.opencontainers.image.url="https://mailpit.axllent.org" \
+  org.opencontainers.image.documentation="https://mailpit.axllent.org/docs/" \
+  org.opencontainers.image.licenses="MIT"
+
 COPY --from=builder /mailpit /mailpit
 
-RUN apk add --no-cache tzdata
+RUN apk upgrade --no-cache && apk add --no-cache tzdata
+
+EXPOSE 1025/tcp 1110/tcp 8025/tcp
+
+HEALTHCHECK --interval=15s --start-period=10s --start-interval=1s CMD /mailpit readyz
 
 ENTRYPOINT ["/mailpit"]

README.md

@@ -1,74 +1,111 @@
-# Mailpit - email testing for developers
+<h1 align="center">
+  Mailpit - email testing for developers
+</h1>
 
-![Tests](https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg)
-![Build status](https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg)
-![Docker builds](https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg)
-![CodeQL](https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg)
-[![Go Report Card](https://goreportcard.com/badge/github.com/axllent/mailpit)](https://goreportcard.com/report/github.com/axllent/mailpit)
+<div align="center">
+    <a href="https://github.com/axllent/mailpit/actions/workflows/tests.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg" alt="CI Tests status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/release-build.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg" alt="CI build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg" alt="CI Docker build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg" alt="Code quality"></a>
+    <a href="https://goreportcard.com/report/github.com/axllent/mailpit"><img src="https://goreportcard.com/badge/github.com/axllent/mailpit" alt="Go Report Card"></a>
+    <br>
+    <a href="https://github.com/axllent/mailpit/releases/latest"><img src="https://img.shields.io/github/v/release/axllent/mailpit.svg" alt="Latest release"></a>
+    <a href="https://hub.docker.com/r/axllent/mailpit"><img src="https://img.shields.io/docker/pulls/axllent/mailpit.svg" alt="Docker pulls"></a>
+</div>
+<br>
+<p align="center">
+  <a href="https://mailpit.axllent.org">Website</a>  •
+  <a href="https://mailpit.axllent.org/docs/">Documentation</a>  •
+  <a href="https://mailpit.axllent.org/docs/api-v1/">API</a>
+</p>
 
-Mailpit is a multi-platform email testing tool for developers.
+<hr>
 
-It acts as both an SMTP server, and provides a web interface to view all captured emails.
+**Mailpit** is a small, fast, low memory, zero-dependency, multi-platform email testing tool & API for developers.
 
-Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster.
+It acts as an SMTP server, provides a modern web interface to view & test captured emails, and includes an API for automated integration testing.
 
-![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/docs/screenshot.png)
+Mailpit was originally **inspired** by MailHog which is [no longer maintained](https://github.com/mailhog/MailHog/issues/442#issuecomment-1493415258) and hasn't seen active development or security updates for a few years now.
+
+![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/server/ui-src/screenshot.png)
 
 
 ## Features
 
-- Runs entirely from a single binary, no installation required
-- SMTP server (default `0.0.0.0:1025`)
-- Web UI to view emails (formatted HTML, highlighted HTML source, text, raw source and MIME attachments including image thumbnails)
-- Advanced mail search ([see wiki](https://github.com/axllent/mailpit/wiki/Mail-search))
-- Real-time web UI updates using web sockets for new mail
-- Optional browser notifications for new mail (HTTPS only)
-- Configurable automatic email pruning (default keeps the most recent 500 emails)
-- Email storage either in a temporary or persistent database ([see wiki](https://github.com/axllent/mailpit/wiki/Email-storage))
-- Fast SMTP processing & storing - approximately 70-100 emails per second depending on CPU, network speed & email size
-- Can handle hundreds of thousands of emails
-- Optional SMTP with STARTTLS & SMTP authentication ([see wiki](https://github.com/axllent/mailpit/wiki/SMTP-with-STARTTLS-and-authentication))
-- Optional HTTPS for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/HTTPS))
-- Optional basic authentication for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
-- A simple REST API allowing ([see docs](docs/apiv1/README.md))
-- Multi-architecture [Docker images](https://github.com/axllent/mailpit/wiki/Docker-images)
+- Runs entirely from a single [static binary](https://mailpit.axllent.org/docs/install/) or multi-architecture [Docker images](https://mailpit.axllent.org/docs/install/docker/)
+- Modern web UI with advanced [mail search](https://mailpit.axllent.org/docs/usage/search-filters/) to view emails (formatted HTML, highlighted HTML source, text, headers, raw source, and MIME attachments
+including image thumbnails), including optional [HTTPS](https://mailpit.axllent.org/docs/configuration/http/) & [authentication](https://mailpit.axllent.org/docs/configuration/http/)
+- [SMTP server](https://mailpit.axllent.org/docs/configuration/smtp/) with optional STARTTLS or SSL/TLS, authentication (including an "accept any" mode)
+- A [REST API](https://mailpit.axllent.org/docs/api-v1/) for integration testing
+- Real-time web UI updates using web sockets for new mail & optional [browser notifications](https://mailpit.axllent.org/docs/usage/notifications/) when new mail is received
+- Optional [POP3 server](https://mailpit.axllent.org/docs/configuration/pop3/) to download captured message directly into your email client
+- [HTML check](https://mailpit.axllent.org/docs/usage/html-check/) to test & score mail client compatibility with HTML emails
+- [Link check](https://mailpit.axllent.org/docs/usage/link-check/) to test message links (HTML & text) & linked images
+- [Spam check](https://mailpit.axllent.org/docs/usage/spamassassin/) to test message "spamminess" using a running SpamAssassin server
+- [Create screenshots](https://mailpit.axllent.org/docs/usage/html-screenshots/) of HTML messages via web UI
+- Mobile and tablet HTML preview toggle in desktop mode
+- [Message tagging](https://mailpit.axllent.org/docs/usage/tagging/) including manual tagging or automated tagging using filtering and "plus addressing"
+- [SMTP relaying](https://mailpit.axllent.org/docs/configuration/smtp-relay/) (message release) - relay messages via a different SMTP server including an optional allowlist of accepted recipients
+- [SMTP forwarding](https://mailpit.axllent.org/docs/configuration/smtp-forward/) - automatically forward messages via a different SMTP server to predefined email addresses
+- Fast message [storing & processing](https://mailpit.axllent.org/docs/configuration/email-storage/) - ingesting 100-200 emails per second over SMTP depending on CPU, network speed & email size,
+easily handling tens of thousands of emails, with automatic email pruning (by default keeping the most recent 500 emails)
+- [Chaos](https://mailpit.axllent.org/docs/integration/chaos/) feature to enable configurable SMTP errors to test application resilience
+- `List-Unsubscribe` syntax validation
+- Optional [webhook](https://mailpit.axllent.org/docs/integration/webhook/) for received messages
 
 
 ## Installation
 
+The Mailpit web UI listens by default on `http://0.0.0.0:8025` and the SMTP port on `0.0.0.0:1025`.
+
+Mailpit runs as a single binary and can be installed in different ways:
+
+
+### Install via package managers
+
+- **Mac**: `brew install mailpit` (to run automatically in the background: `brew services start mailpit`)
+- **Arch Linux**: available in the AUR as `mailpit`
+- **FreeBSD**: `pkg install mailpit`
+
+
+### Install via bash script (Linux & Mac)
+
 Linux & Mac users can install it directly to `/usr/local/bin/mailpit` with:
 
 ```bash
 sudo bash < <(curl -sL https://raw.githubusercontent.com/axllent/mailpit/develop/install.sh)
 ```
 
-Or download a pre-built binary in the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` can be placed in your `$PATH`, or simply run as `./mailpit`. See `mailpit -h` for options, or see [the wiki](https://github.com/axllent/mailpit/wiki/Runtime-options) for additional information.
 
-To build Mailpit from source see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source).
+### Download static binary (Windows, Linux and Mac)
 
-The Mailpit web UI listens by default on `http://0.0.0.0:8025`, and the SMTP port on `0.0.0.0:1025`.
+Static binaries can always be found on the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` binary can be extracted and copied to your `$PATH`, or simply run as `./mailpit`.
+
+
+### Docker
+
+See [Docker instructions](https://mailpit.axllent.org/docs/install/docker/) for 386, amd64 & arm64 images.
+
+
+### Compile from source
+
+To build Mailpit from source, see [Building from source](https://mailpit.axllent.org/docs/install/source/).
+
+
+## Usage
+
+Run `mailpit -h` to see options. More information can be seen in [the docs](https://mailpit.axllent.org/docs/configuration/runtime-options/).
+
+If installed using homebrew, you may run `brew services start mailpit` to always run mailpit automatically.
+
+
+### Testing Mailpit
+
+Please refer to [the documentation](https://mailpit.axllent.org/docs/install/testing/) on how to easily test email delivery to Mailpit.
 
 
 ### Configuring sendmail
 
-There are several different options available:
-
-You can use `mailpit sendmail` as your sendmail configuration in `php.ini`:
-```
-sendmail_path = /usr/local/bin/mailpit sendmail
-```
-
-If Mailpit is found on the same host as sendmail, you can symlink the Mailpit binary to sendmail, eg: `ln -s /usr/local/bin/mailpit /usr/sbin/sendmail`  (only if Mailpit is running on default 1025 port).
-
-You can use your default system `sendmail` binary to route directly to port `1025` (configurable) by calling `/usr/sbin/sendmail -S localhost:1025`.
-
-You can build a Mailpit-specific sendmail binary from source (see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source)).
-
-
-## Why rewrite MailHog?
-
-I had been using MailHog for a few years to intercept and test emails generated from several projects. MailHog has a number of severe performance issues, many of the modules are horribly out of date, and other than a few accepted MRs, it is not actively developed.
-
-Initially I started trying to upgrade a fork of MailHog (both the UI as well as the HTTP server & API), but soon discovered that it is (with all due respect) very poorly designed. It is over-engineered (split over 9 separate projects) and has too many unnecessary features for my purpose. It performs exceptionally poorly when dealing with large amounts of emails or processing any email with an attachment (a single email with a 3MB attachment can take over a minute to ingest). The API also transmits a lot of duplicate and unnecessary data on every message request for all web calls, and there is no HTTP compression.
-
-In order to improve it I felt it needed to be completely rewritten, and so Mailpit was born.
+Mailpit's SMTP server (default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. 
+A common MTA (Mail Transfer Agent) that delivers system emails to an SMTP server is `sendmail`, used by many applications, including PHP. 
+Mailpit can also act as substitute for sendmail. For instructions on how to set this up, please refer to the [sendmail documentation](https://mailpit.axllent.org/docs/install/sendmail/).

SECURITY.md

@@ -0,0 +1,19 @@
+# Reporting security vulnerabilities
+
+Your efforts to responsibly disclose your findings are appreciated.
+
+** **Please do _not_ report security vulnerabilities through public GitHub issues.** **
+
+If you believe you have found a **security vulnerability**, then please report it to security@axllent.org so 
+your findings can be investigated, and if confirmed, fixed and released in a timely manner.
+
+Your report should include:
+
+- Mailpit version
+- A vulnerability description
+- Reproduction steps (if applicable)
+- Any other details you think are likely to be important
+
+You should receive an initial acknowledgement within 24 hours in most cases, and will kept updated throughout the process. 
+
+With your consent, your contributions will be publicly acknowledged.

cmd/dump.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/dump"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/spf13/cobra"
+)
+
+// dumpCmd represents the dump command
+var dumpCmd = &cobra.Command{
+	Use:   "dump <database> <output-dir>",
+	Short: "Dump all messages from a database to a directory",
+	Long: `Dump all messages stored in Mailpit into a local directory as individual files.
+
+The database can either be the database file (eg: --database /var/lib/mailpit/mailpit.db) or a
+URL of a running Mailpit instance (eg: --http http://127.0.0.1/). If dumping over HTTP, the URL
+should be the base URL of your running Mailpit instance, not the link to the API itself.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := dump.Sync(args[0]); err != nil {
+			logger.Log().Fatal(err)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(dumpCmd)
+
+	dumpCmd.Flags().SortFlags = false
+
+	dumpCmd.Flags().StringVar(&config.Database, "database", config.Database, "Dump messages directly from a database file")
+	dumpCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data (optional)")
+	dumpCmd.Flags().StringVar(&dump.URL, "http", dump.URL, "Dump messages via HTTP API (base URL of running Mailpit instance)")
+	dumpCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
+}

cmd/ingest.go

@@ -0,0 +1,173 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/logger"
+	sendmail "github.com/axllent/mailpit/sendmail/cmd"
+	"github.com/spf13/cobra"
+)
+
+var (
+	ingestRecent int
+)
+
+// ingestCmd represents the ingest command
+var ingestCmd = &cobra.Command{
+	Use:   "ingest <file|folder> ...[file|folder]",
+	Short: "Ingest a file or folder of emails for testing",
+	Long: `Ingest a file or folder of emails for testing.
+
+This command will scan the folder for emails and deliver them via SMTP to a running 
+Mailpit server. Each email must be a separate file (eg: Maildir format, not mbox).
+The --recent flag will only consider files with a modification date within the last X days.`,
+	// Hidden: true,
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var count int
+		var total int
+		var per100start = time.Now()
+
+		for _, a := range args {
+			err := filepath.Walk(a,
+				func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						logger.Log().Error(err)
+						return nil
+					}
+					if !isFile(path) {
+						return nil
+					}
+
+					if ingestRecent > 0 && time.Since(info.ModTime()) > time.Duration(ingestRecent)*24*time.Hour {
+						return nil
+					}
+
+					f, err := os.Open(filepath.Clean(path))
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+					defer f.Close() // #nosec
+
+					body, err := io.ReadAll(f)
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+
+					msg, err := mail.ReadMessage(bytes.NewReader(body))
+					if err != nil {
+						logger.Log().Errorf("error parsing message body: %s", err.Error())
+						return nil
+					}
+
+					recipients := []string{}
+					// get all recipients in To, Cc and Bcc
+					if to, err := msg.Header.AddressList("To"); err == nil {
+						for _, a := range to {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if cc, err := msg.Header.AddressList("Cc"); err == nil {
+						for _, a := range cc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if bcc, err := msg.Header.AddressList("Bcc"); err == nil {
+						for _, a := range bcc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+
+					if sendmail.FromAddr == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							sendmail.FromAddr = fromAddresses[0].Address
+						}
+					}
+
+					if len(recipients) == 0 {
+						// Bcc
+						recipients = []string{sendmail.FromAddr}
+					}
+
+					returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
+					if returnPath == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							returnPath = fromAddresses[0].Address
+						}
+					}
+
+					err = sendmail.Send(sendmail.SMTPAddr, returnPath, recipients, body)
+					if err != nil {
+						logger.Log().Errorf("error sending mail: %s (%s)", err.Error(), path)
+						return nil
+					}
+
+					count++
+					total++
+					if count%100 == 0 {
+						logger.Log().Infof("[%s] 100 messages in %s", format(total), time.Since(per100start))
+
+						per100start = time.Now()
+					}
+
+					return nil
+				})
+			if err != nil {
+				logger.Log().Error(err)
+			}
+		}
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(ingestCmd)
+
+	ingestCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
+	ingestCmd.Flags().IntVarP(&ingestRecent, "recent", "r", 0, "Only ingest messages from the last X days (default all)")
+}
+
+// IsFile returns if a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// Format a an integer 10000 => 10,000
+func format(n int) string {
+	in := fmt.Sprintf("%d", n)
+	numOfDigits := len(in)
+	if n < 0 {
+		numOfDigits-- // First character is the - sign (not a digit)
+	}
+	numOfCommas := (numOfDigits - 1) / 3
+
+	out := make([]byte, len(in)+numOfCommas)
+	if n < 0 {
+		in, out[0] = in[1:], '-'
+	}
+
+	for i, j, k := len(in)-1, len(out)-1, 0; ; i, j = i-1, j-1 {
+		out[j] = in[i]
+		if i == 0 {
+			return string(out)
+		}
+		if k++; k == 3 {
+			j, k = j-1, 0
+			out[j] = ','
+		}
+	}
+}

cmd/readyz.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/spf13/cobra"
+)
+
+var (
+	useHTTPS bool
+)
+
+// readyzCmd represents the healthcheck command
+var readyzCmd = &cobra.Command{
+	Use:   "readyz",
+	Short: "Run a healthcheck to test if Mailpit is running",
+	Long: `This command connects to the /readyz endpoint of a running Mailpit server
+and exits with a status of 0 if the connection is successful, else with a 
+status 1 if unhealthy.
+
+If running within Docker, it should automatically detect environment
+settings to determine the HTTP bind interface & port.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		webroot := strings.TrimRight(path.Join("/", config.Webroot, "/"), "/") + "/"
+		proto := "http"
+		if useHTTPS {
+			proto = "https"
+		}
+
+		uri := fmt.Sprintf("%s://%s%sreadyz", proto, config.HTTPListen, webroot)
+
+		conf := &http.Transport{
+			IdleConnTimeout:       time.Second * 5,
+			ExpectContinueTimeout: time.Second * 5,
+			TLSHandshakeTimeout:   time.Second * 5,
+			// do not verify TLS in case this instance is using HTTPS
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec
+		}
+		client := &http.Client{Transport: conf}
+
+		res, err := client.Get(uri)
+		if err != nil || res.StatusCode != 200 {
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(readyzCmd)
+
+	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
+		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+	}
+
+	if len(os.Getenv("MP_WEBROOT")) > 0 {
+		config.Webroot = os.Getenv("MP_WEBROOT")
+	}
+
+	config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
+
+	if config.UITLSCert != "" {
+		useHTTPS = true
+	}
+
+	readyzCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "Set the HTTP bind interface & port")
+	readyzCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
+	readyzCmd.Flags().BoolVar(&useHTTPS, "https", useHTTPS, "Connect via HTTPS (ignores HTTPS validation)")
+}

cmd/reindex.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/spf13/cobra"
+)
+
+// reindexCmd represents the reindex command
+var reindexCmd = &cobra.Command{
+	Use:   "reindex <database>",
+	Short: "Reindex the database",
+	Long: `This will reindex all messages in the entire database.
+
+If you have several thousand messages in your mailbox, then it is advised to shut down
+Mailpit while you reindex as this process will likely result in database locking issues.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		config.Database = args[0]
+		config.MaxMessages = 0
+
+		if err := storage.InitDB(); err != nil {
+			logger.Log().Error(err)
+			os.Exit(1)
+		}
+
+		storage.ReindexAll()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(reindexCmd)
+}

cmd/root.go

@@ -1,20 +1,23 @@
+// Package cmd is the main application
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/logger"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/axllent/mailpit/server"
-	"github.com/axllent/mailpit/smtpd"
-	"github.com/axllent/mailpit/storage"
+	"github.com/axllent/mailpit/server/webhook"
 	"github.com/spf13/cobra"
 )
 
-var cfgFile string
-
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "mailpit",
@@ -25,21 +28,22 @@ It acts as an SMTP server, and provides a web interface to view all captured ema
 
 Documentation:
   https://github.com/axllent/mailpit
-  https://github.com/axllent/mailpit/wiki`,
+  https://mailpit.axllent.org/docs/`,
 	Run: func(_ *cobra.Command, _ []string) {
 		if err := config.VerifyConfig(); err != nil {
 			logger.Log().Error(err.Error())
 			os.Exit(1)
 		}
 		if err := storage.InitDB(); err != nil {
-			logger.Log().Error(err.Error())
+			logger.Log().Fatal(err.Error())
 			os.Exit(1)
 		}
 
 		go server.Listen()
 
 		if err := smtpd.Listen(); err != nil {
-			logger.Log().Error(err.Error())
+			storage.Close()
+			logger.Log().Fatal(err.Error())
 			os.Exit(1)
 		}
 	},
@@ -72,86 +76,319 @@ func init() {
 	rootCmd.PersistentFlags().BoolP("help", "h", false, "This help")
 	rootCmd.PersistentFlags().Lookup("help").Hidden = true
 
-	// defaults from envars if provided
-	if len(os.Getenv("MP_DATA_FILE")) > 0 {
-		config.DataFile = os.Getenv("MP_DATA_FILE")
-	}
-	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
-		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
-	}
-	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
-		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+	// load and warn deprecated ENV vars
+	initDeprecatedConfigFromEnv()
+
+	// load environment variables
+	initConfigFromEnv()
+
+	rootCmd.Flags().StringVarP(&config.Database, "database", "d", config.Database, "Database to store persistent data")
+	rootCmd.Flags().StringVar(&config.Label, "label", config.Label, "Optional label identify this Mailpit instance")
+	rootCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data")
+	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
+	rootCmd.Flags().StringVar(&config.MaxAge, "max-age", config.MaxAge, "Max age of messages in either (h)ours or (d)ays (eg: 3d)")
+	rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates")
+	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)")
+	rootCmd.Flags().StringVar(&logger.LogFile, "log-file", logger.LogFile, "Log output to file instead of stdout")
+	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
+	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
+
+	// Web UI / API
+	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface & port for UI")
+	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
+	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI & API authentication")
+	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-tls-cert", config.UITLSCert, "TLS certificate for web UI (HTTPS) - requires ui-tls-key")
+	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-tls-key", config.UITLSKey, "TLS key for web UI (HTTPS) - requires ui-tls-cert")
+	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
+	rootCmd.Flags().BoolVar(&config.BlockRemoteCSSAndFonts, "block-remote-css-and-fonts", config.BlockRemoteCSSAndFonts, "Block access to remote CSS & fonts")
+	rootCmd.Flags().StringVar(&config.EnableSpamAssassin, "enable-spamassassin", config.EnableSpamAssassin, "Enable integration with SpamAssassin")
+	rootCmd.Flags().BoolVar(&config.AllowUntrustedTLS, "allow-untrusted-tls", config.AllowUntrustedTLS, "Do not verify HTTPS certificates (link checker & screenshots)")
+
+	// SMTP server
+	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
+	rootCmd.Flags().StringVar(&config.SMTPAuthFile, "smtp-auth-file", config.SMTPAuthFile, "A password file for SMTP authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPAuthAcceptAny, "smtp-auth-accept-any", config.SMTPAuthAcceptAny, "Accept any SMTP username and password, including none")
+	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key")
+	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert")
+	rootCmd.Flags().BoolVar(&config.SMTPRequireSTARTTLS, "smtp-require-starttls", config.SMTPRequireSTARTTLS, "Require SMTP client use STARTTLS")
+	rootCmd.Flags().BoolVar(&config.SMTPRequireTLS, "smtp-require-tls", config.SMTPRequireTLS, "Require client use SSL/TLS")
+	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Allow insecure PLAIN & LOGIN SMTP authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain <CR><CR><LF>")
+	rootCmd.Flags().IntVar(&config.SMTPMaxRecipients, "smtp-max-recipients", config.SMTPMaxRecipients, "Maximum SMTP recipients allowed")
+	rootCmd.Flags().StringVar(&config.SMTPAllowedRecipients, "smtp-allowed-recipients", config.SMTPAllowedRecipients, "Only allow SMTP recipients matching a regular expression (default allow all)")
+	rootCmd.Flags().BoolVar(&smtpd.DisableReverseDNS, "smtp-disable-rdns", smtpd.DisableReverseDNS, "Disable SMTP reverse DNS lookups")
+
+	// SMTP relay
+	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP relay configuration file to allow releasing messages")
+	rootCmd.Flags().BoolVar(&config.SMTPRelayAll, "smtp-relay-all", config.SMTPRelayAll, "Auto-relay all new messages via external SMTP server (caution!)")
+	rootCmd.Flags().StringVar(&config.SMTPRelayMatching, "smtp-relay-matching", config.SMTPRelayMatching, "Auto-relay new messages to only matching recipients (regular expression)")
+
+	// SMTP forwarding
+	rootCmd.Flags().StringVar(&config.SMTPForwardConfigFile, "smtp-forward-config", config.SMTPForwardConfigFile, "SMTP forwarding configuration file for all messages")
+
+	// Chaos
+	rootCmd.Flags().BoolVar(&chaos.Enabled, "enable-chaos", chaos.Enabled, "Enable Chaos functionality (API / web UI)")
+	rootCmd.Flags().StringVar(&config.ChaosTriggers, "chaos-triggers", config.ChaosTriggers, "Enable Chaos & set the triggers for SMTP server")
+
+	// POP3 server
+	rootCmd.Flags().StringVar(&config.POP3Listen, "pop3", config.POP3Listen, "POP3 server bind interface and port")
+	rootCmd.Flags().StringVar(&config.POP3AuthFile, "pop3-auth-file", config.POP3AuthFile, "A password file for POP3 server authentication (enables POP3 server)")
+	rootCmd.Flags().StringVar(&config.POP3TLSCert, "pop3-tls-cert", config.POP3TLSCert, "Optional TLS certificate for POP3 server - requires pop3-tls-key")
+	rootCmd.Flags().StringVar(&config.POP3TLSKey, "pop3-tls-key", config.POP3TLSKey, "Optional TLS key for POP3 server - requires pop3-tls-cert")
+
+	// Tagging
+	rootCmd.Flags().StringVarP(&config.CLITagsArg, "tag", "t", config.CLITagsArg, "Tag new messages matching filters")
+	rootCmd.Flags().StringVar(&config.TagsConfig, "tags-config", config.TagsConfig, "Load tags filters from yaml configuration file")
+	rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "TitleCase new tags generated from plus-addresses and X-Tags")
+	rootCmd.Flags().StringVar(&config.TagsDisable, "tags-disable", config.TagsDisable, "Disable auto-tagging, comma separated (eg: plus-addresses,x-tags)")
+
+	// Webhook
+	rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages")
+	rootCmd.Flags().IntVar(&webhook.RateLimit, "webhook-limit", webhook.RateLimit, "Limit webhook requests per second")
+
+	// DEPRECATED FLAG 2024/04/12 - but will not be removed to maintain backwards compatibility
+	rootCmd.Flags().StringVar(&config.Database, "db-file", config.Database, "Database file to store persistent data")
+	rootCmd.Flags().Lookup("db-file").Hidden = true
+
+	// DEPRECATED FLAGS 2023/03/12
+	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-ssl-cert", config.UITLSCert, "SSL certificate for web UI - requires ui-ssl-key")
+	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-ssl-key", config.UITLSKey, "SSL key for web UI - requires ui-ssl-cert")
+	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-ssl-cert", config.SMTPTLSCert, "SSL certificate for SMTP - requires smtp-ssl-key")
+	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-ssl-key", config.SMTPTLSKey, "SSL key for SMTP - requires smtp-ssl-cert")
+	rootCmd.Flags().Lookup("ui-ssl-cert").Hidden = true
+	rootCmd.Flags().Lookup("ui-ssl-cert").Deprecated = "use --ui-tls-cert"
+	rootCmd.Flags().Lookup("ui-ssl-key").Hidden = true
+	rootCmd.Flags().Lookup("ui-ssl-key").Deprecated = "use --ui-tls-key"
+	rootCmd.Flags().Lookup("smtp-ssl-cert").Hidden = true
+	rootCmd.Flags().Lookup("smtp-ssl-cert").Deprecated = "use --smtp-tls-cert"
+	rootCmd.Flags().Lookup("smtp-ssl-key").Hidden = true
+	rootCmd.Flags().Lookup("smtp-ssl-key").Deprecated = "use --smtp-tls-key"
+
+	// DEPRECATED FLAGS 2024/03/16
+	rootCmd.Flags().BoolVar(&config.SMTPRequireSTARTTLS, "smtp-tls-required", config.SMTPRequireSTARTTLS, "smtp-require-starttls")
+	rootCmd.Flags().Lookup("smtp-tls-required").Hidden = true
+	rootCmd.Flags().Lookup("smtp-tls-required").Deprecated = "use --smtp-require-starttls"
+
+	// DEPRECATED FLAG 2024/04/13 - no longer used
+	rootCmd.Flags().BoolVar(&config.DisableHTMLCheck, "disable-html-check", config.DisableHTMLCheck, "Disable the HTML check functionality (web UI & API)")
+	rootCmd.Flags().Lookup("disable-html-check").Hidden = true
+}
+
+// Load settings from environment
+func initConfigFromEnv() {
+	// General
+	if len(os.Getenv("MP_DATABASE")) > 0 {
+		config.Database = os.Getenv("MP_DATABASE")
 	}
+
+	config.TenantID = os.Getenv("MP_TENANT_ID")
+
+	config.Label = os.Getenv("MP_LABEL")
+
 	if len(os.Getenv("MP_MAX_MESSAGES")) > 0 {
 		config.MaxMessages, _ = strconv.Atoi(os.Getenv("MP_MAX_MESSAGES"))
 	}
-	if len(os.Getenv("MP_UI_AUTH_FILE")) > 0 {
-		config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
+	if len(os.Getenv("MP_MAX_AGE")) > 0 {
+		config.MaxAge = os.Getenv("MP_MAX_AGE")
 	}
-	if len(os.Getenv("MP_UI_SSL_CERT")) > 0 {
-		config.UISSLCert = os.Getenv("MP_UI_SSL_CERT")
+	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
+		config.UseMessageDates = true
 	}
-	if len(os.Getenv("MP_UI_SSL_KEY")) > 0 {
-		config.UISSLKey = os.Getenv("MP_UI_SSL_KEY")
+	if getEnabledFromEnv("MP_IGNORE_DUPLICATE_IDS") {
+		config.IgnoreDuplicateIDs = true
 	}
-	if len(os.Getenv("MP_SMTP_AUTH_FILE")) > 0 {
-		config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
+	if len(os.Getenv("MP_LOG_FILE")) > 0 {
+		logger.LogFile = os.Getenv("MP_LOG_FILE")
 	}
-	if len(os.Getenv("MP_SMTP_SSL_CERT")) > 0 {
-		config.SMTPSSLCert = os.Getenv("MP_SMTP_SSL_CERT")
+	if getEnabledFromEnv("MP_QUIET") {
+		logger.QuietLogging = true
 	}
-	if len(os.Getenv("MP_SMTP_SSL_KEY")) > 0 {
-		config.SMTPSSLKey = os.Getenv("MP_SMTP_SSL_KEY")
+	if getEnabledFromEnv("MP_VERBOSE") {
+		logger.VerboseLogging = true
 	}
 
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_AUTH_FILE")) > 0 {
-		config.UIAuthFile = os.Getenv("MP_AUTH_FILE")
+	// Web UI & API
+	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
+		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
 	}
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_SSL_CERT")) > 0 {
-		config.UISSLCert = os.Getenv("MP_SSL_CERT")
+	if len(os.Getenv("MP_WEBROOT")) > 0 {
+		config.Webroot = os.Getenv("MP_WEBROOT")
 	}
-	// deprecated 2022/08/06
-	if len(os.Getenv("MP_SSL_KEY")) > 0 {
-		config.UISSLKey = os.Getenv("MP_SSL_KEY")
+	config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
+	if err := auth.SetUIAuth(os.Getenv("MP_UI_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
+	config.UITLSKey = os.Getenv("MP_UI_TLS_KEY")
+	if len(os.Getenv("MP_API_CORS")) > 0 {
+		server.AccessControlAllowOrigin = os.Getenv("MP_API_CORS")
+	}
+	if getEnabledFromEnv("MP_BLOCK_REMOTE_CSS_AND_FONTS") {
+		config.BlockRemoteCSSAndFonts = true
+	}
+	if len(os.Getenv("MP_ENABLE_SPAMASSASSIN")) > 0 {
+		config.EnableSpamAssassin = os.Getenv("MP_ENABLE_SPAMASSASSIN")
+	}
+	if getEnabledFromEnv("MP_ALLOW_UNTRUSTED_TLS") {
+		config.AllowUntrustedTLS = true
 	}
 
-	// deprecated 2022/08/28
-	if len(os.Getenv("MP_DATA_DIR")) > 0 {
-		fmt.Println("MP_DATA_DIR has been deprecated, use MP_DATA_FILE")
-		config.DataFile = os.Getenv("MP_DATA_DIR")
+	// SMTP server
+	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
+		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
+	}
+	config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
+	if err := auth.SetSMTPAuth(os.Getenv("MP_SMTP_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ACCEPT_ANY") {
+		config.SMTPAuthAcceptAny = true
+	}
+	config.SMTPTLSCert = os.Getenv("MP_SMTP_TLS_CERT")
+	config.SMTPTLSKey = os.Getenv("MP_SMTP_TLS_KEY")
+	if getEnabledFromEnv("MP_SMTP_REQUIRE_STARTTLS") {
+		config.SMTPRequireSTARTTLS = true
+	}
+	if getEnabledFromEnv("MP_SMTP_REQUIRE_TLS") {
+		config.SMTPRequireTLS = true
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ALLOW_INSECURE") {
+		config.SMTPAuthAllowInsecure = true
+	}
+	if getEnabledFromEnv("MP_SMTP_STRICT_RFC_HEADERS") {
+		config.SMTPStrictRFCHeaders = true
+	}
+	if len(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) > 0 {
+		config.SMTPMaxRecipients, _ = strconv.Atoi(os.Getenv("MP_SMTP_MAX_RECIPIENTS"))
+	}
+	if len(os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")) > 0 {
+		config.SMTPAllowedRecipients = os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")
+	}
+	if getEnabledFromEnv("MP_SMTP_DISABLE_RDNS") {
+		smtpd.DisableReverseDNS = true
 	}
 
-	rootCmd.Flags().StringVarP(&config.DataFile, "db-file", "d", config.DataFile, "Database file to store persistent data")
-	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
-	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
-	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
+	// SMTP relay
+	config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")
+	if getEnabledFromEnv("MP_SMTP_RELAY_ALL") {
+		config.SMTPRelayAll = true
+	}
+	config.SMTPRelayMatching = os.Getenv("MP_SMTP_RELAY_MATCHING")
+	config.SMTPRelayConfig = config.SMTPRelayConfigStruct{}
+	config.SMTPRelayConfig.Host = os.Getenv("MP_SMTP_RELAY_HOST")
+	if len(os.Getenv("MP_SMTP_RELAY_PORT")) > 0 {
+		config.SMTPRelayConfig.Port, _ = strconv.Atoi(os.Getenv("MP_SMTP_RELAY_PORT"))
+	}
+	config.SMTPRelayConfig.STARTTLS = getEnabledFromEnv("MP_SMTP_RELAY_STARTTLS")
+	config.SMTPRelayConfig.AllowInsecure = getEnabledFromEnv("MP_SMTP_RELAY_ALLOW_INSECURE")
+	config.SMTPRelayConfig.Auth = os.Getenv("MP_SMTP_RELAY_AUTH")
+	config.SMTPRelayConfig.Username = os.Getenv("MP_SMTP_RELAY_USERNAME")
+	config.SMTPRelayConfig.Password = os.Getenv("MP_SMTP_RELAY_PASSWORD")
+	config.SMTPRelayConfig.Secret = os.Getenv("MP_SMTP_RELAY_SECRET")
+	config.SMTPRelayConfig.ReturnPath = os.Getenv("MP_SMTP_RELAY_RETURN_PATH")
+	config.SMTPRelayConfig.OverrideFrom = os.Getenv("MP_SMTP_RELAY_OVERRIDE_FROM")
+	config.SMTPRelayConfig.AllowedRecipients = os.Getenv("MP_SMTP_RELAY_ALLOWED_RECIPIENTS")
+	config.SMTPRelayConfig.BlockedRecipients = os.Getenv("MP_SMTP_RELAY_BLOCKED_RECIPIENTS")
 
-	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI authentication")
-	rootCmd.Flags().StringVar(&config.UISSLCert, "ui-ssl-cert", config.UISSLCert, "SSL certificate for web UI - requires ui-ssl-key")
-	rootCmd.Flags().StringVar(&config.UISSLKey, "ui-ssl-key", config.UISSLKey, "SSL key for web UI - requires ui-ssl-cert")
+	// SMTP forwarding
+	config.SMTPForwardConfigFile = os.Getenv("MP_SMTP_FORWARD_CONFIG")
+	config.SMTPForwardConfig = config.SMTPForwardConfigStruct{}
+	config.SMTPForwardConfig.Host = os.Getenv("MP_SMTP_FORWARD_HOST")
+	if len(os.Getenv("MP_SMTP_FORWARD_PORT")) > 0 {
+		config.SMTPForwardConfig.Port, _ = strconv.Atoi(os.Getenv("MP_SMTP_FORWARD_PORT"))
+	}
+	config.SMTPForwardConfig.STARTTLS = getEnabledFromEnv("MP_SMTP_FORWARD_STARTTLS")
+	config.SMTPForwardConfig.AllowInsecure = getEnabledFromEnv("MP_SMTP_FORWARD_ALLOW_INSECURE")
+	config.SMTPForwardConfig.Auth = os.Getenv("MP_SMTP_FORWARD_AUTH")
+	config.SMTPForwardConfig.Username = os.Getenv("MP_SMTP_FORWARD_USERNAME")
+	config.SMTPForwardConfig.Password = os.Getenv("MP_SMTP_FORWARD_PASSWORD")
+	config.SMTPForwardConfig.Secret = os.Getenv("MP_SMTP_FORWARD_SECRET")
+	config.SMTPForwardConfig.ReturnPath = os.Getenv("MP_SMTP_FORWARD_RETURN_PATH")
+	config.SMTPForwardConfig.OverrideFrom = os.Getenv("MP_SMTP_FORWARD_OVERRIDE_FROM")
+	config.SMTPForwardConfig.To = os.Getenv("MP_SMTP_FORWARD_TO")
 
-	rootCmd.Flags().StringVar(&config.SMTPAuthFile, "smtp-auth-file", config.SMTPAuthFile, "A password file for SMTP authentication")
-	rootCmd.Flags().StringVar(&config.SMTPSSLCert, "smtp-ssl-cert", config.SMTPSSLCert, "SSL certificate for SMTP - requires smtp-ssl-key")
-	rootCmd.Flags().StringVar(&config.SMTPSSLKey, "smtp-ssl-key", config.SMTPSSLKey, "SSL key for SMTP - requires smtp-ssl-cert")
+	// Chaos
+	chaos.Enabled = getEnabledFromEnv("MP_ENABLE_CHAOS")
+	config.ChaosTriggers = os.Getenv("MP_CHAOS_TRIGGERS")
 
-	rootCmd.Flags().BoolVarP(&config.QuietLogging, "quiet", "q", false, "Quiet logging (errors only)")
-	rootCmd.Flags().BoolVarP(&config.VerboseLogging, "verbose", "v", false, "Verbose logging")
+	// POP3 server
+	if len(os.Getenv("MP_POP3_BIND_ADDR")) > 0 {
+		config.POP3Listen = os.Getenv("MP_POP3_BIND_ADDR")
+	}
+	config.POP3AuthFile = os.Getenv("MP_POP3_AUTH_FILE")
+	if err := auth.SetPOP3Auth(os.Getenv("MP_POP3_AUTH")); err != nil {
+		logger.Log().Error(err.Error())
+	}
+	config.POP3TLSCert = os.Getenv("MP_POP3_TLS_CERT")
+	config.POP3TLSKey = os.Getenv("MP_POP3_TLS_KEY")
 
-	// deprecated 2022/08/06
-	rootCmd.Flags().StringVarP(&config.UIAuthFile, "auth-file", "a", config.UIAuthFile, "A password file for web UI authentication")
-	rootCmd.Flags().StringVar(&config.UISSLCert, "ssl-cert", config.UISSLCert, "SSL certificate - requires ssl-key")
-	rootCmd.Flags().StringVar(&config.UISSLKey, "ssl-key", config.UISSLKey, "SSL key - requires ssl-cert")
-	rootCmd.Flags().Lookup("auth-file").Hidden = true
-	rootCmd.Flags().Lookup("auth-file").Deprecated = "use --ui-auth-file"
-	rootCmd.Flags().Lookup("ssl-cert").Hidden = true
-	rootCmd.Flags().Lookup("ssl-cert").Deprecated = "use --ui-ssl-cert"
-	rootCmd.Flags().Lookup("ssl-key").Hidden = true
-	rootCmd.Flags().Lookup("ssl-key").Deprecated = "use --ui-ssl-key"
+	// Tagging
+	config.CLITagsArg = os.Getenv("MP_TAG")
+	config.TagsConfig = os.Getenv("MP_TAGS_CONFIG")
+	tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE")
+	config.TagsDisable = os.Getenv("MP_TAGS_DISABLE")
 
-	// deprecated 2022/08/30
-	rootCmd.Flags().StringVar(&config.DataFile, "data", config.DataFile, "Database file to store persistent data")
-	rootCmd.Flags().Lookup("data").Hidden = true
-	rootCmd.Flags().Lookup("data").Deprecated = "use --db-file"
+	// Webhook
+	if len(os.Getenv("MP_WEBHOOK_URL")) > 0 {
+		config.WebhookURL = os.Getenv("MP_WEBHOOK_URL")
+	}
+	if len(os.Getenv("MP_WEBHOOK_LIMIT")) > 0 {
+		webhook.RateLimit, _ = strconv.Atoi(os.Getenv("MP_WEBHOOK_LIMIT"))
+	}
+
+	// Demo mode
+	config.DemoMode = getEnabledFromEnv("MP_DEMO_MODE")
+}
+
+// load deprecated settings from environment and warn
+func initDeprecatedConfigFromEnv() {
+	// deprecated 2024/04/12 - but will not be removed to maintain backwards compatibility
+	if len(os.Getenv("MP_DATA_FILE")) > 0 {
+		config.Database = os.Getenv("MP_DATA_FILE")
+	}
+
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_UI_SSL_CERT")) > 0 {
+		logger.Log().Warn("ENV MP_UI_SSL_CERT has been deprecated, use MP_UI_TLS_CERT")
+		config.UITLSCert = os.Getenv("MP_UI_SSL_CERT")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_UI_SSL_KEY")) > 0 {
+		logger.Log().Warn("ENV MP_UI_SSL_KEY has been deprecated, use MP_UI_TLS_KEY")
+		config.UITLSKey = os.Getenv("MP_UI_SSL_KEY")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_SMTP_SSL_CERT")) > 0 {
+		logger.Log().Warn("ENV MP_SMTP_CERT has been deprecated, use MP_SMTP_TLS_CERT")
+		config.SMTPTLSCert = os.Getenv("MP_SMTP_SSL_CERT")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_SMTP_SSL_KEY")) > 0 {
+		logger.Log().Warn("ENV MP_SMTP_KEY has been deprecated, use MP_SMTP_TLS_KEY")
+		config.SMTPTLSKey = os.Getenv("MP_SMTP_SMTP_KEY")
+	}
+	// deprecated 2023/12/10
+	if getEnabledFromEnv("MP_STRICT_RFC_HEADERS") {
+		logger.Log().Warn("ENV MP_STRICT_RFC_HEADERS has been deprecated, use MP_SMTP_STRICT_RFC_HEADERS")
+		config.SMTPStrictRFCHeaders = true
+	}
+	// deprecated 2024/03.16
+	if getEnabledFromEnv("MP_SMTP_TLS_REQUIRED") {
+		logger.Log().Warn("ENV MP_SMTP_TLS_REQUIRED has been deprecated, use MP_SMTP_REQUIRE_STARTTLS")
+		config.SMTPRequireSTARTTLS = true
+	}
+	if getEnabledFromEnv("MP_DISABLE_HTML_CHECK") {
+		logger.Log().Warn("ENV MP_DISABLE_HTML_CHECK has been deprecated and is no longer used")
+		config.DisableHTMLCheck = true
+	}
+}
+
+// Wrapper to get a boolean from an environment variable
+func getEnabledFromEnv(k string) bool {
+	if len(os.Getenv(k)) > 0 {
+		v := strings.ToLower(os.Getenv(k))
+		return v == "1" || v == "true" || v == "yes"
+	}
+
+	return false
 }

cmd/sendmail.go

@@ -1,22 +1,16 @@
 package cmd
 
 import (
+	"os"
+
 	sendmail "github.com/axllent/mailpit/sendmail/cmd"
 	"github.com/spf13/cobra"
 )
 
-var (
-	smtpAddr = "localhost:1025"
-	fromAddr string
-)
-
 // sendmailCmd represents the sendmail command
 var sendmailCmd = &cobra.Command{
-	Use:   "sendmail",
-	Short: "A sendmail command replacement",
-	Long: `A sendmail command replacement.
-	
-You can optionally create a symlink called 'sendmail' to the main binary.`,
+	Use:   "sendmail [flags] [recipients]",
+	Short: "A sendmail command replacement for Mailpit",
 	Run: func(_ *cobra.Command, _ []string) {
 		sendmail.Run()
 	},
@@ -24,10 +18,22 @@ You can optionally create a symlink called 'sendmail' to the main binary.`,
 
 func init() {
 	rootCmd.AddCommand(sendmailCmd)
+	var ignored string
 
-	// these are simply repeated for cli consistency
-	sendmailCmd.Flags().StringVar(&smtpAddr, "smtp-addr", smtpAddr, "SMTP server address")
-	sendmailCmd.Flags().StringVarP(&fromAddr, "from", "f", "", "SMTP sender")
-	sendmailCmd.Flags().BoolP("long-i", "i", false, "Ignored. This flag exists for sendmail compatibility.")
-	sendmailCmd.Flags().BoolP("long-t", "t", false, "Ignored. This flag exists for sendmail compatibility.")
+	// print out manual help screen
+	sendmailCmd.SetHelpTemplate(sendmail.HelpTemplate([]string{os.Args[0], "sendmail"}))
+
+	// these are simply repeated for cli consistency as cobra/viper does not allow
+	// multi-letter single-dash variables (-bs)
+	sendmailCmd.Flags().StringVarP(&sendmail.FromAddr, "from", "f", sendmail.FromAddr, "SMTP sender")
+	sendmailCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseB, "ignored-b", "b", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseS, "ignored-s", "s", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolP("verbose", "v", false, "Verbose mode (sends debug output to stderr)")
+	sendmailCmd.Flags().BoolP("ignored-i", "i", false, "Ignored")
+	sendmailCmd.Flags().BoolP("ignored-o", "o", false, "Ignored")
+	sendmailCmd.Flags().BoolP("ignored-t", "t", false, "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-name", "F", "", "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-bits", "B", "", "Ignored")
+	sendmailCmd.Flags().StringVarP(&ignored, "ignored-errors", "e", "", "Ignored")
 }

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/updater"
+	"github.com/axllent/mailpit/internal/updater"
 	"github.com/spf13/cobra"
 )
 

config/config.go

@@ -1,63 +1,175 @@
+// Package config handles the application configuration
 package config
 
 import (
 	"errors"
 	"fmt"
+	"net"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
+	"strings"
 
-	"github.com/tg123/go-htpasswd"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"github.com/axllent/mailpit/internal/spamassassin"
+	"github.com/axllent/mailpit/internal/tools"
 )
 
 var (
 	// SMTPListen to listen on <interface>:<port>
-	SMTPListen = "0.0.0.0:1025"
+	SMTPListen = "[::]:1025"
 
 	// HTTPListen to listen on <interface>:<port>
-	HTTPListen = "0.0.0.0:8025"
+	HTTPListen = "[::]:8025"
 
-	// DataFile for mail (optional)
-	DataFile string
+	// Database for mail (optional)
+	Database string
+
+	// TenantID is an optional prefix to be applied to all database tables,
+	// allowing multiple isolated instances of Mailpit to share a database.
+	TenantID string
+
+	// Label to identify this Mailpit instance (optional).
+	// This gets applied to web UI, SMTP and optional POP3 server.
+	Label string
 
 	// MaxMessages is the maximum number of messages a mailbox can have (auto-pruned every minute)
 	MaxMessages = 500
 
-	// VerboseLogging for console output
-	VerboseLogging = false
+	// MaxAge is the maximum age of messages (auto-pruned every hour).
+	// Value can be either <int>h for hours or <int>d for days
+	MaxAge string
 
-	// QuietLogging for console output (errors only)
-	QuietLogging = false
+	// MaxAgeInHours is the maximum age of messages in hours, set with parseMaxAge() using MaxAge value
+	MaxAgeInHours int
 
-	// NoLogging for tests
-	NoLogging = false
+	// UseMessageDates sets the Created date using the message date, not the delivered date
+	UseMessageDates bool
 
-	// UISSLCert file
-	UISSLCert string
+	// UITLSCert file
+	UITLSCert string
 
-	// UISSLKey file
-	UISSLKey string
+	// UITLSKey file
+	UITLSKey string
 
-	// UIAuthFile for basic authentication
+	// UIAuthFile for UI & API authentication
 	UIAuthFile string
 
-	// UIAuth used for euthentication
-	UIAuth *htpasswd.File
+	// Webroot to define the base path for the UI and API
+	Webroot = "/"
 
-	// SMTPSSLCert file
-	SMTPSSLCert string
+	// SMTPTLSCert file
+	SMTPTLSCert string
 
-	// SMTPSSLKey file
-	SMTPSSLKey string
+	// SMTPTLSKey file
+	SMTPTLSKey string
+
+	// SMTPRequireSTARTTLS to enforce the use of STARTTLS
+	// The only allowed commands are NOOP, EHLO, STARTTLS and QUIT (as specified in RFC 3207) until
+	// the connection is upgraded to TLS i.e. until STARTTLS is issued.
+	SMTPRequireSTARTTLS bool
+
+	// SMTPRequireTLS to allow only SSL/TLS connections for all connections
+	//
+	SMTPRequireTLS bool
 
 	// SMTPAuthFile for SMTP authentication
 	SMTPAuthFile string
 
-	// SMTPAuth used for euthentication
-	SMTPAuth *htpasswd.File
+	// SMTPAuthAllowInsecure allows PLAIN & LOGIN unencrypted authentication
+	SMTPAuthAllowInsecure bool
 
-	// ContentSecurityPolicy for HTTP server
-	ContentSecurityPolicy = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src 'self' data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';"
+	// SMTPAuthAcceptAny accepts any username/password including none
+	SMTPAuthAcceptAny bool
+
+	// SMTPMaxRecipients is the maximum number of recipients a message may have.
+	// The SMTP RFC states that an server must handle a minimum of 100 recipients
+	// however some servers accept more.
+	SMTPMaxRecipients = 100
+
+	// IgnoreDuplicateIDs will skip messages with the same ID
+	IgnoreDuplicateIDs bool
+
+	// BlockRemoteCSSAndFonts used to disable remote CSS & fonts
+	BlockRemoteCSSAndFonts = false
+
+	// CLITagsArg is used to map the CLI args
+	CLITagsArg string
+
+	// ValidTagRegexp represents a valid tag
+	ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_\.]){1,}$`)
+
+	// TagsConfig is a yaml file to pre-load tags
+	TagsConfig string
+
+	// TagFilters are used to apply tags to new mail
+	TagFilters []autoTag
+
+	// TagsDisable accepts a comma-separated list of tag types to disable
+	// including x-tags & plus-addresses
+	TagsDisable string
+
+	// SMTPRelayConfigFile to parse a yaml file and store config of the relay SMTP server
+	SMTPRelayConfigFile string
+
+	// SMTPRelayConfig to parse a yaml file and store config of the the relay SMTP server
+	SMTPRelayConfig SMTPRelayConfigStruct
+
+	// ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile
+	ReleaseEnabled = false
+
+	// SMTPRelayAll is whether to relay all incoming messages via pre-configured SMTP server.
+	// Use with extreme caution!
+	SMTPRelayAll = false
+
+	// SMTPRelayMatching if set, will auto-release to recipients matching this regular expression
+	SMTPRelayMatching string
+
+	// SMTPRelayMatchingRegexp is the compiled version of SMTPRelayMatching
+	SMTPRelayMatchingRegexp *regexp.Regexp
+
+	// SMTPForwardConfigFile to parse a yaml file and store config of the forwarding SMTP server
+	SMTPForwardConfigFile string
+
+	// SMTPForwardConfig to parse a yaml file and store config of the forwarding SMTP server
+	SMTPForwardConfig SMTPForwardConfigStruct
+
+	// SMTPStrictRFCHeaders will return an error if the email headers contain <CR><CR><LF> (\r\r\n)
+	// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
+	SMTPStrictRFCHeaders bool
+
+	// SMTPAllowedRecipients if set, will only accept recipients matching this regular expression
+	SMTPAllowedRecipients string
+
+	// SMTPAllowedRecipientsRegexp is the compiled version of SMTPAllowedRecipients
+	SMTPAllowedRecipientsRegexp *regexp.Regexp
+
+	// POP3Listen address - if set then Mailpit will start the POP3 server and listen on this address
+	POP3Listen = "[::]:1110"
+
+	// POP3AuthFile for POP3 authentication
+	POP3AuthFile string
+
+	// POP3TLSCert TLS certificate
+	POP3TLSCert string
+
+	// POP3TLSKey TLS certificate key
+	POP3TLSKey string
+
+	// EnableSpamAssassin must be either <host>:<port> or "postmark"
+	EnableSpamAssassin string
+
+	// WebhookURL for calling
+	WebhookURL string
+
+	// ContentSecurityPolicy for HTTP server - set via VerifyConfig()
+	ContentSecurityPolicy string
+
+	// AllowUntrustedTLS allows untrusted HTTPS connections link checking & screenshot generation
+	AllowUntrustedTLS bool
 
 	// Version is the default application version, updated on release
 	Version = "dev"
@@ -67,97 +179,337 @@ var (
 
 	// RepoBinaryName on Github for updater
 	RepoBinaryName = "mailpit"
+
+	// ChaosTriggers are parsed and set in the chaos module
+	ChaosTriggers string
+
+	// DisableHTMLCheck DEPRECATED 2024/04/13 - kept here to display console warning only
+	DisableHTMLCheck = false
+
+	// DemoMode disables SMTP relay, link checking & HTTP send functionality
+	DemoMode = false
 )
 
+// AutoTag struct for auto-tagging
+type autoTag struct {
+	Match string
+	Tags  []string
+}
+
+// SMTPRelayConfigStruct struct for parsing yaml & storing variables
+type SMTPRelayConfigStruct struct {
+	Host                    string         `yaml:"host"`
+	Port                    int            `yaml:"port"`
+	STARTTLS                bool           `yaml:"starttls"`
+	AllowInsecure           bool           `yaml:"allow-insecure"`
+	Auth                    string         `yaml:"auth"`               // none, plain, login, cram-md5
+	Username                string         `yaml:"username"`           // plain & cram-md5
+	Password                string         `yaml:"password"`           // plain
+	Secret                  string         `yaml:"secret"`             // cram-md5
+	ReturnPath              string         `yaml:"return-path"`        // allow overriding the bounce address
+	OverrideFrom            string         `yaml:"override-from"`      // allow overriding of the from address
+	AllowedRecipients       string         `yaml:"allowed-recipients"` // regex, if set needs to match for mails to be relayed
+	AllowedRecipientsRegexp *regexp.Regexp // compiled regexp using AllowedRecipients
+	BlockedRecipients       string         `yaml:"blocked-recipients"` // regex, if set prevents relating to these addresses
+	BlockedRecipientsRegexp *regexp.Regexp // compiled regexp using BlockedRecipients
+
+	// DEPRECATED 2024/03/12
+	RecipientAllowlist string `yaml:"recipient-allowlist"`
+}
+
+// SMTPForwardConfigStruct struct for parsing yaml & storing variables
+type SMTPForwardConfigStruct struct {
+	To            string `yaml:"to"`             // comma-separated list of email addresses
+	Host          string `yaml:"host"`           // SMTP host
+	Port          int    `yaml:"port"`           // SMTP port
+	STARTTLS      bool   `yaml:"starttls"`       // whether to use STARTTLS
+	AllowInsecure bool   `yaml:"allow-insecure"` // allow insecure authentication
+	Auth          string `yaml:"auth"`           // none, plain, login, cram-md5
+	Username      string `yaml:"username"`       // plain & cram-md5
+	Password      string `yaml:"password"`       // plain
+	Secret        string `yaml:"secret"`         // cram-md5
+	ReturnPath    string `yaml:"return-path"`    // allow overriding the bounce address
+	OverrideFrom  string `yaml:"override-from"`  // allow overriding of the from address
+}
+
 // VerifyConfig wil do some basic checking
 func VerifyConfig() error {
-	if DataFile != "" && isDir(DataFile) {
-		DataFile = filepath.Join(DataFile, "mailpit.db")
+	cssFontRestriction := "*"
+	if BlockRemoteCSSAndFonts {
+		cssFontRestriction = "'self'"
 	}
 
-	re := regexp.MustCompile(`^[a-zA-Z0-9\.\-]{3,}:\d{2,}$`)
-	if !re.MatchString(SMTPListen) {
-		return errors.New("SMTP bind should be in the format of <ip>:<port>")
+	// The default Content Security Policy is updates on every application page load to replace script-src 'self'
+	// with a random nonce ID to prevent XSS. This applies to the Mailpit app & API.
+	// See server.middleWareFunc()
+	ContentSecurityPolicy = fmt.Sprintf("default-src 'self'; script-src 'self'; style-src %s 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src %s data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';",
+		cssFontRestriction, cssFontRestriction,
+	)
+
+	if Database != "" && isDir(Database) {
+		Database = filepath.Join(Database, "mailpit.db")
 	}
-	if !re.MatchString(HTTPListen) {
-		return errors.New("HTTP bind should be in the format of <ip>:<port>")
+
+	Label = tools.Normalize(Label)
+
+	if err := parseMaxAge(); err != nil {
+		return err
+	}
+
+	TenantID = DBTenantID(TenantID)
+	if TenantID != "" {
+		logger.Log().Infof("[db] using tenant \"%s\"", TenantID)
+	}
+
+	re := regexp.MustCompile(`.*:\d+$`)
+	if _, _, isSocket := tools.UnixSocket(SMTPListen); !isSocket && !re.MatchString(SMTPListen) {
+		return errors.New("[smtp] bind should be in the format of <ip>:<port>")
+	}
+	if _, _, isSocket := tools.UnixSocket(HTTPListen); !isSocket && !re.MatchString(HTTPListen) {
+		return errors.New("[ui] HTTP bind should be in the format of <ip>:<port>")
 	}
 
 	if UIAuthFile != "" {
+		UIAuthFile = filepath.Clean(UIAuthFile)
+
 		if !isFile(UIAuthFile) {
-			return fmt.Errorf("HTTP password file not found: %s", UIAuthFile)
+			return fmt.Errorf("[ui] HTTP password file not found or readable: %s", UIAuthFile)
 		}
 
-		a, err := htpasswd.New(UIAuthFile, htpasswd.DefaultSystems, nil)
+		b, err := os.ReadFile(UIAuthFile)
 		if err != nil {
 			return err
 		}
-		UIAuth = a
-	}
 
-	if UISSLCert != "" && UISSLKey == "" || UISSLCert == "" && UISSLKey != "" {
-		return errors.New("you must provide both a UI SSL certificate and a key")
-	}
-
-	if UISSLCert != "" {
-		if !isFile(UISSLCert) {
-			return fmt.Errorf("SSL certificate not found: %s", UISSLCert)
-		}
-
-		if !isFile(UISSLKey) {
-			return fmt.Errorf("SSL key not found: %s", UISSLKey)
+		if err := auth.SetUIAuth(string(b)); err != nil {
+			return err
 		}
 	}
 
-	if SMTPSSLCert != "" && SMTPSSLKey == "" || SMTPSSLCert == "" && SMTPSSLKey != "" {
-		return errors.New("you must provide both an SMTP SSL certificate and a key")
+	if UITLSCert != "" && UITLSKey == "" || UITLSCert == "" && UITLSKey != "" {
+		return errors.New("[ui] you must provide both a UI TLS certificate and a key")
 	}
 
-	if SMTPSSLCert != "" {
-		if !isFile(SMTPSSLCert) {
-			return fmt.Errorf("SMTP SSL certificate not found: %s", SMTPSSLCert)
+	if UITLSCert != "" {
+		UITLSCert = filepath.Clean(UITLSCert)
+		UITLSKey = filepath.Clean(UITLSKey)
+
+		if !isFile(UITLSCert) {
+			return fmt.Errorf("[ui] TLS certificate not found or readable: %s", UITLSCert)
 		}
 
-		if !isFile(SMTPSSLKey) {
-			return fmt.Errorf("SMTP SSL key not found: %s", SMTPSSLKey)
+		if !isFile(UITLSKey) {
+			return fmt.Errorf("[ui] TLS key not found or readable: %s", UITLSKey)
 		}
 	}
 
+	if SMTPTLSCert != "" && SMTPTLSKey == "" || SMTPTLSCert == "" && SMTPTLSKey != "" {
+		return errors.New("[smtp] You must provide both an SMTP TLS certificate and a key")
+	}
+
+	if SMTPTLSCert != "" {
+		SMTPTLSCert = filepath.Clean(SMTPTLSCert)
+		SMTPTLSKey = filepath.Clean(SMTPTLSKey)
+
+		if !isFile(SMTPTLSCert) {
+			return fmt.Errorf("[smtp] TLS certificate not found or readable: %s", SMTPTLSCert)
+		}
+
+		if !isFile(SMTPTLSKey) {
+			return fmt.Errorf("[smtp] TLS key not found or readable: %s", SMTPTLSKey)
+		}
+	} else if SMTPRequireTLS {
+		return errors.New("[smtp] TLS cannot be required without an SMTP TLS certificate and key")
+	} else if SMTPRequireSTARTTLS {
+		return errors.New("[smtp] STARTTLS cannot be required without an SMTP TLS certificate and key")
+	}
+	if SMTPRequireSTARTTLS && SMTPAuthAllowInsecure || SMTPRequireTLS && SMTPAuthAllowInsecure {
+		return errors.New("[smtp] TLS cannot be required with --smtp-auth-allow-insecure")
+	}
+	if SMTPRequireSTARTTLS && SMTPRequireTLS {
+		return errors.New("[smtp] TLS & STARTTLS cannot be required together")
+	}
+
 	if SMTPAuthFile != "" {
+		SMTPAuthFile = filepath.Clean(SMTPAuthFile)
+
 		if !isFile(SMTPAuthFile) {
-			return fmt.Errorf("SMTP password file not found: %s", SMTPAuthFile)
+			return fmt.Errorf("[smtp] password file not found or readable: %s", SMTPAuthFile)
 		}
 
-		if SMTPSSLCert == "" {
-			return errors.New("SMTP authentication requires SMTP encryption")
-		}
-
-		a, err := htpasswd.New(SMTPAuthFile, htpasswd.DefaultSystems, nil)
+		b, err := os.ReadFile(SMTPAuthFile)
 		if err != nil {
 			return err
 		}
-		SMTPAuth = a
+
+		if err := auth.SetSMTPAuth(string(b)); err != nil {
+			return err
+		}
+
+		if !SMTPAuthAllowInsecure {
+			// https://www.rfc-editor.org/rfc/rfc4954
+			// A server implementation MUST implement a configuration in which
+			// it does NOT permit any plaintext password mechanisms, unless either
+			// the STARTTLS [SMTP-TLS] command has been negotiated or some other
+			// mechanism that protects the session from password snooping has been
+			// provided.  Server sites SHOULD NOT use any configuration which
+			// permits a plaintext password mechanism without such a protection
+			// mechanism against password snooping.
+			SMTPRequireSTARTTLS = true
+		}
+	}
+
+	if auth.SMTPCredentials != nil && SMTPAuthAcceptAny {
+		return errors.New("[smtp] authentication cannot use both credentials and --smtp-auth-accept-any")
+	}
+
+	if SMTPTLSCert == "" && (auth.SMTPCredentials != nil || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure {
+		return errors.New("[smtp] authentication requires STARTTLS or TLS encryption, run with `--smtp-auth-allow-insecure` to allow insecure authentication")
+	}
+
+	if err := parseChaosTriggers(); err != nil {
+		return fmt.Errorf("[chaos] %s", err.Error())
+	}
+
+	if chaos.Enabled {
+		logger.Log().Info("[chaos] is enabled")
+	}
+
+	// POP3 server
+	if POP3TLSCert != "" {
+		POP3TLSCert = filepath.Clean(POP3TLSCert)
+		POP3TLSKey = filepath.Clean(POP3TLSKey)
+
+		if !isFile(POP3TLSCert) {
+			return fmt.Errorf("[pop3] TLS certificate not found or readable: %s", POP3TLSCert)
+		}
+
+		if !isFile(POP3TLSKey) {
+			return fmt.Errorf("[pop3] TLS key not found or readable: %s", POP3TLSKey)
+		}
+	}
+	if POP3TLSCert != "" && POP3TLSKey == "" || POP3TLSCert == "" && POP3TLSKey != "" {
+		return errors.New("[pop3] You must provide both a POP3 TLS certificate and a key")
+	}
+	if POP3Listen != "" {
+		_, err := net.ResolveTCPAddr("tcp", POP3Listen)
+		if err != nil {
+			return fmt.Errorf("[pop3] %s", err.Error())
+		}
+	}
+	if POP3AuthFile != "" {
+		POP3AuthFile = filepath.Clean(POP3AuthFile)
+
+		if !isFile(POP3AuthFile) {
+			return fmt.Errorf("[pop3] password file not found or readable: %s", POP3AuthFile)
+		}
+
+		b, err := os.ReadFile(POP3AuthFile)
+		if err != nil {
+			return err
+		}
+
+		if err := auth.SetPOP3Auth(string(b)); err != nil {
+			return err
+		}
+	}
+
+	// Web root
+	validWebrootRe := regexp.MustCompile(`[^0-9a-zA-Z\/\-\_\.@]`)
+	if validWebrootRe.MatchString(Webroot) {
+		return fmt.Errorf("invalid characters in Webroot (%s). Valid chars include: [a-z A-Z 0-9 _ . - / @]", Webroot)
+	}
+
+	s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/"
+	Webroot = s
+
+	if WebhookURL != "" && !isValidURL(WebhookURL) {
+		return fmt.Errorf("webhook URL does not appear to be a valid URL (%s)", WebhookURL)
+	}
+
+	// DEPRECATED 2024/04/13
+	if DisableHTMLCheck {
+		logger.Log().Warn("--disable-html-check has been deprecated and is no longer used")
+	}
+
+	if EnableSpamAssassin != "" {
+		spamassassin.SetService(EnableSpamAssassin)
+		logger.Log().Infof("[spamassassin] enabled via %s", EnableSpamAssassin)
+
+		if err := spamassassin.Ping(); err != nil {
+			logger.Log().Warnf("[spamassassin] ping: %s", err.Error())
+		}
+	}
+
+	// load tag filters & options
+	TagFilters = []autoTag{}
+	if err := loadTagsFromArgs(CLITagsArg); err != nil {
+		return err
+	}
+	if err := loadTagsFromConfig(TagsConfig); err != nil {
+		return err
+	}
+	if err := parseTagsDisable(TagsDisable); err != nil {
+		return err
+	}
+
+	if SMTPAllowedRecipients != "" {
+		restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients)
+		if err != nil {
+			return fmt.Errorf("[smtp] failed to compile smtp-allowed-recipients regexp: %s", err.Error())
+		}
+
+		SMTPAllowedRecipientsRegexp = restrictRegexp
+		logger.Log().Infof("[smtp] only allowing recipients matching regexp: %s", SMTPAllowedRecipients)
+	}
+
+	if err := parseRelayConfig(SMTPRelayConfigFile); err != nil {
+		return err
+	}
+
+	// separate relay config validation to account for environment variables
+	if err := validateRelayConfig(); err != nil {
+		return err
+	}
+
+	if !ReleaseEnabled && SMTPRelayAll || !ReleaseEnabled && SMTPRelayMatching != "" {
+		return errors.New("[relay] a relay configuration must be set to auto-relay any messages")
+	}
+
+	if SMTPRelayMatching != "" {
+		if SMTPRelayAll {
+			logger.Log().Warnf("[relay] ignoring smtp-relay-matching when smtp-relay-all is enabled")
+		} else {
+			re, err := regexp.Compile(SMTPRelayMatching)
+			if err != nil {
+				return fmt.Errorf("[relay] failed to compile smtp-relay-matching regexp: %s", err.Error())
+			}
+
+			SMTPRelayMatchingRegexp = re
+			logger.Log().Infof("[relay] auto-relaying new messages to recipients matching \"%s\" via %s:%d",
+				SMTPRelayMatching, SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+		}
+	}
+
+	if SMTPRelayAll {
+		// this deserves a warning
+		logger.Log().Warnf("[relay] auto-relaying all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+	}
+
+	if err := parseForwardConfig(SMTPForwardConfigFile); err != nil {
+		return err
+	}
+
+	// separate forwarding config validation to account for environment variables
+	if err := validateForwardConfig(); err != nil {
+		return err
+	}
+
+	if DemoMode {
+		MaxMessages = 1000
+		// this deserves a warning
+		logger.Log().Info("demo mode enabled")
 	}
 
 	return nil
 }
-
-// IsFile returns if a path is a file
-func isFile(path string) bool {
-	info, err := os.Stat(path)
-	if os.IsNotExist(err) || !info.Mode().IsRegular() {
-		return false
-	}
-
-	return true
-}
-
-// IsDir returns whether a path is a directory
-func isDir(path string) bool {
-	info, err := os.Stat(path)
-	if os.IsNotExist(err) || !info.IsDir() {
-		return false
-	}
-
-	return true
-}

config/tags.go

@@ -0,0 +1,111 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	// TagsDisablePlus disables message tagging using plus-addresses (user+tag@example.com) - set via verifyConfig()
+	TagsDisablePlus bool
+
+	// TagsDisableXTags disables message tagging via the X-Tags header - set via verifyConfig()
+	TagsDisableXTags bool
+)
+
+type yamlTags struct {
+	Filters []yamlTag `yaml:"filters"`
+}
+
+type yamlTag struct {
+	Match string `yaml:"match"`
+	Tags  string `yaml:"tags"`
+}
+
+// Load tags from a configuration from a file, if set
+func loadTagsFromConfig(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[tags] configuration file not found or unreadable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return fmt.Errorf("[tags] %s", err.Error())
+	}
+
+	conf := yamlTags{}
+
+	if err := yaml.Unmarshal(data, &conf); err != nil {
+		return err
+	}
+
+	if conf.Filters == nil {
+		return fmt.Errorf("[tags] missing tag: array in %s", c)
+	}
+
+	for _, t := range conf.Filters {
+		tags := strings.Split(t.Tags, ",")
+		TagFilters = append(TagFilters, autoTag{Match: t.Match, Tags: tags})
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from config %s", tools.Plural(len(conf.Filters), "tag filter", "tag filters"), c)
+
+	return nil
+}
+
+func loadTagsFromArgs(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	args := tools.ArgsParser(c)
+
+	for _, a := range args {
+		t := strings.Split(a, "=")
+		if len(t) > 1 {
+			match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
+			tags := strings.Split(t[0], ",")
+			TagFilters = append(TagFilters, autoTag{Match: match, Tags: tags})
+		} else {
+			return fmt.Errorf("[tag] error parsing tags (%s)", a)
+		}
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from CLI args", tools.Plural(len(args), "tag filter", "tag filters"))
+
+	return nil
+}
+
+func parseTagsDisable(s string) error {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil
+	}
+
+	parts := strings.Split(strings.ToLower(s), ",")
+
+	for _, p := range parts {
+		switch strings.TrimSpace(p) {
+		case "x-tags", "xtags":
+			TagsDisableXTags = true
+		case "plus-addresses", "plus-addressing":
+			TagsDisablePlus = true
+		default:
+			return fmt.Errorf("[tags] invalid --tags-disable option: %s", p)
+		}
+	}
+
+	return nil
+}

config/utils.go

@@ -0,0 +1,51 @@
+package config
+
+import (
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// IsFile returns whether a file exists and is readable
+func isFile(path string) bool {
+	f, err := os.Open(filepath.Clean(path))
+	defer f.Close()
+	return err == nil
+}
+
+// IsDir returns whether a path is a directory
+func isDir(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || os.IsNotExist(err) || !info.IsDir() {
+		return false
+	}
+
+	return true
+}
+
+func isValidURL(s string) bool {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(u.Scheme, "http")
+}
+
+// DBTenantID converts a tenant ID to a DB-friendly value if set
+func DBTenantID(s string) string {
+	s = tools.Normalize(s)
+	if s != "" {
+		re := regexp.MustCompile(`[^a-zA-Z0-9\_]`)
+		s = re.ReplaceAllString(s, "_")
+		if !strings.HasSuffix(s, "_") {
+			s = s + "_"
+		}
+	}
+
+	return s
+}

config/validators.go

@@ -0,0 +1,282 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+	"gopkg.in/yaml.v3"
+)
+
+// Parse the --max-age value (if set)
+func parseMaxAge() error {
+	if MaxAge == "" {
+		return nil
+	}
+
+	re := regexp.MustCompile(`^\d+(h|d)$`)
+	if !re.MatchString(MaxAge) {
+		return fmt.Errorf("max-age must be either <int>h for hours or <int>d for days: %s", MaxAge)
+	}
+
+	if strings.HasSuffix(MaxAge, "h") {
+		hours, err := strconv.Atoi(strings.TrimSuffix(MaxAge, "h"))
+		if err != nil {
+			return err
+		}
+
+		MaxAgeInHours = hours
+
+		return nil
+	}
+
+	days, err := strconv.Atoi(strings.TrimSuffix(MaxAge, "d"))
+	if err != nil {
+		return err
+	}
+
+	logger.Log().Debugf("[db] auto-deleting messages older than %s", MaxAge)
+
+	MaxAgeInHours = days * 24
+	return nil
+}
+
+// Parse the SMTPRelayConfigFile (if set)
+func parseRelayConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[relay] configuration not found or readable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil {
+		return err
+	}
+
+	if SMTPRelayConfig.Host == "" {
+		return errors.New("[relay] host not set")
+	}
+
+	// DEPRECATED 2024/03/12
+	if SMTPRelayConfig.RecipientAllowlist != "" {
+		logger.Log().Warn("[relay] 'recipient-allowlist' is deprecated, use 'allowed-recipients' instead")
+		if SMTPRelayConfig.AllowedRecipients == "" {
+			SMTPRelayConfig.AllowedRecipients = SMTPRelayConfig.RecipientAllowlist
+		}
+	}
+
+	return nil
+}
+
+// Validate the SMTPRelayConfig (if Host is set)
+func validateRelayConfig() error {
+	if SMTPRelayConfig.Host == "" {
+		return nil
+	}
+
+	if SMTPRelayConfig.Port == 0 {
+		SMTPRelayConfig.Port = 25 // default
+	}
+
+	SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth)
+
+	if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" {
+		SMTPRelayConfig.Auth = "none"
+	} else if SMTPRelayConfig.Auth == "plain" {
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[relay] host username or password not set for PLAIN authentication")
+		}
+	} else if SMTPRelayConfig.Auth == "login" {
+		SMTPRelayConfig.Auth = "login"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[relay] host username or password not set for LOGIN authentication")
+		}
+	} else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") {
+		SMTPRelayConfig.Auth = "cram-md5"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" {
+			return fmt.Errorf("[relay] host username or secret not set for CRAM-MD5 authentication")
+		}
+	} else {
+		return fmt.Errorf("[relay] authentication method not supported: %s", SMTPRelayConfig.Auth)
+	}
+
+	if SMTPRelayConfig.AllowedRecipients != "" {
+		re, err := regexp.Compile(SMTPRelayConfig.AllowedRecipients)
+		if err != nil {
+			return fmt.Errorf("[relay] failed to compile recipient allowlist regexp: %s", err.Error())
+		}
+
+		SMTPRelayConfig.AllowedRecipientsRegexp = re
+		logger.Log().Infof("[relay] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.AllowedRecipients)
+	}
+
+	if SMTPRelayConfig.BlockedRecipients != "" {
+		re, err := regexp.Compile(SMTPRelayConfig.BlockedRecipients)
+		if err != nil {
+			return fmt.Errorf("[relay] failed to compile recipient blocklist regexp: %s", err.Error())
+		}
+
+		SMTPRelayConfig.BlockedRecipientsRegexp = re
+		logger.Log().Infof("[relay] recipient blocklist is active with the following regexp: %s", SMTPRelayConfig.BlockedRecipients)
+	}
+
+	if SMTPRelayConfig.OverrideFrom != "" {
+		m, err := mail.ParseAddress(SMTPRelayConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("[relay] override-from is not a valid email address: %s", SMTPRelayConfig.OverrideFrom)
+		}
+
+		SMTPRelayConfig.OverrideFrom = m.Address
+	}
+
+	ReleaseEnabled = true
+
+	logger.Log().Infof("[relay] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+
+	return nil
+}
+
+// Parse the SMTPForwardConfigFile (if set)
+func parseForwardConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[forward] configuration not found or readable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPForwardConfig); err != nil {
+		return err
+	}
+
+	if SMTPForwardConfig.Host == "" {
+		return errors.New("[forward] host not set")
+	}
+
+	return nil
+}
+
+// Validate the SMTPForwardConfig (if Host is set)
+func validateForwardConfig() error {
+	if SMTPForwardConfig.Host == "" {
+		return nil
+	}
+
+	if SMTPForwardConfig.Port == 0 {
+		SMTPForwardConfig.Port = 25 // default
+	}
+
+	SMTPForwardConfig.Auth = strings.ToLower(SMTPForwardConfig.Auth)
+
+	if SMTPForwardConfig.Auth == "" || SMTPForwardConfig.Auth == "none" || SMTPForwardConfig.Auth == "false" {
+		SMTPForwardConfig.Auth = "none"
+	} else if SMTPForwardConfig.Auth == "plain" {
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Password == "" {
+			return fmt.Errorf("[forward] host username or password not set for PLAIN authentication")
+		}
+	} else if SMTPForwardConfig.Auth == "login" {
+		SMTPForwardConfig.Auth = "login"
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Password == "" {
+			return fmt.Errorf("[forward] host username or password not set for LOGIN authentication")
+		}
+	} else if strings.HasPrefix(SMTPForwardConfig.Auth, "cram") {
+		SMTPForwardConfig.Auth = "cram-md5"
+		if SMTPForwardConfig.Username == "" || SMTPForwardConfig.Secret == "" {
+			return fmt.Errorf("[forward] host username or secret not set for CRAM-MD5 authentication")
+		}
+	} else {
+		return fmt.Errorf("[forward] authentication method not supported: %s", SMTPForwardConfig.Auth)
+	}
+
+	if SMTPForwardConfig.To == "" {
+		return errors.New("[forward] To addresses missing")
+	}
+
+	to := []string{}
+	addresses := strings.Split(SMTPForwardConfig.To, ",")
+	for _, a := range addresses {
+		a = strings.TrimSpace(a)
+		m, err := mail.ParseAddress(a)
+		if err != nil {
+			return fmt.Errorf("[forward] To address is not a valid email address: %s", a)
+		}
+		to = append(to, m.Address)
+	}
+
+	if len(to) == 0 {
+		return errors.New("[forward] no valid To addresses found")
+	}
+
+	// overwrite the To field with the cleaned up list
+	SMTPForwardConfig.To = strings.Join(to, ",")
+
+	if SMTPForwardConfig.OverrideFrom != "" {
+		m, err := mail.ParseAddress(SMTPForwardConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("[forward] override-from is not a valid email address: %s", SMTPForwardConfig.OverrideFrom)
+		}
+
+		SMTPForwardConfig.OverrideFrom = m.Address
+	}
+
+	logger.Log().Infof("[forward] enabling message forwarding to %s via %s:%d", SMTPForwardConfig.To, SMTPForwardConfig.Host, SMTPForwardConfig.Port)
+
+	return nil
+}
+
+func parseChaosTriggers() error {
+	if ChaosTriggers == "" {
+		return nil
+	}
+
+	re := regexp.MustCompile(`^([a-zA-Z0-0]+):(\d\d\d):(\d+(\.\d)?)$`)
+
+	parts := strings.Split(ChaosTriggers, ",")
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if !re.MatchString(p) {
+			return fmt.Errorf("invalid argument: %s", p)
+		}
+
+		matches := re.FindAllStringSubmatch(p, 1)
+		key := matches[0][1]
+		errorCode, err := strconv.Atoi(matches[0][2])
+		if err != nil {
+			return err
+		}
+		probability, err := strconv.Atoi(matches[0][3])
+		if err != nil {
+			return err
+		}
+
+		if err := chaos.Set(key, errorCode, probability); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

docs/apiv1/Message.md

@@ -1,115 +0,0 @@
-# Message
-
-## Message summary
-
-Returns a JSON summary of the message and attachments.
-
-**URL** : `api/v1/message/<ID>`
-
-**Method** : `GET`
-
-## Response
-
-**Status** : `200`
-
-```json
-{
-  "ID": "d7a5543b-96dd-478b-9b60-2b465c9884de",
-  "Read": true,
-  "From": {
-    "Name": "John Doe",
-    "Address": "john@example.com"
-  },
-  "To": [
-    {
-      "Name": "Jane Smith",
-      "Address": "jane@example.com"
-    }
-  ],
-  "Cc": null,
-  "Bcc": null,
-  "Subject": "Message subject",
-  "Date": "2016-09-07T16:46:00+13:00",
-  "Text": "Plain text MIME part of the email",
-  "HTML": "HTML MIME part (if exists)",
-  "Size": 79499,
-  "Inline": [
-    {
-      "PartID": "1.2",
-      "FileName": "filename.gif",
-      "ContentType": "image/gif",
-      "ContentID": "919564503@07092006-1525",
-      "Size": 7760
-    }
-  ],
-  "Attachments": [
-    {
-      "PartID": "2",
-      "FileName": "filename.doc",
-      "ContentType": "application/msword",
-      "ContentID": "",
-      "Size": 43520
-    }
-  ]
-}
-```
-### Notes
-
-- `Read` - always true (message marked read on open)
-- `From` - Name & Address, or null
-- `To`, `CC`, `BCC` - Array of Names & Address, or null
-- `Date` - Parsed email local date & time from headers
-- `Size` - Total size of raw email
-- `Inline`, `Attachments` - Array of attachments and inline images.
-
-
----
-## Attachments
-
-**URL** : `api/v1/message/<ID>/part/<PartID>`
-
-**Method** : `GET`
-
-Returns the attachment using the MIME type provided by the attachment `ContentType`.
-
----
-## Headers
-
-**URL** : `api/v1/message/<ID>/headers`
-
-**Method** : `GET`
-
-Returns all message headers as a JSON array.
-Each unique header key contains an array of one or more values (email headers can be listed multiple times.)
-
-```json
-{
-  "Content-Type": [
-    "multipart/related; type=\"multipart/alternative\"; boundary=\"----=_NextPart_000_0013_01C6A60C.47EEAB80\""
-  ],
-  "Date": [
-    "Wed, 12 Jul 2006 23:38:30 +1200"
-  ],
-  "Delivered-To": [
-    "user@example.com",
-    "user-alias@example.com"
-  ],
-  "From": [
-    "\"User Name\" \\u003remote@example.com\\u003e"
-  ],
-  "Message-Id": [
-    "\\u003c001701c6a5a7$b3205580$0201010a@HomeOfficeSM\\u003e"
-  ],
-....
-}
-```
-
-
----
-## Raw (source) email
-
-**URL** : `api/v1/message/<ID>/raw`
-
-**Method** : `GET`
-
-Returns the original email source including headers and attachments.

docs/apiv1/Messages.md

@@ -1,166 +0,0 @@
-# Messages
-
-List & delete messages.
-
-
----
-## List
-
-List messages in the mailbox. Messages are returned in the order of latest received to oldest.
-
-**URL** : `api/v1/messages`
-
-**Method** : `GET`
-
-
-### Query parameters
-
-| Parameter | Type    | Required | Description                |
-|-----------|---------|----------|----------------------------|
-| limit     | integer | false    | Limit results (default 50) |
-| start     | integer | false    | Pagination offset          |
-
-
-### Response
-
-**Status** : `200`
-
-```json
-{
-  "total": 500,
-  "unread": 500,
-  "count": 50,
-  "start": 0,
-  "messages": [
-    {
-      "ID": "1c575821-70ba-466f-8cee-2e1cf0fcdd0f",
-      "Read": false,
-      "From": {
-        "Name": "John Doe",
-        "Address": "john@example.com"
-      },
-      "To": [
-        {
-          "Name": "Jane Smith",
-          "Address": "jane@example.com"
-        }
-      ],
-      "Cc": [
-        {
-          "Name": "Accounts",
-          "Address": "accounts@example.com"
-        }
-      ],
-      "Bcc": null,
-      "Subject": "Message subject",
-      "Created": "2022-10-03T21:35:32.228605299+13:00",
-      "Size": 6144,
-      "Attachments": 0
-    },
-    ...
-  ]
-}
-```
-
-### Notes
-
-- `total` - Total messages in mailbox
-- `unread` - Total unread messages in mailbox
-- `count` - Number of messages returned in request
-- `start` - The offset (default `0`) for pagination
-- `Read` - The read/unread status of the message
-- `From` - Name & Address, or null if none
-- `To`, `CC`, `BCC` - Array of Names & Address, or null if none
-- `Created` - Local date & time the message was received
-- `Size` - Total size of raw email in bytes
-
-
----
-## Delete individual messages
-
-Delete one or more messages by ID.
-
-**URL** : `api/v1/messages`
-
-**Method** : `DELETE`
-
-### Request
-
-```json
-{
-  "ids": ["<ID>","<ID>"...]
-}
-```
-
-### Response
-
-**Status** : `200`
-
-
----
-## Delete all messages
-
-Delete all messages (same as deleting individual messages, but with the "ids" either empty or omitted entirely).
-
-**URL** : `api/v1/messages`
-
-**Method** : `DELETE`
-
-### Request
-
-```json
-{
-  "ids": []
-}
-```
-
-### Response
-
-**Status** : `200`
-
-
----
-## Update individual read statuses
-
-Set the read status of one or more messages. 
-The `read` status can be `true` or `false`.
-
-**URL** : `api/v1/messages`
-
-**Method** : `PUT`
-
-### Request
-
-```json
-{
-  "ids": ["<ID>","<ID>"...],
-  "read": false
-}
-```
-
-### Response
-
-**Status** : `200`
-
----
-## Update all messages read status
-
-Set the read status of all messages. 
-The `read` status can be `true` or `false`.
-
-**URL** : `api/v1/messages`
-
-**Method** : `PUT`
-
-### Request
-
-```json
-{
-  "ids": [],
-  "read": false
-}
-```
-
-### Response
-
-**Status** : `200`

docs/apiv1/README.md

@@ -1,11 +0,0 @@
-# API v1
-
-Mailpit provides a simple REST API to access and delete stored messages.
-
-If the Mailpit server is set to use Basic Authentication, then API requests must use Basic Authentication too.
-
-The API is split into three main parts:
-
-- [Messages](Messages.md) - Listing, deleting & marking messages as read/unread.
-- [Message](Message.md) - Return message data & attachments
-- [Search](Search.md) - Searching messages

docs/apiv1/Search.md

@@ -1,69 +0,0 @@
-# Search
-
-**URL** : `api/v1/search?query=<string>`
-
-**Method** : `GET`
-
-The search returns the most recent matches (default 50).
-Matching messages are returned in the order of latest received to oldest.
-
-
-## Query parameters
-
-| Parameter | Type    | Required | Description                |
-|-----------|---------|----------|----------------------------|
-| query     | string  | true     | Search query               |
-| limit     | integer | false    | Limit results (default 50) |
-| start     | integer | false    | Pagination offset          |
-
-
-## Response
-
-**Status** : `200`
-
-```json
-{
-  "total": 500,
-  "unread": 500,
-  "count": 25,
-  "start": 0,
-  "messages": [
-    {
-      "ID": "1c575821-70ba-466f-8cee-2e1cf0fcdd0f",
-      "Read": false,
-      "From": {
-        "Name": "John Doe",
-        "Address": "john@example.com"
-      },
-      "To": [
-        {
-          "Name": "Jane Smith",
-          "Address": "jane@example.com"
-        }
-      ],
-      "Cc": [
-        {
-          "Name": "Accounts",
-          "Address": "accounts@example.com"
-        }
-      ],
-      "Bcc": null,
-      "Subject": "Test email",
-      "Created": "2022-10-03T21:35:32.228605299+13:00",
-      "Size": 6144,
-      "Attachments": 0
-    },
-    ...
-  ]
-}
-```
-
-### Notes
-
-- `total` - Total messages in mailbox (all messages, not search)
-- `unread` - Total unread messages in mailbox (all messages, not search)
-- `count` - Number of messages returned in request
-- `start` - The offset (default `0`) for pagination
-- `From` - Singular Name & Address, or null if none
-- `To`, `CC`, `BCC` - Array of Name & Address, or null if none
-- `Size` - Total size of raw email in bytes

esbuild.config.js

@@ -1,22 +0,0 @@
-const { build } = require('esbuild')
-const pluginVue = require('esbuild-plugin-vue-next')
-const { sassPlugin } = require('esbuild-sass-plugin');
-
-const doWatch = process.env.WATCH == 'true' ? true : false;
-const doMinify = process.env.MINIFY == 'true' ? true : false;
-
-build({
-    entryPoints: ["server/ui-src/app.js"],
-    bundle: true,
-    watch: doWatch,
-    minify: doMinify,
-    sourcemap: false,
-    outfile: "server/ui/dist/app.js",
-    plugins: [pluginVue(), sassPlugin()],
-    loader: {
-        ".svg": "file",
-        ".woff": "file",
-        ".woff2": "file",
-    },
-    logLevel: "info"
-})

esbuild.config.mjs

@@ -0,0 +1,44 @@
+import * as esbuild from 'esbuild'
+import pluginVue from 'esbuild-plugin-vue-next'
+import { sassPlugin } from 'esbuild-sass-plugin'
+
+const doWatch = process.env.WATCH == 'true' ? true : false;
+const doMinify = process.env.MINIFY == 'true' ? true : false;
+
+const ctx = await esbuild.context(
+    {
+        entryPoints: [
+            "server/ui-src/app.js",
+            "server/ui-src/docs.js"
+        ],
+        bundle: true,
+        minify: doMinify,
+        sourcemap: false,
+        define: {
+            '__VUE_OPTIONS_API__': 'true',
+            '__VUE_PROD_DEVTOOLS__': 'false',
+            '__VUE_PROD_HYDRATION_MISMATCH_DETAILS__': 'false',
+        },
+        outdir: "server/ui/dist/",
+        plugins: [
+            pluginVue(),
+            sassPlugin({
+                silenceDeprecations: ['import'],
+                quietDeps: true,
+            })
+        ],
+        loader: {
+            ".svg": "file",
+            ".woff": "file",
+            ".woff2": "file",
+        },
+        logLevel: "info"
+    }
+)
+
+if (doWatch) {
+    await ctx.watch()
+} else {
+    await ctx.rebuild()
+    ctx.dispose()
+}

go.mod

@@ -1,62 +1,63 @@
 module github.com/axllent/mailpit
 
-go 1.18
+go 1.23.0
+
+toolchain go1.23.2
 
 require (
-	github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244
+	github.com/PuerkitoBio/goquery v1.10.2
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/axllent/semver v0.0.1
-	github.com/disintegration/imaging v1.6.2
-	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/websocket v1.5.0
-	github.com/jhillyerd/enmime v0.10.1
-	github.com/k3a/html2text v1.0.8
-	github.com/klauspost/compress v1.15.11
-	github.com/leporo/sqlf v1.3.0
-	github.com/mattn/go-shellwords v1.0.12
-	github.com/mhale/smtpd v0.8.0
-	github.com/satori/go.uuid v1.2.0
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.5.0
-	github.com/spf13/pflag v1.0.5
-	github.com/tg123/go-htpasswd v1.2.0
-	golang.org/x/text v0.3.7
-	modernc.org/sqlite v1.19.1
+	github.com/gomarkdown/markdown v0.0.0-20250207164621-7a1f277a159e
+	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/websocket v1.5.3
+	github.com/jhillyerd/enmime v1.3.0
+	github.com/klauspost/compress v1.17.11
+	github.com/kovidgoyal/imaging v1.6.4
+	github.com/leporo/sqlf v1.4.0
+	github.com/lithammer/shortuuid/v4 v4.2.0
+	github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62
+	github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.9.0
+	github.com/spf13/pflag v1.0.6
+	github.com/tg123/go-htpasswd v1.2.3
+	github.com/vanng822/go-premailer v1.23.0
+	golang.org/x/net v0.35.0
+	golang.org/x/text v0.22.0
+	golang.org/x/time v0.10.0
+	gopkg.in/yaml.v3 v3.0.1
+	modernc.org/sqlite v1.35.0
 )
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect
-	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 // indirect
+	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
-	github.com/cznic/ql v1.2.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect
-	github.com/rivo/uniseg v0.4.2 // indirect
+	github.com/reiver/go-oi v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b // indirect
-	golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect
-	golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	github.com/vanng822/css v1.0.1 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/exp v0.0.0-20250215185904-eff6e970281f // indirect
+	golang.org/x/image v0.24.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.9 // indirect
-	modernc.org/libc v1.20.2 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.4.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/libc v1.61.13 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.8.2 // indirect
 )

go.sum

@@ -1,76 +1,52 @@
-github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
-github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw=
-github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo=
-github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 h1:dqzm54OhCqY8RinR/cx+Ppb0y56Ds5I3wwWhx4XybDg=
-github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244/go.mod h1:3sqgkckuISJ5rs1EpOp6vCvwOUKe/z9vPmyuIlq8Q/A=
+github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI=
+github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/PuerkitoBio/goquery v1.10.2 h1:7fh2BdHcG6VFZsK7toXBT/Bh1z5Wmy8Q9MV9HqT2AM8=
+github.com/PuerkitoBio/goquery v1.10.2/go.mod h1:0guWGjcLu9AYC7C1GHnpysHy056u9aEkUHwhdnePMCU=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E=
 github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk=
-github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
-github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas=
-github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
-github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4 h1:CVAqftqbj+exlab+8KJQrE+kNIVlQfJt58j4GxCMF1s=
-github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
-github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00 h1:FHpbUtp2K8X53/b4aFNj4my5n+i3x+CQCZWNuHWH/+E=
-github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4=
-github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs=
-github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A=
-github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk=
-github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
-github.com/cznic/ql v1.2.0 h1:lcKp95ZtdF0XkWhGnVIXGF8dVD2X+ClS08tglKtf+ak=
-github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE=
-github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
-github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ=
-github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA=
-github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
-github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc h1:YKKpTb2BrXN2GYyGaygIdis1vXbE7SSAG9axGWIMClg=
-github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
-github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
-github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
-github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
-github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
-github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg=
-github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jhillyerd/enmime v0.10.1 h1:3VP8gFhK7R948YJBrna5bOgnTXEuPAoICo79kKkBKfA=
-github.com/jhillyerd/enmime v0.10.1/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/z6mkcDHsA=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/k3a/html2text v1.0.8 h1:rVanLhKilpnJUJs/CNKWzMC4YaQINGxK0rSG8ssmnV0=
-github.com/k3a/html2text v1.0.8/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
-github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/gomarkdown/markdown v0.0.0-20250207164621-7a1f277a159e h1:ESHlT0RVZphh4JGBz49I5R6nTdC8Qyc08vU25GQHzzQ=
+github.com/gomarkdown/markdown v0.0.0-20250207164621-7a1f277a159e/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/jhillyerd/enmime v1.3.0 h1:LV5kzfLidiOr8qRGIpYYmUZCnhrPbcFAnAFUnWn99rw=
+github.com/jhillyerd/enmime v1.3.0/go.mod h1:6c6jg5HdRRV2FtvVL69LjiX1M8oE0xDX9VEhV3oy4gs=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/kovidgoyal/imaging v1.6.4 h1:K0idhRPXnRrJBKnBYcTfI1HTWSNDeAn7hYDvf9I0dCk=
+github.com/kovidgoyal/imaging v1.6.4/go.mod h1:bEIgsaZmXlvFfkv/CUxr9rJook6AQkJnpB5EPosRfRY=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -78,149 +54,199 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/leporo/sqlf v1.3.0 h1:nAkuPYxMIJg/sUmcd1h4avV5iYo8tBTGEGOIR4BIZO8=
-github.com/leporo/sqlf v1.3.0/go.mod h1:f4dHqIi1+nLl6k1IsNQ8QIEbGWK49th2ei1IxTXk+2E=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/leporo/sqlf v1.4.0 h1:SyWnX/8GSGOzVmanG0Ub1c04mR9nNl6Tq3IeFKX2/4c=
+github.com/leporo/sqlf v1.4.0/go.mod h1:pgN9yKsAnQ+2ewhbZogr98RcasUjPsHF3oXwPPhHvBw=
+github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c=
+github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
-github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
-github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
-github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=
-github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62 h1:XMG5DklHoioVYysfYglOB7vRBg/LOUJZy2mq2QyedLg=
+github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62/go.mod h1:niAM5cni0I/47IFA995xQfeK58Mkbb7FHJjacY4OGQg=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY=
-github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/reiver/go-oi v1.0.0 h1:nvECWD7LF+vOs8leNGV/ww+F2iZKf3EYjYZ527turzM=
+github.com/reiver/go-oi v1.0.0/go.mod h1:RrDBct90BAhoDTxB1fenZwfykqeGvhI6LsNfStJoEkI=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
-github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a h1:9O8zgGrMBuTsnA3yyFd+JWhFSflQwzSUEB4AMnFHKhU=
+github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.9.0 h1:Py5fIuq/lJsRYxcxfOtsJqpmwJWCMOUy2tMJYV8TNHE=
+github.com/spf13/cobra v1.9.0/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
-github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0=
-github.com/tg123/go-htpasswd v1.2.0/go.mod h1:h7IzlfpvIWnVJhNZ0nQ9HaFxHb7pn5uFJYLlEUJa2sM=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tg123/go-htpasswd v1.2.3 h1:ALR6ZBIc2m9u70m+eAWUFt5p43ISbIvAvRFYzZPTOY8=
+github.com/tg123/go-htpasswd v1.2.3/go.mod h1:FcIrK0J+6zptgVwK1JDlqyajW/1B4PtuJ/FLWl7nx8A=
+github.com/unrolled/render v1.7.0/go.mod h1:LwQSeDhjml8NLjIO9GJO1/1qpFJxtfVIpzxXKjfVkoI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/vanng822/css v1.0.1 h1:10yiXc4e8NI8ldU6mSrWmSWMuyWgPr9DZ63RSlsgDw8=
+github.com/vanng822/css v1.0.1/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w=
+github.com/vanng822/go-premailer v1.23.0 h1:vZp2wuz1jb4q/DurUV18VGjXWtTFYZHwTCw2EAWKO74=
+github.com/vanng822/go-premailer v1.23.0/go.mod h1:0+z0UJ6ZGQatzkWlaQNl50M7fLz5f6FcP8V2p0oie88=
+github.com/vanng822/r2router v0.0.0-20150523112421-1023140a4f30/go.mod h1:1BVq8p2jVr55Ost2PkZWDrG86PiJ/0lxqcXoAcGxvWU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0=
-golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 h1:Lj6HJGCSn5AjxRAH2+r35Mir4icalbqku+CLUtjnvXY=
-golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg=
+golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
+golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ=
+golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM=
-golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
+golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.9 h1:AXquSwg7GuMk11pIdw7fmO1Y/ybgazVkMhsZWCV0mHM=
-modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
-modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
-modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
-modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
-modernc.org/libc v1.20.2 h1:9/C6hYLe+SNLricCd+WYkIGatWrQTZegOfmOcz5fPmY=
-modernc.org/libc v1.20.2/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0=
-modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
-modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
-modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.19.1 h1:8xmS5oLnZtAK//vnd4aTVj8VOeTAccEFOtUnIzfSw+4=
-modernc.org/sqlite v1.19.1/go.mod h1:UfQ83woKMaPW/ZBruK0T7YaFCrI+IE0LeWVY6pmnVms=
-modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.14.0 h1:cO7oyRWEXweSJmjdbs1L86P52D9QmBy/CPFKmFvNYTU=
-modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.6.0 h1:gLwAw6aS973K/k9EOJGlofauyMk4YOUiPDYzWnq/oXo=
+modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
+modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.23.16 h1:Z2N+kk38b7SfySC1ZkpGLN2vthNJP1+ZzGZIlH7uBxo=
+modernc.org/ccgo/v4 v4.23.16/go.mod h1:nNma8goMTY7aQZQNTyN9AIoJfxav4nvTnvKThAeMDdo=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.6.3 h1:aJVhcqAte49LF+mGveZ5KPlsp4tdGdAOT4sipJXADjw=
+modernc.org/gc/v2 v2.6.3/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8=
+modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI=
+modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.35.0 h1:yQps4fegMnZFdphtzlfQTCNBWtS0CZv48pRpW3RFHRw=
+modernc.org/sqlite v1.35.0/go.mod h1:9cr2sicr7jIaWTBKQmAxQLfBv9LL0su4ZTEV+utt3ic=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

install.sh

@@ -7,7 +7,7 @@ set -e
 
 VERSION=$(curl --silent --location --max-time "${TIMEOUT}" "https://api.github.com/repos/${GH_REPO}/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
 if [ $? -ne 0 ]; then
-    echo -ne "\nThere was an error trying to check what is the latest version of ssbak.\nPlease try again later.\n"
+    echo -ne "\nThere was an error trying to check what is the latest version of Mailpit.\nPlease try again later.\n"
     exit 1
 fi
 

internal/auth/auth.go

@@ -0,0 +1,90 @@
+// Package auth handles the web UI and SMTP authentication
+package auth
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/tg123/go-htpasswd"
+)
+
+var (
+	// UICredentials passwords
+	UICredentials *htpasswd.File
+	// SMTPCredentials passwords
+	SMTPCredentials *htpasswd.File
+	// POP3Credentials passwords
+	POP3Credentials *htpasswd.File
+)
+
+// SetUIAuth will set Basic Auth credentials required for the UI & API
+func SetUIAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	UICredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetSMTPAuth will set SMTP credentials
+func SetSMTPAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	SMTPCredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetPOP3Auth will set POP3 server credentials
+func SetPOP3Auth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	POP3Credentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func credentialsFromString(s string) []string {
+	// split string by any whitespace character
+	re := regexp.MustCompile(`\s+`)
+
+	words := re.Split(s, -1)
+	credentials := []string{}
+	for _, w := range words {
+		if w != "" {
+			credentials = append(credentials, w)
+		}
+	}
+
+	return credentials
+}

internal/dump/dump.go

@@ -0,0 +1,163 @@
+// Package dump is used to export all messages from mailpit into a directory
+package dump
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/apiv1"
+)
+
+var (
+	linkRe = regexp.MustCompile(`(?i)^https?:\/\/`)
+
+	outDir string
+
+	// Base URL of mailpit instance
+	base string
+
+	// URL is the base URL of a remove Mailpit instance
+	URL string
+
+	summary = []storage.MessageSummary{}
+)
+
+// Sync will sync all messages from the specified database or API to the specified output directory
+func Sync(d string) error {
+
+	outDir = path.Clean(d)
+
+	if URL != "" {
+		if !linkRe.MatchString(URL) {
+			return errors.New("Invalid URL")
+		}
+
+		base = strings.TrimRight(URL, "/") + "/"
+	}
+
+	if base == "" && config.Database == "" {
+		return errors.New("No database or API URL specified")
+	}
+
+	if !tools.IsDir(outDir) {
+		if err := os.MkdirAll(outDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	if err := loadIDs(); err != nil {
+		return err
+	}
+
+	if err := saveMessages(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// LoadIDs will load all message IDs from the specified database or API
+func loadIDs() error {
+	if base != "" {
+		// remote
+		logger.Log().Debugf("Fetching messages summary from %s", base)
+		res, err := http.Get(base + "api/v1/messages?limit=0")
+
+		if err != nil {
+			return err
+		}
+
+		body, err := io.ReadAll(res.Body)
+
+		if err != nil {
+			return err
+		}
+
+		var data apiv1.MessagesSummary
+		if err := json.Unmarshal(body, &data); err != nil {
+			return err
+		}
+
+		summary = data.Messages
+
+	} else {
+		// make sure the database isn't pruned while open
+		config.MaxMessages = 0
+
+		var err error
+		// local database
+		if err = storage.InitDB(); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("Fetching messages summary from %s", config.Database)
+
+		summary, err = storage.List(0, 0, 0)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(summary) == 0 {
+		return errors.New("No messages found")
+	}
+
+	return nil
+}
+
+func saveMessages() error {
+	for _, m := range summary {
+		out := path.Join(outDir, m.ID+".eml")
+
+		// skip if message exists
+		if tools.IsFile(out) {
+			continue
+		}
+
+		var b []byte
+
+		if base != "" {
+			res, err := http.Get(base + "api/v1/message/" + m.ID + "/raw")
+
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+
+			b, err = io.ReadAll(res.Body)
+
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+		} else {
+			var err error
+			b, err = storage.GetMessageRaw(m.ID)
+			if err != nil {
+				logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error())
+				continue
+			}
+		}
+
+		if err := os.WriteFile(out, b, 0644); err != nil {
+			logger.Log().Errorf("Error writing message %s: %s", m.ID, err.Error())
+			continue
+		}
+
+		_ = os.Chtimes(out, m.Created, m.Created)
+
+		logger.Log().Debugf("Saved message %s to %s", m.ID, out)
+	}
+
+	return nil
+}

internal/html2text/html2text.go

@@ -0,0 +1,83 @@
+// Package html2text is a simple library to convert HTML to plain text
+package html2text
+
+import (
+	"bytes"
+	"log"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"golang.org/x/net/html"
+)
+
+var (
+	re      = regexp.MustCompile(`\s+`)
+	spaceRe = regexp.MustCompile(`(?mi)<\/(div|p|td|th|h[1-6]|ul|ol|li|address|article|aside|blockquote|dl|dt|footer|header|hr|main|nav|pre|table|thead|tfoot|video)><`)
+	brRe    = regexp.MustCompile(`(?mi)<(br /|br)>`)
+	imgRe   = regexp.MustCompile(`(?mi)<(img)`)
+	skip    = make(map[string]bool)
+)
+
+func init() {
+	skip["script"] = true
+	skip["title"] = true
+	skip["head"] = true
+	skip["link"] = true
+	skip["meta"] = true
+	skip["style"] = true
+	skip["noscript"] = true
+}
+
+// Strip will convert a HTML string to plain text
+func Strip(h string, includeLinks bool) string {
+	h = spaceRe.ReplaceAllString(h, "</$1> <")
+	h = brRe.ReplaceAllString(h, " ")
+	h = imgRe.ReplaceAllString(h, " <$1")
+	var buffer bytes.Buffer
+	doc, err := html.Parse(strings.NewReader(h))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	extract(doc, &buffer, includeLinks)
+	return clean(buffer.String())
+}
+
+func extract(node *html.Node, buff *bytes.Buffer, includeLinks bool) {
+	if node.Type == html.TextNode {
+		data := node.Data
+		if data != "" {
+			buff.WriteString(data)
+		}
+	}
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		if _, skip := skip[c.Data]; !skip {
+			if includeLinks && c.Data == "a" {
+				for _, a := range c.Attr {
+					if a.Key == "href" && strings.HasPrefix(strings.ToLower(a.Val), "http") {
+						buff.WriteString(" " + a.Val + " ")
+					}
+				}
+			}
+			extract(c, buff, includeLinks)
+		}
+	}
+}
+
+func clean(text string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	text = strings.ReplaceAll(text, string('\uFEFF'), " ")
+
+	// remove non-printable characters
+	text = strings.Map(func(r rune) rune {
+		if unicode.IsPrint(r) {
+			return r
+		}
+		return []rune(" ")[0]
+	}, text)
+
+	text = re.ReplaceAllString(text, " ")
+
+	return strings.TrimSpace(text)
+}

internal/html2text/html2text_test.go

@@ -0,0 +1,250 @@
+package html2text
+
+import "testing"
+
+func TestPlain(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, false)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestWithLinks(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading https://github.com linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading https://github.com linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, true)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func BenchmarkPlain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, false)
+	}
+}
+
+func BenchmarkLinks(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, true)
+	}
+}
+
+var htmlTestData = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en" style="font-family: sans-serif; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; box-sizing: border-box;" xml:lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>[axllent/mailpit] Run failed: .github/workflows/tests.yml - feature/swagger (284335a)</title>
+    
+  </head>
+  <body style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;; font-size: 14px; line-height: 1.5; color: #24292e; background-color: #fff; margin: 0;" bgcolor="#fff">
+    <table align="center" class="container-sm width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 544px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <td class="center p-3" align="center" valign="top" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 16px;">
+          <center style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full container-md" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 768px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                  <td class="text-left" style="box-sizing: border-box; text-align: left !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;" align="left">
+                    <img src="https://github.githubassets.com/images/email/global/octocat-logo.png" alt="GitHub" width="32" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+                    <h2 class="lh-condensed mt-2 text-normal" style="box-sizing: border-box; margin-top: 8px !important; margin-bottom: 0; font-size: 24px; font-weight: 400 !important; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                        [axllent/mailpit] .github/workflows/tests.yml workflow run
+
+                    </h2>
+                  </td>
+                </tr>
+              </table>
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+</td>
+  </tr>
+</table>
+            <table width="100%" class="width-full" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+              <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <td class="border rounded-2 d-block" style="box-sizing: border-box; border-radius: 6px !important; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0; border: 1px solid #e1e4e8;">
+                  <table align="center" class="width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                      <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                        <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                          
+<table align="center" class="border-bottom width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; border-bottom-width: 1px !important; border-bottom-color: #e1e4e8 !important; border-bottom-style: solid !important; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td class="d-block px-3 pt-3 p-sm-4" style="box-sizing: border-box; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 24px;">
+      <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+        
+    <img src="https://github.githubassets.com/images/email/icons/actions.png" width="56" height="56" alt="" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="12" style="font-size: 12px; line-height: 12px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+<h3 class="lh-condensed" style="box-sizing: border-box; margin-top: 0; margin-bottom: 0; font-size: 20px; font-weight: 600; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">.github/workflows/tests.yml: No jobs were run</h3>
+<table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+
+  <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+    <table width="100%" border="0" cellspacing="0" cellpadding="0" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+      <table border="0" cellspacing="0" cellpadding="0" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+          <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <!--[if mso]> <table><tr><td align="center" bgcolor="#28a745"> <![endif]-->
+                <a href="https://github.com/axllent/mailpit/actions/runs/6522820865" target="_blank" rel="noopener noreferrer" class="btn btn-large btn-primary" style="background-color: #1f883d !important; box-sizing: border-box; color: #fff; text-decoration: none; position: relative; display: inline-block; font-size: inherit; font-weight: 500; line-height: 1.5; white-space: nowrap; vertical-align: middle; cursor: pointer; -webkit-user-select: none; user-select: none; border-radius: .5em; -webkit-appearance: none; appearance: none; box-shadow: 0 1px 0 rgba(27,31,35,.1),inset 0 1px 0 rgba(255,255,255,.03); transition: background-color .2s cubic-bezier(0.3, 0, 0.5, 1); font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: .75em 1.5em; border: 1px solid #1f883d;">View workflow run</a>
+              <!--[if mso]> </td></tr></table> <![endif]-->
+          </td>
+        </tr>
+      </table>
+    </td>
+  </tr>
+</table>
+
+</td>
+  </tr>
+</table>
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="32" style="font-size: 32px; line-height: 32px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+</td>
+  </tr>
+</table>
+    </td>
+  </tr>
+</table>
+
+
+
+
+</td>
+  </tr>
+</table>
+                      </td>
+                    </tr>
+                  </table>
+                </td>
+              </tr>
+            </table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <p class="f5 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 14px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">  </p><p style="font-size: small; -webkit-text-size-adjust: none; color: #666; box-sizing: border-box; margin-top: 0; margin-bottom: 10px; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">&#8212;<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" />You are receiving this because you are subscribed to this thread.<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" /><a href="https://github.com/settings/notifications" style="background-color: transparent; box-sizing: border-box; color: #0366d6; text-decoration: none; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">Manage your GitHub Actions notifications</a></p>
+
+</td>
+  </tr>
+</table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+  <p class="f6 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 12px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">GitHub, Inc. &#12539;88 Colin P Kelly Jr Street &#12539;San Francisco, CA 94107</p>
+</td>
+  </tr>
+</table>
+
+          </center>
+        </td>
+      </tr>
+    </table>
+    <!-- prevent Gmail on iOS font size manipulation -->
+   <div style="display: none; white-space: nowrap; box-sizing: border-box; font: 15px/0 apple-system, BlinkMacSystemFont, &quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;"> &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; </div>
+  </body>
+</html>`

internal/htmlcheck/README.md

@@ -0,0 +1,5 @@
+# HTML check
+
+The database used for HTML support tests is based on [can I email](https://www.caniemail.com/).
+
+The `caniemail-data.json` file used to determine client support is copied from the [API](https://www.caniemail.com/api/data.json)

internal/htmlcheck/caniemail.go

@@ -0,0 +1,74 @@
+// Package htmlcheck is used for parsing HTML and returning
+// HTML compatibility errors and warnings
+package htmlcheck
+
+import (
+	"embed"
+	"encoding/json"
+	"regexp"
+)
+
+//go:embed caniemail-data.json
+var embeddedFS embed.FS
+
+var (
+	cie = CanIEmail{}
+
+	noteMatch = regexp.MustCompile(` #(\d)+$`)
+
+	// LimitFamilies will limit results to families if set
+	LimitFamilies = []string{}
+
+	// LimitPlatforms will limit results to platforms if set
+	LimitPlatforms = []string{}
+
+	// LimitClients will limit results to clients if set
+	LimitClients = []string{}
+)
+
+// CanIEmail struct for JSON data
+type CanIEmail struct {
+	APIVersion     string `json:"api_version"`
+	LastUpdateDate string `json:"last_update_date"`
+	// NiceNames map[string]string `json:"last_update_date"`
+	NiceNames struct {
+		Family   map[string]string `json:"family"`
+		Platform map[string]string `json:"platform"`
+		Support  map[string]string `json:"support"`
+		Category map[string]string `json:"category"`
+	} `json:"nicenames"`
+	Data []JSONResult `json:"data"`
+}
+
+// JSONResult struct for CanIEmail Data
+type JSONResult struct {
+	Slug           string                 `json:"slug"`
+	Title          string                 `json:"title"`
+	Description    string                 `json:"description"`
+	URL            string                 `json:"url"`
+	Category       string                 `json:"category"`
+	Tags           []string               `json:"tags"`
+	Keywords       string                 `json:"keywords"`
+	LastTestDate   string                 `json:"last_test_date"`
+	TestURL        string                 `json:"test_url"`
+	TestResultsURL string                 `json:"test_results_url"`
+	Stats          map[string]interface{} `json:"stats"`
+	Notes          string                 `json:"notes"`
+	NotesByNumber  map[string]string      `json:"notes_by_num"`
+}
+
+// Load the JSON data
+func loadJSONData() error {
+	if cie.APIVersion != "" {
+		return nil
+	}
+
+	b, err := embeddedFS.ReadFile("caniemail-data.json")
+	if err != nil {
+		return err
+	}
+
+	cie = CanIEmail{}
+
+	return json.Unmarshal(b, &cie)
+}

internal/htmlcheck/config.go

@@ -0,0 +1,204 @@
+package htmlcheck
+
+import "regexp"
+
+// HTML tests
+var htmlTests = map[string]string{
+	// body check is manually done because it always exists in *goquery.Document
+	"html-body": "body",
+	// HTML tests
+	"html-object":         "object, embed, image, pdf",
+	"html-link":           "link",
+	"html-hr":             "hr",
+	"html-dialog":         "dialog",
+	"html-srcset":         "[srcset]",
+	"html-picture":        "picture",
+	"html-svg":            "svg",
+	"html-progress":       "progress",
+	"html-required":       "[required]",
+	"html-meter":          "meter",
+	"html-audio":          "audio",
+	"html-form":           "form",
+	"html-input-submit":   "submit",
+	"html-button-reset":   "button[type=\"reset\"]",
+	"html-button-submit":  "submit, button[type=\"submit\"]",
+	"html-base":           "base",
+	"html-input-checkbox": "checkbox",
+	"html-input-hidden":   "[type=\"hidden\"]",
+	"html-input-radio":    "radio",
+	"html-input-text":     "input[type=\"text\"]",
+	"html-video":          "video",
+	"html-semantics":      "article, aside, details, figcaption, figure, footer, header, main, mark, nav, section, summary, time",
+	"html-select":         "select",
+	"html-textarea":       "textarea",
+	"html-anchor-links":   "a[href^=\"#\"]",
+	"html-style":          "style",
+	"html-image-maps":     "map, img[usemap]",
+}
+
+// Image tests using regex to match against img[src]
+var imageRegexpTests = map[string]*regexp.Regexp{
+	"image-apng":   regexp.MustCompile(`(?i)\.apng$`),       // 78.723404
+	"image-avif":   regexp.MustCompile(`(?i)\.avif$`),       // 14.864864
+	"image-base64": regexp.MustCompile(`^(?i)data:image\/`), // 61.702126
+	"image-bmp":    regexp.MustCompile(`(?i)\.bmp$`),        // 89.3617
+	"image-gif":    regexp.MustCompile(`(?i)\.gif$`),        // 89.3617
+	"image-hdr":    regexp.MustCompile(`(?i)\.hdr$`),        // 12.5
+	"image-heif":   regexp.MustCompile(`(?i)\.heif$`),       // 0
+	"image-ico":    regexp.MustCompile(`(?i)\.ico$`),        // 87.23404
+	"image-mp4":    regexp.MustCompile(`(?i)\.mp4$`),        // 26.53061
+	"image-ppm":    regexp.MustCompile(`(?i)\.ppm$`),        // 2.0833282
+	"image-svg":    regexp.MustCompile(`(?i)\.svg$`),        // 64.91228
+	"image-tiff":   regexp.MustCompile(`(?i)\.tiff?$`),      // 38.29787
+	"image-webp":   regexp.MustCompile(`(?i)\.webp$`),       // 59.649124
+}
+
+var cssInlineTests = map[string]string{
+	"css-accent-color":                   "[style*=\"accent-color:\"]",                                           // 6.6666718
+	"css-align-items":                    "[style*=\"align-items:\"]",                                            // 60.784313
+	"css-aspect-ratio":                   "[style*=\"aspect-ratio:\"]",                                           // 30
+	"css-background-blend-mode":          "[style*=\"background-blend-mode:\"]",                                  // 61.70213
+	"css-background-clip":                "[style*=\"background-clip:\"]",                                        // 61.70213
+	"css-background-color":               "[style*=\"background-color:\"], [bgcolor]",                            // 90
+	"css-background-image":               "[style*=\"background-image:\"]",                                       // 57.62712
+	"css-background-origin":              "[style*=\"background-origin:\"]",                                      // 61.70213
+	"css-background-position":            "[style*=\"background-position:\"]",                                    // 61.224487
+	"css-background-repeat":              "[style*=\"background-repeat:\"]",                                      // 67.34694
+	"css-background-size":                "[style*=\"background-size:\"]",                                        // 61.702126
+	"css-background":                     "[style*=\"background:\"], [background]",                               // 57.407406
+	"css-block-inline-size":              "[style*=\"block-inline-size:\"]",                                      // 46.93877
+	"css-border-image":                   "[style*=\"border-image:\"]",                                           // 52.173912
+	"css-border-inline-block-individual": "[style*=\"border-inline:\"]",                                          // 18.518517
+	"css-border-radius":                  "[style*=\"border-radius:\"]",                                          // 67.34694
+	"css-border":                         "[style*=\"border:\"], [border]",                                       // 86.95652
+	"css-box-shadow":                     "[style*=\"box-shadow:\"]",                                             // 43.103447
+	"css-box-sizing":                     "[style*=\"box-sizing:\"]",                                             // 71.739136
+	"css-caption-side":                   "[style*=\"caption-side:\"]",                                           // 84
+	"css-clip-path":                      "[style*=\"clip-path:\"]",                                              // 43.396225
+	"css-column-count":                   "[style*=\"column-count:\"]",                                           // 67.391304
+	"css-column-layout-properties":       "[style*=\"column-layout-properties:\"]",                               // 47.368423
+	"css-conic-gradient":                 "[style*=\"conic-gradient:\"]",                                         // 38.461536
+	"css-direction":                      "[style*=\"direction:\"]",                                              // 97.77778
+	"css-display-flex":                   "[style*=\"display:flex\"]",                                            // 53.448277
+	"css-display-grid":                   "[style*=\"display:grid\"]",                                            // 54.347824
+	"css-display-none":                   "[style*=\"display:none\"]",                                            // 84.78261
+	"css-display":                        "[style*=\"display:\"]",                                                // 55.555553
+	"css-filter":                         "[style*=\"filter:\"]",                                                 // 50
+	"css-flex-direction":                 "[style*=\"flex-direction:\"]",                                         // 50
+	"css-flex-wrap":                      "[style*=\"flex-wrap:\"]",                                              // 49.09091
+	"css-float":                          "[style*=\"float:\"]",                                                  // 85.10638
+	"css-font-kerning":                   "[style*=\"font-kerning:\"]",                                           // 66.666664
+	"css-font-weight":                    "[style*=\"font-weight:\"]",                                            // 76.666664
+	"css-font":                           "[style*=\"font:\"]",                                                   // 95.833336
+	"css-gap":                            "[style*=\"gap:\"]",                                                    // 40
+	"css-grid-template":                  "[style*=\"grid-template:\"]",                                          // 34.042553
+	"css-height":                         "[style*=\"height:\"], [height]",                                       // 77.08333
+	"css-hyphens":                        "[style*=\"hyphens:\"]",                                                // 31.111107
+	"css-important":                      "[style*=\"!important\"]",                                              // 43.478264
+	"css-inline-size":                    "[style*=\"inline-size:\"]",                                            // 43.478264
+	"css-intrinsic-size":                 "[style*=\"intrinsic-size:\"]",                                         // 40.54054
+	"css-justify-content":                "[style*=\"justify-content:\"]",                                        // 59.25926
+	"css-letter-spacing":                 "[style*=\"letter-spacing:\"]",                                         // 87.23404
+	"css-line-height":                    "[style*=\"line-height:\"]",                                            // 82.608696
+	"css-list-style-image":               "[style*=\"list-style-image:\"]",                                       // 54.16667
+	"css-list-style-position":            "[style*=\"list-style-position:\"]",                                    // 87.5
+	"css-list-style":                     "[style*=\"list-style:\"]",                                             // 62.500004
+	"css-margin-block-start-end":         "[style*=\"margin-block-start:\"], [style*=\"margin-block-end:\"]",     // 32.07547
+	"css-margin-inline-block":            "[style*=\"margin-inline-block:\"]",                                    // 16.981125
+	"css-margin-inline-start-end":        "[style*=\"margin-inline-start:\"], [style*=\"margin-inline-end:\"]",   // 32.07547
+	"css-margin-inline":                  "[style*=\"margin-inline:\"]",                                          // 43.39623
+	"css-margin":                         "[style*=\"margin:\"]",                                                 // 71.42857
+	"css-max-block-size":                 "[style*=\"max-block-size:\"]",                                         // 35.714287
+	"css-max-height":                     "[style*=\"max-height:\"]",                                             // 86.95652
+	"css-max-width":                      "[style*=\"max-width:\"]",                                              // 76.47058
+	"css-min-height":                     "[style*=\"min-height:\"]",                                             // 82.608696
+	"css-min-inline-size":                "[style*=\"min-inline-size:\"]",                                        // 33.33333
+	"css-min-width":                      "[style*=\"min-width:\"]",                                              // 86.95652
+	"css-mix-blend-mode":                 "[style*=\"mix-blend-mode:\"]",                                         // 62.745094
+	"css-modern-color":                   "[style*=\"modern-color:\"]",                                           // 10.81081
+	"css-object-fit":                     "[style*=\"object-fit:\"]",                                             // 57.142857
+	"css-object-position":                "[style*=\"object-position:\"]",                                        // 55.10204
+	"css-opacity":                        "[style*=\"opacity:\"]",                                                // 63.04348
+	"css-outline-offset":                 "[style*=\"outline-offset:\"]",                                         // 42.5
+	"css-outline":                        "[style*=\"outline:\"]",                                                // 80.85106
+	"css-overflow-wrap":                  "[style*=\"overflow-wrap:\"]",                                          // 6.6666603
+	"css-overflow":                       "[style*=\"overflow:\"]",                                               // 78.26087
+	"css-padding-block-start-end":        "[style*=\"padding-block-start:\"], [style*=\"padding-block-end:\"]",   // 32.07547
+	"css-padding-inline-block":           "[style*=\"padding-inline-block:\"]",                                   // 16.981125
+	"css-padding-inline-start-end":       "[style*=\"padding-inline-start:\"], [style*=\"padding-inline-end:\"]", // 32.07547
+	"css-padding":                        "[style*=\"padding:\"], [padding]",                                     // 87.755104
+	"css-position":                       "[style*=\"position:\"]",                                               // 19.56522
+	"css-radial-gradient":                "[style*=\"radial-gradient:\"]",                                        // 64.583336
+	"css-rgb":                            "[style*=\"rgb(\"]",                                                    // 53.846153
+	"css-rgba":                           "[style*=\"rgba(\"]",                                                   // 56
+	"css-scroll-snap":                    "[style*=\"roll-snap:\"]",                                              // 38.88889
+	"css-tab-size":                       "[style*=\"tab-size:\"]",                                               // 32.075474
+	"css-table-layout":                   "[style*=\"table-layout:\"]",                                           // 53.33333
+	"css-text-align-last":                "[style*=\"text-align-last:\"]",                                        // 42.307693
+	"css-text-align":                     "[style*=\"text-align:\"]",                                             // 60.416664
+	"css-text-decoration-color":          "[style*=\"text-decoration-color:\"]",                                  // 67.34695
+	"css-text-decoration-thickness":      "[style*=\"text-decoration-thickness:\"]",                              // 38.333336
+	"css-text-decoration":                "[style*=\"text-decoration:\"]",                                        // 67.391304
+	"css-text-emphasis-position":         "[style*=\"text-emphasis-position:\"]",                                 // 28.571434
+	"css-text-emphasis":                  "[style*=\"text-emphasis:\"]",                                          // 36.734695
+	"css-text-indent":                    "[style*=\"text-indent:\"]",                                            // 78.43137
+	"css-text-overflow":                  "[style*=\"text-overflow:\"]",                                          // 58.695656
+	"css-text-shadow":                    "[style*=\"text-shadow:\"]",                                            // 69.565216
+	"css-text-transform":                 "[style*=\"text-transform:\"]",                                         // 86.666664
+	"css-text-underline-offset":          "[style*=\"text-underline-offset:\"]",                                  // 39.285713
+	"css-transform":                      "[style*=\"transform:\"]",                                              // 50
+	"css-unit-calc":                      "[style*=\"calc(:\"]",                                                  // 56.25
+	"css-variables":                      "[style*=\"variables:\"]",                                              // 46.551727
+	"css-visibility":                     "[style*=\"visibility:\"]",                                             // 52.173916
+	"css-white-space":                    "[style*=\"white-space:\"]",                                            // 58.69565
+	"css-width":                          "[style*=\"width:\"], [width]",                                         // 87.5
+	"css-word-break":                     "[style*=\"word-break:\"]",                                             // 28.888887
+	"css-writing-mode":                   "[style*=\"writing-mode:\"]",                                           // 56.25
+	"css-z-index":                        "[style*=\"z-index:\"]",                                                // 76.08696
+}
+
+// some CSS tests using regex for things that can't be merged inline
+var cssRegexpTests = map[string]*regexp.Regexp{
+	"css-at-font-face":                  regexp.MustCompile(`(?mi)@font\-face\s+?{`),        // 26.923073
+	"css-at-import":                     regexp.MustCompile(`(?mi)@import\s`),               // 36.170216
+	"css-at-keyframes":                  regexp.MustCompile(`(?mi)@keyframes\s`),            // 31.914898
+	"css-at-media":                      regexp.MustCompile(`(?mi)@media\s?\(`),             // 47.05882
+	"css-at-supports":                   regexp.MustCompile(`(?mi)@supports\s?\(`),          // 40.81633
+	"css-pseudo-class-active":           regexp.MustCompile(`:active`),                      // 52.173912
+	"css-pseudo-class-checked":          regexp.MustCompile(`:checked`),                     // 31.91489
+	"css-pseudo-class-first-child":      regexp.MustCompile(`:first\-child`),                // 66.666664
+	"css-pseudo-class-first-of-type":    regexp.MustCompile(`:first\-of\-type`),             // 62.5
+	"css-pseudo-class-focus":            regexp.MustCompile(`:focus`),                       // 47.826088
+	"css-pseudo-class-has":              regexp.MustCompile(`:has`),                         // 25.531914
+	"css-pseudo-class-hover":            regexp.MustCompile(`:hover`),                       // 60.41667
+	"css-pseudo-class-lang":             regexp.MustCompile(`:lang\s?\(`),                   // 18.918922
+	"css-pseudo-class-last-child":       regexp.MustCompile(`:last\-child`),                 // 64.58333
+	"css-pseudo-class-last-of-type":     regexp.MustCompile(`:last\-of\-type`),              // 60.416664
+	"css-pseudo-class-link":             regexp.MustCompile(`:link`),                        // 81.63265
+	"css-pseudo-class-not":              regexp.MustCompile(`:not(\s+)?\(`),                 // 44.89796
+	"css-pseudo-class-nth-child":        regexp.MustCompile(`:nth\-child(\s+)?\(`),          // 44.89796
+	"css-pseudo-class-nth-last-child":   regexp.MustCompile(`:nth\-last\-child(\s+)?\(`),    // 44.89796
+	"css-pseudo-class-nth-last-of-type": regexp.MustCompile(`:nth\-last\-of\-type(\s+)?\(`), // 42.857143
+	"css-pseudo-class-nth-of-type":      regexp.MustCompile(`:nth\-of\-type(\s+)?\(`),       // 42.857143
+	"css-pseudo-class-only-child":       regexp.MustCompile(`:only\-child(\s+)?\(`),         // 64.58333
+	"css-pseudo-class-only-of-type":     regexp.MustCompile(`:only\-of\-type(\s+)?\(`),      // 64.58333
+	"css-pseudo-class-target":           regexp.MustCompile(`:target`),                      // 39.13044
+	"css-pseudo-class-visited":          regexp.MustCompile(`:visited`),                     // 39.13044
+	"css-pseudo-element-after":          regexp.MustCompile(`:after`),                       // 40
+	"css-pseudo-element-before":         regexp.MustCompile(`:before`),                      // 40
+	"css-pseudo-element-first-letter":   regexp.MustCompile(`::first\-letter`),              // 60
+	"css-pseudo-element-first-line":     regexp.MustCompile(`::first\-line`),                // 60
+	"css-pseudo-element-marker":         regexp.MustCompile(`::marker`),                     // 50
+	"css-pseudo-element-placeholder":    regexp.MustCompile(`::placeholder`),                // 32
+}
+
+// some CSS tests using regex for units
+var cssRegexpUnitTests = map[string]*regexp.Regexp{
+	"css-unit-ch":      regexp.MustCompile(`\b\d+ch\b`),     // 66.666664
+	"css-unit-initial": regexp.MustCompile(`:\s?initial\b`), // 58.33333
+	"css-unit-rem":     regexp.MustCompile(`\b\d+rem\b`),    // 66.666664
+	"css-unit-vh":      regexp.MustCompile(`\b\d+vh\b`),     // 68.75
+	"css-unit-vmax":    regexp.MustCompile(`\b\d+vmax\b`),   // 60.416664
+	"css-unit-vmin":    regexp.MustCompile(`\b\d+vmin\b`),   // 58.333336
+	"css-unit-vw":      regexp.MustCompile(`\b\d+vw\b`),     // 77.08333
+}

internal/htmlcheck/css.go

@@ -0,0 +1,217 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/vanng822/go-premailer/premailer"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Go cannot calculate any rendered CSS attributes, so we merge all styles
+// into the HTML and detect elements with styles containing the keywords.
+func runCSSTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	inlined, err := inlineRemoteCSS(html)
+	if err != nil {
+		inlined = html
+	}
+
+	// merge all CSS inline
+	merged, err := mergeInlineCSS(inlined)
+	if err != nil {
+		merged = inlined
+	}
+
+	reader := strings.NewReader(merged)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	for key, test := range cssInlineTests {
+		totalTests++
+		found := len(doc.Find(test).Nodes)
+		if found > 0 {
+			result, err := cie.getTest(key)
+			if err != nil {
+				return results, totalTests, err
+			}
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	// get a list of all generated styles from all nodes
+	allNodeStyles := []string{}
+	for _, n := range doc.Find("*[style]").Nodes {
+		style, err := tools.GetHTMLAttributeVal(n, "style")
+		if err == nil {
+			allNodeStyles = append(allNodeStyles, style)
+		}
+	}
+
+	for key, re := range cssRegexpUnitTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := 0
+		// loop through all styles to count total
+		for _, styles := range allNodeStyles {
+			found = found + len(re.FindAllString(styles, -1))
+		}
+
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	// get all inline CSS block data
+	reader = strings.NewReader(inlined)
+
+	// Load the HTML document
+	doc, _ = goquery.NewDocumentFromReader(reader)
+
+	cssCode := ""
+	for _, n := range doc.Find("style").Nodes {
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			cssCode = cssCode + c.Data
+		}
+	}
+
+	for key, re := range cssRegexpTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := len(re.FindAllString(cssCode, -1))
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	return results, totalTests, nil
+}
+
+// MergeInlineCSS merges header CSS into element attributes
+func mergeInlineCSS(html string) (string, error) {
+	options := premailer.NewOptions()
+	// options.RemoveClasses = true
+	// options.CssToAttributes = false
+	options.KeepBangImportant = true
+	pre, err := premailer.NewPremailerFromString(html, options)
+	if err != nil {
+		return "", err
+	}
+
+	return pre.Transform()
+}
+
+// InlineRemoteCSS searches the HTML for linked stylesheets, downloads the content, and
+// inserts new <style> blocks into the head, unless BlockRemoteCSSAndFonts is set
+func inlineRemoteCSS(h string) (string, error) {
+	reader := strings.NewReader(h)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return h, err
+	}
+
+	remoteCSS := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range remoteCSS {
+		attributes := link.Attr
+		for _, a := range attributes {
+			if a.Key == "href" {
+				if !isURL(a.Val) {
+					// skip invalid URL
+					continue
+				}
+
+				if config.BlockRemoteCSSAndFonts {
+					logger.Log().Debugf("[html-check] skip testing remote CSS content: %s (--block-remote-css-and-fonts)", a.Val)
+					return h, nil
+				}
+
+				resp, err := downloadToBytes(a.Val)
+				if err != nil {
+					logger.Log().Warnf("[html-check] failed to download %s", a.Val)
+					continue
+				}
+
+				// create new <style> block and insert downloaded CSS
+				styleBlock := &html.Node{
+					Type:     html.ElementNode,
+					Data:     "style",
+					DataAtom: atom.Style,
+				}
+				styleBlock.AppendChild(&html.Node{
+					Type: html.TextNode,
+					Data: string(resp),
+				})
+
+				link.Parent.AppendChild(styleBlock)
+			}
+		}
+	}
+
+	newDoc, err := doc.Html()
+	if err != nil {
+		logger.Log().Warnf("[html-check] failed to download %s", err.Error())
+		return h, err
+	}
+
+	return newDoc, nil
+}
+
+// DownloadToBytes returns a []byte slice from a URL
+func downloadToBytes(url string) ([]byte, error) {
+	client := http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	// Get the link response data
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		err := fmt.Errorf("Error downloading %s", url)
+		return nil, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+// Test if str is a URL
+func isURL(str string) bool {
+	u, err := url.Parse(str)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}

internal/htmlcheck/html.go

@@ -0,0 +1,102 @@
+package htmlcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// HTML tests
+func runHTMLTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	reader := strings.NewReader(html)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	// Almost all <script> is bad
+	scripts := len(doc.Find("script:not([type=\"application/ld+json\"]):not([type=\"application/json\"])").Nodes)
+	if scripts > 0 {
+		var result = Warning{}
+		result.Title = "<script> element"
+		result.Slug = "html-script"
+		result.Category = "html"
+		result.Description = "JavaScript is not supported in any email client."
+		result.Tags = []string{}
+		result.Results = []Result{}
+		result.NotesByNumber = map[string]string{}
+		result.Score.Found = scripts
+		result.Score.Supported = 0
+		result.Score.Partial = 0
+		result.Score.Unsupported = 100
+		results = append(results, result)
+		totalTests++
+	}
+
+	for key, test := range htmlTests {
+		totalTests++
+		if test == "body" {
+			re := regexp.MustCompile(`(?im)</body>`)
+			if re.MatchString(html) {
+				result, err := cie.getTest(key)
+				if err != nil {
+					return results, totalTests, err
+				}
+
+				result.Score.Found = 1
+				results = append(results, result)
+			}
+		} else if len(doc.Find(test).Nodes) > 0 {
+			result, err := cie.getTest(key)
+			if err != nil {
+				return results, totalTests, err
+			}
+			totalTests++
+
+			result.Score.Found = len(doc.Find(test).Nodes)
+
+			results = append(results, result)
+		}
+	}
+
+	// find all images
+	images := doc.Find("img[src]").Nodes
+	imageResults := make(map[string]int)
+	totalTests = totalTests + len(imageRegexpTests)
+
+	for _, image := range images {
+		src, err := tools.GetHTMLAttributeVal(image, "src")
+		if err != nil {
+			continue
+		}
+		for key, test := range imageRegexpTests {
+			if test.MatchString(src) {
+				matches, exists := imageResults[key]
+				if exists {
+					imageResults[key] = matches + 1
+				} else {
+					imageResults[key] = 1
+				}
+
+			}
+		}
+	}
+
+	for key, found := range imageResults {
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+		result.Score.Found = found
+		results = append(results, result)
+	}
+
+	return results, totalTests, nil
+}

internal/htmlcheck/main.go

@@ -0,0 +1,201 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+)
+
+// RunTests will run all tests on an HTML string
+func RunTests(html string) (Response, error) {
+	s := Response{}
+	s.Warnings = []Warning{}
+	if platforms, err := Platforms(); err == nil {
+		s.Platforms = platforms
+	}
+
+	s.Total = Total{}
+
+	// crude way to determine whether the HTML contains a <body> structure
+	// or whether it's just plain HTML content
+	re := regexp.MustCompile(`(?im)</body>`)
+	nodeMatch := "body *, script"
+	if re.MatchString(html) {
+		nodeMatch = "*:not(html):not(head):not(meta), script"
+	}
+	reader := strings.NewReader(html)
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return s, err
+	}
+	// calculate the number of nodes in HTML
+	s.Total.Nodes = len(doc.Find(nodeMatch).Nodes)
+
+	if err := loadJSONData(); err != nil {
+		return s, err
+	}
+
+	// HTML tests
+	htmlResults, totalTests, err := runHTMLTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add html test totals
+	s.Warnings = append(s.Warnings, htmlResults...)
+
+	// CSS tests
+	cssResults, totalTests, err := runCSSTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add css test totals
+	s.Warnings = append(s.Warnings, cssResults...)
+
+	// calculate total score
+	var partial, unsupported float32
+	partial = 0
+	unsupported = 0
+
+	for _, w := range s.Warnings {
+		if w.Score.Found == 0 {
+			continue
+		}
+
+		// supported is calculated by subtracting partial and unsupported from 100%
+		if w.Score.Partial > 0 {
+			weighted := w.Score.Partial * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > partial {
+				partial = weighted
+			}
+		}
+		if w.Score.Unsupported > 0 {
+			weighted := w.Score.Unsupported * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > unsupported {
+				unsupported = weighted
+			}
+		}
+	}
+
+	s.Total.Supported = 100 - partial - unsupported
+	s.Total.Partial = partial
+	s.Total.Unsupported = unsupported
+
+	// sort slice to get lowest scores first
+	sort.Slice(s.Warnings, func(i, j int) bool {
+		return (s.Warnings[i].Score.Unsupported+s.Warnings[i].Score.Partial)*float32(s.Warnings[i].Score.Found)/float32(s.Total.Nodes) >
+			(s.Warnings[j].Score.Unsupported+s.Warnings[j].Score.Partial)*float32(s.Warnings[j].Score.Found)/float32(s.Total.Nodes)
+	})
+
+	return s, nil
+}
+
+// Test returns a test
+func (c CanIEmail) getTest(k string) (Warning, error) {
+	warning := Warning{}
+	exists := false
+	found := JSONResult{}
+	for _, r := range cie.Data {
+		if r.Slug == k {
+			found = r
+			exists = true
+			break
+		}
+	}
+
+	if !exists {
+		return warning, fmt.Errorf("%s does not exist", k)
+	}
+
+	warning.Slug = found.Slug
+	warning.Title = found.Title
+	warning.Description = mdToHTML(found.Description)
+	warning.Category = found.Category
+	warning.URL = found.URL
+	warning.Tags = found.Tags
+	// warning.Keywords = found.Keywords
+	// warning.Notes = found.Notes
+	warning.NotesByNumber = make(map[string]string, len(found.NotesByNumber))
+	for nr, note := range found.NotesByNumber {
+		warning.NotesByNumber[nr] = mdToHTML(note)
+	}
+	warning.Results = []Result{}
+
+	var y, n, p float32
+
+	for family, stats := range found.Stats {
+		if len(LimitFamilies) != 0 && !tools.InArray(family, LimitFamilies) {
+			continue
+		}
+
+		for platform, clients := range stats.(map[string]interface{}) {
+			if len(LimitPlatforms) != 0 && !tools.InArray(platform, LimitPlatforms) {
+				continue
+			}
+			for version, support := range clients.(map[string]interface{}) {
+				s := Result{}
+				s.Name = fmt.Sprintf("%s %s (%s)", c.NiceNames.Family[family], c.NiceNames.Platform[platform], version)
+				s.Family = family
+				s.Platform = platform
+				s.Version = version
+
+				if support == "y" {
+					y++
+					s.Support = "yes"
+				} else if support == "n" {
+					n++
+					s.Support = "no"
+				} else {
+					p++
+					s.Support = "partial"
+
+					noteIDS := noteMatch.FindStringSubmatch(fmt.Sprintf("%s", support))
+
+					for _, id := range noteIDS {
+						s.NoteNumber = id
+					}
+				}
+
+				warning.Results = append(warning.Results, s)
+			}
+		}
+
+	}
+
+	total := y + n + p
+	warning.Score.Supported = y / total * 100
+	warning.Score.Unsupported = n / total * 100
+	warning.Score.Partial = p / total * 100
+
+	return warning, nil
+}
+
+// Convert markdown to HTML, stripping <p> & </p>
+func mdToHTML(str string) string {
+	md := []byte(str)
+	// create markdown parser with extensions
+	extensions := parser.CommonExtensions | parser.AutoHeadingIDs | parser.NoEmptyLineBeforeBlock
+	// extensions := parser.NoExtensions
+	p := parser.NewWithExtensions(extensions)
+	doc := p.Parse(md)
+
+	// create HTML renderer with extensions
+	htmlFlags := html.CommonFlags | html.HrefTargetBlank
+	opts := html.RendererOptions{Flags: htmlFlags}
+	renderer := html.NewRenderer(opts)
+
+	return strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(string(markdown.Render(doc, renderer))), "<p>"), "</p>")
+}

internal/htmlcheck/platforms.go

@@ -0,0 +1,42 @@
+package htmlcheck
+
+import (
+	"sort"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Platforms returns all platforms with their respective email clients
+func Platforms() (map[string][]string, error) {
+	// [platform]clients
+	data := make(map[string][]string)
+
+	if err := loadJSONData(); err != nil {
+		return data, err
+	}
+
+	for _, t := range cie.Data {
+		for family, stats := range t.Stats {
+			niceFamily := cie.NiceNames.Family[family]
+			for platform := range stats.(map[string]interface{}) {
+				c, found := data[platform]
+				if !found {
+					data[platform] = []string{}
+				}
+				if !tools.InArray(niceFamily, c) {
+					c = append(c, niceFamily)
+					data[platform] = c
+				}
+			}
+		}
+	}
+
+	for group, clients := range data {
+		sort.Slice(clients, func(i, j int) bool {
+			return clients[i] < clients[j]
+		})
+		data[group] = clients
+	}
+
+	return data, nil
+}

internal/htmlcheck/structs.go

@@ -0,0 +1,87 @@
+package htmlcheck
+
+// Response represents the HTML check response struct
+//
+// swagger:model HTMLCheckResponse
+type Response struct {
+	// List of warnings from tests
+	Warnings []Warning `json:"Warnings"`
+	// All platforms tested, mainly for the web UI
+	Platforms map[string][]string `json:"Platforms"`
+	// Total overall score
+	Total Total `json:"Total"`
+}
+
+// Warning represents a failed test
+//
+// swagger:model HTMLCheckWarning
+type Warning struct {
+	// Slug identifier
+	Slug string `json:"Slug"`
+	// Friendly title
+	Title string `json:"Title"`
+	// Description
+	Description string `json:"Description"`
+	// URL to caniemail.com
+	URL string `json:"URL"`
+	// Category [css, html]
+	Category string `json:"Category"`
+	// Tags
+	Tags []string `json:"Tags"`
+	// Keywords
+	Keywords string `json:"Keywords"`
+	// Test results
+	Results []Result `json:"Results"`
+	// Notes based on results
+	NotesByNumber map[string]string `json:"NotesByNumber"`
+	// Test score calculated from results
+	Score Score `json:"Score"`
+}
+
+// Result struct
+//
+// swagger:model HTMLCheckResult
+type Result struct {
+	// Friendly name of result, combining family, platform & version
+	Name string `json:"Name"`
+	// Platform eg: ios, android, windows
+	Platform string `json:"Platform"`
+	// Family eg: Outlook, Mozilla Thunderbird
+	Family string `json:"Family"`
+	// Family version eg: 4.7.1, 2019-10, 10.3
+	Version string `json:"Version"`
+	// Support [yes, no, partial]
+	Support string `json:"Support"`
+	// Note number for partially supported if applicable
+	NoteNumber string `json:"NoteNumber"` // where applicable
+}
+
+// Score struct
+//
+// swagger:model HTMLCheckScore
+type Score struct {
+	// Number of matches in the document
+	Found int `json:"Found"`
+	// Total percentage supported
+	Supported float32 `json:"Supported"`
+	// Total percentage partially supported
+	Partial float32 `json:"Partial"`
+	// Total percentage unsupported
+	Unsupported float32 `json:"Unsupported"`
+}
+
+// Total weighted result for all scores
+//
+// swagger:model HTMLCheckTotal
+type Total struct {
+	// Total number of tests done
+	Tests int `json:"Tests"`
+	// Total number of HTML nodes detected in message
+	Nodes int `json:"Nodes"`
+	// Overall percentage supported
+	Supported float32 `json:"Supported"`
+	// Overall percentage partially supported
+	Partial float32 `json:"Partial"` // total percentage
+	// Overall percentage unsupported
+	Unsupported float32 `json:"Unsupported"` // total percentage
+}

internal/linkcheck/linkcheck_test.go

@@ -0,0 +1,71 @@
+package linkcheck
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/axllent/mailpit/internal/storage"
+)
+
+var (
+	testHTML = `
+	<html>
+	<head>
+		<link rel=stylesheet href="http://remote-host/style.css"></link>
+		<script async src="https://www.googletagmanager.com/gtag/js?id=ignored"></script>
+	</head>
+	<body>
+		<div>
+			<p><a href="http://example.com">HTTP link</a></p>
+			<p><a href="https://example.com">HTTPS link</a></p>
+			<p><a href="HTTPS://EXAMPLE.COM">HTTPS link</a></p>
+			<p><a href="http://localhost">Localhost link</a> (ignored)</p>
+			<p><a href="https://localhost">Localhost link</a> (ignored)</p>
+			<p><a href='https://127.0.0.1'>Single quotes link</a> (ignored)</p>
+			<p><img src=https://example.com/image.jpg></p>
+			<p href="http://invalid-link.com">This should be ignored</p>
+			<p><a href="http://link with spaces">Link with spaces</a></p>
+			<p><a href="http://example.com/?blaah=yes&amp;test=true">URL-encoded characters</a></p>
+		</div>
+	</body>
+	</html>`
+
+	expectedHTMLLinks = []string{
+		"http://example.com", "https://example.com", "HTTPS://EXAMPLE.COM", "http://localhost", "https://localhost", "https://127.0.0.1", "http://link with spaces", "http://example.com/?blaah=yes&test=true",
+		"http://remote-host/style.css",  // css
+		"https://example.com/image.jpg", // images
+	}
+
+	testTextLinks = `This is a line with http://example.com https://example.com
+		HTTPS://EXAMPLE.COM
+		[http://localhost]
+		www.google.com < ignored
+		|||http://example.com/?some=query-string|||
+	`
+
+	expectedTextLinks = []string{
+		"http://example.com", "https://example.com", "HTTPS://EXAMPLE.COM", "http://localhost", "http://example.com/?some=query-string",
+	}
+)
+
+func TestLinkDetection(t *testing.T) {
+
+	t.Log("Testing HTML link detection")
+
+	m := storage.Message{}
+
+	m.Text = testTextLinks
+	m.HTML = testHTML
+
+	textLinks := extractTextLinks(&m)
+
+	if !reflect.DeepEqual(textLinks, expectedTextLinks) {
+		t.Fatalf("Failed to detect text links correctly")
+	}
+
+	htmlLinks := extractHTMLLinks(&m)
+
+	if !reflect.DeepEqual(htmlLinks, expectedHTMLLinks) {
+		t.Fatalf("Failed to detect HTML links correctly")
+	}
+}

internal/linkcheck/main.go

@@ -0,0 +1,90 @@
+// Package linkcheck handles message links checking
+package linkcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+var linkRe = regexp.MustCompile(`(?im)\b(http|https):\/\/([\-\w@:%_\+'!.~#?,&\/\/=;]+)`)
+
+// RunTests will run all tests on an HTML string
+func RunTests(msg *storage.Message, followRedirects bool) (Response, error) {
+	s := Response{}
+
+	allLinks := extractHTMLLinks(msg)
+	allLinks = strUnique(append(allLinks, extractTextLinks(msg)...))
+	s.Links = getHTTPStatuses(allLinks, followRedirects)
+
+	for _, l := range s.Links {
+		if l.StatusCode >= 400 || l.StatusCode == 0 {
+			s.Errors++
+		}
+	}
+
+	return s, nil
+}
+
+func extractTextLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	links = append(links, linkRe.FindAllString(msg.Text, -1)...)
+
+	return links
+}
+
+func extractHTMLLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	reader := strings.NewReader(msg.HTML)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return links
+	}
+
+	aLinks := doc.Find("a[href]").Nodes
+	for _, link := range aLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	cssLinks := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range cssLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	imgLinks := doc.Find("img[src]").Nodes
+	for _, link := range imgLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "src")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	return links
+}
+
+// strUnique return a slice of unique strings from a slice
+func strUnique(strSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range strSlice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	return list
+}

internal/linkcheck/status.go

@@ -0,0 +1,114 @@
+package linkcheck
+
+import (
+	"crypto/tls"
+	"net/http"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+func getHTTPStatuses(links []string, followRedirects bool) []Link {
+	// allow 5 threads
+	threads := make(chan int, 5)
+
+	results := make(map[string]Link, len(links))
+	resultsMutex := sync.RWMutex{}
+
+	output := []Link{}
+
+	var wg sync.WaitGroup
+
+	for _, l := range links {
+		wg.Add(1)
+		go func(link string, w *sync.WaitGroup) {
+			threads <- 1 // will block if MAX threads
+			defer w.Done()
+
+			code, err := doHead(link, followRedirects)
+			l := Link{}
+			l.URL = link
+			if err != nil {
+				l.StatusCode = 0
+				l.Status = httpErrorSummary(err)
+			} else {
+				l.StatusCode = code
+				l.Status = http.StatusText(code)
+			}
+			resultsMutex.Lock()
+			results[link] = l
+			resultsMutex.Unlock()
+
+			<-threads // remove from threads
+		}(l, &wg)
+	}
+
+	wg.Wait()
+
+	for _, l := range results {
+		output = append(output, l)
+	}
+
+	return output
+}
+
+// Do a HEAD request to return HTTP status code
+func doHead(link string, followRedirects bool) (int, error) {
+
+	timeout := time.Duration(10 * time.Second)
+
+	tr := &http.Transport{}
+
+	if config.AllowUntrustedTLS {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec
+	}
+
+	client := http.Client{
+		Timeout:   timeout,
+		Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if followRedirects {
+				return nil
+			}
+			return http.ErrUseLastResponse
+		},
+	}
+
+	req, err := http.NewRequest("HEAD", link, nil)
+	if err != nil {
+		logger.Log().Errorf("[link-check] %s", err.Error())
+		return 0, err
+	}
+
+	req.Header.Set("User-Agent", "Mailpit/"+config.Version)
+
+	res, err := client.Do(req)
+	if err != nil {
+		if res != nil {
+			return res.StatusCode, err
+		}
+
+		return 0, err
+
+	}
+
+	return res.StatusCode, nil
+}
+
+// HTTP errors include a lot more info that just the actual error, so this
+// tries to take the final part of it, eg: `no such host`
+func httpErrorSummary(err error) string {
+	var re = regexp.MustCompile(`.*: (.*)$`)
+
+	e := err.Error()
+	if !re.MatchString(e) {
+		return e
+	}
+
+	parts := re.FindAllStringSubmatch(e, -1)
+
+	return parts[0][len(parts[0])-1]
+}

internal/linkcheck/structs.go

@@ -0,0 +1,21 @@
+package linkcheck
+
+// Response represents the Link check response
+//
+// swagger:model LinkCheckResponse
+type Response struct {
+	// Total number of errors
+	Errors int `json:"Errors"`
+	// Tested links
+	Links []Link `json:"Links"`
+}
+
+// Link struct
+type Link struct {
+	// Link URL
+	URL string `json:"URL"`
+	// HTTP status code
+	StatusCode int `json:"StatusCode"`
+	// HTTP status definition
+	Status string `json:"Status"`
+}

internal/logger/logger.go

@@ -0,0 +1,89 @@
+// Package logger handles the logging
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	log *logrus.Logger
+	// VerboseLogging for verbose logging
+	VerboseLogging bool
+	// QuietLogging shows only errors
+	QuietLogging bool
+	// NoLogging shows only fatal errors
+	NoLogging bool
+	// LogFile sets a log file
+	LogFile string
+)
+
+// Log returns the logger instance
+func Log() *logrus.Logger {
+	if log == nil {
+		log = logrus.New()
+		log.SetLevel(logrus.InfoLevel)
+		if VerboseLogging {
+			// verbose logging (debug)
+			log.SetLevel(logrus.DebugLevel)
+		} else if QuietLogging {
+			// show errors only
+			log.SetLevel(logrus.ErrorLevel)
+		} else if NoLogging {
+			// disable all logging (tests)
+			log.SetLevel(logrus.PanicLevel)
+		}
+
+		if LogFile != "" {
+			file, err := os.OpenFile(filepath.Clean(LogFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0664) // #nosec
+			if err == nil {
+				log.Out = file
+			} else {
+				log.Out = os.Stdout
+				log.Warn("Failed to log to file, using default stderr")
+			}
+		} else {
+			log.Out = os.Stdout
+		}
+
+		log.SetFormatter(&logrus.TextFormatter{
+			FullTimestamp:   true,
+			TimestampFormat: "2006/01/02 15:04:05",
+		})
+	}
+
+	return log
+}
+
+// PrettyPrint for debugging
+func PrettyPrint(i interface{}) {
+	s, _ := json.MarshalIndent(i, "", "\t")
+	fmt.Println(string(s))
+}
+
+// CleanIP returns a human-readable IP for the logging interface
+// when starting services. It translates [::]:<port> to "0.0.0.0:<port>"
+func CleanIP(s string) string {
+	re := regexp.MustCompile(`^\[\:\:\]\:\d+`)
+	if re.MatchString(s) {
+		return "0.0.0.0:" + s[5:]
+	}
+
+	return s
+}
+
+// CleanHTTPIP returns a human-readable IP for the logging interface
+// when starting services. It translates [::]:<port> to "localhost:<port>"
+func CleanHTTPIP(s string) string {
+	re := regexp.MustCompile(`^\[\:\:\]\:\d+`)
+	if re.MatchString(s) {
+		return "localhost:" + s[5:]
+	}
+
+	return s
+}

internal/pop3/functions.go

@@ -0,0 +1,84 @@
+package pop3
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+func authUser(username, password string) bool {
+	return auth.POP3Credentials.Match(username, password)
+}
+
+// Send a response with debug logging
+func sendResponse(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+	logger.Log().Debugf("[pop3] response: %s", m)
+
+	if strings.HasPrefix(m, "-ERR ") {
+		sub, _ := strings.CutPrefix(m, "-ERR ")
+		websockets.BroadCastClientError("error", "pop3", c.RemoteAddr().String(), sub)
+	}
+}
+
+// Send a response without debug logging (for data)
+func sendData(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+}
+
+// Get the latest 100 messages
+func getMessages() ([]message, error) {
+	messages := []message{}
+	list, err := storage.List(0, 0, 100)
+	if err != nil {
+		return messages, err
+	}
+
+	for _, m := range list {
+		msg := message{}
+		msg.ID = m.ID
+		msg.Size = m.Size
+		messages = append(messages, msg)
+	}
+
+	return messages, nil
+}
+
+// POP3 TOP command returns the headers, followed by the next x lines
+func getTop(id string, nr int) (string, string, error) {
+	var header, body string
+	raw, err := storage.GetMessageRaw(id)
+	if err != nil {
+		return header, body, errors.New("-ERR no such message")
+	}
+
+	parts := strings.SplitN(string(raw), "\r\n\r\n", 2)
+	header = parts[0]
+	lines := []string{}
+	if nr > 0 && len(parts) == 2 {
+		lines = strings.SplitN(parts[1], "\r\n", nr)
+	}
+
+	return header, strings.Join(lines, "\r\n"), nil
+}
+
+// cuts the line into command and arguments
+func getCommand(line string) (string, []string) {
+	line = strings.Trim(line, "\r \n")
+	cmd := strings.Split(line, " ")
+	return cmd[0], cmd[1:]
+}
+
+func getSafeArg(args []string, nr int) (string, error) {
+	if nr < len(args) {
+		return args[nr], nil
+	}
+
+	return "", errors.New("-ERR out of range")
+}

internal/pop3/pop3_test.go

@@ -0,0 +1,365 @@
+package pop3
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/pop3client"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	testingPort int
+)
+
+func TestPOP3(t *testing.T) {
+	t.Log("Testing POP3 server")
+	setup()
+	defer storage.Close()
+
+	// connect with bad password
+	t.Log("Testing invalid login")
+	c, err := connectBadAuth()
+	if err == nil {
+		t.Error("invalid login gained access")
+		return
+	}
+
+	t.Log("Testing valid login")
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, size, err := c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 0, "incorrect message count")
+	assertEqual(t, size, 0, "incorrect size")
+
+	// quit else we get old data
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Inserting 50 messages")
+
+	insertEmailData(t) // insert 50 messages
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 50, "incorrect message count")
+
+	t.Log("Fetching 20 messages")
+
+	for i := 1; i <= 20; i++ {
+		_, err := c.Retr(i)
+		if err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	t.Log("Deleting 25 messages")
+
+	for i := 1; i <= 25; i++ {
+		if err := c.Dele(i); err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	// messages get deleted after a QUIT
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Fetching message count")
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 25, "incorrect message count")
+
+	// messages get deleted after a QUIT
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Deleting 25 messages")
+
+	for i := 1; i <= 25; i++ {
+		if err := c.Dele(i); err != nil {
+			t.Errorf(err.Error())
+			return
+		}
+	}
+
+	t.Log("Undeleting messages")
+
+	if err := c.Rset(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	count, _, err = c.Stat()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, 25, "incorrect message count")
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+}
+
+func TestAuthentication(t *testing.T) {
+	// commands only allowed after authentication
+	authCommands := make(map[string]bool)
+	authCommands["STAT"] = false
+	authCommands["LIST"] = true
+	authCommands["NOOP"] = false
+	authCommands["RSET"] = false
+	authCommands["RETR 1"] = true
+
+	t.Log("Testing authenticated commands while not logged in")
+	setup()
+	defer storage.Close()
+
+	insertEmailData(t) // insert 50 messages
+
+	// non-authenticated connection
+	c, err := connect()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	for cmd, multi := range authCommands {
+		if _, err := c.Cmd(cmd, multi); err == nil {
+			t.Errorf("%s should require authentication", cmd)
+			return
+		}
+
+		if _, err := c.Cmd(strings.ToLower(cmd), multi); err == nil {
+			t.Errorf("%s should require authentication", cmd)
+			return
+		}
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	t.Log("Testing authenticated commands while logged in")
+
+	// authenticated connection
+	c, err = connectAuth()
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	for cmd, multi := range authCommands {
+		if _, err := c.Cmd(cmd, multi); err != nil {
+			t.Errorf("%s should work when authenticated", cmd)
+			return
+		}
+
+		if _, err := c.Cmd(strings.ToLower(cmd), multi); err != nil {
+			t.Errorf("%s should work when authenticated", cmd)
+			return
+		}
+	}
+
+	if err := c.Quit(); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+}
+
+func setup() {
+	auth.SetPOP3Auth("username:password")
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.Database = os.Getenv("MP_DATABASE")
+	var foundPort bool
+	for !foundPort {
+		testingPort = randRange(1111, 2000)
+		if portFree(testingPort) {
+			foundPort = true
+		}
+	}
+
+	config.POP3Listen = fmt.Sprintf("localhost:%d", testingPort)
+
+	if err := storage.InitDB(); err != nil {
+		panic(err)
+	}
+
+	if err := storage.DeleteAllMessages(); err != nil {
+		panic(err)
+	}
+
+	go Run()
+
+	time.Sleep(time.Second)
+}
+
+// connect and authenticate
+func connectAuth() (*pop3client.Conn, error) {
+	c, err := connect()
+	if err != nil {
+		return c, err
+	}
+
+	err = c.Auth("username", "password")
+
+	return c, err
+}
+
+// connect and authenticate
+func connectBadAuth() (*pop3client.Conn, error) {
+	c, err := connect()
+	if err != nil {
+		return c, err
+	}
+
+	err = c.Auth("username", "notPassword")
+
+	return c, err
+}
+
+// connect but do not authenticate
+func connect() (*pop3client.Conn, error) {
+	p := pop3client.New(pop3client.Opt{
+		Host:       "localhost",
+		Port:       testingPort,
+		TLSEnabled: false,
+	})
+
+	c, err := p.NewConn()
+	if err != nil {
+		return c, err
+	}
+
+	return c, err
+}
+
+func portFree(port int) bool {
+	ln, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
+	if err != nil {
+		return false
+	}
+
+	if err := ln.Close(); err != nil {
+		panic(err)
+	}
+
+	return true
+}
+
+func randRange(min, max int) int {
+	return rand.IntN(max-min) + min
+}
+
+func insertEmailData(t *testing.T) {
+	for i := 0; i < 50; i++ {
+		msg := enmime.Builder().
+			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+			Subject(fmt.Sprintf("Subject line %d end", i)).
+			Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+			To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i))
+
+		env, err := msg.Build()
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := env.Encode(buf); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		bufBytes := buf.Bytes()
+
+		id, err := storage.Store(&bufBytes)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		if _, err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}

internal/pop3/server.go

@@ -0,0 +1,319 @@
+// Package pop3 is a simple POP3 server for Mailpit.
+// By default it is disabled unless password credentials have been loaded.
+//
+// References: https://github.com/r0stig/golang-pop3 | https://github.com/inbucket/inbucket
+// See RFC: https://datatracker.ietf.org/doc/html/rfc1939
+package pop3
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+const (
+	// AUTHORIZATION is the initial state
+	AUTHORIZATION = 1
+	// TRANSACTION is the state after login
+	TRANSACTION = 2
+	// UPDATE is the state before closing
+	UPDATE = 3
+)
+
+// Run will start the POP3 server if enabled
+func Run() {
+	if auth.POP3Credentials == nil || config.POP3Listen == "" {
+		// POP3 server is disabled without authentication
+		return
+	}
+
+	var listener net.Listener
+	var err error
+
+	if config.POP3TLSCert != "" {
+		cer, err2 := tls.LoadX509KeyPair(config.POP3TLSCert, config.POP3TLSKey)
+		if err2 != nil {
+			logger.Log().Errorf("[pop3] %s", err2.Error())
+			return
+		}
+
+		tlsConfig := &tls.Config{
+			Certificates: []tls.Certificate{cer},
+			MinVersion:   tls.VersionTLS12,
+		}
+
+		listener, err = tls.Listen("tcp", config.POP3Listen, tlsConfig)
+	} else {
+		// unencrypted
+		listener, err = net.Listen("tcp", config.POP3Listen)
+	}
+
+	if err != nil {
+		logger.Log().Errorf("[pop3] %s", err.Error())
+		return
+	}
+
+	logger.Log().Infof("[pop3] starting on %s", config.POP3Listen)
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			logger.Log().Errorf("[pop3] accept error: %s", err.Error())
+			continue
+		}
+
+		// run as goroutine
+		go handleClient(conn)
+	}
+}
+
+type message struct {
+	ID   string
+	Size float64
+}
+
+func handleClient(conn net.Conn) {
+	var (
+		user     = ""
+		state    = AUTHORIZATION // Start with AUTHORIZATION state
+		toDelete []string        // Track messages marked for deletion
+		messages []message
+	)
+
+	defer func() {
+		if state == UPDATE {
+			if len(toDelete) > 0 {
+				if err := storage.DeleteMessages(toDelete); err != nil {
+					logger.Log().Errorf("[pop3] error deleting: %s", err.Error())
+				}
+				// Update web UI to remove deleted messages
+				websockets.Broadcast("prune", nil)
+			}
+		}
+
+		if err := conn.Close(); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+		}
+	}()
+
+	reader := bufio.NewReader(conn)
+
+	logger.Log().Debugf("[pop3] connection opened by %s", conn.RemoteAddr().String())
+
+	// First welcome the new connection
+	serverName := "Mailpit"
+	if config.Label != "" {
+		serverName = fmt.Sprintf("Mailpit (%s)", config.Label)
+	}
+	sendResponse(conn, fmt.Sprintf("+OK %s POP3 server", serverName))
+
+	// Set 10 minutes timeout according to RFC1939
+	timeoutDuration := 600 * time.Second
+
+	for {
+		// Set read deadline
+		if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+			return
+		}
+
+		// Reads a line from the client
+		rawLine, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				logger.Log().Debugf("[pop3] client disconnected: %s", conn.RemoteAddr().String())
+			} else {
+				logger.Log().Errorf("[pop3] read error: %s", err.Error())
+			}
+			return
+		}
+
+		// Parses the command
+		cmd, args := getCommand(rawLine)
+		cmd = strings.ToUpper(cmd) // Commands in the POP3 are case-insensitive
+
+		logger.Log().Debugf("[pop3] received: %s (%s)", strings.TrimSpace(rawLine), conn.RemoteAddr().String())
+
+		switch cmd {
+		case "CAPA":
+			// List our capabilities per RFC2449
+			sendResponse(conn, "+OK capability list follows")
+			sendResponse(conn, "TOP")
+			sendResponse(conn, "USER")
+			sendResponse(conn, "UIDL")
+			sendResponse(conn, "IMPLEMENTATION Mailpit")
+			sendResponse(conn, ".")
+		case "USER":
+			if state == AUTHORIZATION {
+				if len(args) != 1 {
+					sendResponse(conn, "-ERR must supply a user")
+					return
+				}
+				sendResponse(conn, "+OK")
+				user = args[0]
+			} else {
+				sendResponse(conn, "-ERR user already specified")
+			}
+		case "PASS":
+			if state == AUTHORIZATION {
+				if user == "" {
+					sendResponse(conn, "-ERR must supply a user")
+					return
+				}
+				if len(args) != 1 {
+					sendResponse(conn, "-ERR must supply a password")
+					return
+				}
+
+				pass := args[0]
+				if authUser(user, pass) {
+					sendResponse(conn, "+OK signed in")
+					var err error
+					messages, err = getMessages()
+					if err != nil {
+						logger.Log().Errorf("[pop3] %s", err.Error())
+					}
+					state = TRANSACTION
+				} else {
+					sendResponse(conn, "-ERR invalid password")
+					logger.Log().Warnf("[pop3] failed login: %s", user)
+				}
+			} else {
+				sendResponse(conn, "-ERR user not specified")
+			}
+		case "STAT", "LIST", "UIDL", "RETR", "TOP", "NOOP", "DELE", "RSET":
+			if state == TRANSACTION {
+				handleTransactionCommand(conn, cmd, args, messages, &toDelete)
+			} else {
+				sendResponse(conn, "-ERR user not authenticated")
+			}
+		case "QUIT":
+			sendResponse(conn, "+OK goodbye")
+			state = UPDATE
+			return
+		default:
+			sendResponse(conn, "-ERR unknown command")
+		}
+	}
+}
+
+func handleTransactionCommand(conn net.Conn, cmd string, args []string, messages []message, toDelete *[]string) {
+	switch cmd {
+	case "STAT":
+		totalSize := float64(0)
+		for _, m := range messages {
+			totalSize += m.Size
+		}
+		sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), int64(totalSize)))
+	case "LIST":
+		totalSize := float64(0)
+		for _, m := range messages {
+			totalSize += m.Size
+		}
+		sendResponse(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), int64(totalSize)))
+
+		for row, m := range messages {
+			sendResponse(conn, fmt.Sprintf("%d %d", row+1, int64(m.Size))) // Convert Size to int64 when printing
+		}
+		sendResponse(conn, ".")
+	case "UIDL":
+		sendResponse(conn, "+OK unique-id listing follows")
+		for row, m := range messages {
+			sendResponse(conn, fmt.Sprintf("%d %s", row+1, m.ID))
+		}
+		sendResponse(conn, ".")
+	case "RETR":
+		if len(args) != 1 {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		nr, err := strconv.Atoi(args[0])
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		m := messages[nr-1]
+		raw, err := storage.GetMessageRaw(m.ID)
+		if err != nil {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+		size := len(raw)
+		sendResponse(conn, fmt.Sprintf("+OK %d octets", size))
+
+		// When all lines of the response have been sent, a
+		// final line is sent, consisting of a termination octet (decimal code
+		// 046, ".") and a CRLF pair. If any line of the multi-line response
+		// begins with the termination octet, the line is "byte-stuffed" by
+		// pre-pending the termination octet to that line of the response.
+		// @see: https://www.ietf.org/rfc/rfc1939.txt
+		sendData(conn, strings.Replace(string(raw), "\n.", "\n..", -1))
+		sendResponse(conn, ".")
+	case "TOP":
+		arg, err := getSafeArg(args, 0)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+		nr, err := strconv.Atoi(arg)
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		arg2, err := getSafeArg(args, 1)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+
+		lines, err := strconv.Atoi(arg2)
+		if err != nil {
+			sendResponse(conn, "-ERR TOP requires two arguments")
+			return
+		}
+
+		m := messages[nr-1]
+		headers, body, err := getTop(m.ID, lines)
+		if err != nil {
+			sendResponse(conn, err.Error())
+			return
+		}
+
+		sendResponse(conn, "+OK top of message follows")
+		sendData(conn, headers+"\r\n")
+		sendData(conn, body)
+		sendResponse(conn, ".")
+	case "NOOP":
+		sendResponse(conn, "+OK")
+	case "DELE":
+		arg, _ := getSafeArg(args, 0)
+		nr, err := strconv.Atoi(arg)
+		if err != nil || nr < 1 || nr > len(messages) {
+			sendResponse(conn, "-ERR no such message")
+			return
+		}
+
+		m := messages[nr-1]
+		*toDelete = append(*toDelete, m.ID)
+		sendResponse(conn, "+OK message marked for deletion")
+	case "RSET":
+		*toDelete = []string{}
+		sendResponse(conn, "+OK")
+	default:
+		sendResponse(conn, "-ERR unknown command")
+	}
+}

internal/pop3client/client.go

@@ -0,0 +1,453 @@
+// Package pop3client is borrowed directly from https://github.com/knadh/go-pop3 to reduce dependencies.
+// This is used solely for testing the POP3 server
+package pop3client
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Client implements a Client e-mail client.
+type Client struct {
+	opt    Opt
+	dialer Dialer
+}
+
+// Conn is a stateful connection with the POP3 server/
+type Conn struct {
+	conn net.Conn
+	r    *bufio.Reader
+	w    *bufio.Writer
+}
+
+// Opt represents the client configuration.
+type Opt struct {
+	// Host name
+	Host string `json:"host"`
+	// Port number
+	Port int `json:"port"`
+	// DialTimeout default is 3 seconds.
+	DialTimeout time.Duration `json:"dial_timeout"`
+	// Dialer
+	Dialer Dialer `json:"-"`
+	// TLSEnabled sets whether SLS is enabled
+	TLSEnabled bool `json:"tls_enabled"`
+	// TLSSkipVerify skips TLS verification (ie: self-signed)
+	TLSSkipVerify bool `json:"tls_skip_verify"`
+}
+
+// Dialer interface
+type Dialer interface {
+	Dial(network, address string) (net.Conn, error)
+}
+
+// MessageID contains the ID and size of an individual message.
+type MessageID struct {
+	// ID is the numerical index (non-unique) of the message.
+	ID int
+	// Size in bytes
+	Size int
+	// UID is only present if the response is to the UIDL command.
+	UID string
+}
+
+var (
+	lineBreak   = []byte("\r\n")
+	respOK      = []byte("+OK")   // `+OK` without additional info
+	respOKInfo  = []byte("+OK ")  // `+OK <info>`
+	respErr     = []byte("-ERR")  // `-ERR` without additional info
+	respErrInfo = []byte("-ERR ") // `-ERR <info>`
+)
+
+// New returns a new client object using an existing connection.
+func New(opt Opt) *Client {
+	if opt.DialTimeout < time.Millisecond {
+		opt.DialTimeout = time.Second * 3
+	}
+
+	c := &Client{
+		opt:    opt,
+		dialer: opt.Dialer,
+	}
+
+	if c.dialer == nil {
+		c.dialer = &net.Dialer{Timeout: opt.DialTimeout}
+	}
+
+	return c
+}
+
+// NewConn creates and returns live POP3 server connection.
+func (c *Client) NewConn() (*Conn, error) {
+	var (
+		addr = fmt.Sprintf("%s:%d", c.opt.Host, c.opt.Port)
+	)
+
+	conn, err := c.dialer.Dial("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+
+	// No TLS.
+	if c.opt.TLSEnabled {
+		// Skip TLS host verification.
+		tlsCfg := tls.Config{} // #nosec
+		if c.opt.TLSSkipVerify {
+			tlsCfg.InsecureSkipVerify = c.opt.TLSSkipVerify // #nosec
+		} else {
+			tlsCfg.ServerName = c.opt.Host
+		}
+
+		conn = tls.Client(conn, &tlsCfg)
+	}
+
+	pCon := &Conn{
+		conn: conn,
+		r:    bufio.NewReader(conn),
+		w:    bufio.NewWriter(conn),
+	}
+
+	// Verify the connection by reading the welcome +OK greeting.
+	if _, err := pCon.ReadOne(); err != nil {
+		return nil, err
+	}
+
+	return pCon, nil
+}
+
+// Send sends a POP3 command to the server. The given comand is suffixed with "\r\n".
+func (c *Conn) Send(b string) error {
+	if _, err := c.w.WriteString(b + "\r\n"); err != nil {
+		return err
+	}
+
+	return c.w.Flush()
+}
+
+// Cmd sends a command to the server. POP3 responses are either single line or multi-line.
+// The first line always with -ERR in case of an error or +OK in case of a successful operation.
+// OK+ is always followed by a response on the same line which is either the actual response data
+// in case of single line responses, or a help message followed by multiple lines of actual response
+// data in case of multiline responses.
+// See https://www.shellhacks.com/retrieve-email-pop3-server-command-line/ for examples.
+func (c *Conn) Cmd(cmd string, isMulti bool, args ...interface{}) (*bytes.Buffer, error) {
+	var cmdLine string
+
+	// Repeat a %v to format each arg.
+	if len(args) > 0 {
+		format := " " + strings.TrimRight(strings.Repeat("%v ", len(args)), " ")
+
+		// CMD arg1 argn ...\r\n
+		cmdLine = fmt.Sprintf(cmd+format, args...)
+	} else {
+		cmdLine = cmd
+	}
+
+	if err := c.Send(cmdLine); err != nil {
+		return nil, err
+	}
+
+	// Read the first line of response to get the +OK/-ERR status.
+	b, err := c.ReadOne()
+	if err != nil {
+		return nil, err
+	}
+
+	// Single line response.
+	if !isMulti {
+		return bytes.NewBuffer(b), err
+	}
+
+	buf, err := c.ReadAll()
+	return buf, err
+}
+
+// ReadOne reads a single line response from the conn.
+func (c *Conn) ReadOne() ([]byte, error) {
+	b, _, err := c.r.ReadLine()
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := parseResp(b)
+	return r, err
+}
+
+// ReadAll reads all lines from the connection until the POP3 multiline terminator "." is encountered
+// and returns a bytes.Buffer of all the read lines.
+func (c *Conn) ReadAll() (*bytes.Buffer, error) {
+	buf := &bytes.Buffer{}
+
+	for {
+		b, _, err := c.r.ReadLine()
+		if err != nil {
+			return nil, err
+		}
+
+		// "." indicates the end of a multi-line response.
+		if bytes.Equal(b, []byte(".")) {
+			break
+		}
+
+		if _, err := buf.Write(b); err != nil {
+			return nil, err
+		}
+		if _, err := buf.Write(lineBreak); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf, nil
+}
+
+// Auth authenticates the given credentials with the server.
+func (c *Conn) Auth(user, password string) error {
+	if err := c.User(user); err != nil {
+		return err
+	}
+
+	if err := c.Pass(password); err != nil {
+		return err
+	}
+
+	// Issue a NOOP to force the server to respond to the auth.
+	// Courtesy: github.com/TheCreeper/go-pop3
+	return c.Noop()
+}
+
+// User sends the username to the server.
+func (c *Conn) User(s string) error {
+	_, err := c.Cmd("USER", false, s)
+
+	return err
+}
+
+// Pass sends the password to the server.
+func (c *Conn) Pass(s string) error {
+	_, err := c.Cmd("PASS", false, s)
+
+	return err
+}
+
+// Stat returns the number of messages and their total size in bytes in the inbox.
+func (c *Conn) Stat() (int, int, error) {
+	b, err := c.Cmd("STAT", false)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// count size
+	f := bytes.Fields(b.Bytes())
+
+	// Total number of messages.
+	count, err := strconv.Atoi(string(f[0]))
+	if err != nil {
+		return 0, 0, err
+	}
+	if count == 0 {
+		return 0, 0, nil
+	}
+
+	// Total size of all messages in bytes.
+	size, err := strconv.Atoi(string(f[1]))
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return count, size, nil
+}
+
+// List returns a list of (message ID, message Size) pairs.
+// If the optional msgID > 0, then only that particular message is listed.
+// The message IDs are sequential, 1 to N.
+func (c *Conn) List(msgID int) ([]MessageID, error) {
+	var (
+		buf *bytes.Buffer
+		err error
+	)
+
+	if msgID <= 0 {
+		// Multiline response listing all messages.
+		buf, err = c.Cmd("LIST", true)
+	} else {
+		// Single line response listing one message.
+		buf, err = c.Cmd("LIST", false, msgID)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		out   []MessageID
+		lines = bytes.Split(buf.Bytes(), lineBreak)
+	)
+
+	for _, l := range lines {
+		// id size
+		f := bytes.Fields(l)
+		if len(f) == 0 {
+			break
+		}
+
+		id, err := strconv.Atoi(string(f[0]))
+		if err != nil {
+			return nil, err
+		}
+
+		size, err := strconv.Atoi(string(f[1]))
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, MessageID{ID: id, Size: size})
+	}
+
+	return out, nil
+}
+
+// Uidl returns a list of (message ID, message UID) pairs. If the optional msgID
+// is > 0, then only that particular message is listed. It works like Top() but only works on
+// servers that support the UIDL command. Messages size field is not available in the UIDL response.
+func (c *Conn) Uidl(msgID int) ([]MessageID, error) {
+	var (
+		buf *bytes.Buffer
+		err error
+	)
+
+	if msgID <= 0 {
+		// Multiline response listing all messages.
+		buf, err = c.Cmd("UIDL", true)
+	} else {
+		// Single line response listing one message.
+		buf, err = c.Cmd("UIDL", false, msgID)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		out   []MessageID
+		lines = bytes.Split(buf.Bytes(), lineBreak)
+	)
+
+	for _, l := range lines {
+		// id size
+		f := bytes.Fields(l)
+		if len(f) == 0 {
+			break
+		}
+
+		id, err := strconv.Atoi(string(f[0]))
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, MessageID{ID: id, UID: string(f[1])})
+	}
+
+	return out, nil
+}
+
+// Retr downloads a message by the given msgID, parses it and returns it as a *mail.Message.
+func (c *Conn) Retr(msgID int) (*mail.Message, error) {
+	b, err := c.Cmd("RETR", true, msgID)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := mail.ReadMessage(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// RetrRaw downloads a message by the given msgID and returns the raw []byte
+// of the entire message.
+func (c *Conn) RetrRaw(msgID int) (*bytes.Buffer, error) {
+	b, err := c.Cmd("RETR", true, msgID)
+	return b, err
+}
+
+// Top retrieves a message by its ID with full headers and numLines lines of the body.
+func (c *Conn) Top(msgID int, numLines int) (*mail.Message, error) {
+	b, err := c.Cmd("TOP", true, msgID, numLines)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := mail.ReadMessage(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// Dele deletes one or more messages. The server only executes the
+// deletions after a successful Quit().
+func (c *Conn) Dele(msgID ...int) error {
+	for _, id := range msgID {
+		_, err := c.Cmd("DELE", false, id)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Rset clears the messages marked for deletion in the current session.
+func (c *Conn) Rset() error {
+	_, err := c.Cmd("RSET", false)
+	return err
+}
+
+// Noop issues a do-nothing NOOP command to the server. This is useful for
+// prolonging open connections.
+func (c *Conn) Noop() error {
+	_, err := c.Cmd("NOOP", false)
+	return err
+}
+
+// Quit sends the QUIT command to server and gracefully closes the connection.
+// Message deletions (DELE command) are only executed by the server on a graceful
+// quit and close.
+func (c *Conn) Quit() error {
+	defer c.conn.Close()
+
+	if _, err := c.Cmd("QUIT", false); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// parseResp checks if the response is an error that starts with `-ERR`
+// and returns an error with the message that succeeds the error indicator.
+// For success `+OK` messages, it returns the remaining response bytes.
+func parseResp(b []byte) ([]byte, error) {
+	if len(b) == 0 {
+		return nil, nil
+	}
+
+	if bytes.Equal(b, respOK) {
+		return nil, nil
+	} else if bytes.HasPrefix(b, respOKInfo) {
+		return bytes.TrimPrefix(b, respOKInfo), nil
+	} else if bytes.Equal(b, respErr) {
+		return nil, errors.New("unknown error (no info specified in response)")
+	} else if bytes.HasPrefix(b, respErrInfo) {
+		return nil, errors.New(string(bytes.TrimPrefix(b, respErrInfo)))
+	}
+
+	return nil, fmt.Errorf("unknown response: %s. Neither -ERR, nor +OK", string(b))
+}

internal/smtpd/chaos/chaos.go

@@ -0,0 +1,121 @@
+// Package chaos is used to simulate Chaos engineering (random failures) in the SMTPD server.
+// See https://en.wikipedia.org/wiki/Chaos_engineering
+// See https://mailpit.axllent.org/docs/integration/chaos/
+package chaos
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var (
+	// Enabled is a flag to enable or disable support for chaos
+	Enabled = false
+
+	// Config is the global Chaos configuration
+	Config = Triggers{
+		Sender:         Trigger{ErrorCode: 451, Probability: 0},
+		Recipient:      Trigger{ErrorCode: 451, Probability: 0},
+		Authentication: Trigger{ErrorCode: 535, Probability: 0},
+	}
+)
+
+// Triggers for the Chaos configuration
+// swagger:model Triggers
+type Triggers struct {
+	// Sender trigger to fail on From, Sender
+	Sender Trigger
+	// Recipient trigger to fail on To, Cc, Bcc
+	Recipient Trigger
+	// Authentication trigger to fail while authenticating (auth must be configured)
+	Authentication Trigger
+}
+
+// Trigger for Chaos
+// swagger:model Trigger
+type Trigger struct {
+	// SMTP error code to return. The value must range from 400 to 599.
+	// required: true
+	// example: 451
+	ErrorCode int
+
+	// Probability (chance) of triggering the error. The value must range from 0 to 100.
+	// required: true
+	// example: 5
+	Probability int
+}
+
+// SetFromStruct will set a whole map of chaos configurations (ie: API)
+func SetFromStruct(c Triggers) error {
+	if c.Sender.ErrorCode == 0 {
+		c.Sender.ErrorCode = 451 // default
+	}
+
+	if c.Recipient.ErrorCode == 0 {
+		c.Recipient.ErrorCode = 451 // default
+	}
+
+	if c.Authentication.ErrorCode == 0 {
+		c.Authentication.ErrorCode = 535 // default
+	}
+
+	if err := Set("Sender", c.Sender.ErrorCode, c.Sender.Probability); err != nil {
+		return err
+	}
+	if err := Set("Recipient", c.Recipient.ErrorCode, c.Recipient.Probability); err != nil {
+		return err
+	}
+	if err := Set("Authentication", c.Authentication.ErrorCode, c.Authentication.Probability); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Set will set the chaos configuration for the given key (CLI & setMap())
+func Set(key string, errorCode int, probability int) error {
+	Enabled = true
+	if errorCode < 400 || errorCode > 599 {
+		return fmt.Errorf("error code must be between 400 and 599")
+	}
+
+	if probability > 100 || probability < 0 {
+		return fmt.Errorf("probability must be between 0 and 100")
+	}
+
+	key = strings.ToLower(key)
+
+	switch key {
+	case "sender":
+		Config.Sender = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Sender to return %d error with %d%% probability", errorCode, probability)
+	case "recipient", "recipients":
+		Config.Recipient = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Recipient to return %d error with %d%% probability", errorCode, probability)
+	case "auth", "authentication":
+		Config.Authentication = Trigger{ErrorCode: errorCode, Probability: probability}
+		logger.Log().Infof("[chaos] Authentication to return %d error with %d%% probability", errorCode, probability)
+	default:
+		return fmt.Errorf("unknown key %s", key)
+	}
+
+	return nil
+}
+
+// Trigger will return whether the Chaos rule is triggered based on the configuration
+// and a randomly-generated percentage value.
+func (c Trigger) Trigger() (bool, int) {
+	if !Enabled || c.Probability == 0 {
+		return false, 0
+	}
+
+	nBig, _ := rand.Int(rand.Reader, big.NewInt(100))
+
+	// rand.IntN(100) will return 0-99, whereas probability is 1-100,
+	// so value must be less than (not <=) to the probability to trigger
+	return int(nBig.Int64()) < c.Probability, c.ErrorCode
+}

internal/smtpd/forward.go

@@ -0,0 +1,111 @@
+package smtpd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/smtp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Wrapper to forward messages if configured
+func autoForwardMessage(from string, data *[]byte) {
+	if config.SMTPForwardConfig.Host == "" {
+		return
+	}
+
+	if err := forward(from, *data); err != nil {
+		logger.Log().Errorf("[forward] error: %s", err.Error())
+	} else {
+		logger.Log().Debugf("[forward] message from %s to %s via %s:%d",
+			from, config.SMTPForwardConfig.To, config.SMTPForwardConfig.Host, config.SMTPForwardConfig.Port)
+	}
+}
+
+// Forward will connect to a pre-configured SMTP server and send a message to one or more recipients.
+func forward(from string, msg []byte) error {
+	addr := fmt.Sprintf("%s:%d", config.SMTPForwardConfig.Host, config.SMTPForwardConfig.Port)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		return fmt.Errorf("error connecting to %s: %s", addr, err.Error())
+	}
+
+	defer c.Close()
+
+	if config.SMTPForwardConfig.STARTTLS {
+		conf := &tls.Config{ServerName: config.SMTPForwardConfig.Host} // #nosec
+
+		conf.InsecureSkipVerify = config.SMTPForwardConfig.AllowInsecure
+
+		if err = c.StartTLS(conf); err != nil {
+			return fmt.Errorf("error creating StartTLS config: %s", err.Error())
+		}
+	}
+
+	auth := forwardAuthFromConfig()
+
+	if auth != nil {
+		if err = c.Auth(auth); err != nil {
+			return fmt.Errorf("error response to AUTH command: %s", err.Error())
+		}
+	}
+
+	if config.SMTPForwardConfig.OverrideFrom != "" {
+		msg, err = tools.OverrideFromHeader(msg, config.SMTPForwardConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("error overriding From header: %s", err.Error())
+		}
+
+		from = config.SMTPForwardConfig.OverrideFrom
+	}
+
+	if err = c.Mail(from); err != nil {
+		return fmt.Errorf("error response to MAIL command: %s", err.Error())
+	}
+
+	to := strings.Split(config.SMTPForwardConfig.To, ",")
+
+	for _, addr := range to {
+		if err = c.Rcpt(addr); err != nil {
+			logger.Log().Warnf("error response to RCPT command for %s: %s", addr, err.Error())
+		}
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return fmt.Errorf("error response to DATA command: %s", err.Error())
+	}
+
+	if _, err := w.Write(msg); err != nil {
+		return fmt.Errorf("error sending message: %s", err.Error())
+	}
+
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("error closing connection: %s", err.Error())
+	}
+
+	return c.Quit()
+}
+
+// Return the SMTP forwarding authentication based on config
+func forwardAuthFromConfig() smtp.Auth {
+	var a smtp.Auth
+
+	if config.SMTPForwardConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Password, config.SMTPForwardConfig.Host)
+	}
+
+	if config.SMTPForwardConfig.Auth == "login" {
+		a = LoginAuth(config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Password)
+	}
+
+	if config.SMTPForwardConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPForwardConfig.Username, config.SMTPForwardConfig.Secret)
+	}
+
+	return a
+}

internal/smtpd/main.go

@@ -0,0 +1,337 @@
+// Package smtpd is the SMTP daemon
+package smtpd
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/mail"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/stats"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+var (
+	// DisableReverseDNS allows rDNS to be disabled
+	DisableReverseDNS bool
+
+	warningResponse = regexp.MustCompile(`^4\d\d `)
+	errorResponse   = regexp.MustCompile(`^5\d\d `)
+)
+
+// MailHandler handles the incoming message to store in the database
+func mailHandler(origin net.Addr, from string, to []string, data []byte) (string, error) {
+	return SaveToDatabase(origin, from, to, data)
+}
+
+// SaveToDatabase will attempt to save a message to the database
+func SaveToDatabase(origin net.Addr, from string, to []string, data []byte) (string, error) {
+	if !config.SMTPStrictRFCHeaders {
+		// replace all <CR><CR><LF> (\r\r\n) with <CR><LF> (\r\n)
+		// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
+		data = bytes.ReplaceAll(data, []byte("\r\r\n"), []byte("\r\n"))
+	}
+
+	msg, err := mail.ReadMessage(bytes.NewReader(data))
+	if err != nil {
+		logger.Log().Warnf("[smtpd] error parsing message: %s", err.Error())
+		stats.LogSMTPRejected()
+		return "", err
+	}
+
+	// check / set the Return-Path based on SMTP from
+	returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
+	if returnPath != from {
+		if returnPath != "" {
+			// replace Return-Path
+			re := regexp.MustCompile(`(?i)(^|\n)(Return\-Path: .*\n)`)
+			replaced := false
+			data = re.ReplaceAllFunc(data, func(r []byte) []byte {
+				if replaced {
+					return r
+				}
+				replaced = true // only replace first occurrence
+
+				return re.ReplaceAll(r, []byte("${1}Return-Path: <"+from+">\r\n"))
+			})
+		} else {
+			// add Return-Path
+			data = append([]byte("Return-Path: <"+from+">\r\n"), data...)
+		}
+	}
+
+	messageID := strings.Trim(msg.Header.Get("Message-Id"), "<>")
+
+	// add a message ID if not set
+	if messageID == "" {
+		// generate unique ID
+		messageID = shortuuid.New() + "@mailpit"
+		// add unique ID
+		data = append([]byte("Message-Id: <"+messageID+">\r\n"), data...)
+	} else if config.IgnoreDuplicateIDs {
+		if storage.MessageIDExists(messageID) {
+			logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID)
+			stats.LogSMTPIgnored()
+			return "", nil
+		}
+	}
+
+	// if enabled, this may conditionally relay the email through to the preconfigured smtp server
+	autoRelayMessage(from, to, &data)
+
+	// if enabled, this will forward a copy to preconfigured addresses
+	autoForwardMessage(from, &data)
+
+	// build array of all addresses in the header to compare to the []to array
+	emails, hasBccHeader := scanAddressesInHeader(msg.Header)
+
+	missingAddresses := []string{}
+	for _, a := range to {
+		// loop through passed email addresses to check if they are in the headers
+		if _, err := mail.ParseAddress(a); err == nil {
+			_, ok := emails[strings.ToLower(a)]
+			if !ok {
+				missingAddresses = append(missingAddresses, a)
+			}
+		} else {
+			logger.Log().Warnf("[smtpd] ignoring invalid email address: %s", a)
+		}
+	}
+
+	// add missing email addresses to Bcc (eg: Laravel doesn't include these in the headers)
+	if len(missingAddresses) > 0 {
+		if hasBccHeader {
+			// email already has Bcc header, add to existing addresses
+			re := regexp.MustCompile(`(?i)(^|\n)(Bcc: )`)
+			replaced := false
+			data = re.ReplaceAllFunc(data, func(r []byte) []byte {
+				if replaced {
+					return r
+				}
+				replaced = true // only replace first occurrence
+
+				return re.ReplaceAll(r, []byte("${1}Bcc: "+strings.Join(missingAddresses, ", ")+", "))
+			})
+
+		} else {
+			// prepend new Bcc header
+			bcc := []byte(fmt.Sprintf("Bcc: %s\r\n", strings.Join(missingAddresses, ", ")))
+			data = append(bcc, data...)
+		}
+
+		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
+	}
+
+	id, err := storage.Store(&data)
+	if err != nil {
+		logger.Log().Errorf("[db] error storing message: %s", err.Error())
+		return "", err
+	}
+
+	stats.LogSMTPAccepted(len(data))
+
+	data = nil // avoid memory leaks
+
+	subject := msg.Header.Get("Subject")
+	logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject)
+
+	return id, err
+}
+
+func authHandler(remoteAddr net.Addr, mechanism string, username []byte, password []byte, _ []byte) (bool, error) {
+	allow := auth.SMTPCredentials.Match(string(username), string(password))
+	if allow {
+		logger.Log().Debugf("[smtpd] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+	} else {
+		logger.Log().Warnf("[smtpd] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+	}
+
+	return allow, nil
+}
+
+// Allow any username and password
+func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, _ []byte, _ []byte) (bool, error) {
+	logger.Log().Debugf("[smtpd] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr))
+
+	return true, nil
+}
+
+// HandlerRcpt used to optionally restrict recipients based on `--smtp-allowed-recipients`
+func handlerRcpt(remoteAddr net.Addr, from string, to string) bool {
+	if config.SMTPAllowedRecipientsRegexp == nil {
+		return true
+	}
+
+	result := config.SMTPAllowedRecipientsRegexp.MatchString(to)
+
+	if !result {
+		logger.Log().Warnf("[smtpd] rejected message to %s from %s (%s)", to, from, cleanIP(remoteAddr))
+		stats.LogSMTPRejected()
+	}
+
+	return result
+}
+
+// Listen starts the SMTPD server
+func Listen() error {
+	if config.SMTPAuthAllowInsecure {
+		if auth.SMTPCredentials != nil {
+			logger.Log().Info("[smtpd] enabling login authentication (insecure)")
+		} else if config.SMTPAuthAcceptAny {
+			logger.Log().Info("[smtpd] enabling any authentication (insecure)")
+		}
+	} else {
+		if auth.SMTPCredentials != nil {
+			logger.Log().Info("[smtpd] enabling login authentication")
+		} else if config.SMTPAuthAcceptAny {
+			logger.Log().Info("[smtpd] enabling any authentication")
+		}
+	}
+
+	return listenAndServe(config.SMTPListen, mailHandler, authHandler)
+}
+
+// Translate the smtpd verb from READ/WRITE
+func verbLogTranslator(verb string) string {
+	if verb == "READ" {
+		return "received"
+	}
+
+	return "response"
+}
+
+func listenAndServe(addr string, handler MsgIDHandler, authHandler AuthHandler) error {
+
+	socketAddr, perm, isSocket := tools.UnixSocket(addr)
+
+	Debug = true // to enable Mailpit logging
+	srv := &Server{
+		Addr:              addr,
+		MsgIDHandler:      handler,
+		HandlerRcpt:       handlerRcpt,
+		AppName:           "Mailpit",
+		Hostname:          "",
+		AuthHandler:       nil,
+		AuthRequired:      false,
+		MaxRecipients:     config.SMTPMaxRecipients,
+		DisableReverseDNS: DisableReverseDNS,
+		LogRead: func(remoteIP, verb, line string) {
+			logger.Log().Debugf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+		},
+		LogWrite: func(remoteIP, verb, line string) {
+			if warningResponse.MatchString(line) {
+				logger.Log().Warnf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+				websockets.BroadCastClientError("warning", "smtpd", remoteIP, line)
+			} else if errorResponse.MatchString(line) {
+				logger.Log().Errorf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+				websockets.BroadCastClientError("error", "smtpd", remoteIP, line)
+			} else {
+				logger.Log().Debugf("[smtpd] %s (%s) %s", verbLogTranslator(verb), remoteIP, line)
+			}
+		},
+	}
+
+	if config.Label != "" {
+		srv.AppName = fmt.Sprintf("Mailpit (%s)", config.Label)
+	}
+
+	if config.SMTPAuthAllowInsecure {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+	}
+
+	if auth.SMTPCredentials != nil {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+		srv.AuthHandler = authHandler
+		srv.AuthRequired = true
+	} else if config.SMTPAuthAcceptAny {
+		srv.AuthMechs = map[string]bool{"CRAM-MD5": false, "PLAIN": true, "LOGIN": true}
+		srv.AuthHandler = authHandlerAny
+	}
+
+	if config.SMTPTLSCert != "" {
+		srv.TLSRequired = config.SMTPRequireSTARTTLS
+		srv.TLSListener = config.SMTPRequireTLS // if true overrules srv.TLSRequired
+		if err := srv.ConfigureTLS(config.SMTPTLSCert, config.SMTPTLSKey); err != nil {
+			return err
+		}
+	}
+
+	if isSocket {
+		srv.Addr = socketAddr
+		srv.Protocol = "unix"
+		srv.SocketPerm = perm
+
+		if err := tools.PrepareSocket(srv.Addr); err != nil {
+			storage.Close()
+			return err
+		}
+
+		// delete the Unix socket file on exit
+		storage.AddTempFile(srv.Addr)
+
+		logger.Log().Infof("[smtpd] starting on %s", config.SMTPListen)
+	} else {
+		smtpType := "no encryption"
+
+		if config.SMTPTLSCert != "" {
+			if config.SMTPRequireSTARTTLS {
+				smtpType = "STARTTLS required"
+			} else if config.SMTPRequireTLS {
+				smtpType = "SSL/TLS required"
+			} else {
+				smtpType = "STARTTLS optional"
+				if !config.SMTPAuthAllowInsecure && auth.SMTPCredentials != nil {
+					smtpType = "STARTTLS required"
+				}
+			}
+		}
+
+		logger.Log().Infof("[smtpd] starting on %s (%s)", config.SMTPListen, smtpType)
+	}
+
+	return srv.ListenAndServe()
+}
+
+func cleanIP(i net.Addr) string {
+	parts := strings.Split(i.String(), ":")
+
+	return parts[0]
+}
+
+// Returns a list of all lowercased emails found in To, Cc and Bcc,
+// as well as whether there is a Bcc field
+func scanAddressesInHeader(h mail.Header) (map[string]bool, bool) {
+	emails := make(map[string]bool)
+	hasBccHeader := false
+
+	if recipients, err := h.AddressList("To"); err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+	}
+
+	if recipients, err := h.AddressList("Cc"); err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+	}
+
+	recipients, err := h.AddressList("Bcc")
+	if err == nil {
+		for _, r := range recipients {
+			emails[strings.ToLower(r.Address)] = true
+		}
+
+		hasBccHeader = true
+	}
+
+	return emails, hasBccHeader
+}

internal/smtpd/relay.go

@@ -0,0 +1,173 @@
+package smtpd
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/smtp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// Wrapper to auto relay messages if configured
+func autoRelayMessage(from string, to []string, data *[]byte) {
+	if config.SMTPRelayConfig.BlockedRecipientsRegexp != nil {
+		filteredTo := []string{}
+		for _, address := range to {
+			if config.SMTPRelayConfig.BlockedRecipientsRegexp.MatchString(address) {
+				logger.Log().Debugf("[relay] ignoring auto-relay to %s: found in blocklist", address)
+				continue
+			}
+
+			filteredTo = append(filteredTo, address)
+		}
+		to = filteredTo
+	}
+
+	if len(to) == 0 {
+		return
+	}
+
+	if config.SMTPRelayAll {
+		if err := Relay(from, to, *data); err != nil {
+			logger.Log().Errorf("[relay] error: %s", err.Error())
+		} else {
+			logger.Log().Debugf("[relay] sent message to %s from %s via %s:%d",
+				strings.Join(to, ", "), from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		}
+	} else if config.SMTPRelayMatchingRegexp != nil {
+		filtered := []string{}
+		for _, t := range to {
+			if config.SMTPRelayMatchingRegexp.MatchString(t) {
+				filtered = append(filtered, t)
+			}
+		}
+
+		if len(filtered) == 0 {
+			return
+		}
+
+		if err := Relay(from, filtered, *data); err != nil {
+			logger.Log().Errorf("[relay] error: %s", err.Error())
+		} else {
+			logger.Log().Debugf("[relay] auto-relay message to %s from %s via %s:%d",
+				strings.Join(filtered, ", "), from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		}
+	}
+}
+
+// Relay will connect to a pre-configured SMTP server and send a message to one or more recipients.
+func Relay(from string, to []string, msg []byte) error {
+	addr := fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		return fmt.Errorf("error connecting to %s: %s", addr, err.Error())
+	}
+
+	defer c.Close()
+
+	if config.SMTPRelayConfig.STARTTLS {
+		conf := &tls.Config{ServerName: config.SMTPRelayConfig.Host} // #nosec
+
+		conf.InsecureSkipVerify = config.SMTPRelayConfig.AllowInsecure
+
+		if err = c.StartTLS(conf); err != nil {
+			return fmt.Errorf("error creating StartTLS config: %s", err.Error())
+		}
+	}
+
+	auth := relayAuthFromConfig()
+
+	if auth != nil {
+		if err = c.Auth(auth); err != nil {
+			return fmt.Errorf("error response to AUTH command: %s", err.Error())
+		}
+	}
+
+	if config.SMTPRelayConfig.OverrideFrom != "" {
+		msg, err = tools.OverrideFromHeader(msg, config.SMTPRelayConfig.OverrideFrom)
+		if err != nil {
+			return fmt.Errorf("error overriding From header: %s", err.Error())
+		}
+
+		from = config.SMTPRelayConfig.OverrideFrom
+	}
+
+	if err = c.Mail(from); err != nil {
+		return fmt.Errorf("error response to MAIL command: %s", err.Error())
+	}
+
+	for _, addr := range to {
+		if err = c.Rcpt(addr); err != nil {
+			logger.Log().Warnf("error response to RCPT command for %s: %s", addr, err.Error())
+		}
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return fmt.Errorf("error response to DATA command: %s", err.Error())
+	}
+
+	if _, err := w.Write(msg); err != nil {
+		return fmt.Errorf("error sending message: %s", err.Error())
+	}
+
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("error closing connection: %s", err.Error())
+	}
+
+	return c.Quit()
+}
+
+// Return the SMTP relay authentication based on config
+func relayAuthFromConfig() smtp.Auth {
+	var a smtp.Auth
+
+	if config.SMTPRelayConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host)
+	}
+
+	if config.SMTPRelayConfig.Auth == "login" {
+		a = LoginAuth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password)
+	}
+
+	if config.SMTPRelayConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret)
+	}
+
+	return a
+}
+
+// Custom implementation of LOGIN SMTP authentication
+// @see https://gist.github.com/andelf/5118732
+type loginAuth struct {
+	username, password string
+}
+
+// LoginAuth authentication
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(_ *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch string(fromServer) {
+		case "Username:":
+			return []byte(a.username), nil
+		case "Password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.New("Unknown fromServer")
+		}
+	}
+
+	return nil, nil
+}

internal/smtpd/smtpd.go

@@ -0,0 +1,979 @@
+// Package smtpd implements a basic SMTP server.
+//
+// This is a modified version of https://github.com/mhale/smtpd to
+// add support for unix sockets and Mailpit Chaos.
+package smtpd
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"net"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/axllent/mailpit/internal/smtpd/chaos"
+)
+
+var (
+	// Debug `true` enables verbose logging.
+	Debug      = false
+	rcptToRE   = regexp.MustCompile(`(?i)TO: ?<([^<>\v]+)>( |$)(.*)?`)
+	mailFromRE = regexp.MustCompile(`(?i)FROM: ?<(|[^<>\v]+)>( |$)(.*)?`) // Delivery Status Notifications are sent with "MAIL FROM:<>"
+
+	// extract mail size from 'MAIL FROM' parameter
+	mailFromSizeRE = regexp.MustCompile(`(?U)(^| |,)[Ss][Ii][Zz][Ee]=(.*)($|,| )`)
+)
+
+// Handler function called upon successful receipt of an email.
+// Results in a "250 2.0.0 Ok: queued" response.
+type Handler func(remoteAddr net.Addr, from string, to []string, data []byte) error
+
+// MsgIDHandler function called upon successful receipt of an email. Returns a message ID.
+// Results in a "250 2.0.0 Ok: queued as <message-id>" response.
+type MsgIDHandler func(remoteAddr net.Addr, from string, to []string, data []byte) (string, error)
+
+// HandlerRcpt function called on RCPT. Return accept status.
+type HandlerRcpt func(remoteAddr net.Addr, from string, to string) bool
+
+// AuthHandler function called when a login attempt is performed. Returns true if credentials are correct.
+type AuthHandler func(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error)
+
+// ErrServerClosed is the default message when a server closes a connection
+var ErrServerClosed = errors.New("Server has been closed")
+
+// ListenAndServe listens on the TCP network address addr
+// and then calls Serve with handler to handle requests
+// on incoming connections.
+func ListenAndServe(addr string, handler Handler, appName string, hostname string) error {
+	srv := &Server{Addr: addr, Handler: handler, AppName: appName, Hostname: hostname}
+	return srv.ListenAndServe()
+}
+
+// ListenAndServeTLS listens on the TCP network address addr
+// and then calls Serve with handler to handle requests
+// on incoming connections. Connections may be upgraded to TLS if the client requests it.
+func ListenAndServeTLS(addr string, certFile string, keyFile string, handler Handler, appName string, hostname string) error {
+	srv := &Server{Addr: addr, Handler: handler, AppName: appName, Hostname: hostname}
+	err := srv.ConfigureTLS(certFile, keyFile)
+	if err != nil {
+		return err
+	}
+	return srv.ListenAndServe()
+}
+
+type maxSizeExceededError struct {
+	limit int
+}
+
+func maxSizeExceeded(limit int) maxSizeExceededError {
+	return maxSizeExceededError{limit}
+}
+
+// Error uses the RFC 5321 response message in preference to RFC 1870.
+// RFC 3463 defines enhanced status code x.3.4 as "Message too big for system".
+func (err maxSizeExceededError) Error() string {
+	return fmt.Sprintf("552 5.3.4 Requested mail action aborted: exceeded storage allocation (%d)", err.limit)
+}
+
+// LogFunc is a function capable of logging the client-server communication.
+type LogFunc func(remoteIP, verb, line string)
+
+// Server is an SMTP server.
+type Server struct {
+	Addr              string // TCP address to listen on, defaults to ":25" (all addresses, port 25) if empty
+	AppName           string
+	AuthHandler       AuthHandler
+	AuthMechs         map[string]bool // Override list of allowed authentication mechanisms. Currently supported: LOGIN, PLAIN, CRAM-MD5. Enabling LOGIN and PLAIN will reduce RFC 4954 compliance.
+	AuthRequired      bool            // Require authentication for every command except AUTH, EHLO, HELO, NOOP, RSET or QUIT as per RFC 4954. Ignored if AuthHandler is not configured.
+	DisableReverseDNS bool            // Disable reverse DNS lookups, enforces "unknown" hostname
+	Handler           Handler
+	HandlerRcpt       HandlerRcpt
+	Hostname          string
+	LogRead           LogFunc
+	LogWrite          LogFunc
+	MaxSize           int // Maximum message size allowed, in bytes
+	MaxRecipients     int // Maximum number of recipients, defaults to 100.
+	MsgIDHandler      MsgIDHandler
+	Timeout           time.Duration
+	TLSConfig         *tls.Config
+	TLSListener       bool        // Listen for incoming TLS connections only (not recommended as it may reduce compatibility). Ignored if TLS is not configured.
+	TLSRequired       bool        // Require TLS for every command except NOOP, EHLO, STARTTLS, or QUIT as per RFC 3207. Ignored if TLS is not configured.
+	Protocol          string      // Default tcp, supports unix
+	SocketPerm        fs.FileMode // if using Unix socket, socket permissions
+
+	inShutdown   int32 // server was closed or shutdown
+	openSessions int32 // count of open sessions
+	mu           sync.Mutex
+	shutdownChan chan struct{} // let the sessions know we are shutting down
+
+	XClientAllowed []string // List of XCLIENT allowed IP addresses
+}
+
+// ConfigureTLS creates a TLS configuration from certificate and key files.
+func (srv *Server) ConfigureTLS(certFile string, keyFile string) error {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return err
+	}
+	srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cert}} // #nosec
+	return nil
+}
+
+// // ConfigureTLSWithPassphrase creates a TLS configuration from a certificate,
+// // an encrypted key file and the associated passphrase:
+// func (srv *Server) ConfigureTLSWithPassphrase(
+// 	certFile string,
+// 	keyFile string,
+// 	passphrase string,
+// ) error {
+// 	certPEMBlock, err := os.ReadFile(certFile)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	keyPEMBlock, err := os.ReadFile(keyFile)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	keyDERBlock, _ := pem.Decode(keyPEMBlock)
+// 	keyPEMDecrypted, err := x509.DecryptPEMBlock(keyDERBlock, []byte(passphrase))
+// 	if err != nil {
+// 		return err
+// 	}
+// 	var pemBlock pem.Block
+// 	pemBlock.Type = keyDERBlock.Type
+// 	pemBlock.Bytes = keyPEMDecrypted
+// 	keyPEMBlock = pem.EncodeToMemory(&pemBlock)
+// 	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cert}}
+// 	return nil
+// }
+
+// ListenAndServe listens on the either a TCP network address srv.Addr or
+// alternatively a Unix socket. and then calls Serve to handle requests on
+// incoming connections. If srv.Addr is blank, ":25" is used.
+func (srv *Server) ListenAndServe() error {
+	if atomic.LoadInt32(&srv.inShutdown) != 0 {
+		return ErrServerClosed
+	}
+
+	if srv.Addr == "" {
+		srv.Addr = ":25"
+	}
+	if srv.AppName == "" {
+		srv.AppName = "smtpd"
+	}
+	if srv.Hostname == "" {
+		srv.Hostname, _ = os.Hostname()
+	}
+	if srv.Timeout == 0 {
+		srv.Timeout = 5 * time.Minute
+	}
+	if srv.Protocol == "" {
+		srv.Protocol = "tcp"
+	}
+
+	var ln net.Listener
+	var err error
+
+	// If TLSListener is enabled, listen for TLS connections only.
+	if srv.TLSConfig != nil && srv.TLSListener {
+		ln, err = tls.Listen(srv.Protocol, srv.Addr, srv.TLSConfig)
+	} else {
+		ln, err = net.Listen(srv.Protocol, srv.Addr)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if srv.Protocol == "unix" {
+		// set permissions
+		if err := os.Chmod(srv.Addr, srv.SocketPerm); err != nil {
+			return err
+		}
+	}
+
+	return srv.Serve(ln)
+}
+
+// Serve creates a new SMTP session after a network connection is established.
+func (srv *Server) Serve(ln net.Listener) error {
+	if atomic.LoadInt32(&srv.inShutdown) != 0 {
+		return ErrServerClosed
+	}
+
+	defer ln.Close()
+
+	for {
+		// if we are shutting down, don't accept new connections
+		select {
+		case <-srv.getShutdownChan():
+			return ErrServerClosed
+		default:
+		}
+
+		conn, err := ln.Accept()
+		if err != nil {
+			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
+				continue
+			}
+			return err
+		}
+
+		session := srv.newSession(conn)
+		atomic.AddInt32(&srv.openSessions, 1)
+		go session.serve()
+	}
+}
+
+type session struct {
+	srv           *Server
+	conn          net.Conn
+	br            *bufio.Reader
+	bw            *bufio.Writer
+	remoteIP      string // Remote IP address
+	remoteHost    string // Remote hostname according to reverse DNS lookup
+	remoteName    string // Remote hostname as supplied with EHLO
+	xClient       string // Information string as supplied with XCLIENT
+	xClientADDR   string // Information string as supplied with XCLIENT ADDR
+	xClientNAME   string // Information string as supplied with XCLIENT NAME
+	xClientTrust  bool   // Trust XCLIENT from current IP address
+	tls           bool
+	authenticated bool
+}
+
+// Create new session from connection.
+func (srv *Server) newSession(conn net.Conn) (s *session) {
+	s = &session{
+		srv:  srv,
+		conn: conn,
+		br:   bufio.NewReader(conn),
+		bw:   bufio.NewWriter(conn),
+	}
+
+	// Get remote end info for the Received header.
+	s.remoteIP, _, _ = net.SplitHostPort(s.conn.RemoteAddr().String())
+	if s.remoteIP == "" {
+		s.remoteIP = "127.0.0.1"
+	}
+	if !s.srv.DisableReverseDNS {
+		names, err := net.LookupAddr(s.remoteIP)
+		if err == nil && len(names) > 0 {
+			s.remoteHost = names[0]
+		} else {
+			s.remoteHost = "unknown"
+		}
+	} else {
+		s.remoteHost = "unknown"
+	}
+
+	// Set tls = true if TLS is already in use.
+	_, s.tls = s.conn.(*tls.Conn)
+
+	for _, checkIP := range srv.XClientAllowed {
+		if s.remoteIP == checkIP {
+			s.xClientTrust = true
+		}
+	}
+	return
+}
+
+func (srv *Server) getShutdownChan() <-chan struct{} {
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+	if srv.shutdownChan == nil {
+		srv.shutdownChan = make(chan struct{})
+	}
+
+	return srv.shutdownChan
+}
+
+func (srv *Server) closeShutdownChan() {
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+	if srv.shutdownChan == nil {
+		srv.shutdownChan = make(chan struct{})
+	}
+
+	select {
+	case <-srv.shutdownChan:
+	default:
+		close(srv.shutdownChan)
+	}
+}
+
+// Close - closes the connection without waiting
+func (srv *Server) Close() error {
+	atomic.StoreInt32(&srv.inShutdown, 1)
+	srv.closeShutdownChan()
+	return nil
+}
+
+// Shutdown - waits for current sessions to complete before closing
+func (srv *Server) Shutdown(ctx context.Context) error {
+	atomic.StoreInt32(&srv.inShutdown, 1)
+	srv.closeShutdownChan()
+
+	// wait for up to 30 seconds to allow the current sessions to
+	// end
+	timer := time.NewTimer(100 * time.Millisecond)
+	defer timer.Stop()
+
+	for i := 0; i < 300; i++ {
+		// wait for open sessions to close
+		if atomic.LoadInt32(&srv.openSessions) == 0 {
+			break
+		}
+
+		select {
+		case <-timer.C:
+			timer.Reset(100 * time.Millisecond)
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+	}
+
+	return nil
+}
+
+// Function called to handle connection requests.
+func (s *session) serve() {
+	defer atomic.AddInt32(&s.srv.openSessions, -1)
+	defer s.conn.Close()
+
+	var from string
+	var gotFrom bool
+	var to []string
+	var buffer bytes.Buffer
+
+	// RFC 5321 specifies support for minimum of 100 recipients is required.
+	if s.srv.MaxRecipients == 0 {
+		s.srv.MaxRecipients = 100
+	}
+
+	// Send banner.
+	s.writef("220 %s %s ESMTP Service ready", s.srv.Hostname, s.srv.AppName)
+
+loop:
+	for {
+		// Attempt to read a line from the socket.
+		// On timeout, send a timeout message and return from serve().
+		// On error, assume the client has gone away i.e. return from serve().
+		line, err := s.readLine()
+		if err != nil {
+			if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+				s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+			}
+			break
+		}
+
+		verb, args := s.parseLine(line)
+
+		switch verb {
+		case "HELO":
+			s.remoteName = args
+			s.writef("250 %s greets %s", s.srv.Hostname, s.remoteName)
+
+			// RFC 2821 section 4.1.4 specifies that EHLO has the same effect as RSET, so reset for HELO too.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "EHLO":
+			s.remoteName = args
+			s.writef("%s", s.makeEHLOResponse())
+
+			// RFC 2821 section 4.1.4 specifies that EHLO has the same effect as RSET.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "MAIL":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+
+			match := mailFromRE.FindStringSubmatch(args)
+			if match == nil {
+				s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid FROM parameter)")
+			} else {
+				// Mailpit Chaos
+				if fail, code := chaos.Config.Sender.Trigger(); fail {
+					s.writef("%d Chaos sender error", code)
+					break
+				}
+
+				// Validate the SIZE parameter if one was sent.
+				if len(match[2]) > 0 { // A parameter is present
+					sizeMatch := mailFromSizeRE.FindStringSubmatch(match[3])
+					if sizeMatch == nil {
+						// ignore other parameter
+						from = match[1]
+						gotFrom = true
+						s.writef("250 2.1.0 Ok")
+					} else {
+						// Enforce the maximum message size if one is set.
+						size, err := strconv.Atoi(sizeMatch[2])
+						if err != nil { // Bad SIZE parameter
+							s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid SIZE parameter)")
+						} else if s.srv.MaxSize > 0 && size > s.srv.MaxSize { // SIZE above maximum size, if set
+							err = maxSizeExceeded(s.srv.MaxSize)
+							s.writef("%s", err.Error())
+						} else { // SIZE ok
+							from = match[1]
+							gotFrom = true
+							s.writef("250 2.1.0 Ok")
+						}
+					}
+				} else { // No parameters after FROM
+					from = match[1]
+					gotFrom = true
+					s.writef("250 2.1.0 Ok")
+				}
+			}
+
+			to = nil
+			buffer.Reset()
+		case "RCPT":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+			if !gotFrom {
+				s.writef("503 5.5.1 Bad sequence of commands (MAIL required before RCPT)")
+				break
+			}
+
+			match := rcptToRE.FindStringSubmatch(args)
+			if match == nil {
+				s.writef("501 5.5.4 Syntax error in parameters or arguments (invalid TO parameter)")
+			} else {
+				// Mailpit Chaos
+				if fail, code := chaos.Config.Recipient.Trigger(); fail {
+					s.writef("%d Chaos recipient error", code)
+					break
+				}
+
+				if len(to) >= s.srv.MaxRecipients {
+					s.writef("452 4.5.3 Too many recipients")
+				} else {
+					accept := true
+					if s.srv.HandlerRcpt != nil {
+						accept = s.srv.HandlerRcpt(s.conn.RemoteAddr(), from, match[1])
+					}
+					if accept {
+						to = append(to, match[1])
+						s.writef("250 2.1.5 Ok")
+					} else {
+						s.writef("550 5.1.0 Requested action not taken: mailbox unavailable")
+					}
+				}
+			}
+		case "DATA":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			if s.srv.AuthHandler != nil && s.srv.AuthRequired && !s.authenticated {
+				s.writef("530 5.7.0 Authentication required")
+				break
+			}
+			if !gotFrom || len(to) == 0 {
+				s.writef("503 5.5.1 Bad sequence of commands (MAIL & RCPT required before DATA)")
+				break
+			}
+
+			s.writef("354 Start mail input; end with <CR><LF>.<CR><LF>")
+
+			// Attempt to read message body from the socket.
+			// On timeout, send a timeout message and return from serve().
+			// On net.Error, assume the client has gone away i.e. return from serve().
+			// On other errors, allow the client to try again.
+			data, err := s.readData()
+			if err != nil {
+				switch err.(type) {
+				case net.Error:
+					if err.(net.Error).Timeout() {
+						s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+					}
+					break loop
+				case maxSizeExceededError:
+					s.writef("%s", err.Error())
+					continue
+				default:
+					s.writef("451 4.3.0 Requested action aborted: local error in processing")
+					continue
+				}
+			}
+
+			// Create Received header & write message body into buffer.
+			buffer.Reset()
+			buffer.Write(s.makeHeaders(to))
+			buffer.Write(data)
+
+			// Pass mail on to handler.
+			if s.srv.Handler != nil {
+				err := s.srv.Handler(s.conn.RemoteAddr(), from, to, buffer.Bytes())
+				if err != nil {
+					checkErrFormat := regexp.MustCompile(`^([2-5][0-9]{2})[\s\-](.+)$`)
+					if checkErrFormat.MatchString(err.Error()) {
+						s.writef("%s", err.Error())
+					} else {
+						s.writef("451 4.3.5 Unable to process mail")
+					}
+					break
+				}
+				s.writef("250 2.0.0 Ok: queued")
+			} else if s.srv.MsgIDHandler != nil {
+				msgID, err := s.srv.MsgIDHandler(s.conn.RemoteAddr(), from, to, buffer.Bytes())
+				if err != nil {
+					checkErrFormat := regexp.MustCompile(`^([2-5][0-9]{2})[\s\-](.+)$`)
+					if checkErrFormat.MatchString(err.Error()) {
+						s.writef("%s", err.Error())
+					} else {
+						s.writef("451 4.3.5 Unable to process mail")
+					}
+					break
+				}
+
+				if msgID != "" {
+					s.writef("250 2.0.0 Ok: queued as %s", msgID)
+				} else {
+					s.writef("250 2.0.0 Ok: queued")
+				}
+			} else {
+				s.writef("250 2.0.0 Ok: queued")
+			}
+
+			// Reset for next mail.
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "QUIT":
+			s.writef("221 2.0.0 %s %s ESMTP Service closing transmission channel", s.srv.Hostname, s.srv.AppName)
+			break loop
+		case "RSET":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			s.writef("250 2.0.0 Ok")
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "NOOP":
+			s.writef("250 2.0.0 Ok")
+		case "XCLIENT":
+			s.xClient = args
+			if s.xClientTrust {
+				xCArgs := strings.Split(args, " ")
+				for _, xCArg := range xCArgs {
+					xCParse := strings.Split(strings.TrimSpace(xCArg), "=")
+					if strings.ToUpper(xCParse[0]) == "ADDR" && (net.ParseIP(xCParse[1]) != nil) {
+						s.xClientADDR = xCParse[1]
+					}
+					if strings.ToUpper(xCParse[0]) == "NAME" && len(xCParse[1]) > 0 {
+						if xCParse[1] != "[UNAVAILABLE]" {
+							s.xClientNAME = xCParse[1]
+						}
+					}
+				}
+				if len(s.xClientADDR) > 7 {
+					s.remoteIP = s.xClientADDR
+					if len(s.xClientNAME) > 4 {
+						s.remoteHost = s.xClientNAME
+					} else {
+						names, err := net.LookupAddr(s.remoteIP)
+						if err == nil && len(names) > 0 {
+							s.remoteHost = names[0]
+						} else {
+							s.remoteHost = "unknown"
+						}
+					}
+				}
+			}
+			s.writef("250 2.0.0 Ok")
+		case "HELP", "VRFY", "EXPN":
+			// See RFC 5321 section 4.2.4 for usage of 500 & 502 response codes.
+			s.writef("502 5.5.1 Command not implemented")
+		case "STARTTLS":
+			// Parameters are not allowed (RFC 3207 section 4).
+			if args != "" {
+				s.writef("501 5.5.2 Syntax error (no parameters allowed)")
+				break
+			}
+
+			// Handle case where TLS is requested but not configured (and therefore not listed as a service extension).
+			if s.srv.TLSConfig == nil {
+				s.writef("502 5.5.1 Command not implemented")
+				break
+			}
+
+			// Handle case where STARTTLS is received when TLS is already in use.
+			if s.tls {
+				s.writef("503 5.5.1 Bad sequence of commands (TLS already in use)")
+				break
+			}
+
+			s.writef("220 2.0.0 Ready to start TLS")
+
+			// Establish a TLS connection with the client.
+			tlsConn := tls.Server(s.conn, s.srv.TLSConfig)
+			err := tlsConn.Handshake()
+			if err != nil {
+				s.writef("403 4.7.0 TLS handshake failed")
+				break
+			}
+
+			// TLS handshake succeeded, switch to using the TLS connection.
+			s.conn = tlsConn
+			s.br = bufio.NewReader(s.conn)
+			s.bw = bufio.NewWriter(s.conn)
+			s.tls = true
+
+			// RFC 3207 specifies that the server must discard any prior knowledge obtained from the client.
+			s.remoteName = ""
+			from = ""
+			gotFrom = false
+			to = nil
+			buffer.Reset()
+		case "AUTH":
+			if s.srv.TLSConfig != nil && s.srv.TLSRequired && !s.tls {
+				s.writef("530 5.7.0 Must issue a STARTTLS command first")
+				break
+			}
+			// Handle case where AUTH is requested but not configured (and therefore not listed as a service extension).
+			if s.srv.AuthHandler == nil {
+				s.writef("502 5.5.1 Command not implemented")
+				break
+			}
+
+			// Handle case where AUTH is received when already authenticated.
+			if s.authenticated {
+				s.writef("503 5.5.1 Bad sequence of commands (already authenticated for this session)")
+				break
+			}
+
+			// RFC 4954 specifies that AUTH is not permitted during mail transactions.
+			if gotFrom || len(to) > 0 {
+				s.writef("503 5.5.1 Bad sequence of commands (AUTH not permitted during mail transaction)")
+				break
+			}
+
+			// RFC 4954 requires a mechanism parameter.
+			authType, authArgs := s.parseLine(args)
+			if authType == "" {
+				s.writef("501 5.5.4 Malformed AUTH input (argument required)")
+				break
+			}
+
+			// RFC 4954 requires rejecting unsupported authentication mechanisms with a 504 response.
+			allowedAuth := s.authMechs()
+			if allowed, found := allowedAuth[authType]; !found || !allowed {
+				s.writef("504 5.5.4 Unrecognized authentication type")
+				break
+			}
+
+			// Mailpit Chaos
+			if fail, code := chaos.Config.Authentication.Trigger(); fail {
+				s.writef("%d Chaos authentication error", code)
+				break
+			}
+
+			// RFC 4954 also specifies that ESMTP code 5.5.4 ("Invalid command arguments") should be returned
+			// when attempting to use an unsupported authentication type.
+			// Many servers return 5.7.4 ("Security features not supported") instead.
+			switch authType {
+			case "PLAIN":
+				s.authenticated, err = s.handleAuthPlain(authArgs)
+			case "LOGIN":
+				s.authenticated, err = s.handleAuthLogin(authArgs)
+			case "CRAM-MD5":
+				s.authenticated, err = s.handleAuthCramMD5()
+			}
+
+			if err != nil {
+				if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+					s.writef("421 4.4.2 %s %s ESMTP Service closing transmission channel after timeout exceeded", s.srv.Hostname, s.srv.AppName)
+					break loop
+				}
+
+				s.writef("%s", err.Error())
+				break
+			}
+
+			if s.authenticated {
+				s.writef("235 2.7.0 Authentication successful")
+			} else {
+				s.writef("535 5.7.8 Authentication credentials invalid")
+			}
+		default:
+			// See RFC 5321 section 4.2.4 for usage of 500 & 502 response codes.
+			s.writef("500 5.5.2 Syntax error, command unrecognized")
+		}
+	}
+}
+
+// Wrapper function for writing a complete line to the socket.
+func (s *session) writef(format string, args ...interface{}) {
+	if s.srv.Timeout > 0 {
+		_ = s.conn.SetWriteDeadline(time.Now().Add(s.srv.Timeout))
+	}
+
+	line := fmt.Sprintf(format, args...)
+	fmt.Fprintf(s.bw, "%s\r\n", line)
+	_ = s.bw.Flush()
+
+	if Debug {
+		verb := "WROTE"
+		if s.srv.LogWrite != nil {
+			s.srv.LogWrite(s.remoteIP, verb, line)
+		} else {
+			log.Println(s.remoteIP, verb, line)
+		}
+	}
+}
+
+// Read a complete line from the socket.
+func (s *session) readLine() (string, error) {
+	if s.srv.Timeout > 0 {
+		_ = s.conn.SetReadDeadline(time.Now().Add(s.srv.Timeout))
+	}
+
+	line, err := s.br.ReadString('\n')
+	if err != nil {
+		return "", err
+	}
+	line = strings.TrimSpace(line) // Strip trailing \r\n
+
+	if Debug {
+		verb := "READ"
+		if s.srv.LogRead != nil {
+			s.srv.LogRead(s.remoteIP, verb, line)
+		} else {
+			log.Println(s.remoteIP, verb, line)
+		}
+	}
+
+	return line, err
+}
+
+// Parse a line read from the socket.
+func (s *session) parseLine(line string) (verb string, args string) {
+	if idx := strings.Index(line, " "); idx != -1 {
+		verb = strings.ToUpper(line[:idx])
+		args = strings.TrimSpace(line[idx+1:])
+	} else {
+		verb = strings.ToUpper(line)
+		args = ""
+	}
+	return verb, args
+}
+
+// Read the message data following a DATA command.
+func (s *session) readData() ([]byte, error) {
+	var data []byte
+	for {
+		if s.srv.Timeout > 0 {
+			_ = s.conn.SetReadDeadline(time.Now().Add(s.srv.Timeout))
+		}
+
+		line, err := s.br.ReadBytes('\n')
+		if err != nil {
+			return nil, err
+		}
+		// Handle end of data denoted by lone period (\r\n.\r\n)
+		if bytes.Equal(line, []byte(".\r\n")) {
+			break
+		}
+		// Remove leading period (RFC 5321 section 4.5.2)
+		if line[0] == '.' {
+			line = line[1:]
+		}
+
+		// Enforce the maximum message size limit.
+		if s.srv.MaxSize > 0 {
+			if len(data)+len(line) > s.srv.MaxSize {
+				_, _ = s.br.Discard(s.br.Buffered()) // Discard the buffer remnants.
+				return nil, maxSizeExceeded(s.srv.MaxSize)
+			}
+		}
+
+		data = append(data, line...)
+	}
+	return data, nil
+}
+
+// Create the Received header to comply with RFC 2821 section 3.8.2.
+// TODO: Work out what to do with multiple to addresses.
+func (s *session) makeHeaders(to []string) []byte {
+	var buffer bytes.Buffer
+	now := time.Now().Format("Mon, 2 Jan 2006 15:04:05 -0700 (MST)")
+	buffer.WriteString(fmt.Sprintf("Received: from %s (%s [%s])\r\n", s.remoteName, s.remoteHost, s.remoteIP))
+	buffer.WriteString(fmt.Sprintf("        by %s (%s) with SMTP\r\n", s.srv.Hostname, s.srv.AppName))
+	buffer.WriteString(fmt.Sprintf("        for <%s>; %s\r\n", to[0], now))
+	return buffer.Bytes()
+}
+
+// Determine allowed authentication mechanisms.
+// RFC 4954 specifies that plaintext authentication mechanisms such as LOGIN and PLAIN require a TLS connection.
+// This can be explicitly overridden e.g. setting s.srv.AuthMechs["LOGIN"] = true.
+func (s *session) authMechs() (mechs map[string]bool) {
+	mechs = map[string]bool{"LOGIN": s.tls, "PLAIN": s.tls, "CRAM-MD5": true}
+
+	for mech := range mechs {
+		allowed, found := s.srv.AuthMechs[mech]
+		if found {
+			mechs[mech] = allowed
+		}
+	}
+
+	return
+}
+
+// Create the greeting string sent in response to an EHLO command.
+func (s *session) makeEHLOResponse() (response string) {
+	response = fmt.Sprintf("250-%s greets %s\r\n", s.srv.Hostname, s.remoteName)
+
+	// RFC 1870 specifies that "SIZE 0" indicates no maximum size is in force.
+	response += fmt.Sprintf("250-SIZE %d\r\n", s.srv.MaxSize)
+
+	// Only list STARTTLS if TLS is configured, but not currently in use.
+	if s.srv.TLSConfig != nil && !s.tls {
+		response += "250-STARTTLS\r\n"
+	}
+
+	// Only list AUTH if an AuthHandler is configured and at least one mechanism is allowed.
+	if s.srv.AuthHandler != nil {
+		var mechs []string
+		for mech, allowed := range s.authMechs() {
+			if allowed {
+				mechs = append(mechs, mech)
+			}
+		}
+		if len(mechs) > 0 {
+			response += "250-AUTH " + strings.Join(mechs, " ") + "\r\n"
+		}
+	}
+
+	response += "250 ENHANCEDSTATUSCODES"
+	return
+}
+
+func (s *session) handleAuthLogin(arg string) (bool, error) {
+	var err error
+
+	if arg == "" {
+		s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte("Username:")))
+		arg, err = s.readLine()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	username, err := base64.StdEncoding.DecodeString(arg)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte("Password:")))
+	line, err := s.readLine()
+	if err != nil {
+		return false, err
+	}
+
+	password, err := base64.StdEncoding.DecodeString(line)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "LOGIN", username, password, nil)
+
+	return authenticated, err
+}
+
+func (s *session) handleAuthPlain(arg string) (bool, error) {
+	var err error
+
+	// If fast mode (AUTH PLAIN [arg]) is not used, prompt for credentials.
+	if arg == "" {
+		s.writef("334 ")
+		arg, err = s.readLine()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	data, err := base64.StdEncoding.DecodeString(arg)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	parts := bytes.Split(data, []byte{0})
+	if len(parts) != 3 {
+		return false, errors.New("501 5.5.2 Syntax error (unable to parse)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "PLAIN", parts[1], parts[2], nil)
+
+	return authenticated, err
+}
+
+func (s *session) handleAuthCramMD5() (bool, error) {
+	shared := "<" + strconv.Itoa(os.Getpid()) + "." + strconv.Itoa(time.Now().Nanosecond()) + "@" + s.srv.Hostname + ">"
+
+	s.writef("334 %s", base64.StdEncoding.EncodeToString([]byte(shared)))
+
+	data, err := s.readLine()
+	if err != nil {
+		return false, err
+	}
+
+	if data == "*" {
+		return false, errors.New("501 5.7.0 Authentication cancelled")
+	}
+
+	buf, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return false, errors.New("501 5.5.2 Syntax error (unable to decode)")
+	}
+
+	fields := strings.Split(string(buf), " ")
+	if len(fields) < 2 {
+		return false, errors.New("501 5.5.2 Syntax error (unable to parse)")
+	}
+
+	// Validate credentials.
+	authenticated, err := s.srv.AuthHandler(s.conn.RemoteAddr(), "CRAM-MD5", []byte(fields[0]), []byte(fields[1]), []byte(shared))
+
+	return authenticated, err
+}

internal/spamassassin/postmark/postmark.go

@@ -0,0 +1,100 @@
+// Package postmark uses the free https://spamcheck.postmarkapp.com/
+// See https://spamcheck.postmarkapp.com/doc/ for more details.
+package postmark
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Response struct
+type Response struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"` // for errors only
+	Score   string `json:"score"`
+	Rules   []Rule `json:"rules"`
+	Report  string `json:"report"` // ignored
+}
+
+// Rule struct
+type Rule struct {
+	Score string `json:"score"`
+	// Name not returned by postmark but rather extracted from description
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// Check will post the email data to Postmark
+func Check(email []byte, timeout int) (Response, error) {
+	r := Response{}
+	// '{"email":"raw dump of email", "options":"short"}'
+	var d struct {
+		// The raw dump of the email to be filtered, including all headers.
+		Email string `json:"email"`
+		// Default "long". Must either be "long" for a full report of processing rules, or "short" for a score request.
+		Options string `json:"options"`
+	}
+
+	d.Email = string(email)
+	d.Options = "long"
+
+	data, err := json.Marshal(d)
+	if err != nil {
+		return r, err
+	}
+
+	client := http.Client{
+		Timeout: time.Duration(timeout) * time.Second,
+	}
+
+	resp, err := client.Post("https://spamcheck.postmarkapp.com/filter", "application/json",
+		bytes.NewBuffer(data))
+
+	if err != nil {
+		return r, err
+	}
+
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&r)
+
+	// remove trailing line spaces for all lines in report
+	re := regexp.MustCompile("\r?\n")
+	lines := re.Split(r.Report, -1)
+	reportLines := []string{}
+	for _, l := range lines {
+		line := strings.TrimRight(l, " ")
+		reportLines = append(reportLines, line)
+	}
+	reportRaw := strings.Join(reportLines, "\n")
+
+	// join description lines to make a single line per rule
+	re2 := regexp.MustCompile("\n                                ")
+	report := re2.ReplaceAllString(reportRaw, "")
+	for i, rule := range r.Rules {
+		// populate rule name
+		r.Rules[i].Name = nameFromReport(rule.Score, rule.Description, report)
+	}
+
+	return r, err
+}
+
+// Extract the name of the test from the report as Postmark does not include this in the JSON reports
+func nameFromReport(score, description, report string) string {
+	score = regexp.QuoteMeta(score)
+	description = regexp.QuoteMeta(description)
+	str := fmt.Sprintf("%s\\s+([A-Z0-9\\_]+)\\s+%s", score, description)
+	re := regexp.MustCompile(str)
+
+	matches := re.FindAllStringSubmatch(report, 1)
+	if len(matches) > 0 && len(matches[0]) == 2 {
+		return strings.TrimSpace(matches[0][1])
+	}
+
+	return ""
+}

internal/spamassassin/spamassassin.go

@@ -0,0 +1,147 @@
+// Package spamassassin will return results from either a SpamAssassin server or
+// Postmark's public API depending on configuration
+package spamassassin
+
+import (
+	"errors"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/spamassassin/postmark"
+	"github.com/axllent/mailpit/internal/spamassassin/spamc"
+)
+
+var (
+	// Service to use, either "<host>:<ip>" for self-hosted SpamAssassin or "postmark"
+	service string
+
+	// SpamScore is the score at which a message is determined to be spam
+	spamScore = 5.0
+
+	// Timeout in seconds
+	timeout = 8
+)
+
+// Result is a SpamAssassin result
+//
+// swagger:model SpamAssassinResponse
+type Result struct {
+	// Whether the message is spam or not
+	IsSpam bool
+	// If populated will return an error string
+	Error string
+	// Total spam score based on triggered rules
+	Score float64
+	// Spam rules triggered
+	Rules []Rule
+}
+
+// Rule struct
+type Rule struct {
+	// Spam rule score
+	Score float64
+	// SpamAssassin rule name
+	Name string
+	// SpamAssassin rule description
+	Description string
+}
+
+// SetService defines which service should be used.
+func SetService(s string) {
+	switch s {
+	case "postmark":
+		service = "postmark"
+	default:
+		service = s
+	}
+}
+
+// SetTimeout defines the timeout
+func SetTimeout(t int) {
+	if t > 0 {
+		timeout = t
+	}
+}
+
+// Ping returns whether a service is active or not
+func Ping() error {
+	if service == "postmark" {
+		return nil
+	}
+
+	var client *spamc.Client
+	if strings.HasPrefix(service, "unix:") {
+		client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+	} else {
+		client = spamc.NewTCP(service, timeout)
+	}
+
+	return client.Ping()
+}
+
+// Check will return a Result
+func Check(msg []byte) (Result, error) {
+	r := Result{Score: 0}
+
+	if service == "" {
+		return r, errors.New("no SpamAssassin service defined")
+	}
+
+	if service == "postmark" {
+		res, err := postmark.Check(msg, timeout)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		resFloat, err := strconv.ParseFloat(res.Score, 32)
+		if err == nil {
+			r.Score = round1dm(resFloat)
+			r.IsSpam = resFloat >= spamScore
+		}
+		r.Error = res.Message
+		for _, pr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(pr.Score, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = pr.Name
+			rule.Description = pr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	} else {
+		var client *spamc.Client
+		if strings.HasPrefix(service, "unix:") {
+			client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+		} else {
+			client = spamc.NewTCP(service, timeout)
+		}
+
+		res, err := client.Report(msg)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		r.IsSpam = res.Score >= spamScore
+		r.Score = round1dm(res.Score)
+		r.Rules = []Rule{}
+		for _, sr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(sr.Points, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = sr.Name
+			rule.Description = sr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	}
+
+	return r, nil
+}
+
+// Round to one decimal place
+func round1dm(n float64) float64 {
+	return math.Floor(n*10) / 10
+}

internal/spamassassin/spamc/spamc.go

@@ -0,0 +1,250 @@
+// Package spamc provides a client for the SpamAssassin spamd protocol.
+// http://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+//
+// Modified to add timeouts from https://github.com/cgt/spamc
+package spamc
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// ProtoVersion is the protocol version
+const ProtoVersion = "1.5"
+
+var (
+	spamInfoRe    = regexp.MustCompile(`(.+)\/(.+) (\d+) (.+)`)
+	spamMainRe    = regexp.MustCompile(`^Spam: (.+) ; (.+) . (.+)$`)
+	spamDetailsRe = regexp.MustCompile(`^\s?(-?[0-9\.]+)\s([a-zA-Z0-9_]*)(\W*)(.*)`)
+)
+
+// connection is like net.Conn except that it also has a CloseWrite method.
+// CloseWrite is implemented by net.TCPConn and net.UnixConn, but for some
+// reason it is not present in the net.Conn interface.
+type connection interface {
+	net.Conn
+	CloseWrite() error
+}
+
+// Client is a spamd client.
+type Client struct {
+	net     string
+	addr    string
+	timeout int
+}
+
+// NewTCP returns a *Client that connects to spamd via the given TCP address.
+func NewTCP(addr string, timeout int) *Client {
+	return &Client{"tcp", addr, timeout}
+}
+
+// NewUnix returns a *Client that connects to spamd via the given Unix socket.
+func NewUnix(addr string) *Client {
+	return &Client{"unix", addr, 0}
+}
+
+// Rule represents a matched SpamAssassin rule.
+type Rule struct {
+	Points      string
+	Name        string
+	Description string
+}
+
+// Result struct
+type Result struct {
+	ResponseCode int
+	Message      string
+	Spam         bool
+	Score        float64
+	Threshold    float64
+	Rules        []Rule
+}
+
+// dial connects to spamd through TCP or a Unix socket.
+func (c *Client) dial() (connection, error) {
+	if c.net == "tcp" {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialTCP("tcp", nil, tcpAddr)
+	} else if c.net == "unix" {
+		unixAddr, err := net.ResolveUnixAddr("unix", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialUnix("unix", nil, unixAddr)
+	}
+
+	panic("Client.net must be either \"tcp\" or \"unix\"")
+}
+
+// Report checks if message is spam or not, and returns score plus report
+func (c *Client) Report(email []byte) (Result, error) {
+	output, err := c.report(email)
+	if err != nil {
+		return Result{}, err
+	}
+
+	return c.parseOutput(output), nil
+}
+
+func (c *Client) report(email []byte) ([]string, error) {
+	conn, err := c.dial()
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return nil, err
+	}
+
+	bw := bufio.NewWriter(conn)
+	if _, err := bw.WriteString("REPORT SPAMC/" + ProtoVersion + "\r\n"); err != nil {
+		return nil, err
+	}
+
+	if _, err := bw.WriteString("Content-length: " + strconv.Itoa(len(email)) + "\r\n\r\n"); err != nil {
+		return nil, err
+	}
+
+	if _, err := bw.Write(email); err != nil {
+		return nil, err
+	}
+
+	if err := bw.Flush(); err != nil {
+		return nil, err
+	}
+
+	// Client is supposed to close its writing side of the connection
+	// after sending its request.
+	if err := conn.CloseWrite(); err != nil {
+		return nil, err
+	}
+
+	var (
+		lines []string
+		br    = bufio.NewReader(conn)
+	)
+
+	for {
+		line, err := br.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		line = strings.TrimRight(line, " \t\r\n")
+		lines = append(lines, line)
+	}
+
+	// join lines, and replace multi-line descriptions with single line for each
+	tmp := strings.Join(lines, "\n")
+	re := regexp.MustCompile("\n                            ")
+	n := re.ReplaceAllString(tmp, " ")
+
+	//split lines again
+	return strings.Split(n, "\n"), nil
+}
+
+func (c *Client) parseOutput(output []string) Result {
+	var result Result
+	var reachedRules bool
+	for _, row := range output {
+		// header
+		if spamInfoRe.MatchString(row) {
+			res := spamInfoRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				resCode, err := strconv.Atoi(res[3])
+				if err == nil {
+					result.ResponseCode = resCode
+				}
+				result.Message = res[4]
+				continue
+			}
+		}
+
+		// summary
+		if spamMainRe.MatchString(row) {
+			res := spamMainRe.FindStringSubmatch(row)
+			if len(res) == 4 {
+				if tools.InArray(res[1], []string{"true", "yes"}) {
+					result.Spam = true
+				} else {
+					result.Spam = false
+				}
+				resFloat, err := strconv.ParseFloat(res[2], 32)
+				if err == nil {
+					result.Score = resFloat
+					continue
+				}
+				resFloat, err = strconv.ParseFloat(res[3], 32)
+				if err == nil {
+					result.Threshold = resFloat
+					continue
+				}
+			}
+		}
+
+		if strings.HasPrefix(row, "Content analysis details") {
+			reachedRules = true
+			continue
+		}
+
+		// details
+		if reachedRules && spamDetailsRe.MatchString(row) {
+			res := spamDetailsRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				rule := Rule{Points: res[1], Name: res[2], Description: res[4]}
+				result.Rules = append(result.Rules, rule)
+			}
+		}
+	}
+
+	return result
+}
+
+// Ping the spamd
+func (c *Client) Ping() error {
+	conn, err := c.dial()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return err
+	}
+
+	if _, err := io.WriteString(conn, fmt.Sprintf("PING SPAMC/%s\r\n\r\n", ProtoVersion)); err != nil {
+		return err
+	}
+
+	if err := conn.CloseWrite(); err != nil {
+		return err
+	}
+
+	br := bufio.NewReader(conn)
+	for {
+		_, err = br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/stats/stats.go

@@ -0,0 +1,131 @@
+// Package stats stores and returns Mailpit statistics
+package stats
+
+import (
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/updater"
+)
+
+var (
+	// to prevent hammering Github for latest version
+	latestVersionCache string
+
+	// StartedAt is set to the current ime when Mailpit starts
+	startedAt time.Time
+
+	mu sync.RWMutex
+
+	smtpAccepted     float64
+	smtpAcceptedSize float64
+	smtpRejected     float64
+	smtpIgnored      float64
+)
+
+// AppInformation struct
+// swagger:model AppInformation
+type AppInformation struct {
+	// Current Mailpit version
+	Version string
+	// Latest Mailpit version
+	LatestVersion string
+	// Database path
+	Database string
+	// Database size in bytes
+	DatabaseSize float64
+	// Total number of messages in the database
+	Messages float64
+	// Total number of messages in the database
+	Unread float64
+	// Tags and message totals per tag
+	Tags map[string]int64
+	// Runtime statistics
+	RuntimeStats struct {
+		// Mailpit server uptime in seconds
+		Uptime float64
+		// Current memory usage in bytes
+		Memory uint64
+		// Database runtime messages deleted
+		MessagesDeleted float64
+		// Accepted runtime SMTP messages
+		SMTPAccepted float64
+		// Total runtime accepted messages size in bytes
+		SMTPAcceptedSize float64
+		// Rejected runtime SMTP messages
+		SMTPRejected float64
+		// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
+		SMTPIgnored float64
+	}
+}
+
+// Load the current statistics
+func Load() AppInformation {
+	info := AppInformation{}
+	info.Version = config.Version
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	info.RuntimeStats.Memory = m.Sys - m.HeapReleased
+	info.RuntimeStats.Uptime = time.Since(startedAt).Seconds()
+	info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
+	info.RuntimeStats.SMTPAccepted = smtpAccepted
+	info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
+	info.RuntimeStats.SMTPRejected = smtpRejected
+	info.RuntimeStats.SMTPIgnored = smtpIgnored
+
+	if latestVersionCache != "" {
+		info.LatestVersion = latestVersionCache
+	} else {
+		latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
+		if err == nil {
+			info.LatestVersion = latest
+			latestVersionCache = latest
+
+			// clear latest version cache after 5 minutes
+			go func() {
+				time.Sleep(5 * time.Minute)
+				latestVersionCache = ""
+			}()
+		}
+	}
+
+	info.Database = config.Database
+	info.DatabaseSize = storage.DbSize()
+	info.Messages = storage.CountTotal()
+	info.Unread = storage.CountUnread()
+	info.Tags = storage.GetAllTagsCount()
+
+	return info
+}
+
+// Track will start the statistics logging in memory
+func Track() {
+	startedAt = time.Now()
+}
+
+// LogSMTPAccepted logs a successful SMTP transaction
+func LogSMTPAccepted(size int) {
+	mu.Lock()
+	smtpAccepted = smtpAccepted + 1
+	smtpAcceptedSize = smtpAcceptedSize + float64(size)
+	mu.Unlock()
+}
+
+// LogSMTPRejected logs a rejected SMTP transaction
+func LogSMTPRejected() {
+	mu.Lock()
+	smtpRejected = smtpRejected + 1
+	mu.Unlock()
+}
+
+// LogSMTPIgnored logs an ignored SMTP transaction
+func LogSMTPIgnored() {
+	mu.Lock()
+	smtpIgnored = smtpIgnored + 1
+	mu.Unlock()
+}

internal/storage/cron.go

@@ -0,0 +1,216 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+// Database cron runs every minute
+func dbCron() {
+	for {
+		time.Sleep(60 * time.Second)
+
+		currentTime := time.Now()
+		sinceLastDbAction := currentTime.Sub(dbLastAction)
+
+		// only run the database has been idle for 5 minutes
+		if math.Floor(sinceLastDbAction.Minutes()) == 5 {
+			deletedSize := getDeletedSize()
+
+			if deletedSize > 0 {
+				total := totalMessagesSize()
+				var deletedPercent float64
+				if total == 0 {
+					deletedPercent = 100
+				} else {
+					deletedPercent = deletedSize * 100 / total
+				}
+				// only vacuum the DB if at least 1% of mail storage size has been deleted
+				if deletedPercent >= 1 {
+					logger.Log().Debugf("[db] deleted messages is %f%% of total size, reclaim space", deletedPercent)
+					vacuumDb()
+				}
+			}
+		}
+
+		pruneMessages()
+	}
+}
+
+// PruneMessages will auto-delete the oldest messages if messages > config.MaxMessages.
+// Set config.MaxMessages to 0 to disable.
+func pruneMessages() {
+	if config.MaxMessages < 1 && config.MaxAgeInHours == 0 {
+		return
+	}
+
+	start := time.Now()
+
+	ids := []string{}
+	var prunedSize int64
+	var size float64
+
+	// prune using `--max` if set
+	if config.MaxMessages > 0 {
+		total := CountTotal()
+		if total > float64(config.MaxAgeInHours) {
+			offset := config.MaxMessages
+			if config.DemoMode {
+				offset = 500
+			}
+			q := sqlf.Select("ID, Size").
+				From(tenant("mailbox")).
+				OrderBy("Created DESC").
+				Limit(5000).
+				Offset(offset)
+
+			if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+				var id string
+
+				if err := row.Scan(&id, &size); err != nil {
+					logger.Log().Errorf("[db] %s", err.Error())
+					return
+				}
+				ids = append(ids, id)
+				prunedSize = prunedSize + int64(size)
+
+			}); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+		}
+	}
+
+	// prune using `--max-age` if set
+	if config.MaxAgeInHours > 0 {
+		// now() minus the number of hours
+		ts := time.Now().Add(time.Duration(-config.MaxAgeInHours) * time.Hour).UnixMilli()
+
+		q := sqlf.Select("ID, Size").
+			From(tenant("mailbox")).
+			Where("Created < ?", ts).
+			Limit(5000)
+
+		if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			var id string
+
+			if err := row.Scan(&id, &size); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+
+			if !tools.InArray(id, ids) {
+				ids = append(ids, id)
+				prunedSize = prunedSize + int64(size)
+			}
+
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+	}
+
+	if len(ids) == 0 {
+		return
+	}
+
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("mailbox_data")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("message_tags")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Exec(`DELETE FROM `+tenant("mailbox")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	err = tx.Commit()
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		if err := tx.Rollback(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	if err := pruneUnusedTags(); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	addDeletedSize(prunedSize)
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed)
+
+	logMessagesDeleted(len(ids))
+
+	if config.DemoMode {
+		vacuumDb()
+	}
+
+	websockets.Broadcast("prune", nil)
+}
+
+// Vacuum the database to reclaim space from deleted messages
+func vacuumDb() {
+	if sqlDriver == "rqlite" {
+		// let rqlite handle vacuuming
+		return
+	}
+
+	start := time.Now()
+
+	// set WAL file checkpoint
+	if _, err := db.Exec("PRAGMA wal_checkpoint"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	// vacuum database
+	if _, err := db.Exec("VACUUM"); err != nil {
+		logger.Log().Errorf("[db] VACUUM: %s", err.Error())
+		return
+	}
+
+	// truncate WAL file
+	if _, err := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] vacuum completed in %s", elapsed)
+}

internal/storage/database.go

@@ -0,0 +1,259 @@
+// Package storage handles all database actions
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"os/signal"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/klauspost/compress/zstd"
+	"github.com/leporo/sqlf"
+
+	// sqlite - https://gitlab.com/cznic/sqlite
+	_ "modernc.org/sqlite"
+
+	// rqlite - https://github.com/rqlite/gorqlite | https://rqlite.io/
+	_ "github.com/rqlite/gorqlite/stdlib"
+)
+
+var (
+	db           *sql.DB
+	dbFile       string
+	sqlDriver    string
+	dbLastAction time.Time
+
+	// zstd compression encoder & decoder
+	dbEncoder, _ = zstd.NewWriter(nil)
+	dbDecoder, _ = zstd.NewReader(nil)
+
+	temporaryFiles = []string{}
+)
+
+// InitDB will initialise the database
+func InitDB() error {
+	var (
+		dsn string
+		err error
+	)
+
+	p := config.Database
+
+	if p == "" {
+		// when no path is provided then we create a temporary file
+		// which will get deleted on Close(), SIGINT or SIGTERM
+		p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano())
+		// delete the Unix socket file on exit
+		AddTempFile(p)
+		sqlDriver = "sqlite"
+		dsn = p
+		logger.Log().Debugf("[db] using temporary database: %s", p)
+	} else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
+		sqlDriver = "rqlite"
+		dsn = p
+		logger.Log().Debugf("[db] opening rqlite database %s", p)
+	} else {
+		p = filepath.Clean(p)
+		sqlDriver = "sqlite"
+		dsn = fmt.Sprintf("file:%s?cache=shared", p)
+		logger.Log().Debugf("[db] opening database %s", p)
+	}
+
+	config.Database = p
+
+	if sqlDriver == "sqlite" {
+		if !isFile(p) {
+			// try create a file to ensure permissions
+			f, err := os.Create(p)
+			if err != nil {
+				return fmt.Errorf("[db] %s", err.Error())
+			}
+			_ = f.Close()
+		}
+	}
+
+	db, err = sql.Open(sqlDriver, dsn)
+	if err != nil {
+		return err
+	}
+
+	for i := 1; i < 6; i++ {
+		if err := Ping(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			logger.Log().Infof("[db] reconnecting in 5 seconds (attempt %d/5)", i)
+			time.Sleep(5 * time.Second)
+		} else {
+			continue
+		}
+	}
+
+	// prevent "database locked" errors
+	// @see https://github.com/mattn/go-sqlite3#faq
+	db.SetMaxOpenConns(1)
+
+	if sqlDriver == "sqlite" {
+		// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
+		_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
+		if err != nil {
+			return err
+		}
+	}
+
+	// create tables if necessary & apply migrations
+	if err := dbApplySchemas(); err != nil {
+		return err
+	}
+
+	LoadTagFilters()
+
+	dbFile = p
+	dbLastAction = time.Now()
+
+	sigs := make(chan os.Signal, 1)
+	// catch all signals since not explicitly listing
+	// Program that will listen to the SIGINT and SIGTERM
+	// SIGINT will listen to CTRL-C.
+	// SIGTERM will be caught if kill command executed
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
+	// method invoked upon seeing signal
+	go func() {
+		s := <-sigs
+		fmt.Printf("[db] got %s signal, shutting down\n", s)
+		Close()
+		os.Exit(0)
+	}()
+
+	// auto-prune & delete
+	go dbCron()
+
+	go dataMigrations()
+
+	return nil
+}
+
+// Tenant applies an optional prefix to the table name
+func tenant(table string) string {
+	return fmt.Sprintf("%s%s", config.TenantID, table)
+}
+
+// Close will close the database, and delete if temporary
+func Close() {
+	// on a fatal exit (eg: ports blocked), allow Mailpit to run migration tasks before closing the DB
+	time.Sleep(200 * time.Millisecond)
+
+	if db != nil {
+		if err := db.Close(); err != nil {
+			logger.Log().Warn("[db] error closing database, ignoring")
+		}
+	}
+
+	// allow SQLite to finish closing DB & write WAL logs if local
+	time.Sleep(100 * time.Millisecond)
+
+	// delete all temporary files
+	deleteTempFiles()
+}
+
+// Ping the database connection and return an error if unsuccessful
+func Ping() error {
+	return db.Ping()
+}
+
+// StatsGet returns the total/unread statistics for a mailbox
+func StatsGet() MailboxStats {
+	var (
+		total  = CountTotal()
+		unread = CountUnread()
+		tags   = GetAllTags()
+	)
+
+	dbLastAction = time.Now()
+
+	return MailboxStats{
+		Total:  total,
+		Unread: unread,
+		Tags:   tags,
+	}
+}
+
+// CountTotal returns the number of emails in the database
+func CountTotal() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// CountUnread returns the number of emails in the database that are unread.
+func CountUnread() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 0).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// CountRead returns the number of emails in the database that are read.
+func CountRead() float64 {
+	var total float64
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 1).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total
+}
+
+// DbSize returns the size of the SQLite database.
+func DbSize() float64 {
+	var total sql.NullFloat64
+
+	err := db.QueryRow("SELECT page_count * page_size AS size FROM pragma_page_count(), pragma_page_size()").Scan(&total)
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return total.Float64
+	}
+
+	return total.Float64
+}
+
+// IsUnread returns whether a message is unread or not.
+func IsUnread(id string) bool {
+	var unread int
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&unread).
+		Where("Read = ?", 0).
+		Where("ID = ?", id).
+		QueryRowAndClose(context.TODO(), db)
+
+	return unread == 1
+}
+
+// MessageIDExists checks whether a Message-ID exists in the DB
+func MessageIDExists(id string) bool {
+	var total int
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		Where("MessageID = ?", id).
+		QueryRowAndClose(context.TODO(), db)
+
+	return total != 0
+}

internal/storage/messages.go

@@ -0,0 +1,731 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/mail"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/webhook"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+// Store will save an email to the database tables.
+// Returns the database ID of the saved message.
+func Store(body *[]byte) (string, error) {
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	// Parse message body with enmime
+	env, err := parser.ReadEnvelope(bytes.NewReader(*body))
+	if err != nil {
+		logger.Log().Warnf("[message] %s", err.Error())
+		return "", nil
+	}
+
+	from := &mail.Address{}
+	fromJSON := addressToSlice(env, "From")
+	if len(fromJSON) > 0 {
+		from = fromJSON[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	obj := DBMailSummary{
+		From:    from,
+		To:      addressToSlice(env, "To"),
+		Cc:      addressToSlice(env, "Cc"),
+		Bcc:     addressToSlice(env, "Bcc"),
+		ReplyTo: addressToSlice(env, "Reply-To"),
+	}
+
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+	created := time.Now()
+
+	// use message date instead of created date
+	if config.UseMessageDates {
+		if mDate, err := env.Date(); err == nil {
+			created = mDate
+		}
+	}
+
+	// generate the search text
+	searchText := createSearchText(env)
+
+	// generate unique ID
+	id := shortuuid.New()
+
+	summaryJSON, err := json.Marshal(obj)
+	if err != nil {
+		return "", err
+	}
+
+	// begin a transaction to ensure both the message
+	// and data are stored successfully
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return "", err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	subject := env.GetHeader("Subject")
+	size := float64(len(*body))
+	inline := len(env.Inlines)
+	attachments := len(env.Attachments)
+	snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+	sql := fmt.Sprintf(`INSERT INTO %s 
+		(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) 
+		VALUES(?,?,?,?,?,?,?,?,?,0,?)`,
+		tenant("mailbox"),
+	) // #nosec
+
+	// insert mail summary data
+	_, err = tx.Exec(sql, created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet)
+	if err != nil {
+		return "", err
+	}
+
+	// insert compressed raw message
+	encoded := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size)))
+	hexStr := hex.EncodeToString(encoded)
+	_, err = tx.Exec(fmt.Sprintf(`INSERT INTO %s (ID, Email) VALUES(?, x'%s')`, tenant("mailbox_data"), hexStr), id) // #nosec
+	if err != nil {
+		return "", err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return "", err
+	}
+
+	// extract tags using pre-set tag filters, empty slice if not set
+	tags := findTagsInRawMessage(body)
+
+	if !config.TagsDisableXTags {
+		xTagsHdr := env.GetHeader("X-Tags")
+		if xTagsHdr != "" {
+			// extract tags from X-Tags header
+			tags = append(tags, tools.SetTagCasing(strings.Split(strings.TrimSpace(xTagsHdr), ","))...)
+		}
+	}
+
+	if !config.TagsDisablePlus {
+		// get tags from plus-addresses
+		tags = append(tags, obj.tagsFromPlusAddresses()...)
+	}
+
+	// extract tags from search matches, and sort and extract unique tags
+	tags = sortedUniqueTags(append(tags, tagFilterMatches(id)...))
+
+	setTags := []string{}
+	if len(tags) > 0 {
+		setTags, err = SetMessageTags(id, tags)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	c := &MessageSummary{}
+	if err := json.Unmarshal(summaryJSON, c); err != nil {
+		return "", err
+	}
+
+	c.Created = created
+	c.ID = id
+	c.MessageID = messageID
+	c.Attachments = attachments
+	c.Subject = subject
+	c.Size = size
+	c.Tags = setTags
+	c.Snippet = snippet
+
+	websockets.Broadcast("new", c)
+	webhook.Send(c)
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	logger.Log().Debugf("[db] saved message %s (%d bytes)", id, int64(size))
+
+	return id, nil
+}
+
+// List returns a subset of messages from the mailbox,
+// sorted latest to oldest
+func List(start int, beforeTS int64, limit int) ([]MessageSummary, error) {
+	results := []MessageSummary{}
+	tsStart := time.Now()
+
+	q := sqlf.From(tenant("mailbox") + " m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet`).
+		OrderBy("m.Created DESC")
+
+	if limit > 0 {
+		q = q.Limit(limit).Offset(start)
+	}
+
+	if beforeTS > 0 {
+		q = q.Where("Created < ?", beforeTS)
+	}
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var read int
+		var snippet string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[json] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(int64(created))
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+		// artificially generate ReplyTo if legacy data is missing Reply-To field
+		if em.ReplyTo == nil {
+			em.ReplyTo = []*mail.Address{}
+		}
+
+		results = append(results, em)
+	}); err != nil {
+		return results, err
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] list INBOX in %s", elapsed)
+
+	return results, nil
+}
+
+// GetMessage returns a Message generated from the mailbox_data collection.
+// If the message lacks a date header, then the received datetime is used.
+func GetMessage(id string) (*Message, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	env, err := parser.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var from *mail.Address
+	fromData := addressToSlice(env, "From")
+	if len(fromData) > 0 {
+		from = fromData[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
+	returnPath := strings.Trim(env.GetHeader("Return-Path"), "<>")
+	if returnPath == "" && from != nil {
+		returnPath = from.Address
+	}
+
+	date, err := env.Date()
+	if err != nil {
+		// return received datetime when message does not contain a date header
+		q := sqlf.From(tenant("mailbox")).
+			Select(`Created`).
+			Where(`ID = ?`, id)
+
+		if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			var created float64
+
+			if err := row.Scan(&created); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+
+			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
+
+			date = time.UnixMilli(int64(created))
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	obj := Message{
+		ID:         id,
+		MessageID:  messageID,
+		From:       from,
+		Date:       date,
+		To:         addressToSlice(env, "To"),
+		Cc:         addressToSlice(env, "Cc"),
+		Bcc:        addressToSlice(env, "Bcc"),
+		ReplyTo:    addressToSlice(env, "Reply-To"),
+		ReturnPath: returnPath,
+		Subject:    env.GetHeader("Subject"),
+		Tags:       getMessageTags(id),
+		Size:       float64(len(raw)),
+		Text:       env.Text,
+	}
+
+	obj.HTML = env.HTML
+	obj.Inline = []Attachment{}
+	obj.Attachments = []Attachment{}
+
+	for _, i := range env.Inlines {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, i := range env.OtherParts {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.FileName != "" || a.ContentID != "" {
+			obj.Attachments = append(obj.Attachments, AttachmentSummary(a))
+		}
+	}
+
+	// get List-Unsubscribe links if set
+	obj.ListUnsubscribe = ListUnsubscribe{}
+	obj.ListUnsubscribe.Links = []string{}
+	if env.GetHeader("List-Unsubscribe") != "" {
+		l := env.GetHeader("List-Unsubscribe")
+		links, err := tools.ListUnsubscribeParser(l)
+		obj.ListUnsubscribe.Header = l
+		obj.ListUnsubscribe.Links = links
+		if err != nil {
+			obj.ListUnsubscribe.Errors = err.Error()
+		}
+		obj.ListUnsubscribe.HeaderPost = env.GetHeader("List-Unsubscribe-Post")
+	}
+
+	// mark message as read
+	if err := MarkRead(id); err != nil {
+		return &obj, err
+	}
+
+	dbLastAction = time.Now()
+
+	return &obj, nil
+}
+
+// GetMessageRaw returns an []byte of the full message
+func GetMessageRaw(id string) ([]byte, error) {
+	var i string
+	var msg string
+	q := sqlf.From(tenant("mailbox_data")).
+		Select(`ID`).To(&i).
+		Select(`Email`).To(&msg).
+		Where(`ID = ?`, id)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return nil, err
+	}
+
+	if i == "" {
+		return nil, errors.New("message not found")
+	}
+
+	var data []byte
+	if sqlDriver == "rqlite" {
+		data, err = base64.StdEncoding.DecodeString(msg)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding base64 message: %w", err)
+		}
+	} else {
+		data = []byte(msg)
+	}
+
+	raw, err := dbDecoder.DecodeAll(data, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error decompressing message: %s", err.Error())
+	}
+
+	dbLastAction = time.Now()
+
+	return raw, err
+}
+
+// GetAttachmentPart returns an *enmime.Part (attachment or inline) from a message
+func GetAttachmentPart(id, partID string) (*enmime.Part, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	env, err := parser.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range env.Inlines {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.OtherParts {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	dbLastAction = time.Now()
+
+	return nil, errors.New("attachment not found")
+}
+
+// AttachmentSummary returns a summary of the attachment without any binary data
+func AttachmentSummary(a *enmime.Part) Attachment {
+	o := Attachment{}
+	o.PartID = a.PartID
+	o.FileName = a.FileName
+	if o.FileName == "" {
+		o.FileName = a.ContentID
+	}
+	o.ContentType = a.ContentType
+	o.ContentID = a.ContentID
+	o.Size = float64(len(a.Content))
+
+	return o
+}
+
+// LatestID returns the latest message ID
+//
+// If a query argument is set in the request the function will return the
+// latest message matching the search
+func LatestID(r *http.Request) (string, error) {
+	var messages []MessageSummary
+	var err error
+
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search != "" {
+		messages, _, err = Search(search, r.URL.Query().Get("tz"), 0, 0, 1)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		messages, err = List(0, 0, 1)
+		if err != nil {
+			return "", err
+		}
+	}
+	if len(messages) == 0 {
+		return "", errors.New("Message not found")
+	}
+
+	return messages[0].ID, nil
+}
+
+// MarkRead will mark a message as read
+func MarkRead(id string) error {
+	if !IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 1).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as read", id)
+	}
+
+	BroadcastMailboxStats()
+
+	d := struct {
+		ID   string
+		Read bool
+	}{ID: id, Read: true}
+
+	websockets.Broadcast("update", d)
+
+	return err
+}
+
+// MarkAllRead will mark all messages as read
+func MarkAllRead() error {
+	var (
+		start = time.Now()
+		total = CountUnread()
+	)
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 1).
+		Where("Read = ?", 0).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %v messages as read in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkAllUnread will mark all messages as unread
+func MarkAllUnread() error {
+	var (
+		start = time.Now()
+		total = CountRead()
+	)
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 0).
+		Where("Read = ?", 1).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %v messages as unread in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkUnread will mark a message as unread
+func MarkUnread(id string) error {
+	if IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update(tenant("mailbox")).
+		Set("Read", 0).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as unread", id)
+	}
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	d := struct {
+		ID   string
+		Read bool
+	}{ID: id, Read: false}
+
+	websockets.Broadcast("update", d)
+
+	return err
+}
+
+// DeleteMessages deletes one or more messages in bulk
+func DeleteMessages(ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	start := time.Now()
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	sql := fmt.Sprintf(`SELECT ID, Size FROM %s WHERE  ID IN (?%s)`, tenant("mailbox"), strings.Repeat(",?", len(args)-1)) // #nosec
+	rows, err := db.Query(sql, args...)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	toDelete := []string{}
+	var totalSize float64
+
+	for rows.Next() {
+		var id string
+		var size float64
+		if err := rows.Scan(&id, &size); err != nil {
+			return err
+		}
+		toDelete = append(toDelete, id)
+		totalSize = totalSize + size
+	}
+
+	if err = rows.Err(); err != nil {
+		return err
+	}
+
+	if len(toDelete) == 0 {
+		return nil // nothing to delete
+	}
+
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	args = make([]interface{}, len(toDelete))
+	for i, id := range toDelete {
+		args[i] = id
+	}
+
+	tables := []string{"mailbox", "mailbox_data", "message_tags"}
+
+	for _, t := range tables {
+		sql = fmt.Sprintf(`DELETE FROM %s WHERE ID IN (?%s)`, tenant(t), strings.Repeat(",?", len(ids)-1))
+
+		_, err = tx.Exec(sql, args...) // #nosec
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	dbLastAction = time.Now()
+	addDeletedSize(int64(totalSize))
+
+	logMessagesDeleted(len(toDelete))
+
+	_ = pruneUnusedTags()
+
+	elapsed := time.Since(start)
+
+	messages := "messages"
+	if len(toDelete) == 1 {
+		messages = "message"
+	}
+
+	logger.Log().Debugf("[db] deleted %d %s in %s", len(toDelete), messages, elapsed)
+
+	BroadcastMailboxStats()
+
+	// broadcast individual message deletions
+	for _, id := range toDelete {
+		d := struct {
+			ID string
+		}{ID: id}
+
+		websockets.Broadcast("delete", d)
+	}
+
+	return nil
+}
+
+// DeleteAllMessages will delete all messages from a mailbox
+func DeleteAllMessages() error {
+	var (
+		start = time.Now()
+		total int
+	)
+
+	_ = sqlf.From(tenant("mailbox")).
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(context.TODO(), db)
+
+	// begin a transaction to ensure both the message
+	// summaries and data are deleted successfully
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	tables := []string{"mailbox", "mailbox_data", "tags", "message_tags"}
+
+	for _, t := range tables {
+		sql := fmt.Sprintf(`DELETE FROM %s`, tenant(t)) // #nosec
+		_, err := tx.Exec(sql)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed)
+
+	vacuumDb()
+
+	dbLastAction = time.Now()
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Warnf("[db] %s", err.Error())
+	}
+
+	logMessagesDeleted(total)
+
+	BroadcastMailboxStats()
+
+	websockets.Broadcast("truncate", nil)
+
+	return err
+}

internal/storage/messages_test.go

@@ -0,0 +1,198 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+)
+
+func TestTextEmailInserts(t *testing.T) {
+	setup("")
+	defer Close()
+
+	t.Log("Testing text email storage")
+
+	start := time.Now()
+
+	for i := 0; i < testRuns; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of text emails stored")
+
+	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+	delStart := time.Now()
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, CountTotal(), float64(0), "incorrect number of text emails deleted")
+
+	t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
+
+	assertEqualStats(t, 0, 0)
+}
+
+func TestMimeEmailInserts(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing mime email storage")
+		} else {
+			t.Logf("Testing mime email storage (tenant %s)", tenantID)
+		}
+
+		start := time.Now()
+
+		for i := 0; i < testRuns; i++ {
+			if _, err := Store(&testMimeEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of mime emails stored")
+
+		t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+		delStart := time.Now()
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, CountTotal(), float64(0), "incorrect number of mime emails deleted")
+
+		t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart))
+
+		Close()
+	}
+}
+
+func TestRetrieveMimeEmail(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing mime email retrieval")
+		} else {
+			t.Logf("Testing mime email retrieval (tenant %s)", tenantID)
+		}
+
+		id, err := Store(&testMimeEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		msg, err := GetMessage(id)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+		assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+		assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+		assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+		assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+		assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+		assertEqual(t, len(msg.Attachments), 1, "incorrect number of attachments")
+		assertEqual(t, msg.Attachments[0].FileName, "Sample PDF.pdf", "attachment filename does not match")
+		assertEqual(t, len(msg.Inline), 1, "incorrect number of inline attachments")
+		assertEqual(t, msg.Inline[0].FileName, "inline-image.jpg", "inline attachment filename does not match")
+
+		attachmentData, err := GetAttachmentPart(id, msg.Attachments[0].PartID)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		assertEqual(t, float64(len(attachmentData.Content)), msg.Attachments[0].Size, "attachment size does not match")
+
+		inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		assertEqual(t, float64(len(inlineData.Content)), msg.Inline[0].Size, "inline attachment size does not match")
+
+		Close()
+	}
+}
+
+func TestMessageSummary(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing message summary")
+		} else {
+			t.Logf("Testing message summary (tenant %s)", tenantID)
+		}
+
+		if _, err := Store(&testMimeEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		summaries, err := List(0, 0, 1)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, len(summaries), 1, "Expected 1 result")
+
+		msg := summaries[0]
+
+		assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+		assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+		assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+		assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+		assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+		assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+		assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match")
+		assertEqual(t, msg.Attachments, 1, "Expected 1 attachment")
+		assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match")
+
+		Close()
+	}
+}
+
+func BenchmarkImportText(b *testing.B) {
+	setup("")
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+}
+
+func BenchmarkImportMime(b *testing.B) {
+	setup("")
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testMimeEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+
+}

internal/storage/notifications.go

@@ -0,0 +1,38 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+var bcStatsDelay = false
+
+// BroadcastMailboxStats broadcasts the total number of messages
+// displayed to the web UI, as well as the total unread messages.
+// The lookup is very fast (< 10ms / 100k messages under load).
+// Rate limited to 4x per second.
+func BroadcastMailboxStats() {
+	if bcStatsDelay {
+		return
+	}
+
+	bcStatsDelay = true
+
+	go func() {
+		time.Sleep(250 * time.Millisecond)
+		bcStatsDelay = false
+		b := struct {
+			Total   float64
+			Unread  float64
+			Version string
+		}{
+			Total:   CountTotal(),
+			Unread:  CountUnread(),
+			Version: config.Version,
+		}
+
+		websockets.Broadcast("stats", b)
+	}()
+}

internal/storage/reindex.go

@@ -0,0 +1,146 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/mail"
+	"os"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+)
+
+// ReindexAll will regenerate the search text and snippet for a message
+// and update the database.
+func ReindexAll() {
+	ids := []string{}
+	var i string
+	chunkSize := 1000
+
+	finished := 0
+
+	err := sqlf.Select("ID").To(&i).
+		From(tenant("mailbox")).
+		OrderBy("Created DESC").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			ids = append(ids, i)
+		})
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		os.Exit(1)
+	}
+
+	total := len(ids)
+
+	chunks := chunkBy(ids, chunkSize)
+
+	logger.Log().Infof("reindexing %d messages", total)
+
+	type updateStruct struct {
+		// ID in database
+		ID string
+		// SearchText for searching
+		SearchText string
+		// Snippet for UI
+		Snippet string
+		// Metadata info
+		Metadata string
+	}
+
+	parser := enmime.NewParser(enmime.DisableCharacterDetection(true))
+
+	for _, ids := range chunks {
+		updates := []updateStruct{}
+
+		for _, id := range ids {
+			raw, err := GetMessageRaw(id)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+
+			r := bytes.NewReader(raw)
+
+			env, err := parser.ReadEnvelope(r)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			from := &mail.Address{}
+			fromJSON := addressToSlice(env, "From")
+			if len(fromJSON) > 0 {
+				from = fromJSON[0]
+			} else if env.GetHeader("From") != "" {
+				from = &mail.Address{Name: env.GetHeader("From")}
+			}
+
+			obj := DBMailSummary{
+				From:    from,
+				To:      addressToSlice(env, "To"),
+				Cc:      addressToSlice(env, "Cc"),
+				Bcc:     addressToSlice(env, "Bcc"),
+				ReplyTo: addressToSlice(env, "Reply-To"),
+			}
+
+			MetadataJSON, err := json.Marshal(obj)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			searchText := createSearchText(env)
+			snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+			u := updateStruct{}
+			u.ID = id
+			u.SearchText = searchText
+			u.Snippet = snippet
+			u.Metadata = string(MetadataJSON)
+
+			updates = append(updates, u)
+		}
+
+		ctx := context.Background()
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		// insert mail summary data
+		for _, u := range updates {
+			_, err = tx.Exec(fmt.Sprintf(`UPDATE %s SET SearchText = ?, Snippet = ?, Metadata = ? WHERE ID = ?`, tenant("mailbox")), u.SearchText, u.Snippet, u.Metadata, u.ID)
+			if err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				continue
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		finished += len(updates)
+
+		logger.Log().Printf("reindexed: %d / %d (%d%%)", finished, total, finished*100/total)
+	}
+}
+
+func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) {
+	for chunkSize < len(items) {
+		items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
+	}
+
+	return append(chunks, items)
+}

internal/storage/schemas.go

@@ -0,0 +1,152 @@
+package storage
+
+import (
+	"bytes"
+	"embed"
+	"log"
+	"path"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/semver"
+)
+
+//go:embed schemas/*
+var schemaScripts embed.FS
+
+// Create tables and apply schemas if required
+func dbApplySchemas() error {
+	if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS ` + tenant("schemas") + ` (Version TEXT PRIMARY KEY NOT NULL)`); err != nil {
+		return err
+	}
+
+	var legacyMigrationTable int
+	err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)`, tenant("darwin_migrations")).Scan(&legacyMigrationTable)
+	if err != nil {
+		return err
+	}
+
+	if legacyMigrationTable == 1 {
+		rows, err := db.Query(`SELECT version FROM ` + tenant("darwin_migrations"))
+		if err != nil {
+			return err
+		}
+
+		legacySchemas := []string{}
+
+		for rows.Next() {
+			var oldID string
+			if err := rows.Scan(&oldID); err == nil {
+				legacySchemas = append(legacySchemas, semver.MajorMinor(oldID)+"."+semver.Patch(oldID))
+			}
+		}
+
+		legacySchemas = semver.SortMin(legacySchemas)
+
+		for _, v := range legacySchemas {
+			var migrated int
+			err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, v).Scan(&migrated)
+			if err != nil {
+				return err
+			}
+			if migrated == 0 {
+				// copy to tenant("schemas")
+				if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, v); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	schemaFiles, err := schemaScripts.ReadDir("schemas")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	temp := template.New("")
+	temp.Funcs(
+		template.FuncMap{
+			"tenant": tenant,
+		},
+	)
+
+	type schema struct {
+		Name   string
+		Semver string
+	}
+
+	scripts := []schema{}
+
+	for _, s := range schemaFiles {
+		if !s.Type().IsRegular() || !strings.HasSuffix(s.Name(), ".sql") {
+			continue
+		}
+
+		schemaID := strings.TrimRight(s.Name(), ".sql")
+
+		if !semver.IsValid(schemaID) {
+			logger.Log().Warnf("[db] invalid schema name: %s", s.Name())
+			continue
+		}
+
+		script := schema{s.Name(), semver.MajorMinor(schemaID) + "." + semver.Patch(schemaID)}
+		scripts = append(scripts, script)
+	}
+
+	// sort schemas by semver, low to high
+	sort.Slice(scripts, func(i, j int) bool {
+		return semver.Compare(scripts[j].Semver, scripts[i].Semver) == 1
+	})
+
+	for _, s := range scripts {
+		var complete int
+		err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, s.Semver).Scan(&complete)
+		if err != nil {
+			return err
+		}
+
+		if complete == 1 {
+			// already completed, ignore
+			continue
+		}
+		// use path.Join for Windows compatibility, see https://github.com/golang/go/issues/44305
+		b, err := schemaScripts.ReadFile(path.Join("schemas", s.Name))
+		if err != nil {
+			return err
+		}
+
+		// parse import script
+		t1, err := temp.Parse(string(b))
+		if err != nil {
+			return err
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := t1.Execute(buf, nil); err != nil {
+			return err
+		}
+
+		if _, err := db.Exec(buf.String()); err != nil {
+			return err
+		}
+
+		if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, s.Semver); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("[db] applied schema: %s", s.Name)
+	}
+
+	return nil
+}
+
+// These functions are used to migrate data formats/structure on startup.
+func dataMigrations() {
+	// ensure DeletedSize has a value if empty
+	if SettingGet("DeletedSize") == "" {
+		_ = SettingPut("DeletedSize", "0")
+	}
+}

internal/storage/schemas/1.0.0.sql

@@ -0,0 +1,19 @@
+-- CREATE TABLES
+CREATE TABLE IF NOT EXISTS {{ tenant "mailbox" }} (
+	Sort INTEGER PRIMARY KEY AUTOINCREMENT,
+	ID TEXT NOT NULL,
+	Data BLOB,
+	Search TEXT,
+	Read INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_sort" }} ON {{ tenant "mailbox" }} (Sort);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read);
+
+CREATE TABLE IF NOT EXISTS {{ tenant "mailbox_data" }} (
+	ID TEXT KEY NOT NULL,
+	Email BLOB
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_data_id" }} ON {{ tenant "mailbox_data" }} (ID);

internal/storage/schemas/1.1.0.sql

@@ -0,0 +1,3 @@
+-- CREATE TAGS COLUMN
+ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Tags Text NOT NULL DEFAULT '[]';
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags);

internal/storage/schemas/1.2.0.sql

@@ -0,0 +1,36 @@
+-- CREATING NEW MAILBOX FORMAT
+CREATE TABLE IF NOT EXISTS {{ tenant "mailboxtmp" }} (
+	Created INTEGER NOT NULL,
+	ID TEXT NOT NULL,
+	MessageID TEXT NOT NULL,
+	Subject TEXT NOT NULL,
+	Metadata TEXT,
+	Size INTEGER NOT NULL,
+	Inline INTEGER NOT NULL,
+	Attachments INTEGER NOT NULL,
+	Read INTEGER,
+	Tags TEXT,
+	SearchText TEXT
+);
+
+INSERT INTO {{ tenant "mailboxtmp" }}
+	(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+	SELECT 
+		Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+		json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+		Search, Read, Tags
+	FROM {{ tenant "mailbox" }};
+
+DROP TABLE IF EXISTS {{ tenant "mailbox" }};
+
+ALTER TABLE {{ tenant "mailboxtmp" }} RENAME TO {{ tenant "mailbox" }};
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_created" }} ON {{ tenant "mailbox" }} (Created);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_id" }} ON {{ tenant "mailbox" }} (MessageID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_subject" }} ON {{ tenant "mailbox" }} (Subject);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_size" }} ON {{ tenant "mailbox" }} (Size);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_inline" }} ON {{ tenant "mailbox" }} (Inline);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_attachments" }} ON {{ tenant "mailbox" }} (Attachments);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags);

internal/storage/schemas/1.21.2.sql

@@ -0,0 +1,6 @@
+-- DROP LEGACY MIGRATION TABLE
+DROP TABLE IF EXISTS {{ tenant "darwin_migrations" }};
+
+-- DROP LEGACY TAGS COLUMN
+DROP INDEX IF EXISTS {{ tenant "idx_tags" }};
+ALTER TABLE {{ tenant "mailbox" }} DROP COLUMN Tags;

internal/storage/schemas/1.21.8.sql

@@ -0,0 +1,22 @@
+-- Rebuild message_tags to remove FOREIGN KEY REFERENCES
+PRAGMA foreign_keys=OFF;
+
+DROP INDEX IF EXISTS {{ tenant "idx_message_tag_id" }};
+DROP INDEX IF EXISTS {{ tenant "idx_message_tag_tagid" }};
+
+ALTER TABLE {{ tenant "message_tags" }} RENAME TO _{{ tenant "message_tags" }}_old;
+
+CREATE TABLE IF NOT EXISTS {{ tenant "message_tags" }} (
+	Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	ID TEXT NOT NULL,
+	TagID INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tags_id" }} ON {{ tenant "message_tags" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tags_tagid" }} ON {{ tenant "message_tags" }} (TagID);
+
+INSERT INTO {{ tenant "message_tags" }} SELECT * FROM _{{ tenant "message_tags" }}_old;
+
+DROP TABLE IF EXISTS _{{ tenant "message_tags" }}_old;
+
+PRAGMA foreign_keys=ON;

internal/storage/schemas/1.3.0.sql

@@ -0,0 +1,2 @@
+-- CREATE SNIPPET COLUMN
+ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Snippet TEXT NOT NULL DEFAULT '';

internal/storage/schemas/1.4.0.sql

@@ -0,0 +1,16 @@
+-- CREATE TAG TABLES
+CREATE TABLE IF NOT EXISTS {{ tenant "tags" }} (
+	ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	Name TEXT COLLATE NOCASE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_tag_name" }} ON {{ tenant "tags" }} (Name);
+
+CREATE TABLE IF NOT EXISTS {{ tenant "message_tags" }} (
+	Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	ID TEXT REFERENCES {{ tenant "mailbox" }} (ID),
+	TagID INT REFERENCES {{ tenant "tags" }} (ID)
+);
+
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_id" }} ON {{ tenant "message_tags" }} (ID);
+CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_tagid" }} ON {{ tenant "message_tags" }} (TagID);

internal/storage/schemas/1.5.0.sql

@@ -0,0 +1,7 @@
+-- CREATE SETTINGS TABLE
+CREATE TABLE IF NOT EXISTS {{ tenant "settings" }} (
+	Key TEXT,
+	Value TEXT
+);
+CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_settings_key" }} ON {{ tenant "settings" }} (Key);
+INSERT INTO {{ tenant "settings" }} (Key, Value) VALUES ("DeletedSize", (SELECT SUM(Size)/2 FROM {{ tenant "mailbox" }}));

internal/storage/schemas/README.md

@@ -0,0 +1,5 @@
+# Migration scripts
+
+- Scripts should be named using semver and have the `.sql` extension.
+- Inline comments should be prefixed with a `--`
+- All references to tables and indexes should be wrapped with a `{{ tenant "<name>" }}`

internal/storage/search.go

@@ -0,0 +1,478 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/araddon/dateparse"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+// Search will search a mailbox for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func Search(search, timezone string, start int, beforeTS int64, limit int) ([]MessageSummary, int, error) {
+	results := []MessageSummary{}
+	allResults := []MessageSummary{}
+	tsStart := time.Now()
+	nrResults := 0
+	if limit < 0 {
+		limit = 50
+	}
+
+	q := searchQueryBuilder(search, timezone)
+
+	if beforeTS > 0 {
+		q = q.Where(`Created < ?`, beforeTS)
+	}
+
+	var err error
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var snippet string
+		var read int
+		var ignore string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(int64(created))
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+
+		allResults = append(allResults, em)
+	}); err != nil {
+		return results, nrResults, err
+	}
+
+	dbLastAction = time.Now()
+
+	nrResults = len(allResults)
+
+	if nrResults > start {
+		end := nrResults
+		if nrResults >= start+limit {
+			end = start + limit
+		}
+
+		results = allResults[start:end]
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
+
+	return results, nrResults, err
+}
+
+// DeleteSearch will delete all messages for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func DeleteSearch(search, timezone string) error {
+	q := searchQueryBuilder(search, timezone)
+
+	ids := []string{}
+	deleteSize := float64(0)
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var created float64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size float64
+		var attachments int
+		var read int
+		var snippet string
+		var ignore string
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		ids = append(ids, id)
+		deleteSize = deleteSize + size
+	}); err != nil {
+		return err
+	}
+
+	if len(ids) > 0 {
+		total := len(ids)
+
+		// split ids into chunks of 1000 ids
+		var chunks [][]string
+		if total > 1000 {
+			chunkSize := 1000
+			chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize)
+			for chunkSize < len(ids) {
+				ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize])
+			}
+			if len(ids) > 0 {
+				// add remaining ids <= 1000
+				chunks = append(chunks, ids)
+			}
+		} else {
+			chunks = append(chunks, ids)
+		}
+
+		// begin a transaction to ensure both the message
+		// and data are deleted successfully
+		tx, err := db.BeginTx(context.Background(), nil)
+		if err != nil {
+			return err
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		for _, ids := range chunks {
+			delIDs := make([]interface{}, len(ids))
+			for i, id := range ids {
+				delIDs[i] = id
+			}
+
+			sqlDelete1 := `DELETE FROM ` + tenant("mailbox") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete1, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete2 := `DELETE FROM ` + tenant("mailbox_data") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete2, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete3 := `DELETE FROM ` + tenant("message_tags") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete3, delIDs...)
+			if err != nil {
+				return err
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			return err
+		}
+
+		if err := pruneUnusedTags(); err != nil {
+			return err
+		}
+
+		logger.Log().Debugf("[db] deleted %d messages matching %s", total, search)
+
+		dbLastAction = time.Now()
+
+		// broadcast changes
+		if len(ids) > 200 {
+			websockets.Broadcast("prune", nil)
+		} else {
+			for _, id := range ids {
+				d := struct {
+					ID string
+				}{ID: id}
+				websockets.Broadcast("delete", d)
+			}
+		}
+
+		addDeletedSize(int64(deleteSize))
+
+		logMessagesDeleted(total)
+
+		BroadcastMailboxStats()
+	}
+
+	return nil
+}
+
+// SearchParser returns the SQL syntax for the database search based on the search arguments
+func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt {
+	// group strings with quotes as a single argument and remove quotes
+	args := tools.ArgsParser(searchString)
+
+	if timezone != "" {
+		loc, err := time.LoadLocation(timezone)
+		if err != nil {
+			logger.Log().Warnf("ignoring invalid timezone:\"%s\"", timezone)
+		} else {
+			time.Local = loc
+		}
+	}
+
+	q := sqlf.From(tenant("mailbox") + " m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read,
+			m.Snippet,
+			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
+			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
+			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
+			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON,
+			IFNULL(json_extract(Metadata, '$.ReplyTo'), '{}') as ReplyToJSON
+		`).
+		OrderBy("m.Created DESC")
+
+	for _, w := range args {
+		if cleanString(w) == "" {
+			continue
+		}
+
+		// lowercase search to try match search prefixes
+		lw := strings.ToLower(w)
+
+		exclude := false
+		// search terms starting with a `-` or `!` imply an exclude
+		if len(w) > 1 && (strings.HasPrefix(w, "-") || strings.HasPrefix(w, "!")) {
+			exclude = true
+			w = w[1:]
+			lw = lw[1:]
+		}
+
+		re := regexp.MustCompile(`[a-zA-Z0-9]+`)
+		if !re.MatchString(w) {
+			continue
+		}
+
+		if strings.HasPrefix(lw, "to:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("ToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "from:") {
+			w = cleanString(w[5:])
+			if w != "" {
+				if exclude {
+					q.Where("FromJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("FromJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "cc:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("CcJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("CcJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "bcc:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where("BccJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("BccJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "reply-to:") {
+			w = cleanString(w[9:])
+			if w != "" {
+				if exclude {
+					q.Where("ReplyToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ReplyToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "addressed:") {
+			w = cleanString(w[10:])
+			arg := "%" + escPercentChar(w) + "%"
+			if w != "" {
+				if exclude {
+					q.Where("(ToJSON NOT LIKE ? AND FromJSON NOT LIKE ? AND CcJSON NOT LIKE ? AND BccJSON NOT LIKE ? AND ReplyToJSON NOT LIKE ?)", arg, arg, arg, arg, arg)
+				} else {
+					q.Where("(ToJSON LIKE ? OR FromJSON LIKE ? OR CcJSON LIKE ? OR BccJSON LIKE ? OR ReplyToJSON LIKE ?)", arg, arg, arg, arg, arg)
+				}
+			}
+		} else if strings.HasPrefix(lw, "subject:") {
+			w = w[8:]
+			if w != "" {
+				if exclude {
+					q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "message-id:") {
+			w = cleanString(w[11:])
+			if w != "" {
+				if exclude {
+					q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "tag:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where(`m.ID NOT IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				} else {
+					q.Where(`m.ID IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				}
+			}
+		} else if lw == "is:read" {
+			if exclude {
+				q.Where("Read = 0")
+			} else {
+				q.Where("Read = 1")
+			}
+		} else if lw == "is:unread" {
+			if exclude {
+				q.Where("Read = 1")
+			} else {
+				q.Where("Read = 0")
+			}
+		} else if lw == "is:tagged" {
+			if exclude {
+				q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`)
+			} else {
+				q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`)
+			}
+		} else if lw == "has:inline" || lw == "has:inlines" {
+			if exclude {
+				q.Where("Inline = 0")
+			} else {
+				q.Where("Inline > 0")
+			}
+		} else if lw == "has:attachment" || lw == "has:attachments" {
+			if exclude {
+				q.Where("Attachments = 0")
+			} else {
+				q.Where("Attachments > 0")
+			}
+		} else if strings.HasPrefix(lw, "after:") {
+			w = cleanString(w[6:])
+			if w != "" {
+				t, err := dateparse.ParseLocal(w)
+				if err != nil {
+					logger.Log().Warnf("ignoring invalid after: date \"%s\"", w)
+				} else {
+					timestamp := t.UnixMilli()
+					if exclude {
+						q.Where(`m.Created <= ?`, timestamp)
+					} else {
+						q.Where(`m.Created >= ?`, timestamp)
+					}
+				}
+			}
+		} else if strings.HasPrefix(lw, "before:") {
+			w = cleanString(w[7:])
+			if w != "" {
+				t, err := dateparse.ParseLocal(w)
+				if err != nil {
+					logger.Log().Warnf("ignoring invalid before: date \"%s\"", w)
+				} else {
+					timestamp := t.UnixMilli()
+					if exclude {
+						q.Where(`m.Created >= ?`, timestamp)
+					} else {
+						q.Where(`m.Created <= ?`, timestamp)
+					}
+				}
+			}
+		} else if strings.HasPrefix(lw, "larger:") && sizeToBytes(cleanString(w[7:])) > 0 {
+			w = cleanString(w[7:])
+			size := sizeToBytes(w)
+			if exclude {
+				q.Where("Size < ?", size)
+			} else {
+				q.Where("Size > ?", size)
+			}
+		} else if strings.HasPrefix(lw, "smaller:") && sizeToBytes(cleanString(w[8:])) > 0 {
+			w = cleanString(w[8:])
+			size := sizeToBytes(w)
+			if exclude {
+				q.Where("Size > ?", size)
+			} else {
+				q.Where("Size < ?", size)
+			}
+		} else {
+			// search text
+			if exclude {
+				q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			} else {
+				q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			}
+		}
+	}
+
+	return q
+}
+
+// Simple function to return a size in bytes, eg 2kb, 4MB or 1.5m.
+//
+// K, k, Kb, KB, kB and kb are treated as Kilobytes.
+// M, m, Mb, MB and mb are treated as Megabytes.
+func sizeToBytes(v string) int64 {
+	v = strings.ToLower(v)
+	re := regexp.MustCompile(`^(\d+)(\.\d+)?\s?([a-z]{1,2})?$`)
+
+	m := re.FindAllStringSubmatch(v, -1)
+	if len(m) == 0 {
+		return 0
+	}
+
+	val := fmt.Sprintf("%s%s", m[0][1], m[0][2])
+	unit := m[0][3]
+
+	i, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
+	if err != nil {
+		return 0
+	}
+
+	if unit == "" {
+		return int64(i)
+	}
+
+	if unit == "k" || unit == "kb" {
+		return int64(i * 1024)
+	}
+
+	if unit == "m" || unit == "mb" {
+		return int64(i * 1024 * 1024)
+	}
+
+	return 0
+}

internal/storage/search_test.go

@@ -0,0 +1,225 @@
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/jhillyerd/enmime"
+)
+
+func TestSearch(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing search")
+		} else {
+			t.Logf("Testing search (tenant %s)", tenantID)
+		}
+
+		for i := 0; i < testRuns; i++ {
+			msg := enmime.Builder().
+				From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+				CC(fmt.Sprintf("CC %d", i), fmt.Sprintf("cc-%d@example.com", i)).
+				CC(fmt.Sprintf("CC2 %d", i), fmt.Sprintf("cc2-%d@example.com", i)).
+				Subject(fmt.Sprintf("Subject line %d end", i)).
+				Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+				To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i)).
+				To(fmt.Sprintf("To2 %d", i), fmt.Sprintf("to2-%d@example.com", i)).
+				ReplyTo(fmt.Sprintf("Reply To %d", i), fmt.Sprintf("reply-to-%d@example.com", i))
+
+			env, err := msg.Build()
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			buf := new(bytes.Buffer)
+
+			if err := env.Encode(buf); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			bufBytes := buf.Bytes()
+
+			if _, err := Store(&bufBytes); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		for i := 1; i < 51; i++ {
+			// search a random something that will return a single result
+			uniqueSearches := []string{
+				fmt.Sprintf("from-%d@example.com", i),
+				fmt.Sprintf("from:from-%d@example.com", i),
+				fmt.Sprintf("to-%d@example.com", i),
+				fmt.Sprintf("to:to-%d@example.com", i),
+				fmt.Sprintf("to2-%d@example.com", i),
+				fmt.Sprintf("to:to2-%d@example.com", i),
+				fmt.Sprintf("cc-%d@example.com", i),
+				fmt.Sprintf("cc:cc-%d@example.com", i),
+				fmt.Sprintf("cc2-%d@example.com", i),
+				fmt.Sprintf("cc:cc2-%d@example.com", i),
+				fmt.Sprintf("reply-to-%d@example.com", i),
+				fmt.Sprintf("reply-to:\"reply-to-%d@example.com\"", i),
+				fmt.Sprintf("\"Subject line %d end\"", i),
+				fmt.Sprintf("subject:\"Subject line %d end\"", i),
+				fmt.Sprintf("\"the email body %d jdsauk dwqmdqw\"", i),
+			}
+			searchIdx := rand.Intn(len(uniqueSearches))
+
+			search := uniqueSearches[searchIdx]
+
+			summaries, _, err := Search(search, "", 0, 0, 100)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			assertEqual(t, len(summaries), 1, "search result expected")
+
+			assertEqual(t, summaries[0].From.Name, fmt.Sprintf("From %d", i), "\"From\" name does not match")
+			assertEqual(t, summaries[0].From.Address, fmt.Sprintf("from-%d@example.com", i), "\"From\" address does not match")
+			assertEqual(t, summaries[0].To[0].Name, fmt.Sprintf("To %d", i), "\"To\" name does not match")
+			assertEqual(t, summaries[0].To[0].Address, fmt.Sprintf("to-%d@example.com", i), "\"To\" address does not match")
+			assertEqual(t, summaries[0].Subject, fmt.Sprintf("Subject line %d end", i), "\"Subject\" does not match")
+		}
+
+		// search something that will return 200 results
+		summaries, _, err := Search("This is the email body", "", 0, 0, testRuns)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		assertEqual(t, len(summaries), testRuns, "search results expected")
+
+		Close()
+	}
+}
+
+func TestSearchDelete100(t *testing.T) {
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing search delete of 100 messages")
+		} else {
+			t.Logf("Testing search delete of 100 messages (tenant %s)", tenantID)
+		}
+
+		for i := 0; i < 100; i++ {
+			if _, err := Store(&testTextEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			if _, err := Store(&testMimeEmail); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		_, total, err := Search("from:sender@example.com", "", 0, 0, 100)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, total, 100, "100 search results expected")
+
+		if err := DeleteSearch("from:sender@example.com", ""); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		_, total, err = Search("from:sender@example.com", "", 0, 0, 100)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, total, 0, "0 search results expected")
+
+		Close()
+	}
+}
+
+func TestSearchDelete1100(t *testing.T) {
+	setup("")
+	defer Close()
+
+	t.Log("Testing search delete of 1100 messages")
+	for i := 0; i < 1100; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	_, total, err := Search("from:sender@example.com", "", 0, 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 1100, "100 search results expected")
+
+	if err := DeleteSearch("from:sender@example.com", ""); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	_, total, err = Search("from:sender@example.com", "", 0, 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 0, "0 search results expected")
+}
+
+func TestEscPercentChar(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a test"] = "this is a test"
+	tests["this is% a test"] = "this is%% a test"
+	tests["this is%% a test"] = "this is%%%% a test"
+	tests["this is%%% a test"] = "this is%%%%%% a test"
+	tests["%this is% a test"] = "%%this is%% a test"
+	tests["Ä"] = "Ä"
+	tests["Ä%"] = "Ä%%"
+
+	for search, expected := range tests {
+		res := escPercentChar(search)
+		assertEqual(t, res, expected, "no match")
+	}
+}
+
+func TestSizeToBytes(t *testing.T) {
+	tests := map[string]int64{}
+	tests["1m"] = 1048576
+	tests["1mb"] = 1048576
+	tests["1 M"] = 1048576
+	tests["1 MB"] = 1048576
+	tests["1k"] = 1024
+	tests["1kb"] = 1024
+	tests["1 K"] = 1024
+	tests["1 kB"] = 1024
+	tests["1.5M"] = 1572864
+	tests["1234567890"] = 1234567890
+	tests["invalid"] = 0
+	tests["1.2.3"] = 0
+	tests["1.2.3M"] = 0
+
+	for search, expected := range tests {
+		res := sizeToBytes(search)
+		assertEqual(t, res, expected, "size does not match")
+	}
+}

internal/storage/settings.go

@@ -0,0 +1,76 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/leporo/sqlf"
+)
+
+// SettingGet returns a setting string value, blank is it does not exist
+func SettingGet(k string) string {
+	var result sql.NullString
+	err := sqlf.From(tenant("settings")).
+		Select("Value").To(&result).
+		Where("Key = ?", k).
+		Limit(1).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return ""
+	}
+
+	return result.String
+}
+
+// SettingPut sets a setting string value, inserting if new
+func SettingPut(k, v string) error {
+	_, err := db.Exec(`INSERT INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?) ON CONFLICT(Key) DO UPDATE SET Value = ?`, k, v, v)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return err
+}
+
+// The total deleted message size as an int64 value
+func getDeletedSize() float64 {
+	var result sql.NullFloat64
+	err := sqlf.From(tenant("settings")).
+		Select("Value").To(&result).
+		Where("Key = ?", "DeletedSize").
+		Limit(1).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Float64
+}
+
+// The total raw non-compressed messages size in bytes of all messages in the database
+func totalMessagesSize() float64 {
+	var result sql.NullFloat64
+	err := sqlf.From(tenant("mailbox")).
+		Select("SUM(Size)").To(&result).
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Float64
+}
+
+// AddDeletedSize will add the value to the DeletedSize setting
+func addDeletedSize(v int64) {
+	if _, err := db.Exec(`INSERT OR IGNORE INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?)`, "DeletedSize", 0); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	if _, err := db.Exec(`UPDATE `+tenant("settings")+` SET Value = Value + ? WHERE Key = ?`, v, "DeletedSize"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+}

internal/storage/structs.go

@@ -0,0 +1,126 @@
+package storage
+
+import (
+	"net/mail"
+	"time"
+)
+
+// Message data excluding physical attachments
+//
+// swagger:model Message
+type Message struct {
+	// Database ID
+	ID string
+	// Message ID
+	MessageID string
+	// From address
+	From *mail.Address
+	// To addresses
+	To []*mail.Address
+	// Cc addresses
+	Cc []*mail.Address
+	// Bcc addresses
+	Bcc []*mail.Address
+	// ReplyTo addresses
+	ReplyTo []*mail.Address
+	// Return-Path
+	ReturnPath string
+	// Message subject
+	Subject string
+	// List-Unsubscribe header information
+	// swagger:ignore
+	ListUnsubscribe ListUnsubscribe
+	// Message date if set, else date received
+	Date time.Time
+	// Message tags
+	Tags []string
+	// Message body text
+	Text string
+	// Message body HTML
+	HTML string
+	// Message size in bytes
+	Size float64
+	// Inline message attachments
+	Inline []Attachment
+	// Message attachments
+	Attachments []Attachment
+}
+
+// Attachment struct for inline and attachments
+//
+// swagger:model Attachment
+type Attachment struct {
+	// Attachment part ID
+	PartID string
+	// File name
+	FileName string
+	// Content type
+	ContentType string
+	// Content ID
+	ContentID string
+	// Size in bytes
+	Size float64
+}
+
+// MessageSummary struct for frontend messages
+//
+// swagger:model MessageSummary
+type MessageSummary struct {
+	// Database ID
+	ID string
+	// Message ID
+	MessageID string
+	// Read status
+	Read bool
+	// From address
+	From *mail.Address
+	// To address
+	To []*mail.Address
+	// Cc addresses
+	Cc []*mail.Address
+	// Bcc addresses
+	Bcc []*mail.Address
+	// Reply-To address
+	ReplyTo []*mail.Address
+	// Email subject
+	Subject string
+	// Created time
+	Created time.Time
+	// Message tags
+	Tags []string
+	// Message size in bytes (total)
+	Size float64
+	// Whether the message has any attachments
+	Attachments int
+	// Message snippet includes up to 250 characters
+	Snippet string
+}
+
+// MailboxStats struct for quick mailbox total/read lookups
+type MailboxStats struct {
+	Total  float64
+	Unread float64
+	Tags   []string
+}
+
+// DBMailSummary struct for storing mail summary
+type DBMailSummary struct {
+	From    *mail.Address
+	To      []*mail.Address
+	Cc      []*mail.Address
+	Bcc     []*mail.Address
+	ReplyTo []*mail.Address
+}
+
+// ListUnsubscribe contains a summary of List-Unsubscribe & List-Unsubscribe-Post headers
+// including validation of the link structure
+type ListUnsubscribe struct {
+	// List-Unsubscribe header value
+	Header string
+	// Detected links, maximum one email and one HTTP(S)
+	Links []string
+	// Validation errors if any
+	Errors string
+	// List-Unsubscribe-Post value if set
+	HeaderPost string
+}

internal/storage/tagfilters.go

@@ -0,0 +1,87 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/leporo/sqlf"
+)
+
+// TagFilter struct
+type TagFilter struct {
+	// Match is the user-defined match
+	Match string
+	// SQL represents the SQL equivalent of Match
+	SQL *sqlf.Stmt
+	// Tags to add on match
+	Tags []string
+}
+
+var tagFilters = []TagFilter{}
+
+// LoadTagFilters loads tag filters from the config and pre-generates the SQL query
+func LoadTagFilters() {
+	tagFilters = []TagFilter{}
+
+	for _, t := range config.TagFilters {
+		match := strings.TrimSpace(t.Match)
+		if match == "" {
+			logger.Log().Warnf("[tags] ignoring tag item with missing 'match'")
+			continue
+		}
+		if t.Tags == nil || len(t.Tags) == 0 {
+			logger.Log().Warnf("[tags] ignoring tag items with missing 'tags' array")
+			continue
+		}
+
+		validTags := []string{}
+		for _, tag := range t.Tags {
+			tagName := tools.CleanTag(tag)
+			if !config.ValidTagRegexp.MatchString(tagName) || len(tagName) == 0 {
+				logger.Log().Warnf("[tags] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tagName)
+				continue
+			}
+			validTags = append(validTags, tagName)
+		}
+
+		if len(validTags) == 0 {
+			continue
+		}
+
+		tagFilters = append(tagFilters, TagFilter{Match: match, Tags: validTags, SQL: searchQueryBuilder(match, "")})
+	}
+}
+
+// TagFilterMatches returns a slice of matching tags from a message
+func tagFilterMatches(id string) []string {
+	tags := []string{}
+
+	if len(tagFilters) == 0 {
+		return tags
+	}
+
+	for _, f := range tagFilters {
+		var matchID string
+		q := f.SQL.Clone().Where("ID = ?", id)
+		if err := q.QueryAndClose(context.Background(), db, func(row *sql.Rows) {
+			var ignore sql.NullString
+
+			if err := row.Scan(&ignore, &matchID, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return tags
+		}
+		if matchID == id {
+			tags = append(tags, f.Tags...)
+		}
+	}
+
+	return tags
+}

internal/storage/tags.go

@@ -0,0 +1,405 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+var (
+	addressPlusRe = regexp.MustCompile(`(?U)^(.*){1,}\+(.*)@`)
+	addTagMutex   sync.RWMutex
+)
+
+// SetMessageTags will set the tags for a given database ID, removing any not in the array
+func SetMessageTags(id string, tags []string) ([]string, error) {
+	applyTags := []string{}
+	for _, t := range tags {
+		t = tools.CleanTag(t)
+		if t != "" && config.ValidTagRegexp.MatchString(t) && !tools.InArray(t, applyTags) {
+			applyTags = append(applyTags, t)
+		}
+	}
+
+	tagNames := []string{}
+	currentTags := getMessageTags(id)
+	origTagCount := len(currentTags)
+
+	for _, t := range applyTags {
+		if t == "" || !config.ValidTagRegexp.MatchString(t) || tools.InArray(t, currentTags) {
+			continue
+		}
+
+		name, err := addMessageTag(id, t)
+		if err != nil {
+			return []string{}, err
+		}
+
+		tagNames = append(tagNames, name)
+	}
+
+	if origTagCount > 0 {
+		currentTags = getMessageTags(id)
+
+		for _, t := range currentTags {
+			if !tools.InArray(t, applyTags) {
+				if err := deleteMessageTag(id, t); err != nil {
+					return []string{}, err
+				}
+			}
+		}
+	}
+
+	d := struct {
+		ID   string
+		Tags []string
+	}{ID: id, Tags: applyTags}
+
+	websockets.Broadcast("update", d)
+
+	return tagNames, nil
+}
+
+// AddMessageTag adds a tag to a message
+func addMessageTag(id, name string) (string, error) {
+	// prevent two identical tags being added at the same time
+	addTagMutex.Lock()
+
+	var tagID int
+	var foundName sql.NullString
+
+	q := sqlf.From(tenant("tags")).
+		Select("ID").To(&tagID).
+		Select("Name").To(&foundName).
+		Where("Name = ?", name)
+
+	// if tag exists - add tag to message
+	if err := q.QueryRowAndClose(context.TODO(), db); err == nil {
+		addTagMutex.Unlock()
+		// check message does not already have this tag
+		var exists int
+
+		if err := sqlf.From(tenant("message_tags")).
+			Select("COUNT(ID)").To(&exists).
+			Where("ID = ?", id).
+			Where("TagID = ?", tagID).
+			QueryRowAndClose(context.Background(), db); err != nil {
+			return "", err
+		}
+		if exists > 0 {
+			// already exists
+			return foundName.String, nil
+		}
+
+		logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
+
+		_, err := sqlf.InsertInto(tenant("message_tags")).
+			Set("ID", id).
+			Set("TagID", tagID).
+			ExecAndClose(context.TODO(), db)
+
+		return foundName.String, err
+	}
+
+	// new tag, add to the database
+	if _, err := sqlf.InsertInto(tenant("tags")).
+		Set("Name", name).
+		ExecAndClose(context.TODO(), db); err != nil {
+		addTagMutex.Unlock()
+		return name, err
+	}
+
+	addTagMutex.Unlock()
+
+	// add tag to the message
+	return addMessageTag(id, name)
+}
+
+// DeleteMessageTag deletes a tag from a message
+func deleteMessageTag(id, name string) error {
+	if _, err := sqlf.DeleteFrom(tenant("message_tags")).
+		Where(tenant("message_tags.ID")+" = ?", id).
+		Where(tenant("message_tags.Key")+` IN (SELECT Key FROM `+tenant("message_tags")+` LEFT JOIN `+tenant("tags")+` ON TagID=`+tenant("tags.ID")+` WHERE Name = ?)`, name).
+		ExecAndClose(context.TODO(), db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// DeleteAllMessageTags deleted all tags from a message
+func DeleteAllMessageTags(id string) error {
+	if _, err := sqlf.DeleteFrom(tenant("message_tags")).
+		Where(tenant("message_tags.ID")+" = ?", id).
+		ExecAndClose(context.TODO(), db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// GetAllTags returns all used tags
+func GetAllTags() []string {
+	var tags = []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`DISTINCT Name`).
+		From(tenant("tags")).To(&name).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// GetAllTagsCount returns all used tags with their total messages
+func GetAllTagsCount() map[string]int64 {
+	var tags = make(map[string]int64)
+	var name string
+	var total int64
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		Select(`COUNT(`+tenant("message_tags.TagID")+`) as total`).To(&total).
+		From(tenant("tags")).
+		LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")).
+		GroupBy(tenant("message_tags.TagID")).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags[name] = total
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// RenameTag renames a tag
+func RenameTag(from, to string) error {
+	to = tools.CleanTag(to)
+	if to == "" || !config.ValidTagRegexp.MatchString(to) {
+		return fmt.Errorf("invalid tag name: %s", to)
+	}
+
+	if from == to {
+		return nil // ignore
+	}
+
+	var id, existsID int
+
+	q := sqlf.From(tenant("tags")).
+		Select(`ID`).To(&id).
+		Where(`Name = ?`, from).
+		Limit(1)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("tag not found: %s", from)
+	}
+
+	// check if another tag by this name already exists
+	q = sqlf.From(tenant("tags")).
+		Select("ID").To(&existsID).
+		Where(`Name = ?`, to).
+		Where(`ID != ?`, id).
+		Limit(1)
+	err = q.QueryRowAndClose(context.Background(), db)
+	if err == nil || existsID != 0 {
+		return fmt.Errorf("tag already exists: %s", to)
+	}
+
+	q = sqlf.Update(tenant("tags")).
+		Set("Name", to).
+		Where("ID = ?", id)
+	_, err = q.ExecAndClose(context.Background(), db)
+
+	return err
+}
+
+// DeleteTag deleted a tag and removed all references to the tag
+func DeleteTag(tag string) error {
+	var id int
+
+	q := sqlf.From(tenant("tags")).
+		Select(`ID`).To(&id).
+		Where(`Name = ?`, tag).
+		Limit(1)
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("tag not found: %s", tag)
+	}
+
+	// delete all references
+	q = sqlf.DeleteFrom(tenant("message_tags")).
+		Where(`TagID = ?`, id)
+	_, err = q.ExecAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("error deleting tag references: %s", err.Error())
+	}
+
+	// delete tag
+	q = sqlf.DeleteFrom(tenant("tags")).
+		Where(`ID = ?`, id)
+	_, err = q.ExecAndClose(context.Background(), db)
+	if err != nil {
+		return fmt.Errorf("error deleting tag: %s", err.Error())
+	}
+
+	return nil
+}
+
+// PruneUnusedTags will delete all unused tags from the database
+func pruneUnusedTags() error {
+	q := sqlf.From(tenant("tags")).
+		Select(tenant("tags.ID")+", "+tenant("tags.Name")+", COUNT("+tenant("message_tags.ID")+") as COUNT").
+		LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")).
+		GroupBy(tenant("tags.ID"))
+
+	toDel := []int{}
+
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+		var n string
+		var id int
+		var c int
+
+		if err := row.Scan(&id, &n, &c); err != nil {
+			logger.Log().Errorf("[tags] %s", err.Error())
+			return
+		}
+
+		if c == 0 {
+			logger.Log().Debugf("[tags] deleting unused tag \"%s\"", n)
+			toDel = append(toDel, id)
+		}
+	}); err != nil {
+		return err
+	}
+
+	if len(toDel) > 0 {
+		for _, id := range toDel {
+			if _, err := sqlf.DeleteFrom(tenant("tags")).
+				Where("ID = ?", id).
+				ExecAndClose(context.TODO(), db); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Find tags set via --tags in raw message, useful for matching all headers etc.
+// This function is largely superseded by the database searching, however this
+// includes literally everything and is kept for backwards compatibility.
+// Returns a comma-separated string.
+func findTagsInRawMessage(message *[]byte) []string {
+	tags := []string{}
+	if len(tagFilters) == 0 {
+		return tags
+	}
+
+	str := bytes.ToLower(*message)
+	for _, t := range tagFilters {
+		if bytes.Contains(str, []byte(t.Match)) {
+			tags = append(tags, t.Tags...)
+		}
+	}
+
+	return tags
+}
+
+// Returns tags found in email plus addresses (eg: test+tagname@example.com)
+func (d DBMailSummary) tagsFromPlusAddresses() []string {
+	tags := []string{}
+	for _, c := range d.To {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Cc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Bcc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	matches := addressPlusRe.FindAllStringSubmatch(d.From.Address, 1)
+	if len(matches) == 1 {
+		tags = append(tags, strings.Split(matches[0][2], "+")...)
+	}
+
+	return tools.SetTagCasing(tags)
+}
+
+// Get message tags from the database for a given database ID
+// Used when parsing a raw email.
+func getMessageTags(id string) []string {
+	tags := []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		From(tenant("Tags")).
+		LeftJoin(tenant("message_tags"), tenant("Tags.ID")+"="+tenant("message_tags.TagID")).
+		Where(tenant("message_tags.ID")+` = ?`, id).
+		OrderBy("Name").
+		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[tags] %s", err.Error())
+		return tags
+	}
+
+	return tags
+}
+
+// SortedUniqueTags will return a unique slice of normalised tags
+func sortedUniqueTags(s []string) []string {
+	tags := []string{}
+	added := make(map[string]bool)
+
+	if len(s) == 0 {
+		return tags
+	}
+
+	for _, p := range s {
+		w := tools.CleanTag(p)
+		if w == "" {
+			continue
+		}
+		lc := strings.ToLower(w)
+		if _, exists := added[lc]; exists {
+			continue
+		}
+		if config.ValidTagRegexp.MatchString(w) {
+			added[lc] = true
+			tags = append(tags, w)
+		} else {
+			logger.Log().Debugf("[tags] ignoring invalid tag: %s", w)
+		}
+	}
+
+	sort.Strings(tags)
+
+	return tags
+}

internal/storage/tags_test.go

@@ -0,0 +1,143 @@
+package storage
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+)
+
+func TestTags(t *testing.T) {
+
+	for _, tenantID := range []string{"", "MyServer 3", "host.example.com"} {
+		tenantID = config.DBTenantID(tenantID)
+
+		setup(tenantID)
+
+		if tenantID == "" {
+			t.Log("Testing tags")
+		} else {
+			t.Logf("Testing tags (tenant %s)", tenantID)
+		}
+
+		ids := []string{}
+
+		for i := 0; i < 10; i++ {
+			id, err := Store(&testMimeEmail)
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+			ids = append(ids, id)
+		}
+
+		for i := 0; i < 10; i++ {
+			if _, err := SetMessageTags(ids[i], []string{fmt.Sprintf("Tag-%d", i)}); err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+		}
+
+		for i := 0; i < 10; i++ {
+			message, err := GetMessage(ids[i])
+			if err != nil {
+				t.Log("error ", err)
+				t.Fail()
+			}
+
+			if len(message.Tags) != 1 || message.Tags[0] != fmt.Sprintf("Tag-%d", i) {
+				t.Fatal("Message tags do not match")
+			}
+		}
+
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// test 20 tags
+		id, err := Store(&testMimeEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		newTags := []string{}
+		for i := 0; i < 20; i++ {
+			// pad number with 0 to ensure they are returned alphabetically
+			newTags = append(newTags, fmt.Sprintf("AnotherTag %02d", i))
+		}
+		if _, err := SetMessageTags(id, newTags); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags := getMessageTags(id)
+		assertEqual(t, strings.Join(newTags, "|"), strings.Join(returnedTags, "|"), "Message tags do not match")
+
+		// remove first tag
+		if err := deleteMessageTag(id, newTags[0]); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, strings.Join(newTags[1:], "|"), strings.Join(returnedTags, "|"), "Message tags do not match after deleting 1")
+
+		// remove all tags
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "", strings.Join(returnedTags, "|"), "Message tags should be empty")
+
+		// apply the same tag twice
+		if _, err := SetMessageTags(id, []string{"Duplicate Tag", "Duplicate Tag"}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "Duplicate Tag", strings.Join(returnedTags, "|"), "Message tags should be duplicated")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// apply tag with invalid characters
+		if _, err := SetMessageTags(id, []string{"Dirty! \"Tag\""}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "Dirty Tag", strings.Join(returnedTags, "|"), "Dirty message tag did not clean as expected")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// Check deleted message tags also prune the tags database
+		allTags := GetAllTags()
+		assertEqual(t, "", strings.Join(allTags, "|"), "Tags did not delete as expected")
+
+		if err := DeleteAllMessages(); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		// test 20 tags
+		id, err = Store(&testTagEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		returnedTags = getMessageTags(id)
+		assertEqual(t, "BccTag|CcTag|FromFag|ToTag|X-tag1|X-tag2", strings.Join(returnedTags, "|"), "Tags not detected correctly")
+		if err := DeleteAllMessageTags(id); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		Close()
+	}
+
+}

internal/storage/testdata/mime-attachment.eml

@@ -1,4 +1,4 @@
-Delivered-To: recipient@example.com
+Delivered-To: recipient2@example.com
 Received: by 2002:a0c:fe87:0:0:0:0:0 with SMTP id d7csp145570qvs;
         Tue, 26 Jul 2022 20:42:36 -0700 (PDT)
 X-Received: by 2002:a17:902:f788:b0:16c:f48b:905e with SMTP id q8-20020a170902f78800b0016cf48b905emr19885972pln.60.1658893355881;
@@ -23,18 +23,18 @@ ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc
          uSfA==
 ARC-Authentication-Results: i=1; mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])
         by mx.google.com with SMTPS id 11-20020aa7914b000000b0052ab192de4fsor8543241pfi.101.2022.07.26.20.42.35
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (Google Transport Security);
         Tue, 26 Jul 2022 20:42:35 -0700 (PDT)
-Received-SPF: pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
+Received-SPF: pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
 Authentication-Results: mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
         d=gmail.com; s=20210112;
@@ -63,10 +63,10 @@ X-Gm-Message-State: AJIora/WUqr3biShTHQBjSlCKazFbrLxeYpxmr1VF0TpBUbjnJrcLT77
 X-Google-Smtp-Source: AGRyM1tai6X1Bx130Y1yHG5w2e0r8wx6bbI+H+YppWmQoT28TV3dSoYCqmeQK5VViW8WuvdOpQzhPQ==
 X-Received: by 2002:a62:29c3:0:b0:52b:f774:7242 with SMTP id p186-20020a6229c3000000b0052bf7747242mr12504553pfp.67.1658893354675;
         Tue, 26 Jul 2022 20:42:34 -0700 (PDT)
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from [192.168.1.2] ([8.8.8.8])
         by smtp.gmail.com with ESMTPSA id oj16-20020a17090b4d9000b001f291c9d3bdsm387578pjb.48.2022.07.26.20.42.32
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);
         Tue, 26 Jul 2022 20:42:33 -0700 (PDT)
 Content-Type: multipart/mixed; boundary="------------ae0qIOkrNQLQHe1YyfTsUXrk"
@@ -76,8 +76,8 @@ MIME-Version: 1.0
 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101
  Thunderbird/91.11.0
 Content-Language: en-NZ
-To: "Recipient Ross" <recipient@example.com>
-From: Sender Smith <sender@example.com>
+To: "Recipient Ross" <recipient2@example.com>
+From: Sender Smith <sender2@example.com>
 Subject: inline + attachment
 
 This is a multi-part message in MIME format.
@@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit
     <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   </head>
   <body>
-    Message with inline image and attachment:<br>
+    <h1>Message with inline image and attachment:</h1>
     <br>
-    <img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"
-      moz-do-not-send="false"><br>
+    <p><img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"></p>
     <br>
     <br>
   </body>

internal/storage/testdata/tags.eml

@@ -0,0 +1,49 @@
+Date: Wed, 27 Jul 2022 15:44:41 +1200
+From: Sender Smith <sender+FromFag@example.com>
+To: Recipient Ross <recipient+ToTag@example.com>
+Cc: Recipient Ross <cc+CcTag@example.com>
+Bcc: <bcc+BccTag@example.com>
+Subject: Plain text message
+X-Tags: X-tag1, X-tag2
+Message-ID: <20220727034441.7za34h6ljuzfpmj3@localhost.localhost>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras non massa lacinia, 
+fringilla ex vel, ornare nulla. Suspendisse dapibus commodo sapien, non 
+hendrerit diam feugiat sit amet. Nulla lorem quam, laoreet vitae nisl volutpat, 
+mollis bibendum felis. In eget ultricies justo. Donec vitae hendrerit tortor, at 
+posuere libero. Fusce a gravida nibh. Nulla ac odio ex.
+
+Aliquam sem turpis, cursus vitae condimentum at, scelerisque pulvinar lectus. 
+Cras tempor nisl ut arcu interdum, et luctus arcu cursus. Maecenas mollis 
+sagittis commodo. Mauris ac lorem nec ex interdum consequat. Morbi congue 
+ultrices ullamcorper. Aenean ex tortor, dapibus quis dapibus iaculis, iaculis 
+eget felis. Vestibulum purus ante, efficitur in turpis ac, tristique laoreet 
+orci. Nulla facilisi. Praesent mollis orci posuere elementum laoreet. 
+Pellentesque enim nibh, varius at ante id, consequat posuere ante.
+
+Cras maximus venenatis nulla nec cursus. Morbi convallis, enim eget viverra 
+vulputate, ipsum arcu tincidunt tortor, ut cursus dui enim commodo quam. Donec 
+et vulputate quam. Vivamus non posuere erat. Nam commodo pellentesque 
+condimentum. Vivamus condimentum eros at odio dictum feugiat. Ut imperdiet 
+tempor luctus. Aenean varius libero ac faucibus dictum. Aliquam sed finibus 
+massa. Morbi dolor lorem, feugiat quis neque et, suscipit posuere ex. Sed auctor 
+et augue at finibus. Vestibulum interdum mi ac justo porta aliquam. Curabitur 
+nec enim sit amet enim aliquet accumsan. Etiam accumsan tellus tortor, interdum 
+sodales odio finibus eu. Integer eget ante eu nisi lobortis pulvinar et vel 
+ipsum. Cras condimentum posuere vulputate.
+
+Cras nulla felis, blandit vitae egestas quis, fringilla ut dolor. Phasellus est 
+augue, feugiat eu risus quis, posuere ultrices libero. Phasellus non nunc eget 
+justo sollicitudin tincidunt. Praesent pretium dui id felis bibendum sodales. 
+Phasellus eget dictum libero, auctor tempor nibh. Suspendisse posuere libero 
+venenatis elit imperdiet porttitor. In condimentum dictum luctus. Nullam in 
+nulla vitae augue blandit posuere. Vestibulum consectetur ultricies tincidunt. 
+Vivamus dolor quam, pharetra sed eros sed, hendrerit ultrices diam. Vestibulum 
+vulputate tellus eget tellus lacinia, a pulvinar velit vulputate. Suspendisse 
+mauris odio, scelerisque eget turpis sed, tincidunt ultrices magna. Nunc arcu 
+arcu, commodo et porttitor quis, accumsan viverra purus. Fusce id libero iaculis 
+lorem tristique commodo porttitor id ipsum. Vestibulum odio dui, tincidunt eget 
+lectus vel, tristique lacinia libero. Aliquam dapibus ac felis vitae cursus.

internal/storage/testing.go

@@ -0,0 +1,69 @@
+package storage
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var (
+	testTextEmail []byte
+	testTagEmail  []byte
+	testMimeEmail []byte
+	testRuns      = 100
+)
+
+func setup(tenantID string) {
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.Database = os.Getenv("MP_DATABASE")
+	config.TenantID = config.DBTenantID(tenantID)
+
+	if err := InitDB(); err != nil {
+		panic(err)
+	}
+
+	var err error
+
+	// ensure DB is empty
+	if err := DeleteAllMessages(); err != nil {
+		panic(err)
+	}
+
+	testTextEmail, err = os.ReadFile("testdata/plain-text.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testTagEmail, err = os.ReadFile("testdata/tags.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testMimeEmail, err = os.ReadFile("testdata/mime-attachment.eml")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}
+
+func assertEqualStats(t *testing.T, total int, unread int) {
+	s := StatsGet()
+	if float64(total) != s.Total {
+		t.Fatalf("Incorrect total mailbox stats: \"%v\" != \"%v\"", total, s.Total)
+	}
+
+	if float64(unread) != s.Unread {
+		t.Fatalf("Incorrect unread mailbox stats: \"%v\" != \"%v\"", unread, s.Unread)
+	}
+}

internal/storage/utils.go

@@ -0,0 +1,108 @@
+package storage
+
+import (
+	"net/mail"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/axllent/mailpit/internal/html2text"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	// for stats to prevent import cycle
+	mu sync.RWMutex
+	// StatsDeleted for counting the number of messages deleted
+	StatsDeleted float64
+)
+
+// AddTempFile adds a file to the slice of files to delete on exit
+func AddTempFile(s string) {
+	temporaryFiles = append(temporaryFiles, s)
+}
+
+// DeleteTempFiles will delete files added via AddTempFiles
+func deleteTempFiles() {
+	for _, f := range temporaryFiles {
+		if err := os.Remove(f); err == nil {
+			logger.Log().Debugf("removed temporary file: %s", f)
+		}
+	}
+}
+
+// Return a header field as a []*mail.Address, or "null" is not found/empty
+func addressToSlice(env *enmime.Envelope, key string) []*mail.Address {
+	data, err := env.AddressList(key)
+	if err != nil || data == nil {
+		return []*mail.Address{}
+	}
+
+	return data
+}
+
+// Generate the search text based on some header fields (to, from, subject etc)
+// and either the stripped HTML body (if exists) or text body
+func createSearchText(env *enmime.Envelope) string {
+	var b strings.Builder
+
+	b.WriteString(env.GetHeader("From") + " ")
+	b.WriteString(env.GetHeader("Subject") + " ")
+	b.WriteString(env.GetHeader("To") + " ")
+	b.WriteString(env.GetHeader("Cc") + " ")
+	b.WriteString(env.GetHeader("Bcc") + " ")
+	b.WriteString(env.GetHeader("Reply-To") + " ")
+	b.WriteString(env.GetHeader("Return-Path") + " ")
+
+	h := html2text.Strip(env.HTML, true)
+	if h != "" {
+		b.WriteString(h + " ")
+	} else {
+		b.WriteString(env.Text + " ")
+	}
+	// add attachment filenames
+	for _, a := range env.Attachments {
+		b.WriteString(a.FileName + " ")
+	}
+
+	d := cleanString(b.String())
+
+	return d
+}
+
+// CleanString removes unwanted characters from stored search text and search queries
+func cleanString(str string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	str = strings.ReplaceAll(str, string('\uFEFF'), " ")
+
+	// remove/replace new lines
+	re := regexp.MustCompile(`(\r?\n|\t|>|<|"|\,|;|\(|\))`)
+	str = re.ReplaceAllString(str, " ")
+
+	// remove duplicate whitespace and trim
+	return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " "))
+}
+
+// LogMessagesDeleted logs the number of messages deleted
+func logMessagesDeleted(n int) {
+	mu.Lock()
+	StatsDeleted = StatsDeleted + float64(n)
+	mu.Unlock()
+}
+
+// IsFile returns whether a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// Convert `%` to `%%` for SQL searches
+func escPercentChar(s string) string {
+	return strings.ReplaceAll(s, "%", "%%")
+}

internal/tools/argsparser.go

@@ -0,0 +1,32 @@
+package tools
+
+import "strings"
+
+// ArgsParser will split a string by new words and quotes phrases
+func ArgsParser(s string) []string {
+	args := []string{}
+	sb := &strings.Builder{}
+	quoted := false
+	for _, r := range s {
+		if r == '"' {
+			quoted = !quoted
+			sb.WriteRune(r) // keep '"' otherwise comment this line
+		} else if !quoted && r == ' ' {
+			v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+			if v != "" {
+				args = append(args, v)
+			}
+			sb.Reset()
+		} else {
+			sb.WriteRune(r)
+		}
+	}
+	if sb.Len() > 0 {
+		v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+		if v != "" {
+			args = append(args, v)
+		}
+	}
+
+	return args
+}

internal/tools/fs.go

@@ -0,0 +1,23 @@
+package tools
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// IsFile returns whether a file exists and is readable
+func IsFile(path string) bool {
+	f, err := os.Open(filepath.Clean(path))
+	defer f.Close()
+	return err == nil
+}
+
+// IsDir returns whether a path is a directory
+func IsDir(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || os.IsNotExist(err) || !info.IsDir() {
+		return false
+	}
+
+	return true
+}

internal/tools/headers.go

@@ -0,0 +1,160 @@
+// Package tools provides various methods for various things
+package tools
+
+import (
+	"bufio"
+	"bytes"
+	"net/mail"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+// RemoveMessageHeaders scans a message for headers, if found them removes them.
+// It will only remove a single instance of any given message header.
+func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	reBlank := regexp.MustCompile(`^\s+`)
+
+	for _, hdr := range headers {
+		// case-insensitive
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(hdr+":"))
+
+		// header := []byte(hdr + ":")
+		if m.Header.Get(hdr) != "" {
+			scanner := bufio.NewScanner(bytes.NewReader(msg))
+			found := false
+			hdr := []byte("")
+			for scanner.Scan() {
+				line := scanner.Bytes()
+				if !found && reHdr.Match(line) {
+					// add the first line starting with <header>:
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+					found = true
+				} else if found && reBlank.Match(line) {
+					// add any following lines starting with a whitespace (tab or space)
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+				} else if found {
+					// stop scanning, we have the full <header>
+					break
+				}
+			}
+
+			if len(hdr) > 0 {
+				logger.Log().Debugf("[relay] removed %s header", hdr)
+				msg = bytes.Replace(msg, hdr, []byte(""), 1)
+			}
+		}
+	}
+
+	return msg, nil
+}
+
+// UpdateMessageHeader scans a message for a header and updates its value if found.
+func UpdateMessageHeader(msg []byte, header, value string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get(header) != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(header+":"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		if len(hdr) > 0 {
+			logger.Log().Debugf("[relay] replaced %s header", hdr)
+			msg = bytes.Replace(msg, hdr, []byte(header+": "+value+"\r\n"), 1)
+		}
+	}
+
+	return msg, nil
+}
+
+// OverrideFromHeader scans a message for the From header and replaces it with a different email address.
+func OverrideFromHeader(msg []byte, address string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get("From") != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta("From:"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		if len(hdr) > 0 {
+			originalFrom := strings.TrimRight(string(hdr[5:]), "\r\n")
+
+			from, err := mail.ParseAddress(originalFrom)
+			if err != nil {
+				// error parsing the from address, so just replace the whole line
+				msg = bytes.Replace(msg, hdr, []byte("From: "+address+"\r\n"), 1)
+			} else {
+				originalFrom = from.Address
+				// replace the from email, but keep the original name
+				from.Address = address
+				msg = bytes.Replace(msg, hdr, []byte("From: "+from.String()+"\r\n"), 1)
+			}
+
+			// insert the original From header as X-Original-From
+			msg = append([]byte("X-Original-From: "+originalFrom+"\r\n"), msg...)
+
+			logger.Log().Debugf("[relay] Replaced From email address with %s", address)
+		}
+	} else {
+		// no From header, so add one
+		msg = append([]byte("From: "+address+"\r\n"), msg...)
+		logger.Log().Debugf("[relay] Added From email: %s", address)
+	}
+
+	return msg, nil
+}

internal/tools/html.go

@@ -0,0 +1,50 @@
+package tools
+
+import (
+	"fmt"
+
+	"golang.org/x/net/html"
+)
+
+// GetHTMLAttributeVal returns the value of an HTML Attribute, else an error.
+// Returns a blank value if the attribute is set but empty.
+func GetHTMLAttributeVal(e *html.Node, key string) (string, error) {
+	for _, a := range e.Attr {
+		if a.Key == key {
+			return a.Val, nil
+		}
+	}
+
+	return "", fmt.Errorf("%s not found", key)
+}
+
+// SetHTMLAttributeVal sets an attribute on a node.
+func SetHTMLAttributeVal(n *html.Node, key, val string) {
+	for i := range n.Attr {
+		a := &n.Attr[i]
+		if a.Key == key {
+			a.Val = val
+			return
+		}
+	}
+	n.Attr = append(n.Attr, html.Attribute{
+		Key: key,
+		Val: val,
+	})
+}
+
+// WalkHTML traverses the entire HTML tree and calls fn on each node.
+func WalkHTML(n *html.Node, fn func(*html.Node)) {
+	if n == nil {
+		return
+	}
+
+	fn(n)
+
+	// Each node has a pointer to its first child and next sibling. To traverse
+	// all children of a node, we need to start from its first child and then
+	// traverse the next sibling until nil.
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		WalkHTML(c, fn)
+	}
+}