Merge branch 'release/v1.14.4'

Release v1.14.2

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,47 @@
+# Changelog
+
+Notable changes to Mailpit will be documented in this file.
+
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+## [Unreleased]
+
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }} 
+{{- if .CommitGroups -}}
+## [{{ .Tag.Name }}]
+
+{{ if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end -}}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end }}
+
+{{- if .MergeCommits -}}
+### Pull Requests
+{{ range .MergeCommits -}}
+- {{ .Header }}
+{{ end }}
+{{ end }}
+{{ end -}}

.chglog/RELEASE.tpl.md

@@ -0,0 +1,12 @@
+{{ if .Versions -}}
+{{ range .Versions }} 
+{{- if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,28 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/axllent/mailpit
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+    title_maps:
+      feature: Feature
+      fix: Fix
+    #   perf: Performance Improvements
+    #   refactor: Code Refactoring
+  header:
+    pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$"
+    pattern_maps:
+      - Type
+      - Scope
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.dockerignore

@@ -0,0 +1,2 @@
+/node_modules
+/mailpit

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [axllent]

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "docker"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/build-docker-edge.yml

@@ -0,0 +1,34 @@
+on:
+  push:
+    branches: [ develop ]
+
+name: Build docker edge images
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        platforms: linux/386,linux/amd64,linux/arm64
+        build-args: |
+          "VERSION=edge-${{ github.sha }}"
+        push: true
+        tags: |
+          axllent/mailpit:edge

.github/workflows/build-docker.yml

@@ -0,0 +1,44 @@
+on:
+  release:
+    types: [created]
+
+name: Build docker images
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+    - name: Parse semver
+      id: semver_parser 
+      uses: booxmedialtd/ws-action-parse-semver@v1.4.7
+      with:
+        input_string: '${{ github.ref_name }}'
+        version_extractor_regex: 'v(.*)$'
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        # platforms: linux/386,linux/amd64,linux/arm,linux/arm64
+        platforms: linux/386,linux/amd64,linux/arm64
+        build-args: |
+          "VERSION=${{ github.ref_name }}"
+        push: true
+        tags: |
+          axllent/mailpit:latest
+          axllent/mailpit:${{ github.ref_name }}
+          axllent/mailpit:v${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}

.github/workflows/close-stale-issues.yml

@@ -0,0 +1,23 @@
+name: Close stale issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9.0.0
+        with:
+          days-before-issue-stale: 14
+          days-before-issue-close: 7
+          exempt-issue-labels: "enhancement,bug,javascript,docker"
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 14 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 7 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "develop" ]
+  schedule:
+    - cron: '34 23 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/workflows/release-build.yml

@@ -5,40 +5,44 @@ on:
 name: Build & release
 jobs:
   releases-matrix:
-    name: Release Go Binary
+    name: Build
     runs-on: ubuntu-latest
     strategy:
       matrix:
         goos: [linux, windows, darwin]
-        goarch: ["386", amd64, arm64]
+        goarch: ["386", amd64, arm, arm64]
         exclude:
           - goarch: "386"
             goos: darwin
-          - goarch: arm64
+          - goarch: "386"
+            goos: windows
+          - goarch: arm
+            goos: darwin
+          - goarch: arm
             goos: windows
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Get tag
-      id: tag
-      uses: dawidd6/action-get-tag@v1
+    - uses: actions/checkout@v4
 
     # build the assets
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 18
         cache: 'npm'
+    - run: echo "Building assets for ${{ github.ref_name }}"
     - run: npm install
     - run: npm run package
 
     # build the binaries
-    - uses: wangyoucao577/go-release-action@v1.30
+    - uses: wangyoucao577/go-release-action@v1.49
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}
         goarch: ${{ matrix.goarch }}
         binary_name: "mailpit"
+        pre_command: export CGO_ENABLED=0
         asset_name: mailpit-${{ matrix.goos }}-${{ matrix.goarch }}
         extra_files: LICENSE README.md
         md5sum: false
-        ldflags: -w -X "github.com/axllent/mailpit/cmd.Version=${{ steps.tag.outputs.tag }}"
+        overwrite: true
+        retry: 5
+        ldflags: -w -X "github.com/axllent/mailpit/config.Version=${{ github.ref_name }}"

.github/workflows/test.yml

@@ -1,28 +0,0 @@
-name: Test
-on:
-  pull_request:
-    branches: [ develop ]
-  push:
-    branches: [ develop ]
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.18.x]
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-    - uses: actions/checkout@v3
-    - run: go test ./storage -v
-    - run: go test ./storage -bench=.
-    
-    # build the assets
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 16
-        cache: 'npm'
-    - run: npm install
-    - run: npm run package

.github/workflows/tests.yml

@@ -0,0 +1,49 @@
+name: Tests
+on:
+  pull_request:
+    branches: [ develop, 'feature/**' ]
+  push:
+    branches: [ develop, 'feature/**' ]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.21.x]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+        cache: false
+    - uses: actions/checkout@v4
+    - name: Run Go tests
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - run: go test ./internal/storage ./server ./internal/tools ./internal/html2text -v
+    - run: go test ./internal/storage ./internal/html2text -bench=.
+    
+    # build the assets
+    - name: Build web UI
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: actions/setup-node@v4
+      with:
+        node-version: 18
+        cache: 'npm'
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm install
+    - if: startsWith(matrix.os, 'ubuntu') == true
+      run: npm run package
+
+    # validate the swagger file
+    - name: Validate OpenAPI definition
+      if: startsWith(matrix.os, 'ubuntu') == true
+      uses: char0n/swagger-editor-validate@v1
+      with:
+        definition-file: server/ui/api/v1/swagger.json

.gitignore

@@ -1,6 +1,9 @@
 /node_modules/
 /send
+/sendmail/sendmail
 /server/ui/dist
 /Makefile
-/mailpit
+/mailpit*
+/.idea
 *.old
+*.db

Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:alpine as builder
+
+ARG VERSION=dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN apk add --no-cache git npm && \
+npm install && npm run package && \
+CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/axllent/mailpit/config.Version=${VERSION}" -o /mailpit
+
+FROM alpine:latest
+
+COPY --from=builder /mailpit /mailpit
+
+RUN apk add --no-cache tzdata
+
+EXPOSE 1025/tcp 1110/tcp 8025/tcp
+
+ENTRYPOINT ["/mailpit"]

README-BUILDING.md

@@ -1,45 +0,0 @@
-# Building Mailpit from source
-
-Go (>= version 1.8) and npm are required to compile mailpit from source.
-
-```
-git clone git@github.com:axllent/mailpit.git
-cd mailpit
-```
-
-## Building the UI
-
-The Mailpit web user interface is built with node. In the project's root (top) directory run the following to install the required node modules:
-
-
-### Installing the node modules
-```
-npm install
-```
-
-
-### Building the web UI
-
-```
-npm run build
-```
-
-You can also run `npm run watch` which will watch for changes and rebuild the HTML/CSS/JS automatically when changes are detected.
-Please note that you must restart Mailpit (`go run .`) to run with the changes.
-
-
-## Build the mailpit binary
-
-One you have the assets compiled, you can build mailpit as follows:
-```
-go build -ldflags "-s -w"
-```
-
-## Building a stand-alone sendmail binary
-
-This step is unnecessary, however if you do not intend to either symlink `sendmail` to mailpit or configure your existing sendmail to route mail to mailpit, you can optionally build a stand-alone sendmail binary.
-
-```
-cd sendmail
-go build -ldflags "-s -w"
-```

README.md

@@ -1,60 +1,116 @@
-# Mailpit
+<h1 align="center">
+  Mailpit - email testing for developers
+</h1>
 
-Mailpit is an email testing tool for developers.
+<div align="center">
+    <a href="https://github.com/axllent/mailpit/actions/workflows/tests.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg" alt="CI Tests status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/release-build.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg" alt="CI build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg" alt="CI Docker build status"></a>
+    <a href="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml"><img src="https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg" alt="Code quality"></a>
+    <a href="https://goreportcard.com/report/github.com/axllent/mailpit"><img src="https://goreportcard.com/badge/github.com/axllent/mailpit" alt="Go Report Card"></a>
+    <br>
+    <a href="https://github.com/axllent/mailpit/releases/latest"><img src="https://img.shields.io/github/v/release/axllent/mailpit.svg" alt="Latest release"></a>
+    <a href="https://hub.docker.com/r/axllent/mailpit"><img src="https://img.shields.io/docker/pulls/axllent/mailpit.svg" alt="Docker pulls"></a>
+</div>
+<br>
+<p align="center">
+  <a href="https://mailpit.axllent.org">Website</a>  •
+  <a href="https://mailpit.axllent.org/docs/">Documentation</a>  •
+  <a href="https://mailpit.axllent.org/docs/api-v1/">API</a>
+</p>
 
-It acts as both an SMTP server, and provides a web interface to view all captured emails.
+<hr>
 
-Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster.
+**Mailpit** is a small, fast, low memory, zero-dependency, multi-platform email testing tool & API for developers.
+
+It acts as an SMTP server, provides a modern web interface to view & test captured emails, and includes an API for automated integration testing.
+
+Mailpit was originally **inspired** by MailHog which is [no longer maintained](https://github.com/mailhog/MailHog/issues/442#issuecomment-1493415258) and hasn't seen active development or security updates for a few years now.
+
+![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/server/ui-src/screenshot.png)
 
 
 ## Features
 
-- Runs completely on a single binary
-- SMTP server (default `0.0.0.0:1025`)
-- Web UI to view emails (HTML format, text, source and MIME attachments, default `0.0.0.0:8025`)
-- Real-time web UI updates using web sockets for new mail
-- Email storage in either memory or disk (using [CloverDB](https://github.com/ostafen/clover)) - note that in-memory has a physical limit of 1MB per email
+- Runs entirely from a single [static binary](https://mailpit.axllent.org/docs/install/)
+- Modern web UI to view emails (formatted HTML, highlighted HTML source, text, headers, raw source, and MIME attachments
+including image thumbnails), including optional [HTTPS](https://mailpit.axllent.org/docs/configuration/https/)
+- Optional [basic authentication](https://mailpit.axllent.org/docs/configuration/frontend-authentication/) for web UI & API
+- [HTML check](https://mailpit.axllent.org/docs/usage/html-check/) to test & score mail client compatibility with HTML emails
+- [Link check](https://mailpit.axllent.org/docs/usage/link-check/) to test message links (HTML & text) & linked images
+- [Spam check](https://mailpit.axllent.org/docs/usage/spamassassin/) to test message "spamminess" using a running SpamAssassin server
+- [Create screenshots](https://mailpit.axllent.org/docs/usage/html-screenshots/) of HTML messages via web UI
+- `List-Unsubscribe` syntax validation
+- Mobile and tablet HTML preview toggle in desktop mode
+- Advanced [mail search](https://mailpit.axllent.org/docs/usage/search-filters/)
+- [Message tagging](https://mailpit.axllent.org/docs/usage/tagging/)
+- Real-time web UI updates using web sockets for new mail & optional browser notifications for new mail (when accessed
+via either HTTPS or `localhost` only)
+- SMTP server with optional [STARTTLS & SMTP authentication](https://mailpit.axllent.org/docs/configuration/smtp-authentication/) (including an
+"accept any" mode)
+- [SMTP relaying](https://mailpit.axllent.org/docs/configuration/smtp-relay/) (message release) - relay messages via a different SMTP server
+including an optional allowlist of accepted recipients
+- Fast SMTP processing & storing - ingesting 100-200 emails per second depending on CPU, network speed & email size,
+easily handling tens of thousands of emails
+- Optional [POP3 server](https://mailpit.axllent.org/docs/configuration/pop3/) to download captured message directly into your email client
 - Configurable automatic email pruning (default keeps the most recent 500 emails)
-- Fast SMTP processing & storing - approximately 300-600 emails per second depending on CPU, network speed & email size
-- Can handle tens of thousands of emails
-
-
-## Planned features
-
-- Optional HTTPS for web UI
-- Optional basic authentication for web UI
-- Optional authentication for SMTP
-- Browser notifications for new mail (HTTPS only)
-- Docker container
+- A simple [REST API](https://mailpit.axllent.org/docs/api-v1/) for integration testing
+- Optional [webhook](https://mailpit.axllent.org/docs/integration/webhook/) for received messages
+- Multi-architecture [Docker images](https://mailpit.axllent.org/docs/install/docker/)
 
 
 ## Installation
 
-Download a pre-built binary in the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` can be placed in your `$PATH`, or simply run as `./mailpit`. See `mailpit -h` for options.
+The Mailpit web UI listens by default on `http://0.0.0.0:8025` and the SMTP port on `0.0.0.0:1025`.
 
-To build mailpit from source see [building from source](README-BUILDING.md).
+Mailpit runs as a single binary and can be installed in different ways:
+
+
+### Install via package managers
+
+- **Mac**: `brew install mailpit` (to run automatically in the background: `brew services start mailpit`)
+- **Arch Linux**: available in the AUR as `mailpit`
+- **FreeBSD**: `pkg install mailpit`
+
+
+### Install via bash script (Linux & Mac)
+
+Linux & Mac users can install it directly to `/usr/local/bin/mailpit` with:
+
+```bash
+sudo bash < <(curl -sL https://raw.githubusercontent.com/axllent/mailpit/develop/install.sh)
+```
+
+
+### Download static binary (Windows, Linux and Mac)
+
+Static binaries can always be found on the [releases](https://github.com/axllent/mailpit/releases/latest). The `mailpit` binary can be extracted and copied to your `$PATH`, or simply run as `./mailpit`.
+
+
+### Docker
+
+See [Docker instructions](https://mailpit.axllent.org/docs/install/docker/) for 386, amd64 & arm64 images.
+
+
+### Compile from source
+
+To build Mailpit from source, see [Building from source](https://mailpit.axllent.org/docs/install/source/).
+
+
+## Usage
+
+Run `mailpit -h` to see options. More information can be seen in [the docs](https://mailpit.axllent.org/docs/configuration/runtime-options/).
+
+If installed using homebrew, you may run `brew services start mailpit` to always run mailpit automatically.
+
+
+### Testing Mailpit
+
+Please refer to [the documentation](https://mailpit.axllent.org/docs/install/testing/) on how to easily test email delivery to Mailpit.
 
 
 ### Configuring sendmail
 
-There are several different options available:
-
-You can use `mailpit sendmail` as your sendmail configuration in `php.ini`:
-```
-sendmail_path = /usr/local/bin/mailpit sendmail
-```
-
-If mailpit is found on the same host as sendmail, you can symlink the mailpit binary to sendmail, eg: `ln -s /usr/local/bin/mailpit /usr/sbin/sendmail`  (only if mailpit is running on default 1025 port).
-
-You can use your default system `sendmail` binary to route directly to port `1025` (configurable) by calling `/usr/sbin/sendmail -S localhost:1025`.
-
-You can build a mailpit-specific sendmail binary from source ( see [building from source](README-BUILDING.md)).
-
-
-## Why rewrite MailHog?
-
-I had been using MailHog for a few years to intercept and test emails generated from several projects. MailHog has a number of severe performance issues, many of the modules are horribly out of date, and other than a few accepted MRs, it is not actively developed.
-
-Initially I started trying to upgrade a fork of MailHog (both the UI as well as the HTTP server & API), but soon discovered that it is (with all due respect) very poorly designed. It is over-engineered (split over 9 separate projects), has too many unnecessary features for my purpose, and performs exceptionally poorly when dealing with large lumbers of emails or processing any email with an attachment (a single email with a 3MB attachment can take over a minute). The API transmits a lot of duplicate and unnecessary data on every message request for all web calls, and there is no HTTP compression.
-
-In order to improve it I felt it needed to be completely rewritten, and so Mailpit was born.
+Mailpit's SMTP server (default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. 
+A common MTA (Mail Transfer Agent) that delivers system emails to an SMTP server is `sendmail`, used by many applications, including PHP. 
+Mailpit can also act as substitute for sendmail. For instructions on how to set this up, please refer to the [sendmail documentation](https://mailpit.axllent.org/docs/install/sendmail/).

cmd/ingest.go

@@ -0,0 +1,153 @@
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"net/mail"
+	"net/smtp"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/logger"
+	sendmail "github.com/axllent/mailpit/sendmail/cmd"
+	"github.com/spf13/cobra"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+)
+
+var (
+	ingestRecent int
+)
+
+// ingestCmd represents the ingest command
+var ingestCmd = &cobra.Command{
+	Use:   "ingest <file|folder> ...[file|folder]",
+	Short: "Ingest a file or folder of emails for testing",
+	Long: `Ingest a file or folder of emails for testing.
+
+This command will scan the folder for emails and deliver them via SMTP to a running 
+Mailpit server. Each email must be a separate file (eg: Maildir format, not mbox).
+The --recent flag will only consider files with a modification date within the last X days.`,
+	// Hidden: true,
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var count int
+		var total int
+		var per100start = time.Now()
+		p := message.NewPrinter(language.English)
+
+		for _, a := range args {
+			err := filepath.Walk(a,
+				func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						logger.Log().Error(err)
+						return nil
+					}
+					if !isFile(path) {
+						return nil
+					}
+
+					info.ModTime()
+
+					if ingestRecent > 0 && time.Now().Sub(info.ModTime()) > time.Duration(ingestRecent)*24*time.Hour {
+						return nil
+					}
+
+					f, err := os.Open(filepath.Clean(path))
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+					defer f.Close() // #nosec
+
+					body, err := io.ReadAll(f)
+					if err != nil {
+						logger.Log().Errorf("%s: %s", path, err.Error())
+						return nil
+					}
+
+					msg, err := mail.ReadMessage(bytes.NewReader(body))
+					if err != nil {
+						logger.Log().Errorf("error parsing message body: %s", err.Error())
+						return nil
+					}
+
+					recipients := []string{}
+					// get all recipients in To, Cc and Bcc
+					if to, err := msg.Header.AddressList("To"); err == nil {
+						for _, a := range to {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if cc, err := msg.Header.AddressList("Cc"); err == nil {
+						for _, a := range cc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+					if bcc, err := msg.Header.AddressList("Bcc"); err == nil {
+						for _, a := range bcc {
+							recipients = append(recipients, a.Address)
+						}
+					}
+
+					if sendmail.FromAddr == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							sendmail.FromAddr = fromAddresses[0].Address
+						}
+					}
+
+					if len(recipients) == 0 {
+						// Bcc
+						recipients = []string{sendmail.FromAddr}
+					}
+
+					returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
+					if returnPath == "" {
+						if fromAddresses, err := msg.Header.AddressList("From"); err == nil {
+							returnPath = fromAddresses[0].Address
+						}
+					}
+
+					err = smtp.SendMail(sendmail.SMTPAddr, nil, returnPath, recipients, body)
+					if err != nil {
+						logger.Log().Errorf("error sending mail: %s (%s)", err.Error(), path)
+						return nil
+					}
+
+					count++
+					total++
+					if count%100 == 0 {
+						formatted := p.Sprintf("%d", total)
+						logger.Log().Infof("[%s] 100 messages in %s", formatted, time.Since(per100start))
+
+						per100start = time.Now()
+					}
+
+					return nil
+				})
+			if err != nil {
+				logger.Log().Error(err)
+			}
+		}
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(ingestCmd)
+
+	ingestCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
+	ingestCmd.Flags().IntVarP(&ingestRecent, "recent", "r", 0, "Only ingest messages from the last X days (default all)")
+}
+
+// IsFile returns if a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}

cmd/reindex.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/spf13/cobra"
+)
+
+// reindexCmd represents the reindex command
+var reindexCmd = &cobra.Command{
+	Use:   "reindex <database>",
+	Short: "Reindex the database",
+	Long: `This will reindex all messages in the entire database.
+
+If you have several thousand messages in your mailbox, then it is advised to shut down
+Mailpit while you reindex as this process will likely result in database locking issues.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		config.DataFile = args[0]
+		config.MaxMessages = 0
+
+		if err := storage.InitDB(); err != nil {
+			logger.Log().Error(err)
+			os.Exit(1)
+		}
+
+		storage.ReindexAll()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(reindexCmd)
+}

cmd/root.go

@@ -1,14 +1,19 @@
+// Package cmd is the main application
 package cmd
 
 import (
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/logger"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/axllent/mailpit/server"
-	"github.com/axllent/mailpit/smtpd"
-	"github.com/axllent/mailpit/storage"
+	"github.com/axllent/mailpit/server/smtpd"
+	"github.com/axllent/mailpit/server/webhook"
 	"github.com/spf13/cobra"
 )
 
@@ -20,7 +25,11 @@ var rootCmd = &cobra.Command{
 	Short: "Mailpit is an email testing tool for developers",
 	Long: `Mailpit is an email testing tool for developers.
 
-It acts as an SMTP server, and provides a web interface to view all captured emails.`,
+It acts as an SMTP server, and provides a web interface to view all captured emails.
+
+Documentation:
+  https://github.com/axllent/mailpit
+  https://mailpit.axllent.org/docs/`,
 	Run: func(_ *cobra.Command, _ []string) {
 		if err := config.VerifyConfig(); err != nil {
 			logger.Log().Error(err.Error())
@@ -60,27 +69,251 @@ func SendmailExecute() {
 func init() {
 	// hide autocompletion
 	rootCmd.CompletionOptions.HiddenDefaultCmd = true
-	// rootCmd.Flags().SortFlags = false
-	// hide help
+	rootCmd.Flags().SortFlags = false
+	// hide help command
 	rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})
+	// hide help flag
+	rootCmd.PersistentFlags().BoolP("help", "h", false, "This help")
+	rootCmd.PersistentFlags().Lookup("help").Hidden = true
 
-	// defaults from envars if provided
-	if len(os.Getenv("MP_DATA_DIR")) > 0 {
-		config.DataDir = os.Getenv("MP_DATA_DIR")
-	}
-	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
-		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
-	}
-	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
-		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
-	}
+	// load and warn deprecated ENV vars
+	initDeprecatedConfigFromEnv()
+
+	// load environment variables
+	initConfigFromEnv()
+
+	rootCmd.Flags().StringVarP(&config.DataFile, "db-file", "d", config.DataFile, "Database file to store persistent data")
+	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
+	rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates")
+	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)")
+	rootCmd.Flags().StringVar(&logger.LogFile, "log-file", logger.LogFile, "Log output to file instead of stdout")
+	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
+	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
+
+	// Web UI / API
+	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
+	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
+	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI & API authentication")
+	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-tls-cert", config.UITLSCert, "TLS certificate for web UI (HTTPS) - requires ui-tls-key")
+	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-tls-key", config.UITLSKey, "TLS key for web UI (HTTPS) - requires ui-tls-cert")
+	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
+	rootCmd.Flags().BoolVar(&config.DisableHTMLCheck, "disable-html-check", config.DisableHTMLCheck, "Disable the HTML check functionality (web UI & API)")
+	rootCmd.Flags().BoolVar(&config.BlockRemoteCSSAndFonts, "block-remote-css-and-fonts", config.BlockRemoteCSSAndFonts, "Block access to remote CSS & fonts")
+	rootCmd.Flags().StringVar(&config.EnableSpamAssassin, "enable-spamassassin", config.EnableSpamAssassin, "Enable integration with SpamAssassin")
+	rootCmd.Flags().BoolVar(&config.AllowUntrustedTLS, "allow-untrusted-tls", config.AllowUntrustedTLS, "Do not verify HTTPS certificates (link checker & screenshots)")
+
+	// SMTP server
+	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
+	rootCmd.Flags().StringVar(&config.SMTPAuthFile, "smtp-auth-file", config.SMTPAuthFile, "A password file for SMTP authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPAuthAcceptAny, "smtp-auth-accept-any", config.SMTPAuthAcceptAny, "Accept any SMTP username and password, including none")
+	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key")
+	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert")
+	rootCmd.Flags().BoolVar(&config.SMTPTLSRequired, "smtp-tls-required", config.SMTPTLSRequired, "Require TLS SMTP encryption")
+	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Allow insecure PLAIN & LOGIN SMTP authentication")
+	rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain <CR><CR><LF>")
+	rootCmd.Flags().IntVar(&config.SMTPMaxRecipients, "smtp-max-recipients", config.SMTPMaxRecipients, "Maximum SMTP recipients allowed")
+	rootCmd.Flags().StringVar(&config.SMTPAllowedRecipients, "smtp-allowed-recipients", config.SMTPAllowedRecipients, "Only allow SMTP recipients matching a regular expression (default allow all)")
+	rootCmd.Flags().BoolVar(&smtpd.DisableReverseDNS, "smtp-disable-rdns", smtpd.DisableReverseDNS, "Disable SMTP reverse DNS lookups")
+
+	// SMTP relay
+	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
+	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
+
+	// POP3 server
+	rootCmd.Flags().StringVar(&config.POP3Listen, "pop3", config.POP3Listen, "POP3 server bind interface and port")
+	rootCmd.Flags().StringVar(&config.POP3AuthFile, "pop3-auth-file", config.POP3AuthFile, "A password file for POP3 server authentication (enables POP3 server)")
+	rootCmd.Flags().StringVar(&config.POP3TLSCert, "pop3-tls-cert", config.POP3TLSCert, "Optional TLS certificate for POP3 server - requires pop3-tls-key")
+	rootCmd.Flags().StringVar(&config.POP3TLSKey, "pop3-tls-key", config.POP3TLSKey, "Optional TLS key for POP3 server - requires pop3-tls-cert")
+
+	// Tagging
+	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
+	rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "Convert new tags automatically to TitleCase")
+
+	// Webhook
+	rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages")
+	rootCmd.Flags().IntVar(&webhook.RateLimit, "webhook-limit", webhook.RateLimit, "Limit webhook requests per second")
+
+	// DEPRECATED FLAGS 2023/03/12
+	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-ssl-cert", config.UITLSCert, "SSL certificate for web UI - requires ui-ssl-key")
+	rootCmd.Flags().StringVar(&config.UITLSKey, "ui-ssl-key", config.UITLSKey, "SSL key for web UI - requires ui-ssl-cert")
+	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-ssl-cert", config.SMTPTLSCert, "SSL certificate for SMTP - requires smtp-ssl-key")
+	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-ssl-key", config.SMTPTLSKey, "SSL key for SMTP - requires smtp-ssl-cert")
+	rootCmd.Flags().Lookup("ui-ssl-cert").Hidden = true
+	rootCmd.Flags().Lookup("ui-ssl-cert").Deprecated = "use --ui-tls-cert"
+	rootCmd.Flags().Lookup("ui-ssl-key").Hidden = true
+	rootCmd.Flags().Lookup("ui-ssl-key").Deprecated = "use --ui-tls-key"
+	rootCmd.Flags().Lookup("smtp-ssl-cert").Hidden = true
+	rootCmd.Flags().Lookup("smtp-ssl-cert").Deprecated = "use --smtp-tls-cert"
+	rootCmd.Flags().Lookup("smtp-ssl-key").Hidden = true
+	rootCmd.Flags().Lookup("smtp-ssl-key").Deprecated = "use --smtp-tls-key"
+}
+
+// Load settings from environment
+func initConfigFromEnv() {
+	// General
+	config.DataFile = os.Getenv("MP_DATA_FILE")
 	if len(os.Getenv("MP_MAX_MESSAGES")) > 0 {
 		config.MaxMessages, _ = strconv.Atoi(os.Getenv("MP_MAX_MESSAGES"))
 	}
+	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
+		config.UseMessageDates = true
+	}
+	if getEnabledFromEnv("MP_IGNORE_DUPLICATE_IDS") {
+		config.IgnoreDuplicateIDs = true
+	}
+	if len(os.Getenv("MP_LOG_FILE")) > 0 {
+		logger.LogFile = os.Getenv("MP_LOG_FILE")
+	}
+	if getEnabledFromEnv("MP_QUIET") {
+		logger.QuietLogging = true
+	}
+	if getEnabledFromEnv("MP_VERBOSE") {
+		logger.VerboseLogging = true
+	}
 
-	rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store peristent data")
-	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
-	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
-	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages per mailbox")
-	rootCmd.Flags().BoolVarP(&config.VerboseLogging, "verbose", "v", false, "Verbose logging")
+	// Web UI & API
+	if len(os.Getenv("MP_UI_BIND_ADDR")) > 0 {
+		config.HTTPListen = os.Getenv("MP_UI_BIND_ADDR")
+	}
+	if len(os.Getenv("MP_WEBROOT")) > 0 {
+		config.Webroot = os.Getenv("MP_WEBROOT")
+	}
+	config.UIAuthFile = os.Getenv("MP_UI_AUTH_FILE")
+	if err := auth.SetUIAuth(os.Getenv("MP_UI_AUTH")); err != nil {
+		logger.Log().Errorf(err.Error())
+	}
+	config.UITLSCert = os.Getenv("MP_UI_TLS_CERT")
+	config.UITLSKey = os.Getenv("MP_UI_TLS_KEY")
+	if len(os.Getenv("MP_API_CORS")) > 0 {
+		server.AccessControlAllowOrigin = os.Getenv("MP_API_CORS")
+	}
+	if getEnabledFromEnv("MP_DISABLE_HTML_CHECK") {
+		config.DisableHTMLCheck = true
+	}
+	if getEnabledFromEnv("MP_BLOCK_REMOTE_CSS_AND_FONTS") {
+		config.BlockRemoteCSSAndFonts = true
+	}
+	if len(os.Getenv("MP_ENABLE_SPAMASSASSIN")) > 0 {
+		config.EnableSpamAssassin = os.Getenv("MP_ENABLE_SPAMASSASSIN")
+	}
+	if getEnabledFromEnv("MP_ALLOW_UNTRUSTED_TLS") {
+		config.AllowUntrustedTLS = true
+	}
+
+	// SMTP server
+	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {
+		config.SMTPListen = os.Getenv("MP_SMTP_BIND_ADDR")
+	}
+	config.SMTPAuthFile = os.Getenv("MP_SMTP_AUTH_FILE")
+	if err := auth.SetSMTPAuth(os.Getenv("MP_SMTP_AUTH")); err != nil {
+		logger.Log().Errorf(err.Error())
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ACCEPT_ANY") {
+		config.SMTPAuthAcceptAny = true
+	}
+	config.SMTPTLSCert = os.Getenv("MP_SMTP_TLS_CERT")
+	config.SMTPTLSKey = os.Getenv("MP_SMTP_TLS_KEY")
+	if getEnabledFromEnv("MP_SMTP_TLS_REQUIRED") {
+		config.SMTPTLSRequired = true
+	}
+	if getEnabledFromEnv("MP_SMTP_AUTH_ALLOW_INSECURE") {
+		config.SMTPAuthAllowInsecure = true
+	}
+	if getEnabledFromEnv("MP_SMTP_STRICT_RFC_HEADERS") {
+		config.SMTPStrictRFCHeaders = true
+	}
+	if len(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) > 0 {
+		config.SMTPMaxRecipients, _ = strconv.Atoi(os.Getenv("MP_SMTP_MAX_RECIPIENTS"))
+	}
+	if len(os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")) > 0 {
+		config.SMTPAllowedRecipients = os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")
+	}
+	if getEnabledFromEnv("MP_SMTP_DISABLE_RDNS") {
+		smtpd.DisableReverseDNS = true
+	}
+
+	// SMTP relay
+	config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")
+	if getEnabledFromEnv("MP_SMTP_RELAY_ALL") {
+		config.SMTPRelayAllIncoming = true
+	}
+	config.SMTPRelayConfig = config.SMTPRelayConfigStruct{}
+	config.SMTPRelayConfig.Host = os.Getenv("MP_SMTP_RELAY_HOST")
+	if len(os.Getenv("MP_SMTP_RELAY_PORT")) > 0 {
+		config.SMTPRelayConfig.Port, _ = strconv.Atoi(os.Getenv("MP_SMTP_RELAY_PORT"))
+	}
+	config.SMTPRelayConfig.STARTTLS = getEnabledFromEnv("MP_SMTP_RELAY_STARTTLS")
+	config.SMTPRelayConfig.AllowInsecure = getEnabledFromEnv("MP_SMTP_RELAY_ALLOW_INSECURE")
+	config.SMTPRelayConfig.Auth = os.Getenv("MP_SMTP_RELAY_AUTH")
+	config.SMTPRelayConfig.Username = os.Getenv("MP_SMTP_RELAY_USERNAME")
+	config.SMTPRelayConfig.Password = os.Getenv("MP_SMTP_RELAY_PASSWORD")
+	config.SMTPRelayConfig.Secret = os.Getenv("MP_SMTP_RELAY_SECRET")
+	config.SMTPRelayConfig.ReturnPath = os.Getenv("MP_SMTP_RELAY_RETURN_PATH")
+	config.SMTPRelayConfig.AllowedRecipients = os.Getenv("MP_SMTP_RELAY_ALLOWED_RECIPIENTS")
+
+	// POP3 server
+	if len(os.Getenv("MP_POP3_BIND_ADDR")) > 0 {
+		config.POP3Listen = os.Getenv("MP_POP3_BIND_ADDR")
+	}
+	config.POP3AuthFile = os.Getenv("MP_POP3_AUTH_FILE")
+	if err := auth.SetPOP3Auth(os.Getenv("MP_POP3_AUTH")); err != nil {
+		logger.Log().Errorf(err.Error())
+	}
+	config.POP3TLSCert = os.Getenv("MP_POP3_TLS_CERT")
+	config.POP3TLSKey = os.Getenv("MP_POP3_TLS_KEY")
+
+	// Tagging
+	if len(os.Getenv("MP_TAG")) > 0 {
+		config.SMTPCLITags = os.Getenv("MP_TAG")
+	}
+	if getEnabledFromEnv("MP_TAGS_TITLE_CASE") {
+		tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE")
+	}
+
+	// Webhook
+	if len(os.Getenv("MP_WEBHOOK_URL")) > 0 {
+		config.WebhookURL = os.Getenv("MP_WEBHOOK_URL")
+	}
+	if len(os.Getenv("MP_WEBHOOK_LIMIT")) > 0 {
+		webhook.RateLimit, _ = strconv.Atoi(os.Getenv("MP_WEBHOOK_LIMIT"))
+	}
+}
+
+// load deprecated settings from environment and warn
+func initDeprecatedConfigFromEnv() {
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_UI_SSL_CERT")) > 0 {
+		logger.Log().Warn("ENV MP_UI_SSL_CERT has been deprecated, use MP_UI_TLS_CERT")
+		config.UITLSCert = os.Getenv("MP_UI_SSL_CERT")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_UI_SSL_KEY")) > 0 {
+		logger.Log().Warn("ENV MP_UI_SSL_KEY has been deprecated, use MP_UI_TLS_KEY")
+		config.UITLSKey = os.Getenv("MP_UI_SSL_KEY")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_SMTP_SSL_CERT")) > 0 {
+		logger.Log().Warn("ENV MP_SMTP_CERT has been deprecated, use MP_SMTP_TLS_CERT")
+		config.SMTPTLSCert = os.Getenv("MP_SMTP_SSL_CERT")
+	}
+	// deprecated 2023/03/12
+	if len(os.Getenv("MP_SMTP_SSL_KEY")) > 0 {
+		logger.Log().Warn("ENV MP_SMTP_KEY has been deprecated, use MP_SMTP_TLS_KEY")
+		config.SMTPTLSKey = os.Getenv("MP_SMTP_SMTP_KEY")
+	}
+	// deprecated 2023/12/10
+	if getEnabledFromEnv("MP_STRICT_RFC_HEADERS") {
+		logger.Log().Warn("ENV MP_STRICT_RFC_HEADERS has been deprecated, use MP_SMTP_STRICT_RFC_HEADERS")
+		config.SMTPStrictRFCHeaders = true
+	}
+}
+
+// Wrapper to get a boolean from an environment variable
+func getEnabledFromEnv(k string) bool {
+	if len(os.Getenv(k)) > 0 {
+		v := strings.ToLower(os.Getenv(k))
+		return v == "1" || v == "true" || v == "yes"
+	}
+
+	return false
 }

cmd/sendmail.go

@@ -1,23 +1,18 @@
 package cmd
 
 import (
+	"os"
+
 	sendmail "github.com/axllent/mailpit/sendmail/cmd"
 	"github.com/spf13/cobra"
 )
 
-var (
-	smtpAddr = "localhost:1025"
-	fromAddr string
-)
-
 // sendmailCmd represents the sendmail command
 var sendmailCmd = &cobra.Command{
-	Use:   "sendmail",
-	Short: "A sendmail command replacement",
-	Long: `A sendmail command replacement.
-	
-You can optionally create a symlink called 'sendmail' to the main binary.`,
+	Use:   "sendmail [flags] [recipients]",
+	Short: "A sendmail command replacement for Mailpit",
 	Run: func(_ *cobra.Command, _ []string) {
+
 		sendmail.Run()
 	},
 }
@@ -25,9 +20,17 @@ You can optionally create a symlink called 'sendmail' to the main binary.`,
 func init() {
 	rootCmd.AddCommand(sendmailCmd)
 
-	// these are simply repeated for cli consistency
-	sendmailCmd.Flags().StringVar(&smtpAddr, "smtp-addr", smtpAddr, "SMTP server address")
-	sendmailCmd.Flags().StringVarP(&fromAddr, "from", "f", "", "SMTP sender")
-	sendmailCmd.Flags().BoolP("long-i", "i", false, "Ignored. This flag exists for sendmail compatibility.")
-	sendmailCmd.Flags().BoolP("long-t", "t", false, "Ignored. This flag exists for sendmail compatibility.")
+	// print out manual help screen
+	sendmailCmd.SetHelpTemplate(sendmail.HelpTemplate([]string{os.Args[0], "sendmail"}))
+
+	// these are simply repeated for cli consistency as cobra/viper does not allow
+	// multi-letter single-dash variables (-bs)
+	sendmailCmd.Flags().StringVarP(&sendmail.FromAddr, "from", "f", sendmail.FromAddr, "SMTP sender")
+	sendmailCmd.Flags().StringVarP(&sendmail.SMTPAddr, "smtp-addr", "S", sendmail.SMTPAddr, "SMTP server address")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseB, "long-b", "b", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolVarP(&sendmail.UseS, "long-s", "s", false, "Handle SMTP commands on standard input (use as -bs)")
+	sendmailCmd.Flags().BoolP("verbose", "v", false, "Verbose mode (sends debug output to stderr)")
+	sendmailCmd.Flags().BoolP("long-i", "i", false, "Ignored")
+	sendmailCmd.Flags().BoolP("long-o", "o", false, "Ignored")
+	sendmailCmd.Flags().BoolP("long-t", "t", false, "Ignored")
 }

cmd/version.go

@@ -5,21 +5,11 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/axllent/mailpit/updater"
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/updater"
 	"github.com/spf13/cobra"
 )
 
-var (
-	// Version is the default application version, updated on release
-	Version = "dev"
-
-	// Repo on Github for updater
-	Repo = "axllent/mailpit"
-
-	// RepoBinaryName on Github for updater
-	RepoBinaryName = "mailpit"
-)
-
 // versionCmd represents the version command
 var versionCmd = &cobra.Command{
 	Use:   "version",
@@ -36,10 +26,10 @@ var versionCmd = &cobra.Command{
 		}
 
 		fmt.Printf("%s %s compiled with %s on %s/%s\n",
-			os.Args[0], Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+			os.Args[0], config.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 
-		latest, _, _, err := updater.GithubLatest(Repo, RepoBinaryName)
-		if err == nil && updater.GreaterThan(latest, Version) {
+		latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
+		if err == nil && updater.GreaterThan(latest, config.Version) {
 			fmt.Printf(
 				"\nUpdate available: %s\nRun `%s version -u` to update (requires read/write access to install directory).\n",
 				latest,
@@ -59,7 +49,7 @@ func init() {
 }
 
 func updateApp() error {
-	rel, err := updater.GithubUpdate(Repo, RepoBinaryName, Version)
+	rel, err := updater.GithubUpdate(config.Repo, config.RepoBinaryName, config.Version)
 	if err != nil {
 		return err
 	}

config/config.go

@@ -1,44 +1,507 @@
+// Package config handles the application configuration
 package config
 
 import (
 	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
 	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/spamassassin"
+	"github.com/axllent/mailpit/internal/tools"
+	"gopkg.in/yaml.v3"
 )
 
 var (
 	// SMTPListen to listen on <interface>:<port>
-	SMTPListen = "0.0.0.0:1025"
+	SMTPListen = "[::]:1025"
 
 	// HTTPListen to listen on <interface>:<port>
-	HTTPListen = "0.0.0.0:8025"
+	HTTPListen = "[::]:8025"
 
-	// DataDir for mail (optional)
-	DataDir string
+	// DataFile for mail (optional)
+	DataFile string
 
 	// MaxMessages is the maximum number of messages a mailbox can have (auto-pruned every minute)
 	MaxMessages = 500
 
-	// VerboseLogging for console output
-	VerboseLogging = false
+	// UseMessageDates sets the Created date using the message date, not the delivered date
+	UseMessageDates bool
 
-	// NoLogging for testing
-	NoLogging = false
+	// UITLSCert file
+	UITLSCert string
 
-	// SSLCert @TODO
-	SSLCert string
-	// SSLKey @TODO
-	SSLKey string
+	// UITLSKey file
+	UITLSKey string
+
+	// UIAuthFile for UI & API authentication
+	UIAuthFile string
+
+	// Webroot to define the base path for the UI and API
+	Webroot = "/"
+
+	// SMTPTLSCert file
+	SMTPTLSCert string
+
+	// SMTPTLSKey file
+	SMTPTLSKey string
+
+	// SMTPTLSRequired to enforce TLS
+	// The only allowed commands are NOOP, EHLO, STARTTLS and QUIT (as specified in RFC 3207) until
+	// the connection is upgraded to TLS i.e. until STARTTLS is issued.
+	SMTPTLSRequired bool
+
+	// SMTPAuthFile for SMTP authentication
+	SMTPAuthFile string
+
+	// SMTPAuthAllowInsecure allows PLAIN & LOGIN unencrypted authentication
+	SMTPAuthAllowInsecure bool
+
+	// SMTPAuthAcceptAny accepts any username/password including none
+	SMTPAuthAcceptAny bool
+
+	// SMTPMaxRecipients is the maximum number of recipients a message may have.
+	// The SMTP RFC states that an server must handle a minimum of 100 recipients
+	// however some servers accept more.
+	SMTPMaxRecipients = 100
+
+	// IgnoreDuplicateIDs will skip messages with the same ID
+	IgnoreDuplicateIDs bool
+
+	// DisableHTMLCheck used to disable the HTML check in bother the API and web UI
+	DisableHTMLCheck = false
+
+	// BlockRemoteCSSAndFonts used to disable remote CSS & fonts
+	BlockRemoteCSSAndFonts = false
+
+	// SMTPCLITags is used to map the CLI args
+	SMTPCLITags string
+
+	// ValidTagRegexp represents a valid tag
+	ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_\.]){1,}$`)
+
+	// SMTPTags are expressions to apply tags to new mail
+	SMTPTags []AutoTag
+
+	// SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server
+	SMTPRelayConfigFile string
+
+	// SMTPRelayConfig to parse a yaml file and store config of relay SMTP server
+	SMTPRelayConfig SMTPRelayConfigStruct
+
+	// SMTPStrictRFCHeaders will return an error if the email headers contain <CR><CR><LF> (\r\r\n)
+	// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
+	SMTPStrictRFCHeaders bool
+
+	// SMTPAllowedRecipients if set, will only accept recipients matching this regular expression
+	SMTPAllowedRecipients string
+
+	// SMTPAllowedRecipientsRegexp is the compiled version of SMTPAllowedRecipients
+	SMTPAllowedRecipientsRegexp *regexp.Regexp
+
+	// ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile
+	ReleaseEnabled = false
+
+	// SMTPRelayAllIncoming is whether to relay all incoming messages via pre-configured SMTP server.
+	// Use with extreme caution!
+	SMTPRelayAllIncoming = false
+
+	// POP3Listen address - if set then Mailpit will start the POP3 server and listen on this address
+	POP3Listen = "[::]:1110"
+
+	// POP3AuthFile for POP3 authentication
+	POP3AuthFile string
+
+	// POP3TLSCert TLS certificate
+	POP3TLSCert string
+
+	// POP3TLSKey TLS certificate key
+	POP3TLSKey string
+
+	// EnableSpamAssassin must be either <host>:<port> or "postmark"
+	EnableSpamAssassin string
+
+	// WebhookURL for calling
+	WebhookURL string
+
+	// ContentSecurityPolicy for HTTP server - set via VerifyConfig()
+	ContentSecurityPolicy string
+
+	// AllowUntrustedTLS allows untrusted HTTPS connections link checking & screenshot generation
+	AllowUntrustedTLS bool
+
+	// Version is the default application version, updated on release
+	Version = "dev"
+
+	// Repo on Github for updater
+	Repo = "axllent/mailpit"
+
+	// RepoBinaryName on Github for updater
+	RepoBinaryName = "mailpit"
 )
 
+// AutoTag struct for auto-tagging
+type AutoTag struct {
+	Tag   string
+	Match string
+}
+
+// SMTPRelayConfigStruct struct for parsing yaml & storing variables
+type SMTPRelayConfigStruct struct {
+	Host                    string         `yaml:"host"`
+	Port                    int            `yaml:"port"`
+	STARTTLS                bool           `yaml:"starttls"`
+	AllowInsecure           bool           `yaml:"allow-insecure"`
+	Auth                    string         `yaml:"auth"`               // none, plain, login, cram-md5
+	Username                string         `yaml:"username"`           // plain & cram-md5
+	Password                string         `yaml:"password"`           // plain
+	Secret                  string         `yaml:"secret"`             // cram-md5
+	ReturnPath              string         `yaml:"return-path"`        // allow overriding the bounce address
+	AllowedRecipients       string         `yaml:"allowed-recipients"` // regex, if set needs to match for mails to be relayed
+	AllowedRecipientsRegexp *regexp.Regexp // compiled regexp using AllowedRecipients
+	// DEPRECATED 2024/03/12
+	RecipientAllowlist string `yaml:"recipient-allowlist"`
+}
+
 // VerifyConfig wil do some basic checking
 func VerifyConfig() error {
-	re := regexp.MustCompile(`^[a-zA-Z0-9\.\-]{3,}:\d{2,}$`)
+	cssFontRestriction := "*"
+	if BlockRemoteCSSAndFonts {
+		cssFontRestriction = "'self'"
+	}
+
+	ContentSecurityPolicy = fmt.Sprintf("default-src 'self'; script-src 'self'; style-src %s 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src %s data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';",
+		cssFontRestriction, cssFontRestriction,
+	)
+
+	if DataFile != "" && isDir(DataFile) {
+		DataFile = filepath.Join(DataFile, "mailpit.db")
+	}
+
+	re := regexp.MustCompile(`.*:\d+$`)
 	if !re.MatchString(SMTPListen) {
-		return errors.New("SMTP bind should be in the format of <ip>:<port>")
+		return errors.New("[smtp] bind should be in the format of <ip>:<port>")
 	}
 	if !re.MatchString(HTTPListen) {
-		return errors.New("HTTP bind should be in the format of <ip>:<port>")
+		return errors.New("[ui] HTTP bind should be in the format of <ip>:<port>")
+	}
+
+	if UIAuthFile != "" {
+		UIAuthFile = filepath.Clean(UIAuthFile)
+
+		if !isFile(UIAuthFile) {
+			return fmt.Errorf("[ui] HTTP password file not found: %s", UIAuthFile)
+		}
+
+		b, err := os.ReadFile(UIAuthFile)
+		if err != nil {
+			return err
+		}
+
+		if err := auth.SetUIAuth(string(b)); err != nil {
+			return err
+		}
+	}
+
+	if UITLSCert != "" && UITLSKey == "" || UITLSCert == "" && UITLSKey != "" {
+		return errors.New("[ui] you must provide both a UI TLS certificate and a key")
+	}
+
+	if UITLSCert != "" {
+		UITLSCert = filepath.Clean(UITLSCert)
+		UITLSKey = filepath.Clean(UITLSKey)
+
+		if !isFile(UITLSCert) {
+			return fmt.Errorf("[ui] TLS certificate not found: %s", UITLSCert)
+		}
+
+		if !isFile(UITLSKey) {
+			return fmt.Errorf("[ui] TLS key not found: %s", UITLSKey)
+		}
+	}
+
+	if SMTPTLSCert != "" && SMTPTLSKey == "" || SMTPTLSCert == "" && SMTPTLSKey != "" {
+		return errors.New("[smtp] You must provide both an SMTP TLS certificate and a key")
+	}
+
+	if SMTPTLSCert != "" {
+		SMTPTLSCert = filepath.Clean(SMTPTLSCert)
+		SMTPTLSKey = filepath.Clean(SMTPTLSKey)
+
+		if !isFile(SMTPTLSCert) {
+			return fmt.Errorf("[smtp] TLS certificate not found: %s", SMTPTLSCert)
+		}
+
+		if !isFile(SMTPTLSKey) {
+			return fmt.Errorf("[smtp] TLS key not found: %s", SMTPTLSKey)
+		}
+	} else if SMTPTLSRequired {
+		return errors.New("[smtp] TLS cannot be required without an SMTP TLS certificate and key")
+	}
+
+	if SMTPTLSRequired && SMTPAuthAllowInsecure {
+		return errors.New("[smtp] TLS cannot be required while also allowing insecure authentication")
+	}
+
+	if SMTPAuthFile != "" {
+		SMTPAuthFile = filepath.Clean(SMTPAuthFile)
+
+		if !isFile(SMTPAuthFile) {
+			return fmt.Errorf("[smtp] password file not found: %s", SMTPAuthFile)
+		}
+
+		b, err := os.ReadFile(SMTPAuthFile)
+		if err != nil {
+			return err
+		}
+
+		if err := auth.SetSMTPAuth(string(b)); err != nil {
+			return err
+		}
+	}
+
+	if auth.SMTPCredentials != nil && SMTPAuthAcceptAny {
+		return errors.New("[smtp] authentication cannot use both credentials and --smtp-auth-accept-any")
+	}
+
+	if SMTPTLSCert == "" && (auth.SMTPCredentials != nil || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure {
+		return errors.New("[smtp] authentication requires TLS encryption, run with `--smtp-auth-allow-insecure` to allow insecure authentication")
+	}
+
+	// POP3 server
+	if POP3TLSCert != "" {
+		POP3TLSCert = filepath.Clean(POP3TLSCert)
+		POP3TLSKey = filepath.Clean(POP3TLSKey)
+
+		if !isFile(POP3TLSCert) {
+			return fmt.Errorf("[pop3] TLS certificate not found: %s", POP3TLSCert)
+		}
+
+		if !isFile(POP3TLSKey) {
+			return fmt.Errorf("[pop3] TLS key not found: %s", POP3TLSKey)
+		}
+	}
+	if POP3TLSCert != "" && POP3TLSKey == "" || POP3TLSCert == "" && POP3TLSKey != "" {
+		return errors.New("[pop3] You must provide both a POP3 TLS certificate and a key")
+	}
+	if POP3Listen != "" {
+		_, err := net.ResolveTCPAddr("tcp", POP3Listen)
+		if err != nil {
+			return fmt.Errorf("[pop3] %s", err.Error())
+		}
+	}
+	if POP3AuthFile != "" {
+		POP3AuthFile = filepath.Clean(POP3AuthFile)
+
+		if !isFile(POP3AuthFile) {
+			return fmt.Errorf("[pop3] password file not found: %s", POP3AuthFile)
+		}
+
+		b, err := os.ReadFile(POP3AuthFile)
+		if err != nil {
+			return err
+		}
+
+		if err := auth.SetPOP3Auth(string(b)); err != nil {
+			return err
+		}
+	}
+
+	// Web root
+	validWebrootRe := regexp.MustCompile(`[^0-9a-zA-Z\/\-\_\.@]`)
+	if validWebrootRe.MatchString(Webroot) {
+		return fmt.Errorf("invalid characters in Webroot (%s). Valid chars include: [a-z A-Z 0-9 _ . - / @]", Webroot)
+	}
+
+	s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/"
+	Webroot = s
+
+	if WebhookURL != "" && !isValidURL(WebhookURL) {
+		return fmt.Errorf("webhook URL does not appear to be a valid URL (%s)", WebhookURL)
+	}
+
+	if EnableSpamAssassin != "" {
+		spamassassin.SetService(EnableSpamAssassin)
+		logger.Log().Infof("[spamassassin] enabled via %s", EnableSpamAssassin)
+
+		if err := spamassassin.Ping(); err != nil {
+			logger.Log().Warnf("[spamassassin] ping: %s", err.Error())
+		}
+	}
+
+	SMTPTags = []AutoTag{}
+
+	if SMTPCLITags != "" {
+		args := tools.ArgsParser(SMTPCLITags)
+
+		for _, a := range args {
+			t := strings.Split(a, "=")
+			if len(t) > 1 {
+				tag := tools.CleanTag(t[0])
+				if !ValidTagRegexp.MatchString(tag) || len(tag) == 0 {
+					return fmt.Errorf("[tag] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tag)
+				}
+				match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
+				if len(match) == 0 {
+					return fmt.Errorf("[tag] invalid tag match (%s) - no search detected", tag)
+				}
+				SMTPTags = append(SMTPTags, AutoTag{Tag: tag, Match: match})
+			} else {
+				return fmt.Errorf("[tag] error parsing tags (%s)", a)
+			}
+		}
+	}
+
+	if SMTPAllowedRecipients != "" {
+		restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients)
+		if err != nil {
+			return fmt.Errorf("[smtp] failed to compile smtp-allowed-recipients regexp: %s", err.Error())
+		}
+
+		SMTPAllowedRecipientsRegexp = restrictRegexp
+		logger.Log().Infof("[smtp] only allowing recipients matching the following regexp: %s", SMTPAllowedRecipients)
+	}
+
+	if err := parseRelayConfig(SMTPRelayConfigFile); err != nil {
+		return err
+	}
+
+	// separate relay config validation to account for environment variables
+	if err := validateRelayConfig(); err != nil {
+		return err
+	}
+
+	if !ReleaseEnabled && SMTPRelayAllIncoming {
+		return errors.New("[smtp] relay config must be set to relay all messages")
+	}
+
+	if SMTPRelayAllIncoming {
+		// this deserves a warning
+		logger.Log().Warnf("[smtp] enabling automatic relay of all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
 	}
 
 	return nil
 }
+
+// Parse the SMTPRelayConfigFile (if set)
+func parseRelayConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[smtp] relay configuration not found: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil {
+		return err
+	}
+
+	if SMTPRelayConfig.Host == "" {
+		return errors.New("[smtp] relay host not set")
+	}
+
+	// DEPRECATED 2024/03/12
+	if SMTPRelayConfig.RecipientAllowlist != "" {
+		logger.Log().Warn("[smtp] relay 'recipient-allowlist' is deprecated, use 'allowed_recipients' instead")
+		if SMTPRelayConfig.AllowedRecipients == "" {
+			SMTPRelayConfig.AllowedRecipients = SMTPRelayConfig.RecipientAllowlist
+		}
+	}
+
+	return nil
+}
+
+// Validate the SMTPRelayConfig (if Host is set)
+func validateRelayConfig() error {
+	if SMTPRelayConfig.Host == "" {
+		return nil
+	}
+
+	if SMTPRelayConfig.Port == 0 {
+		SMTPRelayConfig.Port = 25 // default
+	}
+
+	SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth)
+
+	if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" {
+		SMTPRelayConfig.Auth = "none"
+	} else if SMTPRelayConfig.Auth == "plain" {
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[smtp] relay host username or password not set for PLAIN authentication")
+		}
+	} else if SMTPRelayConfig.Auth == "login" {
+		SMTPRelayConfig.Auth = "login"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("[smtp] relay host username or password not set for LOGIN authentication")
+		}
+	} else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") {
+		SMTPRelayConfig.Auth = "cram-md5"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" {
+			return fmt.Errorf("[smtp] relay host username or secret not set for CRAM-MD5 authentication")
+		}
+	} else {
+		return fmt.Errorf("[smtp] relay authentication method not supported: %s", SMTPRelayConfig.Auth)
+	}
+
+	ReleaseEnabled = true
+
+	logger.Log().Infof("[smtp] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+
+	allowlistRegexp, err := regexp.Compile(SMTPRelayConfig.AllowedRecipients)
+
+	if SMTPRelayConfig.AllowedRecipients != "" {
+		if err != nil {
+			return fmt.Errorf("[smtp] failed to compile relay recipient allowlist regexp: %s", err.Error())
+		}
+
+		SMTPRelayConfig.AllowedRecipientsRegexp = allowlistRegexp
+		logger.Log().Infof("[smtp] relay recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.AllowedRecipients)
+
+	}
+
+	return nil
+}
+
+// IsFile returns if a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// IsDir returns whether a path is a directory
+func isDir(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.IsDir() {
+		return false
+	}
+
+	return true
+}
+
+func isValidURL(s string) bool {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(u.Scheme, "http")
+}

data/mailbox.go

@@ -1,24 +0,0 @@
-package data
-
-import "time"
-
-// MailboxSummary struct
-type MailboxSummary struct {
-	Name        string
-	Slug        string
-	Total       int
-	Unread      int
-	LastMessage time.Time
-}
-
-// WebsocketNotification struct for responses
-type WebsocketNotification struct {
-	Type string
-	Data interface{}
-}
-
-// MailboxStats struct for quick mailbox total/read lookups
-type MailboxStats struct {
-	Total  int
-	Unread int
-}

data/message.go

@@ -1,64 +0,0 @@
-package data
-
-import (
-	"net/mail"
-	"time"
-
-	"github.com/jhillyerd/enmime"
-)
-
-// Message struct for loading messages. It does not include physical attachments.
-type Message struct {
-	ID          string
-	Read        bool
-	From        *mail.Address
-	To          []*mail.Address
-	Cc          []*mail.Address
-	Bcc         []*mail.Address
-	Subject     string
-	Date        time.Time
-	Created     time.Time
-	Text        string
-	HTML        string
-	Size        int
-	Inline      []Attachment
-	Attachments []Attachment
-}
-
-// Attachment struct for inline and attachments
-type Attachment struct {
-	PartID      string
-	FileName    string
-	ContentType string
-	ContentID   string
-	Size        int
-}
-
-// Summary struct for frontend messages
-type Summary struct {
-	ID          string
-	Read        bool
-	From        *mail.Address
-	To          []*mail.Address
-	Cc          []*mail.Address
-	Bcc         []*mail.Address
-	Subject     string
-	Created     time.Time
-	Size        int
-	Attachments int
-}
-
-// AttachmentSummary returns a summary of the attachment without any binary data
-func AttachmentSummary(a *enmime.Part) Attachment {
-	o := Attachment{}
-	o.PartID = a.PartID
-	o.FileName = a.FileName
-	if o.FileName == "" {
-		o.FileName = a.ContentID
-	}
-	o.ContentType = a.ContentType
-	o.ContentID = a.ContentID
-	o.Size = len(a.Content)
-
-	return o
-}

esbuild.config.js

@@ -1,22 +0,0 @@
-const { build } = require('esbuild')
-const pluginVue = require('esbuild-plugin-vue-next')
-const sassPlugin = require("esbuild-plugin-sass");
-
-const doWatch = process.env.WATCH == 'true' ? true : false;
-const doMinify = process.env.MINIFY == 'true' ? true : false;
-
-build({
-    entryPoints: ["server/ui-src/app.js"],
-    bundle: true,
-    watch: doWatch,
-    minify: doMinify,
-    sourcemap: false,
-    outfile: "server/ui/dist/app.js",
-    plugins: [pluginVue(), sassPlugin()],
-    loader: {
-        ".svg": "file",
-        ".woff": "file",
-        ".woff2": "file",
-    },
-    logLevel: "info"
-})

esbuild.config.mjs

@@ -0,0 +1,38 @@
+import * as esbuild from 'esbuild'
+import pluginVue from 'esbuild-plugin-vue-next'
+import { sassPlugin } from 'esbuild-sass-plugin'
+
+const doWatch = process.env.WATCH == 'true' ? true : false;
+const doMinify = process.env.MINIFY == 'true' ? true : false;
+
+const ctx = await esbuild.context(
+    {
+        entryPoints: [
+            "server/ui-src/app.js",
+            "server/ui-src/docs.js"
+        ],
+        bundle: true,
+        minify: doMinify,
+        sourcemap: false,
+        define: {
+            '__VUE_OPTIONS_API__': 'true',
+            '__VUE_PROD_DEVTOOLS__': 'false',
+            '__VUE_PROD_HYDRATION_MISMATCH_DETAILS__': 'false',
+        },
+        outdir: "server/ui/dist/",
+        plugins: [pluginVue(), sassPlugin()],
+        loader: {
+            ".svg": "file",
+            ".woff": "file",
+            ".woff2": "file",
+        },
+        logLevel: "info"
+    }
+)
+
+if (doWatch) {
+    await ctx.watch()
+} else {
+    await ctx.rebuild()
+    ctx.dispose()
+}

go.mod

@@ -1,52 +1,67 @@
 module github.com/axllent/mailpit
 
-go 1.18
+go 1.20
 
 require (
+	github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244
+	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/axllent/semver v0.0.1
-	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/websocket v1.5.0
-	github.com/jhillyerd/enmime v0.10.0
-	github.com/k3a/html2text v1.0.8
-	github.com/mhale/smtpd v0.8.0
-	github.com/ostafen/clover/v2 v2.0.0-alpha.1
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.5.0
+	github.com/disintegration/imaging v1.6.2
+	github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47
+	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/websocket v1.5.1
+	github.com/jhillyerd/enmime v1.2.0
+	github.com/klauspost/compress v1.17.7
+	github.com/leporo/sqlf v1.4.0
+	github.com/lithammer/shortuuid/v4 v4.0.0
+	github.com/mhale/smtpd v0.8.2
+	github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
+	github.com/tg123/go-htpasswd v1.2.2
+	github.com/vanng822/go-premailer v1.20.2
+	golang.org/x/net v0.22.0
+	golang.org/x/text v0.14.0
+	golang.org/x/time v0.5.0
+	gopkg.in/yaml.v3 v3.0.1
+	modernc.org/sqlite v1.29.3
 )
 
 require (
+	github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect
+	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/dgraph-io/badger/v3 v3.2103.2 // indirect
-	github.com/dgraph-io/ristretto v0.1.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/cznic/ql v1.2.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
-	github.com/golang/glog v1.0.0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v2.0.6+incompatible // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
-	github.com/google/orderedcode v0.0.1 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rivo/uniseg v0.3.0 // indirect
-	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/reiver/go-oi v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
-	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vanng822/css v1.0.1 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/image v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	modernc.org/gc/v3 v3.0.0-20240304020402-f0dba7c97c2b // indirect
+	modernc.org/libc v1.43.1 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 )

go.sum

@@ -1,287 +1,229 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI=
+github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
+github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 h1:dqzm54OhCqY8RinR/cx+Ppb0y56Ds5I3wwWhx4XybDg=
+github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244/go.mod h1:3sqgkckuISJ5rs1EpOp6vCvwOUKe/z9vPmyuIlq8Q/A=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E=
 github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc=
-github.com/brianvoe/gofakeit/v6 v6.17.0 h1:obbQTJeHfktJtiZzq0Q1bEpsNUs+yHrYlPVWt7BtmJ4=
-github.com/brianvoe/gofakeit/v6 v6.17.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk=
+github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
+github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas=
+github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
+github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4 h1:CVAqftqbj+exlab+8KJQrE+kNIVlQfJt58j4GxCMF1s=
+github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
+github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00 h1:FHpbUtp2K8X53/b4aFNj4my5n+i3x+CQCZWNuHWH/+E=
+github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4=
+github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs=
+github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A=
+github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk=
+github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
+github.com/cznic/ql v1.2.0 h1:lcKp95ZtdF0XkWhGnVIXGF8dVD2X+ClS08tglKtf+ak=
+github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE=
+github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
+github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ=
+github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA=
+github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
+github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc h1:YKKpTb2BrXN2GYyGaygIdis1vXbE7SSAG9axGWIMClg=
+github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8=
-github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M=
-github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
-github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
-github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
+github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
-github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw=
-github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/orderedcode v0.0.1 h1:UzfcAexk9Vhv8+9pNOgRu41f16lHq725vPwnSeiG/Us=
-github.com/google/orderedcode v0.0.1/go.mod h1:iVyU4/qPKHY5h/wSd6rZZCDcLJNxiWO6dvsYES2Sb20=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg=
-github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jhillyerd/enmime v0.10.0 h1:DZEzhptPRBesvN3gf7K1BOh4rfpqdsdrEoxW1Edr/3s=
-github.com/jhillyerd/enmime v0.10.0/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/z6mkcDHsA=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/k3a/html2text v1.0.8 h1:rVanLhKilpnJUJs/CNKWzMC4YaQINGxK0rSG8ssmnV0=
-github.com/k3a/html2text v1.0.8/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47 h1:k4Tw0nt6lwro3Uin8eqoET7MDA4JnT8YgbCjc/g5E3k=
+github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/jhillyerd/enmime v1.2.0 h1:dIu1IPEymQgoT2dzuB//ttA/xcV40NMPpQtmd4wslHk=
+github.com/jhillyerd/enmime v1.2.0/go.mod h1:FRFuUPCLh8PByQv+8xRcLO9QHqaqTqreYhopv5eyk4I=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/leporo/sqlf v1.4.0 h1:SyWnX/8GSGOzVmanG0Ub1c04mR9nNl6Tq3IeFKX2/4c=
+github.com/leporo/sqlf v1.4.0/go.mod h1:pgN9yKsAnQ+2ewhbZogr98RcasUjPsHF3oXwPPhHvBw=
+github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c=
+github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
-github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=
-github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mhale/smtpd v0.8.2 h1:rHKOMHeFoDvcq8Na9ErCbNcjlWTSyGtznOmJpWsOzuc=
+github.com/mhale/smtpd v0.8.2/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/ostafen/clover/v2 v2.0.0-alpha.1 h1:Oa+N1xXvEXirUUmwssN+ASd/Xf9DikITT8Id+zc0l+o=
-github.com/ostafen/clover/v2 v2.0.0-alpha.1/go.mod h1:7UyIG46NglzTDRKB4LJiS/enXpuo67Lj05eM8mdhL6M=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/reiver/go-oi v1.0.0 h1:nvECWD7LF+vOs8leNGV/ww+F2iZKf3EYjYZ527turzM=
+github.com/reiver/go-oi v1.0.0/go.mod h1:RrDBct90BAhoDTxB1fenZwfykqeGvhI6LsNfStJoEkI=
+github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e h1:quuzZLi72kkJjl+f5AQ93FMcadG19WkS7MO6TXFOSas=
+github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e/go.mod h1:+5vNVvEWwEIx86DB9Ke/+a5wBI464eDRo3eF0LcfpWg=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.3.0 h1:eyC18g7xB83Dv/xlJXLgNkRidVoR7nqFZBJvqo/K188=
-github.com/rivo/uniseg v0.3.0/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
-github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
-github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
-github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
-github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/tg123/go-htpasswd v1.2.2 h1:tmNccDsQ+wYsoRfiONzIhDm5OkVHQzN3w4FOBAlN6BY=
+github.com/tg123/go-htpasswd v1.2.2/go.mod h1:FcIrK0J+6zptgVwK1JDlqyajW/1B4PtuJ/FLWl7nx8A=
+github.com/unrolled/render v1.0.3/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/vanng822/css v1.0.1 h1:10yiXc4e8NI8ldU6mSrWmSWMuyWgPr9DZ63RSlsgDw8=
+github.com/vanng822/css v1.0.1/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w=
+github.com/vanng822/go-premailer v1.20.2 h1:vKs4VdtfXDqL7IXC2pkiBObc1bXM9bYH3Wa+wYw2DnI=
+github.com/vanng822/go-premailer v1.20.2/go.mod h1:RAxbRFp6M/B171gsKu8dsyq+Y5NGsUUvYfg+WQWusbE=
+github.com/vanng822/r2router v0.0.0-20150523112421-1023140a4f30/go.mod h1:1BVq8p2jVr55Ost2PkZWDrG86PiJ/0lxqcXoAcGxvWU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
+golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 h1:UreQrH7DbFXSi9ZFox6FNT3WBooWmdANpU+IfkT1T4I=
-golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+modernc.org/cc/v4 v4.19.3 h1:vE9kmJqUcyvNOf8F2Hn8od14SOMq34BiqcZ2tMzLk5c=
+modernc.org/ccgo/v4 v4.9.9 h1:HGZLD/Ws06nfcbjAKpnR5On3KQRSKUrD+oiHgEvRANI=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v3 v3.0.0-20240304020402-f0dba7c97c2b h1:BnN1t+pb1cy61zbvSUV7SeI0PwosMhlAEi/vBY4qxp8=
+modernc.org/gc/v3 v3.0.0-20240304020402-f0dba7c97c2b/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.43.1 h1:t1JKWKv2dxw3xj3OXmA/abCLTyZGEWSjUcOMh8kZ8zc=
+modernc.org/libc v1.43.1/go.mod h1:KpVOBS+2L3K2i2oZac6eycs//ukjVzwrhobyw+mi81c=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sqlite v1.29.3 h1:6L71d3zXVB8oubdVSuwiurNyYRetQ3It8l1FSwylwQ0=
+modernc.org/sqlite v1.29.3/go.mod h1:MjUIBKZ+tU/lqjNLbVAAMjsQPdWdA/ciwdhsT9kBwk8=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

install.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+GH_REPO="axllent/mailpit"
+TIMEOUT=90
+
+set -e
+
+VERSION=$(curl --silent --location --max-time "${TIMEOUT}" "https://api.github.com/repos/${GH_REPO}/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+if [ $? -ne 0 ]; then
+    echo -ne "\nThere was an error trying to check what is the latest version of Mailpit.\nPlease try again later.\n"
+    exit 1
+fi
+
+# detect the platform
+OS="$(uname)"
+case $OS in
+Linux)
+    OS='linux'
+    ;;
+FreeBSD)
+    OS='freebsd'
+    echo 'OS not supported'
+    exit 2
+    ;;
+NetBSD)
+    OS='netbsd'
+    echo 'OS not supported'
+    exit 2
+    ;;
+OpenBSD)
+    OS='openbsd'
+    echo 'OS not supported'
+    exit 2
+    ;;
+Darwin)
+    OS='darwin'
+    ;;
+SunOS)
+    OS='solaris'
+    echo 'OS not supported'
+    exit 2
+    ;;
+*)
+    echo 'OS not supported'
+    exit 2
+    ;;
+esac
+
+# detect the arch
+OS_type="$(uname -m)"
+case "$OS_type" in
+x86_64 | amd64)
+    OS_type='amd64'
+    ;;
+i?86 | x86)
+    OS_type='386'
+    ;;
+aarch64 | arm64)
+    OS_type='arm64'
+    ;;
+*)
+    echo 'OS type not supported'
+    exit 2
+    ;;
+esac
+
+GH_REPO_BIN="mailpit-${OS}-${OS_type}.tar.gz"
+
+#create tmp directory and move to it with macOS compatibility fallback
+tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mailpit-install.XXXXXXXXXX')
+cd "$tmp_dir"
+
+echo "Downloading Mailpit $VERSION"
+LINK="https://github.com/${GH_REPO}/releases/download/${VERSION}/${GH_REPO_BIN}"
+
+curl --silent --location --max-time "${TIMEOUT}" "${LINK}" | tar zxf - || {
+    echo "Error downloading"
+    exit 2
+}
+
+mkdir -p /usr/local/bin || exit 2
+cp mailpit /usr/local/bin/ || exit 2
+chmod 755 /usr/local/bin/mailpit || exit 2
+case "$OS" in
+'linux')
+    chown root:root /usr/local/bin/mailpit || exit 2
+    ;;
+'freebsd' | 'openbsd' | 'netbsd' | 'darwin')
+    chown root:wheel /usr/local/bin/mailpit || exit 2
+    ;;
+*)
+    echo 'OS not supported'
+    exit 2
+    ;;
+esac
+
+rm -rf "$tmp_dir"
+echo "Installed successfully to /usr/local/bin/mailpit"

internal/auth/auth.go

@@ -0,0 +1,90 @@
+// Package auth handles the web UI and SMTP authentication
+package auth
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/tg123/go-htpasswd"
+)
+
+var (
+	// UICredentials passwords
+	UICredentials *htpasswd.File
+	// SMTPCredentials passwords
+	SMTPCredentials *htpasswd.File
+	// POP3Credentials passwords
+	POP3Credentials *htpasswd.File
+)
+
+// SetUIAuth will set Basic Auth credentials required for the UI & API
+func SetUIAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	UICredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetSMTPAuth will set SMTP credentials
+func SetSMTPAuth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	SMTPCredentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetPOP3Auth will set POP3 server credentials
+func SetPOP3Auth(s string) error {
+	var err error
+
+	credentials := credentialsFromString(s)
+	if len(credentials) == 0 {
+		return nil
+	}
+
+	r := strings.NewReader(strings.Join(credentials, "\n"))
+
+	POP3Credentials, err = htpasswd.NewFromReader(r, htpasswd.DefaultSystems, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func credentialsFromString(s string) []string {
+	// split string by any whitespace character
+	re := regexp.MustCompile(`\s+`)
+
+	words := re.Split(s, -1)
+	credentials := []string{}
+	for _, w := range words {
+		if w != "" {
+			credentials = append(credentials, w)
+		}
+	}
+
+	return credentials
+}

internal/html2text/html2text.go

@@ -0,0 +1,82 @@
+// Package html2text is a simple library to convert HTML to plain text
+package html2text
+
+import (
+	"bytes"
+	"log"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"golang.org/x/net/html"
+)
+
+var (
+	re      = regexp.MustCompile(`\s+`)
+	spaceRe = regexp.MustCompile(`(?mi)<\/(div|p|td|th|h[1-6]|ul|ol|li|address|article|aside|blockquote|dl|dt|footer|header|hr|main|nav|pre|table|thead|tfoot|video)><`)
+	brRe    = regexp.MustCompile(`(?mi)<(br /|br)>`)
+	imgRe   = regexp.MustCompile(`(?mi)<(img)`)
+	skip    = make(map[string]bool)
+)
+
+func init() {
+	skip["script"] = true
+	skip["title"] = true
+	skip["head"] = true
+	skip["link"] = true
+	skip["meta"] = true
+	skip["style"] = true
+	skip["noscript"] = true
+}
+
+// Strip will convert a HTML string to plain text
+func Strip(h string, includeLinks bool) string {
+	h = spaceRe.ReplaceAllString(h, "</$1> <")
+	h = brRe.ReplaceAllString(h, " ")
+	h = imgRe.ReplaceAllString(h, " <$1")
+	var buffer bytes.Buffer
+	doc, err := html.Parse(strings.NewReader(h))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	extract(doc, &buffer, includeLinks)
+	return clean(buffer.String())
+}
+
+func extract(node *html.Node, buff *bytes.Buffer, includeLinks bool) {
+	if node.Type == html.TextNode {
+		data := node.Data
+		if data != "" {
+			buff.WriteString(data)
+		}
+	}
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		if _, skip := skip[c.Data]; !skip {
+			if includeLinks && c.Data == "a" {
+				for _, a := range c.Attr {
+					if a.Key == "href" && strings.HasPrefix(strings.ToLower(a.Val), "http") {
+						buff.WriteString(" " + a.Val + " ")
+					}
+				}
+			}
+			extract(c, buff, includeLinks)
+		}
+	}
+}
+
+func clean(text string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	text = strings.ReplaceAll(text, string('\uFEFF'), " ")
+
+	// remove non-printable characters
+	text = strings.Map(func(r rune) rune {
+		if unicode.IsPrint(r) {
+			return r
+		}
+		return []rune(" ")[0]
+	}, text)
+
+	text = re.ReplaceAllString(text, " ")
+	return strings.TrimSpace(text)
+}

internal/html2text/html2text_test.go

@@ -0,0 +1,250 @@
+package html2text
+
+import "testing"
+
+func TestPlain(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, false)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestWithLinks(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["<p>Paragraph 1</p><p>Paragraph 2</p>"] = "Paragraph 1 Paragraph 2"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests["<span>Alpha</span>bet <strong>chars</strong>"] = "Alphabet chars"
+	tests["<span><b>A</b>lpha</span>bet  chars."] = "Alphabet chars."
+	tests["<table><tr><td>First</td><td>Second</td></table>"] = "First Second"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading https://github.com linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading https://github.com linked text."
+
+	for str, expected := range tests {
+		res := Strip(str, true)
+		if res != expected {
+			t.Log("error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func BenchmarkPlain(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, false)
+	}
+}
+
+func BenchmarkLinks(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Strip(htmlTestData, true)
+	}
+}
+
+var htmlTestData = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en" style="font-family: sans-serif; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; box-sizing: border-box;" xml:lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>[axllent/mailpit] Run failed: .github/workflows/tests.yml - feature/swagger (284335a)</title>
+    
+  </head>
+  <body style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;; font-size: 14px; line-height: 1.5; color: #24292e; background-color: #fff; margin: 0;" bgcolor="#fff">
+    <table align="center" class="container-sm width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 544px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <td class="center p-3" align="center" valign="top" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 16px;">
+          <center style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full container-md" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; max-width: 768px; margin-right: auto; margin-left: auto; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                  <td class="text-left" style="box-sizing: border-box; text-align: left !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;" align="left">
+                    <img src="https://github.githubassets.com/images/email/global/octocat-logo.png" alt="GitHub" width="32" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+                    <h2 class="lh-condensed mt-2 text-normal" style="box-sizing: border-box; margin-top: 8px !important; margin-bottom: 0; font-size: 24px; font-weight: 400 !important; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                        [axllent/mailpit] .github/workflows/tests.yml workflow run
+
+                    </h2>
+                  </td>
+                </tr>
+              </table>
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+</td>
+  </tr>
+</table>
+            <table width="100%" class="width-full" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+              <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                <td class="border rounded-2 d-block" style="box-sizing: border-box; border-radius: 6px !important; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0; border: 1px solid #e1e4e8;">
+                  <table align="center" class="width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+                      <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                        <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+                          
+<table align="center" class="border-bottom width-full text-center" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; border-bottom-width: 1px !important; border-bottom-color: #e1e4e8 !important; border-bottom-style: solid !important; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td class="d-block px-3 pt-3 p-sm-4" style="box-sizing: border-box; display: block !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 24px;">
+      <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+        
+    <img src="https://github.githubassets.com/images/email/icons/actions.png" width="56" height="56" alt="" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; border-style: none;" />
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="12" style="font-size: 12px; line-height: 12px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+<h3 class="lh-condensed" style="box-sizing: border-box; margin-top: 0; margin-bottom: 0; font-size: 20px; font-weight: 600; line-height: 1.25 !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">.github/workflows/tests.yml: No jobs were run</h3>
+<table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+
+  <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+    <table width="100%" border="0" cellspacing="0" cellpadding="0" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+      <table border="0" cellspacing="0" cellpadding="0" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+        <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+          <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <!--[if mso]> <table><tr><td align="center" bgcolor="#28a745"> <![endif]-->
+                <a href="https://github.com/axllent/mailpit/actions/runs/6522820865" target="_blank" rel="noopener noreferrer" class="btn btn-large btn-primary" style="background-color: #1f883d !important; box-sizing: border-box; color: #fff; text-decoration: none; position: relative; display: inline-block; font-size: inherit; font-weight: 500; line-height: 1.5; white-space: nowrap; vertical-align: middle; cursor: pointer; -webkit-user-select: none; user-select: none; border-radius: .5em; -webkit-appearance: none; appearance: none; box-shadow: 0 1px 0 rgba(27,31,35,.1),inset 0 1px 0 rgba(255,255,255,.03); transition: background-color .2s cubic-bezier(0.3, 0, 0.5, 1); font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: .75em 1.5em; border: 1px solid #1f883d;">View workflow run</a>
+              <!--[if mso]> </td></tr></table> <![endif]-->
+          </td>
+        </tr>
+      </table>
+    </td>
+  </tr>
+</table>
+
+</td>
+  </tr>
+</table>
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="32" style="font-size: 32px; line-height: 32px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
+</td>
+  </tr>
+</table>
+    </td>
+  </tr>
+</table>
+
+
+
+
+</td>
+  </tr>
+</table>
+                      </td>
+                    </tr>
+                  </table>
+                </td>
+              </tr>
+            </table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+              <p class="f5 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 14px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">  </p><p style="font-size: small; -webkit-text-size-adjust: none; color: #666; box-sizing: border-box; margin-top: 0; margin-bottom: 10px; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">&#8212;<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" />You are receiving this because you are subscribed to this thread.<br style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;" /><a href="https://github.com/settings/notifications" style="background-color: transparent; box-sizing: border-box; color: #0366d6; text-decoration: none; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">Manage your GitHub Actions notifications</a></p>
+
+</td>
+  </tr>
+</table>
+            <table border="0" cellspacing="0" cellpadding="0" align="center" class="width-full text-center" width="100%" style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; width: 100% !important; text-align: center !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <td align="center" style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">
+  <table style="box-sizing: border-box; border-spacing: 0; border-collapse: collapse; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+  <tbody style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+    <tr style="box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">
+      <td height="16" style="font-size: 16px; line-height: 16px; box-sizing: border-box; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important; padding: 0;">&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+  <p class="f6 text-gray-light" style="box-sizing: border-box; margin-top: 0; margin-bottom: 10px; color: #6a737d !important; font-size: 12px !important; font-family: -apple-system,BlinkMacSystemFont,&quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot; !important;">GitHub, Inc. &#12539;88 Colin P Kelly Jr Street &#12539;San Francisco, CA 94107</p>
+</td>
+  </tr>
+</table>
+
+          </center>
+        </td>
+      </tr>
+    </table>
+    <!-- prevent Gmail on iOS font size manipulation -->
+   <div style="display: none; white-space: nowrap; box-sizing: border-box; font: 15px/0 apple-system, BlinkMacSystemFont, &quot;Segoe UI&quot;,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;"> &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; </div>
+  </body>
+</html>`

internal/htmlcheck/README.md

@@ -0,0 +1,5 @@
+# HTML check
+
+The database used for HTML support tests is based on [can I email](https://www.caniemail.com/).
+
+The `caniemail-data.json` file used to determine client support is copied from the [API](https://www.caniemail.com/api/data.json)

internal/htmlcheck/caniemail.go

@@ -0,0 +1,74 @@
+// Package htmlcheck is used for parsing HTML and returning
+// HTML compatibility errors and warnings
+package htmlcheck
+
+import (
+	"embed"
+	"encoding/json"
+	"regexp"
+)
+
+//go:embed caniemail-data.json
+var embeddedFS embed.FS
+
+var (
+	cie = CanIEmail{}
+
+	noteMatch = regexp.MustCompile(` #(\d)+$`)
+
+	// LimitFamilies will limit results to families if set
+	LimitFamilies = []string{}
+
+	// LimitPlatforms will limit results to platforms if set
+	LimitPlatforms = []string{}
+
+	// LimitClients will limit results to clients if set
+	LimitClients = []string{}
+)
+
+// CanIEmail struct for JSON data
+type CanIEmail struct {
+	APIVersion     string `json:"api_version"`
+	LastUpdateDate string `json:"last_update_date"`
+	// NiceNames map[string]string `json:"last_update_date"`
+	NiceNames struct {
+		Family   map[string]string `json:"family"`
+		Platform map[string]string `json:"platform"`
+		Support  map[string]string `json:"support"`
+		Category map[string]string `json:"category"`
+	} `json:"nicenames"`
+	Data []JSONResult `json:"data"`
+}
+
+// JSONResult struct for CanIEmail Data
+type JSONResult struct {
+	Slug           string                 `json:"slug"`
+	Title          string                 `json:"title"`
+	Description    string                 `json:"description"`
+	URL            string                 `json:"url"`
+	Category       string                 `json:"category"`
+	Tags           []string               `json:"tags"`
+	Keywords       string                 `json:"keywords"`
+	LastTestDate   string                 `json:"last_test_date"`
+	TestURL        string                 `json:"test_url"`
+	TestResultsURL string                 `json:"test_results_url"`
+	Stats          map[string]interface{} `json:"stats"`
+	Notes          string                 `json:"notes"`
+	NotesByNumber  map[string]string      `json:"notes_by_num"`
+}
+
+// Load the JSON data
+func loadJSONData() error {
+	if cie.APIVersion != "" {
+		return nil
+	}
+
+	b, err := embeddedFS.ReadFile("caniemail-data.json")
+	if err != nil {
+		return err
+	}
+
+	cie = CanIEmail{}
+
+	return json.Unmarshal(b, &cie)
+}

internal/htmlcheck/config.go

@@ -0,0 +1,204 @@
+package htmlcheck
+
+import "regexp"
+
+// HTML tests
+var htmlTests = map[string]string{
+	// body check is manually done because it always exists in *goquery.Document
+	"html-body": "body",
+	// HTML tests
+	"html-object":         "object, embed, image, pdf",
+	"html-link":           "link",
+	"html-hr":             "hr",
+	"html-dialog":         "dialog",
+	"html-srcset":         "[srcset]",
+	"html-picture":        "picture",
+	"html-svg":            "svg",
+	"html-progress":       "progress",
+	"html-required":       "[required]",
+	"html-meter":          "meter",
+	"html-audio":          "audio",
+	"html-form":           "form",
+	"html-input-submit":   "submit",
+	"html-button-reset":   "button[type=\"reset\"]",
+	"html-button-submit":  "submit, button[type=\"submit\"]",
+	"html-base":           "base",
+	"html-input-checkbox": "checkbox",
+	"html-input-hidden":   "[type=\"hidden\"]",
+	"html-input-radio":    "radio",
+	"html-input-text":     "input[type=\"text\"]",
+	"html-video":          "video",
+	"html-semantics":      "article, aside, details, figcaption, figure, footer, header, main, mark, nav, section, summary, time",
+	"html-select":         "select",
+	"html-textarea":       "textarea",
+	"html-anchor-links":   "a[href^=\"#\"]",
+	"html-style":          "style",
+	"html-image-maps":     "map, img[usemap]",
+}
+
+// Image tests using regex to match against img[src]
+var imageRegexpTests = map[string]*regexp.Regexp{
+	"image-apng":   regexp.MustCompile(`(?i)\.apng$`),       // 78.723404
+	"image-avif":   regexp.MustCompile(`(?i)\.avif$`),       // 14.864864
+	"image-base64": regexp.MustCompile(`^(?i)data:image\/`), // 61.702126
+	"image-bmp":    regexp.MustCompile(`(?i)\.bmp$`),        // 89.3617
+	"image-gif":    regexp.MustCompile(`(?i)\.gif$`),        // 89.3617
+	"image-hdr":    regexp.MustCompile(`(?i)\.hdr$`),        // 12.5
+	"image-heif":   regexp.MustCompile(`(?i)\.heif$`),       // 0
+	"image-ico":    regexp.MustCompile(`(?i)\.ico$`),        // 87.23404
+	"image-mp4":    regexp.MustCompile(`(?i)\.mp4$`),        // 26.53061
+	"image-ppm":    regexp.MustCompile(`(?i)\.ppm$`),        // 2.0833282
+	"image-svg":    regexp.MustCompile(`(?i)\.svg$`),        // 64.91228
+	"image-tiff":   regexp.MustCompile(`(?i)\.tiff?$`),      // 38.29787
+	"image-webp":   regexp.MustCompile(`(?i)\.webp$`),       // 59.649124
+}
+
+var cssInlineTests = map[string]string{
+	"css-accent-color":                   "[style*=\"accent-color:\"]",                                           // 6.6666718
+	"css-align-items":                    "[style*=\"align-items:\"]",                                            // 60.784313
+	"css-aspect-ratio":                   "[style*=\"aspect-ratio:\"]",                                           // 30
+	"css-background-blend-mode":          "[style*=\"background-blend-mode:\"]",                                  // 61.70213
+	"css-background-clip":                "[style*=\"background-clip:\"]",                                        // 61.70213
+	"css-background-color":               "[style*=\"background-color:\"], [bgcolor]",                            // 90
+	"css-background-image":               "[style*=\"background-image:\"]",                                       // 57.62712
+	"css-background-origin":              "[style*=\"background-origin:\"]",                                      // 61.70213
+	"css-background-position":            "[style*=\"background-position:\"]",                                    // 61.224487
+	"css-background-repeat":              "[style*=\"background-repeat:\"]",                                      // 67.34694
+	"css-background-size":                "[style*=\"background-size:\"]",                                        // 61.702126
+	"css-background":                     "[style*=\"background:\"], [background]",                               // 57.407406
+	"css-block-inline-size":              "[style*=\"block-inline-size:\"]",                                      // 46.93877
+	"css-border-image":                   "[style*=\"border-image:\"]",                                           // 52.173912
+	"css-border-inline-block-individual": "[style*=\"border-inline:\"]",                                          // 18.518517
+	"css-border-radius":                  "[style*=\"border-radius:\"]",                                          // 67.34694
+	"css-border":                         "[style*=\"border:\"], [border]",                                       // 86.95652
+	"css-box-shadow":                     "[style*=\"box-shadow:\"]",                                             // 43.103447
+	"css-box-sizing":                     "[style*=\"box-sizing:\"]",                                             // 71.739136
+	"css-caption-side":                   "[style*=\"caption-side:\"]",                                           // 84
+	"css-clip-path":                      "[style*=\"clip-path:\"]",                                              // 43.396225
+	"css-column-count":                   "[style*=\"column-count:\"]",                                           // 67.391304
+	"css-column-layout-properties":       "[style*=\"column-layout-properties:\"]",                               // 47.368423
+	"css-conic-gradient":                 "[style*=\"conic-gradient:\"]",                                         // 38.461536
+	"css-direction":                      "[style*=\"direction:\"]",                                              // 97.77778
+	"css-display-flex":                   "[style*=\"display:flex\"]",                                            // 53.448277
+	"css-display-grid":                   "[style*=\"display:grid\"]",                                            // 54.347824
+	"css-display-none":                   "[style*=\"display:none\"]",                                            // 84.78261
+	"css-display":                        "[style*=\"display:\"]",                                                // 55.555553
+	"css-filter":                         "[style*=\"filter:\"]",                                                 // 50
+	"css-flex-direction":                 "[style*=\"flex-direction:\"]",                                         // 50
+	"css-flex-wrap":                      "[style*=\"flex-wrap:\"]",                                              // 49.09091
+	"css-float":                          "[style*=\"float:\"]",                                                  // 85.10638
+	"css-font-kerning":                   "[style*=\"font-kerning:\"]",                                           // 66.666664
+	"css-font-weight":                    "[style*=\"font-weight:\"]",                                            // 76.666664
+	"css-font":                           "[style*=\"font:\"]",                                                   // 95.833336
+	"css-gap":                            "[style*=\"gap:\"]",                                                    // 40
+	"css-grid-template":                  "[style*=\"grid-template:\"]",                                          // 34.042553
+	"css-height":                         "[style*=\"height:\"], [height]",                                       // 77.08333
+	"css-hyphens":                        "[style*=\"hyphens:\"]",                                                // 31.111107
+	"css-important":                      "[style*=\"!important\"]",                                              // 43.478264
+	"css-inline-size":                    "[style*=\"inline-size:\"]",                                            // 43.478264
+	"css-intrinsic-size":                 "[style*=\"intrinsic-size:\"]",                                         // 40.54054
+	"css-justify-content":                "[style*=\"justify-content:\"]",                                        // 59.25926
+	"css-letter-spacing":                 "[style*=\"letter-spacing:\"]",                                         // 87.23404
+	"css-line-height":                    "[style*=\"line-height:\"]",                                            // 82.608696
+	"css-list-style-image":               "[style*=\"list-style-image:\"]",                                       // 54.16667
+	"css-list-style-position":            "[style*=\"list-style-position:\"]",                                    // 87.5
+	"css-list-style":                     "[style*=\"list-style:\"]",                                             // 62.500004
+	"css-margin-block-start-end":         "[style*=\"margin-block-start:\"], [style*=\"margin-block-end:\"]",     // 32.07547
+	"css-margin-inline-block":            "[style*=\"margin-inline-block:\"]",                                    // 16.981125
+	"css-margin-inline-start-end":        "[style*=\"margin-inline-start:\"], [style*=\"margin-inline-end:\"]",   // 32.07547
+	"css-margin-inline":                  "[style*=\"margin-inline:\"]",                                          // 43.39623
+	"css-margin":                         "[style*=\"margin:\"]",                                                 // 71.42857
+	"css-max-block-size":                 "[style*=\"max-block-size:\"]",                                         // 35.714287
+	"css-max-height":                     "[style*=\"max-height:\"]",                                             // 86.95652
+	"css-max-width":                      "[style*=\"max-width:\"]",                                              // 76.47058
+	"css-min-height":                     "[style*=\"min-height:\"]",                                             // 82.608696
+	"css-min-inline-size":                "[style*=\"min-inline-size:\"]",                                        // 33.33333
+	"css-min-width":                      "[style*=\"min-width:\"]",                                              // 86.95652
+	"css-mix-blend-mode":                 "[style*=\"mix-blend-mode:\"]",                                         // 62.745094
+	"css-modern-color":                   "[style*=\"modern-color:\"]",                                           // 10.81081
+	"css-object-fit":                     "[style*=\"object-fit:\"]",                                             // 57.142857
+	"css-object-position":                "[style*=\"object-position:\"]",                                        // 55.10204
+	"css-opacity":                        "[style*=\"opacity:\"]",                                                // 63.04348
+	"css-outline-offset":                 "[style*=\"outline-offset:\"]",                                         // 42.5
+	"css-outline":                        "[style*=\"outline:\"]",                                                // 80.85106
+	"css-overflow-wrap":                  "[style*=\"overflow-wrap:\"]",                                          // 6.6666603
+	"css-overflow":                       "[style*=\"overflow:\"]",                                               // 78.26087
+	"css-padding-block-start-end":        "[style*=\"padding-block-start:\"], [style*=\"padding-block-end:\"]",   // 32.07547
+	"css-padding-inline-block":           "[style*=\"padding-inline-block:\"]",                                   // 16.981125
+	"css-padding-inline-start-end":       "[style*=\"padding-inline-start:\"], [style*=\"padding-inline-end:\"]", // 32.07547
+	"css-padding":                        "[style*=\"padding:\"], [padding]",                                     // 87.755104
+	"css-position":                       "[style*=\"position:\"]",                                               // 19.56522
+	"css-radial-gradient":                "[style*=\"radial-gradient:\"]",                                        // 64.583336
+	"css-rgb":                            "[style*=\"rgb(\"]",                                                    // 53.846153
+	"css-rgba":                           "[style*=\"rgba(\"]",                                                   // 56
+	"css-scroll-snap":                    "[style*=\"roll-snap:\"]",                                              // 38.88889
+	"css-tab-size":                       "[style*=\"tab-size:\"]",                                               // 32.075474
+	"css-table-layout":                   "[style*=\"table-layout:\"]",                                           // 53.33333
+	"css-text-align-last":                "[style*=\"text-align-last:\"]",                                        // 42.307693
+	"css-text-align":                     "[style*=\"text-align:\"]",                                             // 60.416664
+	"css-text-decoration-color":          "[style*=\"text-decoration-color:\"]",                                  // 67.34695
+	"css-text-decoration-thickness":      "[style*=\"text-decoration-thickness:\"]",                              // 38.333336
+	"css-text-decoration":                "[style*=\"text-decoration:\"]",                                        // 67.391304
+	"css-text-emphasis-position":         "[style*=\"text-emphasis-position:\"]",                                 // 28.571434
+	"css-text-emphasis":                  "[style*=\"text-emphasis:\"]",                                          // 36.734695
+	"css-text-indent":                    "[style*=\"text-indent:\"]",                                            // 78.43137
+	"css-text-overflow":                  "[style*=\"text-overflow:\"]",                                          // 58.695656
+	"css-text-shadow":                    "[style*=\"text-shadow:\"]",                                            // 69.565216
+	"css-text-transform":                 "[style*=\"text-transform:\"]",                                         // 86.666664
+	"css-text-underline-offset":          "[style*=\"text-underline-offset:\"]",                                  // 39.285713
+	"css-transform":                      "[style*=\"transform:\"]",                                              // 50
+	"css-unit-calc":                      "[style*=\"calc(:\"]",                                                  // 56.25
+	"css-variables":                      "[style*=\"variables:\"]",                                              // 46.551727
+	"css-visibility":                     "[style*=\"visibility:\"]",                                             // 52.173916
+	"css-white-space":                    "[style*=\"white-space:\"]",                                            // 58.69565
+	"css-width":                          "[style*=\"width:\"], [width]",                                         // 87.5
+	"css-word-break":                     "[style*=\"word-break:\"]",                                             // 28.888887
+	"css-writing-mode":                   "[style*=\"writing-mode:\"]",                                           // 56.25
+	"css-z-index":                        "[style*=\"z-index:\"]",                                                // 76.08696
+}
+
+// some CSS tests using regex for things that can't be merged inline
+var cssRegexpTests = map[string]*regexp.Regexp{
+	"css-at-font-face":                  regexp.MustCompile(`(?mi)@font\-face\s+?{`),        // 26.923073
+	"css-at-import":                     regexp.MustCompile(`(?mi)@import\s`),               // 36.170216
+	"css-at-keyframes":                  regexp.MustCompile(`(?mi)@keyframes\s`),            // 31.914898
+	"css-at-media":                      regexp.MustCompile(`(?mi)@media\s?\(`),             // 47.05882
+	"css-at-supports":                   regexp.MustCompile(`(?mi)@supports\s?\(`),          // 40.81633
+	"css-pseudo-class-active":           regexp.MustCompile(`:active`),                      // 52.173912
+	"css-pseudo-class-checked":          regexp.MustCompile(`:checked`),                     // 31.91489
+	"css-pseudo-class-first-child":      regexp.MustCompile(`:first\-child`),                // 66.666664
+	"css-pseudo-class-first-of-type":    regexp.MustCompile(`:first\-of\-type`),             // 62.5
+	"css-pseudo-class-focus":            regexp.MustCompile(`:focus`),                       // 47.826088
+	"css-pseudo-class-has":              regexp.MustCompile(`:has`),                         // 25.531914
+	"css-pseudo-class-hover":            regexp.MustCompile(`:hover`),                       // 60.41667
+	"css-pseudo-class-lang":             regexp.MustCompile(`:lang\s?\(`),                   // 18.918922
+	"css-pseudo-class-last-child":       regexp.MustCompile(`:last\-child`),                 // 64.58333
+	"css-pseudo-class-last-of-type":     regexp.MustCompile(`:last\-of\-type`),              // 60.416664
+	"css-pseudo-class-link":             regexp.MustCompile(`:link`),                        // 81.63265
+	"css-pseudo-class-not":              regexp.MustCompile(`:not(\s+)?\(`),                 // 44.89796
+	"css-pseudo-class-nth-child":        regexp.MustCompile(`:nth\-child(\s+)?\(`),          // 44.89796
+	"css-pseudo-class-nth-last-child":   regexp.MustCompile(`:nth\-last\-child(\s+)?\(`),    // 44.89796
+	"css-pseudo-class-nth-last-of-type": regexp.MustCompile(`:nth\-last\-of\-type(\s+)?\(`), // 42.857143
+	"css-pseudo-class-nth-of-type":      regexp.MustCompile(`:nth\-of\-type(\s+)?\(`),       // 42.857143
+	"css-pseudo-class-only-child":       regexp.MustCompile(`:only\-child(\s+)?\(`),         // 64.58333
+	"css-pseudo-class-only-of-type":     regexp.MustCompile(`:only\-of\-type(\s+)?\(`),      // 64.58333
+	"css-pseudo-class-target":           regexp.MustCompile(`:target`),                      // 39.13044
+	"css-pseudo-class-visited":          regexp.MustCompile(`:visited`),                     // 39.13044
+	"css-pseudo-element-after":          regexp.MustCompile(`:after`),                       // 40
+	"css-pseudo-element-before":         regexp.MustCompile(`:before`),                      // 40
+	"css-pseudo-element-first-letter":   regexp.MustCompile(`::first\-letter`),              // 60
+	"css-pseudo-element-first-line":     regexp.MustCompile(`::first\-line`),                // 60
+	"css-pseudo-element-marker":         regexp.MustCompile(`::marker`),                     // 50
+	"css-pseudo-element-placeholder":    regexp.MustCompile(`::placeholder`),                // 32
+}
+
+// some CSS tests using regex for units
+var cssRegexpUnitTests = map[string]*regexp.Regexp{
+	"css-unit-ch":      regexp.MustCompile(`\b\d+ch\b`),     // 66.666664
+	"css-unit-initial": regexp.MustCompile(`:\s?initial\b`), // 58.33333
+	"css-unit-rem":     regexp.MustCompile(`\b\d+rem\b`),    // 66.666664
+	"css-unit-vh":      regexp.MustCompile(`\b\d+vh\b`),     // 68.75
+	"css-unit-vmax":    regexp.MustCompile(`\b\d+vmax\b`),   // 60.416664
+	"css-unit-vmin":    regexp.MustCompile(`\b\d+vmin\b`),   // 58.333336
+	"css-unit-vw":      regexp.MustCompile(`\b\d+vw\b`),     // 77.08333
+}

internal/htmlcheck/css.go

@@ -0,0 +1,217 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/vanng822/go-premailer/premailer"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Go cannot calculate any rendered CSS attributes, so we merge all styles
+// into the HTML and detect elements with styles containing the keywords.
+func runCSSTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	inlined, err := inlineRemoteCSS(html)
+	if err != nil {
+		inlined = html
+	}
+
+	// merge all CSS inline
+	merged, err := mergeInlineCSS(inlined)
+	if err != nil {
+		merged = inlined
+	}
+
+	reader := strings.NewReader(merged)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	for key, test := range cssInlineTests {
+		totalTests++
+		found := len(doc.Find(test).Nodes)
+		if found > 0 {
+			result, err := cie.getTest(key)
+			if err != nil {
+				return results, totalTests, err
+			}
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	// get a list of all generated styles from all nodes
+	allNodeStyles := []string{}
+	for _, n := range doc.Find("*[style]").Nodes {
+		style, err := tools.GetHTMLAttributeVal(n, "style")
+		if err == nil {
+			allNodeStyles = append(allNodeStyles, style)
+		}
+	}
+
+	for key, re := range cssRegexpUnitTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := 0
+		// loop through all styles to count total
+		for _, styles := range allNodeStyles {
+			found = found + len(re.FindAllString(styles, -1))
+		}
+
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	// get all inline CSS block data
+	reader = strings.NewReader(inlined)
+
+	// Load the HTML document
+	doc, _ = goquery.NewDocumentFromReader(reader)
+
+	cssCode := ""
+	for _, n := range doc.Find("style").Nodes {
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			cssCode = cssCode + c.Data
+		}
+	}
+
+	for key, re := range cssRegexpTests {
+		totalTests++
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+
+		found := len(re.FindAllString(cssCode, -1))
+		if found > 0 {
+			result.Score.Found = found
+			results = append(results, result)
+		}
+	}
+
+	return results, totalTests, nil
+}
+
+// MergeInlineCSS merges header CSS into element attributes
+func mergeInlineCSS(html string) (string, error) {
+	options := premailer.NewOptions()
+	// options.RemoveClasses = true
+	// options.CssToAttributes = false
+	options.KeepBangImportant = true
+	pre, err := premailer.NewPremailerFromString(html, options)
+	if err != nil {
+		return "", err
+	}
+
+	return pre.Transform()
+}
+
+// InlineRemoteCSS searches the HTML for linked stylesheets, downloads the content, and
+// inserts new <style> blocks into the head, unless BlockRemoteCSSAndFonts is set
+func inlineRemoteCSS(h string) (string, error) {
+	reader := strings.NewReader(h)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return h, err
+	}
+
+	remoteCSS := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range remoteCSS {
+		attributes := link.Attr
+		for _, a := range attributes {
+			if a.Key == "href" {
+				if !isURL(a.Val) {
+					// skip invalid URL
+					continue
+				}
+
+				if config.BlockRemoteCSSAndFonts {
+					logger.Log().Debugf("[html-check] skip testing remote CSS content: %s (--block-remote-css-and-fonts)", a.Val)
+					return h, nil
+				}
+
+				resp, err := downloadToBytes(a.Val)
+				if err != nil {
+					logger.Log().Warnf("[html-check] failed to download %s", a.Val)
+					continue
+				}
+
+				// create new <style> block and insert downloaded CSS
+				styleBlock := &html.Node{
+					Type:     html.ElementNode,
+					Data:     "style",
+					DataAtom: atom.Style,
+				}
+				styleBlock.AppendChild(&html.Node{
+					Type: html.TextNode,
+					Data: string(resp),
+				})
+
+				link.Parent.AppendChild(styleBlock)
+			}
+		}
+	}
+
+	newDoc, err := doc.Html()
+	if err != nil {
+		logger.Log().Warnf("[html-check] failed to download %s", err.Error())
+		return h, err
+	}
+
+	return newDoc, nil
+}
+
+// DownloadToBytes returns a []byte slice from a URL
+func downloadToBytes(url string) ([]byte, error) {
+	client := http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	// Get the link response data
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		err := fmt.Errorf("Error downloading %s", url)
+		return nil, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+// Test if str is a URL
+func isURL(str string) bool {
+	u, err := url.Parse(str)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}

internal/htmlcheck/html.go

@@ -0,0 +1,102 @@
+package htmlcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+// HTML tests
+func runHTMLTests(html string) ([]Warning, int, error) {
+	results := []Warning{}
+	totalTests := 0
+
+	reader := strings.NewReader(html)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return results, totalTests, err
+	}
+
+	// Almost all <script> is bad
+	scripts := len(doc.Find("script:not([type=\"application/ld+json\"]):not([type=\"application/json\"])").Nodes)
+	if scripts > 0 {
+		var result = Warning{}
+		result.Title = "<script> element"
+		result.Slug = "html-script"
+		result.Category = "html"
+		result.Description = "JavaScript is not supported in any email client."
+		result.Tags = []string{}
+		result.Results = []Result{}
+		result.NotesByNumber = map[string]string{}
+		result.Score.Found = scripts
+		result.Score.Supported = 0
+		result.Score.Partial = 0
+		result.Score.Unsupported = 100
+		results = append(results, result)
+		totalTests++
+	}
+
+	for key, test := range htmlTests {
+		totalTests++
+		if test == "body" {
+			re := regexp.MustCompile(`(?im)</body>`)
+			if re.MatchString(html) {
+				result, err := cie.getTest(key)
+				if err != nil {
+					return results, totalTests, err
+				}
+
+				result.Score.Found = 1
+				results = append(results, result)
+			}
+		} else if len(doc.Find(test).Nodes) > 0 {
+			result, err := cie.getTest(key)
+			if err != nil {
+				return results, totalTests, err
+			}
+			totalTests++
+
+			result.Score.Found = len(doc.Find(test).Nodes)
+
+			results = append(results, result)
+		}
+	}
+
+	// find all images
+	images := doc.Find("img[src]").Nodes
+	imageResults := make(map[string]int)
+	totalTests = totalTests + len(imageRegexpTests)
+
+	for _, image := range images {
+		src, err := tools.GetHTMLAttributeVal(image, "src")
+		if err != nil {
+			continue
+		}
+		for key, test := range imageRegexpTests {
+			if test.MatchString(src) {
+				matches, exists := imageResults[key]
+				if exists {
+					imageResults[key] = matches + 1
+				} else {
+					imageResults[key] = 1
+				}
+
+			}
+		}
+	}
+
+	for key, found := range imageResults {
+		result, err := cie.getTest(key)
+		if err != nil {
+			return results, totalTests, err
+		}
+		result.Score.Found = found
+		results = append(results, result)
+	}
+
+	return results, totalTests, nil
+}

internal/htmlcheck/main.go

@@ -0,0 +1,212 @@
+package htmlcheck
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+)
+
+// RunTests will run all tests on an HTML string
+func RunTests(html string) (Response, error) {
+	s := Response{}
+	s.Warnings = []Warning{}
+	if platforms, err := Platforms(); err == nil {
+		s.Platforms = platforms
+	}
+
+	s.Total = Total{}
+
+	// crude way to determine whether the HTML contains a <body> structure
+	// or whether it's just plain HTML content
+	re := regexp.MustCompile(`(?im)</body>`)
+	nodeMatch := "body *, script"
+	if re.MatchString(html) {
+		nodeMatch = "*:not(html):not(head):not(meta), script"
+	}
+	reader := strings.NewReader(html)
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return s, err
+	}
+	// calculate the number of nodes in HTML
+	s.Total.Nodes = len(doc.Find(nodeMatch).Nodes)
+
+	if err := loadJSONData(); err != nil {
+		return s, err
+	}
+
+	// HTML tests
+	htmlResults, totalTests, err := runHTMLTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add html test totals
+	s.Warnings = append(s.Warnings, htmlResults...)
+
+	// CSS tests
+	cssResults, totalTests, err := runCSSTests(html)
+	if err != nil {
+		return s, err
+	}
+
+	s.Total.Tests = s.Total.Tests + totalTests
+
+	// add css test totals
+	s.Warnings = append(s.Warnings, cssResults...)
+
+	// calculate total score
+	var partial, unsupported float32
+	partial = 0
+	unsupported = 0
+
+	for _, w := range s.Warnings {
+		if w.Score.Found == 0 {
+			continue
+		}
+
+		// supported is calculated by subtracting partial and unsupported from 100%
+		if w.Score.Partial > 0 {
+			weighted := w.Score.Partial * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > partial {
+				partial = weighted
+			}
+		}
+		if w.Score.Unsupported > 0 {
+			weighted := w.Score.Unsupported * float32(w.Score.Found) / float32(s.Total.Nodes)
+			if weighted > unsupported {
+				unsupported = weighted
+			}
+		}
+	}
+
+	s.Total.Supported = 100 - partial - unsupported
+	s.Total.Partial = partial
+	s.Total.Unsupported = unsupported
+
+	// sort slice to get lowest scores first
+	sort.Slice(s.Warnings, func(i, j int) bool {
+		return (s.Warnings[i].Score.Unsupported+s.Warnings[i].Score.Partial)*float32(s.Warnings[i].Score.Found)/float32(s.Total.Nodes) >
+			(s.Warnings[j].Score.Unsupported+s.Warnings[j].Score.Partial)*float32(s.Warnings[j].Score.Found)/float32(s.Total.Nodes)
+	})
+
+	return s, nil
+}
+
+// Test returns a test
+func (c CanIEmail) getTest(k string) (Warning, error) {
+	warning := Warning{}
+	exists := false
+	found := JSONResult{}
+	for _, r := range cie.Data {
+		if r.Slug == k {
+			found = r
+			exists = true
+			break
+		}
+	}
+
+	if !exists {
+		return warning, fmt.Errorf("%s does not exist", k)
+	}
+
+	warning.Slug = found.Slug
+	warning.Title = found.Title
+	warning.Description = mdToHTML(found.Description)
+	warning.Category = found.Category
+	warning.URL = found.URL
+	warning.Tags = found.Tags
+	// warning.Keywords = found.Keywords
+	// warning.Notes = found.Notes
+	warning.NotesByNumber = make(map[string]string, len(found.NotesByNumber))
+	for nr, note := range found.NotesByNumber {
+		warning.NotesByNumber[nr] = mdToHTML(note)
+	}
+	warning.Results = []Result{}
+
+	var y, n, p float32
+
+	for family, stats := range found.Stats {
+		if len(LimitFamilies) != 0 && !inArray(family, LimitFamilies) {
+			continue
+		}
+
+		for platform, clients := range stats.(map[string]interface{}) {
+			if len(LimitPlatforms) != 0 && !inArray(platform, LimitPlatforms) {
+				continue
+			}
+			for version, support := range clients.(map[string]interface{}) {
+				s := Result{}
+				s.Name = fmt.Sprintf("%s %s (%s)", c.NiceNames.Family[family], c.NiceNames.Platform[platform], version)
+				s.Family = family
+				s.Platform = platform
+				s.Version = version
+
+				if support == "y" {
+					y++
+					s.Support = "yes"
+				} else if support == "n" {
+					n++
+					s.Support = "no"
+				} else {
+					p++
+					s.Support = "partial"
+
+					noteIDS := noteMatch.FindStringSubmatch(fmt.Sprintf("%s", support))
+
+					for _, id := range noteIDS {
+						s.NoteNumber = id
+					}
+				}
+
+				warning.Results = append(warning.Results, s)
+			}
+		}
+
+	}
+
+	total := y + n + p
+	warning.Score.Supported = y / total * 100
+	warning.Score.Unsupported = n / total * 100
+	warning.Score.Partial = p / total * 100
+
+	return warning, nil
+}
+
+func inArray(n string, h []string) bool {
+	n = strings.ToLower(n)
+
+	for _, v := range h {
+		if strings.ToLower(v) == n {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Convert markdown to HTML, stripping <p> & </p>
+func mdToHTML(str string) string {
+	md := []byte(str)
+	// create markdown parser with extensions
+	extensions := parser.CommonExtensions | parser.AutoHeadingIDs | parser.NoEmptyLineBeforeBlock
+	// extensions := parser.NoExtensions
+	p := parser.NewWithExtensions(extensions)
+	doc := p.Parse(md)
+
+	// create HTML renderer with extensions
+	htmlFlags := html.CommonFlags | html.HrefTargetBlank
+	opts := html.RendererOptions{Flags: htmlFlags}
+	renderer := html.NewRenderer(opts)
+
+	return strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(string(markdown.Render(doc, renderer))), "<p>"), "</p>")
+}

internal/htmlcheck/platforms.go

@@ -0,0 +1,38 @@
+package htmlcheck
+
+import "sort"
+
+// Platforms returns all platforms with their respective email clients
+func Platforms() (map[string][]string, error) {
+	// [platform]clients
+	data := make(map[string][]string)
+
+	if err := loadJSONData(); err != nil {
+		return data, err
+	}
+
+	for _, t := range cie.Data {
+		for family, stats := range t.Stats {
+			niceFamily := cie.NiceNames.Family[family]
+			for platform := range stats.(map[string]interface{}) {
+				c, found := data[platform]
+				if !found {
+					data[platform] = []string{}
+				}
+				if !inArray(niceFamily, c) {
+					c = append(c, niceFamily)
+					data[platform] = c
+				}
+			}
+		}
+	}
+
+	for group, clients := range data {
+		sort.Slice(clients, func(i, j int) bool {
+			return clients[i] < clients[j]
+		})
+		data[group] = clients
+	}
+
+	return data, nil
+}

internal/htmlcheck/structs.go

@@ -0,0 +1,87 @@
+package htmlcheck
+
+// Response represents the HTML check response struct
+//
+// swagger:model HTMLCheckResponse
+type Response struct {
+	// List of warnings from tests
+	Warnings []Warning `json:"Warnings"`
+	// All platforms tested, mainly for the web UI
+	Platforms map[string][]string `json:"Platforms"`
+	// Total overall score
+	Total Total `json:"Total"`
+}
+
+// Warning represents a failed test
+//
+// swagger:model HTMLCheckWarning
+type Warning struct {
+	// Slug identifier
+	Slug string `json:"Slug"`
+	// Friendly title
+	Title string `json:"Title"`
+	// Description
+	Description string `json:"Description"`
+	// URL to caniemail.com
+	URL string `json:"URL"`
+	// Category [css, html]
+	Category string `json:"Category"`
+	// Tags
+	Tags []string `json:"Tags"`
+	// Keywords
+	Keywords string `json:"Keywords"`
+	// Test results
+	Results []Result `json:"Results"`
+	// Notes based on results
+	NotesByNumber map[string]string `json:"NotesByNumber"`
+	// Test score calculated from results
+	Score Score `json:"Score"`
+}
+
+// Result struct
+//
+// swagger:model HTMLCheckResult
+type Result struct {
+	// Friendly name of result, combining family, platform & version
+	Name string `json:"Name"`
+	// Platform eg: ios, android, windows
+	Platform string `json:"Platform"`
+	// Family eg: Outlook, Mozilla Thunderbird
+	Family string `json:"Family"`
+	// Family version eg: 4.7.1, 2019-10, 10.3
+	Version string `json:"Version"`
+	// Support [yes, no, partial]
+	Support string `json:"Support"`
+	// Note number for partially supported if applicable
+	NoteNumber string `json:"NoteNumber"` // where applicable
+}
+
+// Score struct
+//
+// swagger:model HTMLCheckScore
+type Score struct {
+	// Number of matches in the document
+	Found int `json:"Found"`
+	// Total percentage supported
+	Supported float32 `json:"Supported"`
+	// Total percentage partially supported
+	Partial float32 `json:"Partial"`
+	// Total percentage unsupported
+	Unsupported float32 `json:"Unsupported"`
+}
+
+// Total weighted result for all scores
+//
+// swagger:model HTMLCheckTotal
+type Total struct {
+	// Total number of tests done
+	Tests int `json:"Tests"`
+	// Total number of HTML nodes detected in message
+	Nodes int `json:"Nodes"`
+	// Overall percentage supported
+	Supported float32 `json:"Supported"`
+	// Overall percentage partially supported
+	Partial float32 `json:"Partial"` // total percentage
+	// Overall percentage unsupported
+	Unsupported float32 `json:"Unsupported"` // total percentage
+}

internal/linkcheck/main.go

@@ -0,0 +1,92 @@
+// Package linkcheck handles message links checking
+package linkcheck
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+)
+
+var linkRe = regexp.MustCompile(`(?m)\b(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:'!\/~+#-]*[\w@?^=%&\/~+#-])`)
+
+// RunTests will run all tests on an HTML string
+func RunTests(msg *storage.Message, followRedirects bool) (Response, error) {
+	s := Response{}
+
+	allLinks := extractHTMLLinks(msg)
+	allLinks = strUnique(append(allLinks, extractTextLinks(msg)...))
+	s.Links = getHTTPStatuses(allLinks, followRedirects)
+
+	for _, l := range s.Links {
+		if l.StatusCode >= 400 || l.StatusCode == 0 {
+			s.Errors++
+		}
+	}
+
+	return s, nil
+}
+
+func extractTextLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	for _, match := range linkRe.FindAllString(msg.Text, -1) {
+		links = append(links, match)
+	}
+
+	return links
+}
+
+func extractHTMLLinks(msg *storage.Message) []string {
+	links := []string{}
+
+	reader := strings.NewReader(msg.HTML)
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return links
+	}
+
+	aLinks := doc.Find("a[href]").Nodes
+	for _, link := range aLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	cssLinks := doc.Find("link[rel=\"stylesheet\"]").Nodes
+	for _, link := range cssLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "href")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	imgLinks := doc.Find("img[src]").Nodes
+	for _, link := range imgLinks {
+		l, err := tools.GetHTMLAttributeVal(link, "src")
+		if err == nil && linkRe.MatchString(l) {
+			links = append(links, l)
+		}
+	}
+
+	return links
+}
+
+// strUnique return a slice of unique strings from a slice
+func strUnique(strSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range strSlice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	return list
+}

internal/linkcheck/status.go

@@ -0,0 +1,114 @@
+package linkcheck
+
+import (
+	"crypto/tls"
+	"net/http"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+func getHTTPStatuses(links []string, followRedirects bool) []Link {
+	// allow 5 threads
+	threads := make(chan int, 5)
+
+	results := make(map[string]Link, len(links))
+	resultsMutex := sync.RWMutex{}
+
+	output := []Link{}
+
+	var wg sync.WaitGroup
+
+	for _, l := range links {
+		wg.Add(1)
+		go func(link string, w *sync.WaitGroup) {
+			threads <- 1 // will block if MAX threads
+			defer w.Done()
+
+			code, err := doHead(link, followRedirects)
+			l := Link{}
+			l.URL = link
+			if err != nil {
+				l.StatusCode = 0
+				l.Status = httpErrorSummary(err)
+			} else {
+				l.StatusCode = code
+				l.Status = http.StatusText(code)
+			}
+			resultsMutex.Lock()
+			results[link] = l
+			resultsMutex.Unlock()
+
+			<-threads // remove from threads
+		}(l, &wg)
+	}
+
+	wg.Wait()
+
+	for _, l := range results {
+		output = append(output, l)
+	}
+
+	return output
+}
+
+// Do a HEAD request to return HTTP status code
+func doHead(link string, followRedirects bool) (int, error) {
+
+	timeout := time.Duration(10 * time.Second)
+
+	tr := &http.Transport{}
+
+	if config.AllowUntrustedTLS {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec
+	}
+
+	client := http.Client{
+		Timeout:   timeout,
+		Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if followRedirects {
+				return nil
+			}
+			return http.ErrUseLastResponse
+		},
+	}
+
+	req, err := http.NewRequest("HEAD", link, nil)
+	if err != nil {
+		logger.Log().Errorf("[link-check] %s", err.Error())
+		return 0, err
+	}
+
+	req.Header.Set("User-Agent", "Mailpit/"+config.Version)
+
+	res, err := client.Do(req)
+	if err != nil {
+		if res != nil {
+			return res.StatusCode, err
+		}
+
+		return 0, err
+
+	}
+
+	return res.StatusCode, nil
+}
+
+// HTTP errors include a lot more info that just the actual error, so this
+// tries to take the final part of it, eg: `no such host`
+func httpErrorSummary(err error) string {
+	var re = regexp.MustCompile(`.*: (.*)$`)
+
+	e := err.Error()
+	if !re.MatchString(e) {
+		return e
+	}
+
+	parts := re.FindAllStringSubmatch(e, -1)
+
+	return parts[0][len(parts[0])-1]
+}

internal/linkcheck/structs.go

@@ -0,0 +1,21 @@
+package linkcheck
+
+// Response represents the Link check response
+//
+// swagger:model LinkCheckResponse
+type Response struct {
+	// Total number of errors
+	Errors int `json:"Errors"`
+	// Tested links
+	Links []Link `json:"Links"`
+}
+
+// Link struct
+type Link struct {
+	// Link URL
+	URL string `json:"URL"`
+	// HTTP status code
+	StatusCode int `json:"StatusCode"`
+	// HTTP status definition
+	Status string `json:"Status"`
+}

internal/logger/logger.go

@@ -0,0 +1,89 @@
+// Package logger handles the logging
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	log *logrus.Logger
+	// VerboseLogging for verbose logging
+	VerboseLogging bool
+	// QuietLogging shows only errors
+	QuietLogging bool
+	// NoLogging shows only fatal errors
+	NoLogging bool
+	// LogFile sets a log file
+	LogFile string
+)
+
+// Log returns the logger instance
+func Log() *logrus.Logger {
+	if log == nil {
+		log = logrus.New()
+		log.SetLevel(logrus.InfoLevel)
+		if VerboseLogging {
+			// verbose logging (debug)
+			log.SetLevel(logrus.DebugLevel)
+		} else if QuietLogging {
+			// show errors only
+			log.SetLevel(logrus.ErrorLevel)
+		} else if NoLogging {
+			// disable all logging (tests)
+			log.SetLevel(logrus.PanicLevel)
+		}
+
+		if LogFile != "" {
+			file, err := os.OpenFile(filepath.Clean(LogFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0664) // #nosec
+			if err == nil {
+				log.Out = file
+			} else {
+				log.Out = os.Stdout
+				log.Warn("Failed to log to file, using default stderr")
+			}
+		} else {
+			log.Out = os.Stdout
+		}
+
+		log.SetFormatter(&logrus.TextFormatter{
+			FullTimestamp:   true,
+			TimestampFormat: "2006/01/02 15:04:05",
+		})
+	}
+
+	return log
+}
+
+// PrettyPrint for debugging
+func PrettyPrint(i interface{}) {
+	s, _ := json.MarshalIndent(i, "", "\t")
+	fmt.Println(string(s))
+}
+
+// CleanIP returns a human-readable IP for the logging interface
+// when starting services. It translates [::]:<port> to "0.0.0.0:<port>"
+func CleanIP(s string) string {
+	re := regexp.MustCompile(`^\[\:\:\]\:\d+`)
+	if re.MatchString(s) {
+		return "0.0.0.0:" + s[5:]
+	}
+
+	return s
+}
+
+// CleanHTTPIP returns a human-readable IP for the logging interface
+// when starting services. It translates [::]:<port> to "localhost:<port>"
+func CleanHTTPIP(s string) string {
+	re := regexp.MustCompile(`^\[\:\:\]\:\d+`)
+	if re.MatchString(s) {
+		return "localhost:" + s[5:]
+	}
+
+	return s
+}

internal/spamassassin/postmark/postmark.go

@@ -0,0 +1,100 @@
+// Package postmark uses the free https://spamcheck.postmarkapp.com/
+// See https://spamcheck.postmarkapp.com/doc/ for more details.
+package postmark
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Response struct
+type Response struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"` // for errors only
+	Score   string `json:"score"`
+	Rules   []Rule `json:"rules"`
+	Report  string `json:"report"` // ignored
+}
+
+// Rule struct
+type Rule struct {
+	Score string `json:"score"`
+	// Name not returned by postmark but rather extracted from description
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// Check will post the email data to Postmark
+func Check(email []byte, timeout int) (Response, error) {
+	r := Response{}
+	// '{"email":"raw dump of email", "options":"short"}'
+	var d struct {
+		// The raw dump of the email to be filtered, including all headers.
+		Email string `json:"email"`
+		// Default "long". Must either be "long" for a full report of processing rules, or "short" for a score request.
+		Options string `json:"options"`
+	}
+
+	d.Email = string(email)
+	d.Options = "long"
+
+	data, err := json.Marshal(d)
+	if err != nil {
+		return r, err
+	}
+
+	client := http.Client{
+		Timeout: time.Duration(timeout) * time.Second,
+	}
+
+	resp, err := client.Post("https://spamcheck.postmarkapp.com/filter", "application/json",
+		bytes.NewBuffer(data))
+
+	if err != nil {
+		return r, err
+	}
+
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&r)
+
+	// remove trailing line spaces for all lines in report
+	re := regexp.MustCompile("\r?\n")
+	lines := re.Split(r.Report, -1)
+	reportLines := []string{}
+	for _, l := range lines {
+		line := strings.TrimRight(l, " ")
+		reportLines = append(reportLines, line)
+	}
+	reportRaw := strings.Join(reportLines, "\n")
+
+	// join description lines to make a single line per rule
+	re2 := regexp.MustCompile("\n                                ")
+	report := re2.ReplaceAllString(reportRaw, "")
+	for i, rule := range r.Rules {
+		// populate rule name
+		r.Rules[i].Name = nameFromReport(rule.Score, rule.Description, report)
+	}
+
+	return r, err
+}
+
+// Extract the name of the test from the report as Postmark does not include this in the JSON reports
+func nameFromReport(score, description, report string) string {
+	score = regexp.QuoteMeta(score)
+	description = regexp.QuoteMeta(description)
+	str := fmt.Sprintf("%s\\s+([A-Z0-9\\_]+)\\s+%s", score, description)
+	re := regexp.MustCompile(str)
+
+	matches := re.FindAllStringSubmatch(report, 1)
+	if len(matches) > 0 && len(matches[0]) == 2 {
+		return strings.TrimSpace(matches[0][1])
+	}
+
+	return ""
+}

internal/spamassassin/spamassassin.go

@@ -0,0 +1,147 @@
+// Package spamassassin will return results from either a SpamAssassin server or
+// Postmark's public API depending on configuration
+package spamassassin
+
+import (
+	"errors"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/spamassassin/postmark"
+	"github.com/axllent/mailpit/internal/spamassassin/spamc"
+)
+
+var (
+	// Service to use, either "<host>:<ip>" for self-hosted SpamAssassin or "postmark"
+	service string
+
+	// SpamScore is the score at which a message is determined to be spam
+	spamScore = 5.0
+
+	// Timeout in seconds
+	timeout = 8
+)
+
+// Result is a SpamAssassin result
+//
+// swagger:model SpamAssassinResponse
+type Result struct {
+	// Whether the message is spam or not
+	IsSpam bool
+	// If populated will return an error string
+	Error string
+	// Total spam score based on triggered rules
+	Score float64
+	// Spam rules triggered
+	Rules []Rule
+}
+
+// Rule struct
+type Rule struct {
+	// Spam rule score
+	Score float64
+	// SpamAssassin rule name
+	Name string
+	// SpamAssassin rule description
+	Description string
+}
+
+// SetService defines which service should be used.
+func SetService(s string) {
+	switch s {
+	case "postmark":
+		service = "postmark"
+	default:
+		service = s
+	}
+}
+
+// SetTimeout defines the timeout
+func SetTimeout(t int) {
+	if t > 0 {
+		timeout = t
+	}
+}
+
+// Ping returns whether a service is active or not
+func Ping() error {
+	if service == "postmark" {
+		return nil
+	}
+
+	var client *spamc.Client
+	if strings.HasPrefix("unix:", service) {
+		client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+	} else {
+		client = spamc.NewTCP(service, timeout)
+	}
+
+	return client.Ping()
+}
+
+// Check will return a Result
+func Check(msg []byte) (Result, error) {
+	r := Result{Score: 0}
+
+	if service == "" {
+		return r, errors.New("no SpamAssassin service defined")
+	}
+
+	if service == "postmark" {
+		res, err := postmark.Check(msg, timeout)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		resFloat, err := strconv.ParseFloat(res.Score, 32)
+		if err == nil {
+			r.Score = round1dm(resFloat)
+			r.IsSpam = resFloat >= spamScore
+		}
+		r.Error = res.Message
+		for _, pr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(pr.Score, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = pr.Name
+			rule.Description = pr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	} else {
+		var client *spamc.Client
+		if strings.HasPrefix("unix:", service) {
+			client = spamc.NewUnix(strings.TrimLeft(service, "unix:"))
+		} else {
+			client = spamc.NewTCP(service, timeout)
+		}
+
+		res, err := client.Report(msg)
+		if err != nil {
+			r.Error = err.Error()
+			return r, nil
+		}
+		r.IsSpam = res.Score >= spamScore
+		r.Score = round1dm(res.Score)
+		r.Rules = []Rule{}
+		for _, sr := range res.Rules {
+			rule := Rule{}
+			value, err := strconv.ParseFloat(sr.Points, 32)
+			if err == nil {
+				rule.Score = round1dm(value)
+			}
+			rule.Name = sr.Name
+			rule.Description = sr.Description
+			r.Rules = append(r.Rules, rule)
+		}
+	}
+
+	return r, nil
+}
+
+// Round to one decimal place
+func round1dm(n float64) float64 {
+	return math.Floor(n*10) / 10
+}

internal/spamassassin/spamc/spamc.go

@@ -0,0 +1,245 @@
+// Package spamc provides a client for the SpamAssassin spamd protocol.
+// http://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+//
+// Modified to add timeouts from https://github.com/cgt/spamc
+package spamc
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ProtoVersion is the protocol version
+const ProtoVersion = "1.5"
+
+var (
+	spamInfoRe    = regexp.MustCompile(`(.+)\/(.+) (\d+) (.+)`)
+	spamMainRe    = regexp.MustCompile(`^Spam: (.+) ; (.+) . (.+)$`)
+	spamDetailsRe = regexp.MustCompile(`^\s?(-?[0-9\.]+)\s([a-zA-Z0-9_]*)(\W*)(.*)`)
+)
+
+// connection is like net.Conn except that it also has a CloseWrite method.
+// CloseWrite is implemented by net.TCPConn and net.UnixConn, but for some
+// reason it is not present in the net.Conn interface.
+type connection interface {
+	net.Conn
+	CloseWrite() error
+}
+
+// Client is a spamd client.
+type Client struct {
+	net     string
+	addr    string
+	timeout int
+}
+
+// NewTCP returns a *Client that connects to spamd via the given TCP address.
+func NewTCP(addr string, timeout int) *Client {
+	return &Client{"tcp", addr, timeout}
+}
+
+// NewUnix returns a *Client that connects to spamd via the given Unix socket.
+func NewUnix(addr string) *Client {
+	return &Client{"unix", addr, 0}
+}
+
+// Rule represents a matched SpamAssassin rule.
+type Rule struct {
+	Points      string
+	Name        string
+	Description string
+}
+
+// Result struct
+type Result struct {
+	ResponseCode int
+	Message      string
+	Spam         bool
+	Score        float64
+	Threshold    float64
+	Rules        []Rule
+}
+
+// dial connects to spamd through TCP or a Unix socket.
+func (c *Client) dial() (connection, error) {
+	if c.net == "tcp" {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialTCP("tcp", nil, tcpAddr)
+	} else if c.net == "unix" {
+		unixAddr, err := net.ResolveUnixAddr("unix", c.addr)
+		if err != nil {
+			return nil, err
+		}
+		return net.DialUnix("unix", nil, unixAddr)
+	}
+	panic("Client.net must be either \"tcp\" or \"unix\"")
+}
+
+// Report checks if message is spam or not, and returns score plus report
+func (c *Client) Report(email []byte) (Result, error) {
+	output, err := c.report(email)
+	if err != nil {
+		return Result{}, err
+	}
+
+	return c.parseOutput(output), nil
+}
+
+func (c *Client) report(email []byte) ([]string, error) {
+	conn, err := c.dial()
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return nil, err
+	}
+
+	bw := bufio.NewWriter(conn)
+	_, err = bw.WriteString("REPORT SPAMC/" + ProtoVersion + "\r\n")
+	if err != nil {
+		return nil, err
+	}
+	_, err = bw.WriteString("Content-length: " + strconv.Itoa(len(email)) + "\r\n\r\n")
+	if err != nil {
+		return nil, err
+	}
+	_, err = bw.Write(email)
+	if err != nil {
+		return nil, err
+	}
+	err = bw.Flush()
+	if err != nil {
+		return nil, err
+	}
+	// Client is supposed to close its writing side of the connection
+	// after sending its request.
+	err = conn.CloseWrite()
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		lines []string
+		br    = bufio.NewReader(conn)
+	)
+	for {
+		line, err := br.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		line = strings.TrimRight(line, " \t\r\n")
+		lines = append(lines, line)
+	}
+
+	// join lines, and replace multi-line descriptions with single line for each
+	tmp := strings.Join(lines, "\n")
+	re := regexp.MustCompile("\n                            ")
+	n := re.ReplaceAllString(tmp, " ")
+
+	//split lines again
+	return strings.Split(n, "\n"), nil
+}
+
+func (c *Client) parseOutput(output []string) Result {
+	var result Result
+	var reachedRules bool
+	for _, row := range output {
+		// header
+		if spamInfoRe.MatchString(row) {
+			res := spamInfoRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				resCode, err := strconv.Atoi(res[3])
+				if err == nil {
+					result.ResponseCode = resCode
+				}
+				result.Message = res[4]
+				continue
+			}
+		}
+		// summary
+		if spamMainRe.MatchString(row) {
+			res := spamMainRe.FindStringSubmatch(row)
+			if len(res) == 4 {
+				if strings.ToLower(res[1]) == "true" || strings.ToLower(res[1]) == "yes" {
+					result.Spam = true
+				} else {
+					result.Spam = false
+				}
+				resFloat, err := strconv.ParseFloat(res[2], 32)
+				if err == nil {
+					result.Score = resFloat
+					continue
+				}
+				resFloat, err = strconv.ParseFloat(res[3], 32)
+				if err == nil {
+					result.Threshold = resFloat
+					continue
+				}
+			}
+		}
+
+		if strings.HasPrefix(row, "Content analysis details") {
+			reachedRules = true
+			continue
+		}
+		// details
+		// row = strings.Trim(row, " \t\r\n")
+		if reachedRules && spamDetailsRe.MatchString(row) {
+			res := spamDetailsRe.FindStringSubmatch(row)
+			if len(res) == 5 {
+				rule := Rule{Points: res[1], Name: res[2], Description: res[4]}
+				result.Rules = append(result.Rules, rule)
+			}
+		}
+	}
+	return result
+}
+
+// Ping the spamd
+func (c *Client) Ping() error {
+	conn, err := c.dial()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if err := conn.SetDeadline(time.Now().Add(time.Duration(c.timeout) * time.Second)); err != nil {
+		return err
+	}
+
+	_, err = io.WriteString(conn, fmt.Sprintf("PING SPAMC/%s\r\n\r\n", ProtoVersion))
+	if err != nil {
+		return err
+	}
+	err = conn.CloseWrite()
+	if err != nil {
+		return err
+	}
+
+	br := bufio.NewReader(conn)
+	for {
+		_, err = br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/stats/stats.go

@@ -0,0 +1,139 @@
+// Package stats stores and returns Mailpit statistics
+package stats
+
+import (
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/updater"
+)
+
+var (
+	// to prevent hammering Github for latest version
+	latestVersionCache string
+
+	// StartedAt is set to the current ime when Mailpit starts
+	startedAt time.Time
+
+	mu sync.RWMutex
+
+	smtpAccepted     int
+	smtpAcceptedSize int
+	smtpRejected     int
+	smtpIgnored      int
+)
+
+// AppInformation struct
+// swagger:model AppInformation
+type AppInformation struct {
+	// Current Mailpit version
+	Version string
+	// Latest Mailpit version
+	LatestVersion string
+	// Database path
+	Database string
+	// Database size in bytes
+	DatabaseSize int64
+	// Total number of messages in the database
+	Messages int
+	// Total number of messages in the database
+	Unread int
+	// Tags and message totals per tag
+	Tags map[string]int64
+	// Runtime statistics
+	RuntimeStats struct {
+		// Mailpit server uptime in seconds
+		Uptime int
+		// Current memory usage in bytes
+		Memory uint64
+		// Database runtime messages deleted
+		MessagesDeleted int
+		// Accepted runtime SMTP messages
+		SMTPAccepted int
+		// Total runtime accepted messages size in bytes
+		SMTPAcceptedSize int
+		// Rejected runtime SMTP messages
+		SMTPRejected int
+		// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
+		SMTPIgnored int
+	}
+}
+
+// Load the current statistics
+func Load() AppInformation {
+	info := AppInformation{}
+	info.Version = config.Version
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	info.RuntimeStats.Memory = m.Sys - m.HeapReleased
+
+	info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds())
+	info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
+	info.RuntimeStats.SMTPAccepted = smtpAccepted
+	info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
+	info.RuntimeStats.SMTPRejected = smtpRejected
+	info.RuntimeStats.SMTPIgnored = smtpIgnored
+
+	if latestVersionCache != "" {
+		info.LatestVersion = latestVersionCache
+	} else {
+		latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
+		if err == nil {
+			info.LatestVersion = latest
+			latestVersionCache = latest
+
+			// clear latest version cache after 5 minutes
+			go func() {
+				time.Sleep(5 * time.Minute)
+				latestVersionCache = ""
+			}()
+		}
+	}
+
+	info.Database = config.DataFile
+
+	db, err := os.Stat(info.Database)
+	if err == nil {
+		info.DatabaseSize = db.Size()
+	}
+
+	info.Messages = storage.CountTotal()
+	info.Unread = storage.CountUnread()
+
+	info.Tags = storage.GetAllTagsCount()
+
+	return info
+}
+
+// Track will start the statistics logging in memory
+func Track() {
+	startedAt = time.Now()
+}
+
+// LogSMTPAccepted logs a successful SMTP transaction
+func LogSMTPAccepted(size int) {
+	mu.Lock()
+	smtpAccepted = smtpAccepted + 1
+	smtpAcceptedSize = smtpAcceptedSize + size
+	mu.Unlock()
+}
+
+// LogSMTPRejected logs a rejected SMTP transaction
+func LogSMTPRejected() {
+	mu.Lock()
+	smtpRejected = smtpRejected + 1
+	mu.Unlock()
+}
+
+// LogSMTPIgnored logs an ignored SMTP transaction
+func LogSMTPIgnored() {
+	mu.Lock()
+	smtpIgnored = smtpIgnored + 1
+	mu.Unlock()
+}

internal/storage/cron.go

@@ -0,0 +1,166 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/leporo/sqlf"
+)
+
+// Database cron runs every minute
+func dbCron() {
+	for {
+		time.Sleep(60 * time.Second)
+
+		currentTime := time.Now()
+		sinceLastDbAction := currentTime.Sub(dbLastAction)
+
+		// only run the database has been idle for 5 minutes
+		if math.Floor(sinceLastDbAction.Minutes()) == 5 {
+			deletedSize := getDeletedSize()
+
+			if deletedSize > 0 {
+				total := totalMessagesSize()
+				var deletedPercent int64
+				if total == 0 {
+					deletedPercent = 100
+				} else {
+					deletedPercent = deletedSize * 100 / total
+				}
+				// only vacuum the DB if at least 1% of mail storage size has been deleted
+				if deletedPercent >= 1 {
+					logger.Log().Debugf("[db] deleted messages is %d%% of total size, reclaim space", deletedPercent)
+					vacuumDb()
+				}
+			}
+		}
+
+		pruneMessages()
+	}
+}
+
+// PruneMessages will auto-delete the oldest messages if messages > config.MaxMessages.
+// Set config.MaxMessages to 0 to disable.
+func pruneMessages() {
+	if config.MaxMessages < 1 {
+		return
+	}
+
+	start := time.Now()
+
+	q := sqlf.Select("ID, Size").
+		From("mailbox").
+		OrderBy("Created DESC").
+		Limit(5000).
+		Offset(config.MaxMessages)
+
+	ids := []string{}
+	var prunedSize int64
+	var size int
+	if err := q.Query(nil, db, func(row *sql.Rows) {
+		var id string
+
+		if err := row.Scan(&id, &size); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+		ids = append(ids, id)
+		prunedSize = prunedSize + int64(size)
+
+	}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	if len(ids) == 0 {
+		return
+	}
+
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	_, err = tx.Query(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Query(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	_, err = tx.Query(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	err = tx.Commit()
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		if err := tx.Rollback(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	if err := pruneUnusedTags(); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	addDeletedSize(prunedSize)
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed)
+
+	logMessagesDeleted(len(ids))
+
+	websockets.Broadcast("prune", nil)
+}
+
+// Vacuum the database to reclaim space from deleted messages
+func vacuumDb() {
+	start := time.Now()
+
+	// set WAL file checkpoint
+	if _, err := db.Exec("PRAGMA wal_checkpoint"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	// vacuum database
+	if _, err := db.Exec("VACUUM"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	// truncate WAL file
+	if _, err := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return
+	}
+
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] vacuumed database in %s", elapsed)
+}

internal/storage/database.go

@@ -0,0 +1,197 @@
+// Package storage handles all database actions
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"os/signal"
+	"path"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/klauspost/compress/zstd"
+	"github.com/leporo/sqlf"
+
+	// sqlite (native) - https://gitlab.com/cznic/sqlite
+	_ "modernc.org/sqlite"
+)
+
+var (
+	db           *sql.DB
+	dbFile       string
+	dbIsTemp     bool
+	dbLastAction time.Time
+
+	// zstd compression encoder & decoder
+	dbEncoder, _ = zstd.NewWriter(nil)
+	dbDecoder, _ = zstd.NewReader(nil)
+)
+
+// InitDB will initialise the database
+func InitDB() error {
+	p := config.DataFile
+
+	if p == "" {
+		// when no path is provided then we create a temporary file
+		// which will get deleted on Close(), SIGINT or SIGTERM
+		p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano())
+		dbIsTemp = true
+		logger.Log().Debugf("[db] using temporary database: %s", p)
+	} else {
+		p = filepath.Clean(p)
+	}
+
+	config.DataFile = p
+
+	logger.Log().Debugf("[db] opening database %s", p)
+
+	var err error
+
+	dsn := fmt.Sprintf("file:%s?cache=shared", p)
+
+	db, err = sql.Open("sqlite", dsn)
+	if err != nil {
+		return err
+	}
+
+	// prevent "database locked" errors
+	// @see https://github.com/mattn/go-sqlite3#faq
+	db.SetMaxOpenConns(1)
+
+	// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
+	_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
+	if err != nil {
+		return err
+	}
+
+	// create tables if necessary & apply migrations
+	if err := dbApplyMigrations(); err != nil {
+		return err
+	}
+
+	dbFile = p
+	dbLastAction = time.Now()
+
+	sigs := make(chan os.Signal, 1)
+	// catch all signals since not explicitly listing
+	// Program that will listen to the SIGINT and SIGTERM
+	// SIGINT will listen to CTRL-C.
+	// SIGTERM will be caught if kill command executed
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
+	// method invoked upon seeing signal
+	go func() {
+		s := <-sigs
+		fmt.Printf("[db] got %s signal, shutting down\n", s)
+		Close()
+		os.Exit(0)
+	}()
+
+	// auto-prune & delete
+	go dbCron()
+
+	go dataMigrations()
+
+	return nil
+}
+
+// Close will close the database, and delete if a temporary table
+func Close() {
+	if db != nil {
+		if err := db.Close(); err != nil {
+			logger.Log().Warn("[db] error closing database, ignoring")
+		}
+	}
+
+	if dbIsTemp && isFile(dbFile) {
+		logger.Log().Debugf("[db] deleting temporary file %s", dbFile)
+		if err := os.Remove(dbFile); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+}
+
+// StatsGet returns the total/unread statistics for a mailbox
+func StatsGet() MailboxStats {
+	var (
+		total  = CountTotal()
+		unread = CountUnread()
+		tags   = GetAllTags()
+	)
+
+	dbLastAction = time.Now()
+
+	return MailboxStats{
+		Total:  total,
+		Unread: unread,
+		Tags:   tags,
+	}
+}
+
+// CountTotal returns the number of emails in the database
+func CountTotal() int {
+	var total int
+
+	_ = sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(nil, db)
+
+	return total
+}
+
+// CountUnread returns the number of emails in the database that are unread.
+func CountUnread() int {
+	var total int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 0)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return total
+}
+
+// CountRead returns the number of emails in the database that are read.
+func CountRead() int {
+	var total int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		Where("Read = ?", 1)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return total
+}
+
+// IsUnread returns the number of emails in the database that are unread.
+// If an ID is supplied, then it is just limited to that message.
+func IsUnread(id string) bool {
+	var unread int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&unread).
+		Where("Read = ?", 0).
+		Where("ID = ?", id)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return unread == 1
+}
+
+// MessageIDExists checks whether a Message-ID exists in the DB
+func MessageIDExists(id string) bool {
+	var total int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		Where("MessageID = ?", id)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return total != 0
+}

internal/storage/messages.go

@@ -0,0 +1,619 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/mail"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/webhook"
+	"github.com/axllent/mailpit/server/websockets"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+// Store will save an email to the database tables.
+// Returns the database ID of the saved message.
+func Store(body *[]byte) (string, error) {
+	// Parse message body with enmime
+	env, err := enmime.ReadEnvelope(bytes.NewReader(*body))
+	if err != nil {
+		logger.Log().Warnf("[message] %s", err.Error())
+		return "", nil
+	}
+
+	from := &mail.Address{}
+	fromJSON := addressToSlice(env, "From")
+	if len(fromJSON) > 0 {
+		from = fromJSON[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	obj := DBMailSummary{
+		From:    from,
+		To:      addressToSlice(env, "To"),
+		Cc:      addressToSlice(env, "Cc"),
+		Bcc:     addressToSlice(env, "Bcc"),
+		ReplyTo: addressToSlice(env, "Reply-To"),
+	}
+
+	messageID := strings.Trim(env.Root.Header.Get("Message-ID"), "<>")
+	created := time.Now()
+
+	// use message date instead of created date
+	if config.UseMessageDates {
+		if mDate, err := env.Date(); err == nil {
+			created = mDate
+		}
+	}
+
+	// generate the search text
+	searchText := createSearchText(env)
+
+	// generate unique ID
+	id := shortuuid.New()
+
+	summaryJSON, err := json.Marshal(obj)
+	if err != nil {
+		return "", err
+	}
+
+	// extract tags from body matches based on --tag, plus addresses & X-Tags header
+	tagStr := findTagsInRawMessage(body) + "," +
+		obj.tagsFromPlusAddresses() + "," +
+		strings.TrimSpace(env.Root.Header.Get("X-Tags"))
+
+	tagData := uniqueTagsFromString(tagStr)
+
+	// begin a transaction to ensure both the message
+	// and data are stored successfully
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return "", err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	subject := env.GetHeader("Subject")
+	size := len(*body)
+	inline := len(env.Inlines)
+	attachments := len(env.Attachments)
+	snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+	// insert mail summary data
+	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) values(?,?,?,?,?,?,?,?,?,0,?)",
+		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet)
+	if err != nil {
+		return "", err
+	}
+
+	// insert compressed raw message
+	compressed := dbEncoder.EncodeAll(*body, make([]byte, 0, size))
+	_, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) values(?,?)", id, string(compressed))
+	if err != nil {
+		return "", err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return "", err
+	}
+
+	if len(tagData) > 0 {
+		// set tags after tx.Commit()
+		if err := SetMessageTags(id, tagData); err != nil {
+			return "", err
+		}
+	}
+
+	c := &MessageSummary{}
+	if err := json.Unmarshal(summaryJSON, c); err != nil {
+		return "", err
+	}
+
+	c.Created = created
+	c.ID = id
+	c.MessageID = messageID
+	c.Attachments = attachments
+	c.Subject = subject
+	c.Size = size
+	c.Tags = tagData
+	c.Snippet = snippet
+
+	websockets.Broadcast("new", c)
+	webhook.Send(c)
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	return id, nil
+}
+
+// List returns a subset of messages from the mailbox,
+// sorted latest to oldest
+func List(start, limit int) ([]MessageSummary, error) {
+	results := []MessageSummary{}
+	tsStart := time.Now()
+
+	q := sqlf.From("mailbox m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet`).
+		OrderBy("m.Created DESC").
+		Limit(limit).
+		Offset(start)
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
+		var read int
+		var snippet string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[json] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(created)
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+		// artificially generate ReplyTo if legacy data is missing Reply-To field
+		if em.ReplyTo == nil {
+			em.ReplyTo = []*mail.Address{}
+		}
+
+		results = append(results, em)
+	}); err != nil {
+		return results, err
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	dbLastAction = time.Now()
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] list INBOX in %s", elapsed)
+
+	return results, nil
+}
+
+// GetMessage returns a Message generated from the mailbox_data collection.
+// If the message lacks a date header, then the received datetime is used.
+func GetMessage(id string) (*Message, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	env, err := enmime.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var from *mail.Address
+	fromData := addressToSlice(env, "From")
+	if len(fromData) > 0 {
+		from = fromData[0]
+	} else if env.GetHeader("From") != "" {
+		from = &mail.Address{Name: env.GetHeader("From")}
+	}
+
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
+	returnPath := strings.Trim(env.GetHeader("Return-Path"), "<>")
+	if returnPath == "" && from != nil {
+		returnPath = from.Address
+	}
+
+	date, err := env.Date()
+	if err != nil {
+		// return received datetime when message does not contain a date header
+		q := sqlf.From("mailbox").
+			Select(`Created`).
+			Where(`ID = ?`, id)
+
+		if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+			var created int64
+
+			if err := row.Scan(&created); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+
+			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
+
+			date = time.UnixMilli(created)
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+		}
+	}
+
+	obj := Message{
+		ID:         id,
+		MessageID:  messageID,
+		From:       from,
+		Date:       date,
+		To:         addressToSlice(env, "To"),
+		Cc:         addressToSlice(env, "Cc"),
+		Bcc:        addressToSlice(env, "Bcc"),
+		ReplyTo:    addressToSlice(env, "Reply-To"),
+		ReturnPath: returnPath,
+		Subject:    env.GetHeader("Subject"),
+		Tags:       getMessageTags(id),
+		Size:       len(raw),
+		Text:       env.Text,
+	}
+
+	obj.HTML = env.HTML
+	obj.Inline = []Attachment{}
+	obj.Attachments = []Attachment{}
+
+	for _, i := range env.Inlines {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, i := range env.OtherParts {
+		if i.FileName != "" || i.ContentID != "" {
+			obj.Inline = append(obj.Inline, AttachmentSummary(i))
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.FileName != "" || a.ContentID != "" {
+			obj.Attachments = append(obj.Attachments, AttachmentSummary(a))
+		}
+	}
+
+	// get List-Unsubscribe links if set
+	obj.ListUnsubscribe = ListUnsubscribe{}
+	obj.ListUnsubscribe.Links = []string{}
+	if env.GetHeader("List-Unsubscribe") != "" {
+		l := env.GetHeader("List-Unsubscribe")
+		links, err := tools.ListUnsubscribeParser(l)
+		obj.ListUnsubscribe.Header = l
+		obj.ListUnsubscribe.Links = links
+		if err != nil {
+			obj.ListUnsubscribe.Errors = err.Error()
+		}
+		obj.ListUnsubscribe.HeaderPost = env.GetHeader("List-Unsubscribe-Post")
+	}
+
+	// mark message as read
+	if err := MarkRead(id); err != nil {
+		return &obj, err
+	}
+
+	dbLastAction = time.Now()
+
+	return &obj, nil
+}
+
+// GetMessageRaw returns an []byte of the full message
+func GetMessageRaw(id string) ([]byte, error) {
+	var i string
+	var msg string
+	q := sqlf.From("mailbox_data").
+		Select(`ID`).To(&i).
+		Select(`Email`).To(&msg).
+		Where(`ID = ?`, id)
+
+	err := q.QueryRowAndClose(context.Background(), db)
+	if err != nil {
+		return nil, err
+	}
+
+	if i == "" {
+		return nil, errors.New("message not found")
+	}
+
+	raw, err := dbDecoder.DecodeAll([]byte(msg), nil)
+	if err != nil {
+		return nil, fmt.Errorf("error decompressing message: %s", err.Error())
+	}
+
+	dbLastAction = time.Now()
+
+	return raw, err
+}
+
+// GetAttachmentPart returns an *enmime.Part (attachment or inline) from a message
+func GetAttachmentPart(id, partID string) (*enmime.Part, error) {
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(raw)
+
+	env, err := enmime.ReadEnvelope(r)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range env.Inlines {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.OtherParts {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	for _, a := range env.Attachments {
+		if a.PartID == partID {
+			return a, nil
+		}
+	}
+
+	dbLastAction = time.Now()
+
+	return nil, errors.New("attachment not found")
+}
+
+// LatestID returns the latest message ID
+//
+// If a query argument is set in the request the function will return the
+// latest message matching the search
+func LatestID(r *http.Request) (string, error) {
+	var messages []MessageSummary
+	var err error
+
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search != "" {
+		messages, _, err = Search(search, 0, 1)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		messages, err = List(0, 1)
+		if err != nil {
+			return "", err
+		}
+	}
+	if len(messages) == 0 {
+		return "", errors.New("Message not found")
+	}
+
+	return messages[0].ID, nil
+}
+
+// MarkRead will mark a message as read
+func MarkRead(id string) error {
+	if !IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update("mailbox").
+		Set("Read", 1).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as read", id)
+	}
+
+	BroadcastMailboxStats()
+
+	return err
+}
+
+// MarkAllRead will mark all messages as read
+func MarkAllRead() error {
+	var (
+		start = time.Now()
+		total = CountUnread()
+	)
+
+	_, err := sqlf.Update("mailbox").
+		Set("Read", 1).
+		Where("Read = ?", 0).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %d messages as read in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkAllUnread will mark all messages as unread
+func MarkAllUnread() error {
+	var (
+		start = time.Now()
+		total = CountRead()
+	)
+
+	_, err := sqlf.Update("mailbox").
+		Set("Read", 0).
+		Where("Read = ?", 1).
+		ExecAndClose(context.Background(), db)
+	if err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] marked %d messages as unread in %s", total, elapsed)
+
+	BroadcastMailboxStats()
+
+	dbLastAction = time.Now()
+
+	return nil
+}
+
+// MarkUnread will mark a message as unread
+func MarkUnread(id string) error {
+	if IsUnread(id) {
+		return nil
+	}
+
+	_, err := sqlf.Update("mailbox").
+		Set("Read", 0).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	if err == nil {
+		logger.Log().Debugf("[db] marked message %s as unread", id)
+	}
+
+	dbLastAction = time.Now()
+
+	BroadcastMailboxStats()
+
+	return err
+}
+
+// DeleteOneMessage will delete a single message from a mailbox
+func DeleteOneMessage(id string) error {
+	m, err := GetMessageRaw(id)
+	if err != nil {
+		return err
+	}
+
+	size := len(m)
+	// begin a transaction to ensure both the message
+	// and data are deleted successfully
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	_, err = tx.Exec("DELETE FROM mailbox WHERE ID  = ?", id)
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec("DELETE FROM mailbox_data WHERE ID  = ?", id)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit()
+
+	if err == nil {
+		logger.Log().Debugf("[db] deleted message %s", id)
+	}
+
+	if err := DeleteAllMessageTags(id); err != nil {
+		return err
+	}
+
+	dbLastAction = time.Now()
+	addDeletedSize(int64(size))
+
+	logMessagesDeleted(1)
+
+	BroadcastMailboxStats()
+
+	return err
+}
+
+// DeleteAllMessages will delete all messages from a mailbox
+func DeleteAllMessages() error {
+	var (
+		start = time.Now()
+		total int
+	)
+
+	_ = sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		QueryRowAndClose(nil, db)
+
+	// begin a transaction to ensure both the message
+	// summaries and data are deleted successfully
+	tx, err := db.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	_, err = tx.Exec("DELETE FROM mailbox")
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec("DELETE FROM mailbox_data")
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec("DELETE FROM tags")
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec("DELETE FROM message_tags")
+	if err != nil {
+		return err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	elapsed := time.Since(start)
+	logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed)
+
+	vacuumDb()
+
+	dbLastAction = time.Now()
+	if err := SettingPut("DeletedSize", "0"); err != nil {
+		logger.Log().Warnf("[db] %s", err.Error())
+	}
+
+	logMessagesDeleted(total)
+
+	websockets.Broadcast("prune", nil)
+	BroadcastMailboxStats()
+
+	return err
+}

internal/storage/messages_test.go

@@ -0,0 +1,175 @@
+package storage
+
+import (
+	"testing"
+	"time"
+)
+
+func TestTextEmailInserts(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing text email storage")
+
+	start := time.Now()
+
+	assertEqualStats(t, 0, 0)
+
+	for i := 0; i < testRuns; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqual(t, CountTotal(), testRuns, "Incorrect number of text emails stored")
+
+	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+	assertEqualStats(t, testRuns, testRuns)
+
+	delStart := time.Now()
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, CountTotal(), 0, "incorrect number of text emails deleted")
+
+	t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
+
+	assertEqualStats(t, 0, 0)
+}
+
+func TestMimeEmailInserts(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing mime email storage")
+
+	start := time.Now()
+
+	for i := 0; i < testRuns; i++ {
+		if _, err := Store(&testMimeEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqual(t, CountTotal(), testRuns, "Incorrect number of mime emails stored")
+
+	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
+
+	assertEqualStats(t, testRuns, testRuns)
+
+	delStart := time.Now()
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, CountTotal(), 0, "incorrect number of mime emails deleted")
+
+	t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart))
+}
+
+func TestRetrieveMimeEmail(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing mime email retrieval")
+
+	id, err := Store(&testMimeEmail)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	msg, err := GetMessage(id)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+	assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+	assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+	assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+	assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+	assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+	assertEqual(t, len(msg.Attachments), 1, "incorrect number of attachments")
+	assertEqual(t, msg.Attachments[0].FileName, "Sample PDF.pdf", "attachment filename does not match")
+	assertEqual(t, len(msg.Inline), 1, "incorrect number of inline attachments")
+	assertEqual(t, msg.Inline[0].FileName, "inline-image.jpg", "inline attachment filename does not match")
+
+	attachmentData, err := GetAttachmentPart(id, msg.Attachments[0].PartID)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	assertEqual(t, len(attachmentData.Content), msg.Attachments[0].Size, "attachment size does not match")
+
+	inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
+}
+
+func TestMessageSummary(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing message summary")
+
+	if _, err := Store(&testMimeEmail); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	summaries, err := List(0, 1)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, len(summaries), 1, "Expected 1 result")
+
+	msg := summaries[0]
+
+	assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+	assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+	assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+	assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+	assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+	assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+	assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match")
+	assertEqual(t, msg.Attachments, 1, "Expected 1 attachment")
+	assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match")
+}
+
+func BenchmarkImportText(b *testing.B) {
+	setup()
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+}
+
+func BenchmarkImportMime(b *testing.B) {
+	setup()
+	defer Close()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Store(&testMimeEmail); err != nil {
+			b.Log("error ", err)
+			b.Fail()
+		}
+	}
+
+}

internal/storage/migrations.go

@@ -0,0 +1,188 @@
+package storage
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/GuiaBolso/darwin"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/leporo/sqlf"
+)
+
+var (
+	dbMigrations = []darwin.Migration{
+		{
+			Version:     1.0,
+			Description: "Creating tables",
+			Script: `CREATE TABLE IF NOT EXISTS mailbox (
+				Sort INTEGER PRIMARY KEY AUTOINCREMENT,
+				ID TEXT NOT NULL,
+				Data BLOB,
+				Search TEXT,
+				Read INTEGER
+			);
+			CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			
+			CREATE TABLE IF NOT EXISTS mailbox_data (
+				ID TEXT KEY NOT NULL,
+				Email BLOB
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`,
+		},
+		{
+			Version:     1.1,
+			Description: "Create tags column",
+			Script: `ALTER TABLE mailbox ADD COLUMN Tags Text  NOT NULL DEFAULT '[]';
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
+		{
+			Version:     1.2,
+			Description: "Creating new mailbox format",
+			Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
+				Created INTEGER NOT NULL,
+				ID TEXT NOT NULL,
+				MessageID TEXT NOT NULL,
+				Subject TEXT NOT NULL,
+				Metadata TEXT,
+				Size INTEGER NOT NULL,
+				Inline INTEGER NOT NULL,
+				Attachments INTEGER NOT NULL,
+				Read INTEGER,
+				Tags TEXT,
+				SearchText TEXT
+			);
+			INSERT INTO mailboxtmp 
+				(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+			SELECT 
+				Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+				json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+				Search, Read, Tags
+			FROM mailbox;
+
+			DROP TABLE IF EXISTS mailbox;
+			ALTER TABLE mailboxtmp RENAME TO mailbox;
+			CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
+			CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
+			CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
+			CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
+			CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
+		{
+			Version:     1.3,
+			Description: "Create snippet column",
+			Script:      `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`,
+		},
+		{
+			Version:     1.4,
+			Description: "Create tag tables",
+			Script: `CREATE TABLE IF NOT EXISTS tags (
+				ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+				Name TEXT COLLATE NOCASE
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_tag_name ON tags (Name);
+
+			CREATE TABLE IF NOT EXISTS message_tags(
+				Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+				ID TEXT REFERENCES mailbox(ID),
+				TagID INT REFERENCES tags(ID)
+			);
+			CREATE INDEX IF NOT EXISTS idx_message_tag_id ON message_tags (ID);
+			CREATE INDEX IF NOT EXISTS idx_message_tag_tagid ON message_tags (TagID);`,
+		},
+		{
+			// assume deleted messages account for 50% of storage
+			// to handle previously-deleted messages
+			Version:     1.5,
+			Description: "Create settings table",
+			Script: `CREATE TABLE IF NOT EXISTS settings (
+				Key TEXT,
+				Value TEXT
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_settings_key ON settings (Key);
+			INSERT INTO settings (Key, Value) VALUES("DeletedSize", (SELECT SUM(Size)/2 FROM mailbox));`,
+		},
+	}
+)
+
+// Create tables and apply migrations if required
+func dbApplyMigrations() error {
+	driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{})
+
+	d := darwin.New(driver, dbMigrations, nil)
+
+	return d.Migrate()
+}
+
+// These functions are used to migrate data formats/structure on startup.
+func dataMigrations() {
+	// ensure DeletedSize has a value if empty
+	if SettingGet("DeletedSize") == "" {
+		_ = SettingPut("DeletedSize", "0")
+	}
+
+	migrateTagsToManyMany()
+}
+
+// Migrate tags to ManyMany structure
+// Migration task implemented 12/2023
+// Can be removed end 06/2024 and Tags column & index dropped from mailbox
+func migrateTagsToManyMany() {
+	toConvert := make(map[string][]string)
+	q := sqlf.
+		Select("ID, Tags").
+		From("mailbox").
+		Where("Tags != ?", "[]").
+		Where("Tags IS NOT NULL")
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var id string
+		var jsonTags string
+		if err := row.Scan(&id, &jsonTags); err != nil {
+			logger.Log().Errorf("[migration] %s", err.Error())
+			return
+		}
+
+		tags := []string{}
+
+		if err := json.Unmarshal([]byte(jsonTags), &tags); err != nil {
+			logger.Log().Errorf("[json] %s", err.Error())
+			return
+		}
+
+		toConvert[id] = tags
+	}); err != nil {
+		logger.Log().Errorf("[migration] %s", err.Error())
+	}
+
+	if len(toConvert) > 0 {
+		logger.Log().Infof("[migration] converting %d message tags", len(toConvert))
+		for id, tags := range toConvert {
+			if err := SetMessageTags(id, tags); err != nil {
+				logger.Log().Errorf("[migration] %s", err.Error())
+			} else {
+				if _, err := sqlf.Update("mailbox").
+					Set("Tags", nil).
+					Where("ID = ?", id).
+					ExecAndClose(nil, db); err != nil {
+					logger.Log().Errorf("[migration] %s", err.Error())
+				}
+			}
+		}
+
+		logger.Log().Info("[migration] tags conversion complete")
+	}
+
+	// set all legacy `[]` tags to NULL
+	if _, err := sqlf.Update("mailbox").
+		Set("Tags", nil).
+		Where("Tags = ?", "[]").
+		ExecAndClose(nil, db); err != nil {
+		logger.Log().Errorf("[migration] %s", err.Error())
+	}
+}

internal/storage/notifications.go

@@ -0,0 +1,38 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+var bcStatsDelay = false
+
+// BroadcastMailboxStats broadcasts the total number of messages
+// displayed to the web UI, as well as the total unread messages.
+// The lookup is very fast (< 10ms / 100k messages under load).
+// Rate limited to 4x per second.
+func BroadcastMailboxStats() {
+	if bcStatsDelay {
+		return
+	}
+
+	bcStatsDelay = true
+
+	go func() {
+		time.Sleep(250 * time.Millisecond)
+		bcStatsDelay = false
+		b := struct {
+			Total   int
+			Unread  int
+			Version string
+		}{
+			Total:   CountTotal(),
+			Unread:  CountUnread(),
+			Version: config.Version,
+		}
+
+		websockets.Broadcast("stats", b)
+	}()
+}

internal/storage/reindex.go

@@ -0,0 +1,138 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"net/mail"
+	"os"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+)
+
+// ReindexAll will regenerate the search text and snippet for a message
+// and update the database.
+func ReindexAll() {
+	ids := []string{}
+	var i string
+	chunkSize := 1000
+
+	finished := 0
+
+	err := sqlf.Select("ID").To(&i).
+		From("mailbox").
+		OrderBy("Created DESC").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			ids = append(ids, i)
+		})
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		os.Exit(1)
+	}
+
+	total := len(ids)
+
+	chunks := chunkBy(ids, chunkSize)
+
+	logger.Log().Infof("reindexing %d messages", total)
+
+	type updateStruct struct {
+		ID         string
+		SearchText string
+		Snippet    string
+		Metadata   string
+	}
+
+	for _, ids := range chunks {
+		updates := []updateStruct{}
+
+		for _, id := range ids {
+			raw, err := GetMessageRaw(id)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+
+			r := bytes.NewReader(raw)
+
+			env, err := enmime.ReadEnvelope(r)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			from := &mail.Address{}
+			fromJSON := addressToSlice(env, "From")
+			if len(fromJSON) > 0 {
+				from = fromJSON[0]
+			} else if env.GetHeader("From") != "" {
+				from = &mail.Address{Name: env.GetHeader("From")}
+			}
+
+			obj := DBMailSummary{
+				From:    from,
+				To:      addressToSlice(env, "To"),
+				Cc:      addressToSlice(env, "Cc"),
+				Bcc:     addressToSlice(env, "Bcc"),
+				ReplyTo: addressToSlice(env, "Reply-To"),
+			}
+
+			MetadataJSON, err := json.Marshal(obj)
+			if err != nil {
+				logger.Log().Errorf("[message] %s", err.Error())
+				continue
+			}
+
+			searchText := createSearchText(env)
+			snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+			u := updateStruct{}
+			u.ID = id
+			u.SearchText = searchText
+			u.Snippet = snippet
+			u.Metadata = string(MetadataJSON)
+
+			updates = append(updates, u)
+		}
+
+		ctx := context.Background()
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		// insert mail summary data
+		for _, u := range updates {
+			_, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ?, Metadata = ? WHERE ID = ?", u.SearchText, u.Snippet, u.Metadata, u.ID)
+			if err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				continue
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			continue
+		}
+
+		finished += len(updates)
+
+		logger.Log().Printf("reindexed: %d / %d (%d%%)", finished, total, finished*100/total)
+	}
+}
+
+func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) {
+	for chunkSize < len(items) {
+		items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
+	}
+	return append(chunks, items)
+}

internal/storage/search.go

@@ -0,0 +1,350 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/leporo/sqlf"
+)
+
+// Search will search a mailbox for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func Search(search string, start, limit int) ([]MessageSummary, int, error) {
+	results := []MessageSummary{}
+	allResults := []MessageSummary{}
+	tsStart := time.Now()
+	nrResults := 0
+	if limit < 0 {
+		limit = 50
+	}
+
+	q := searchQueryBuilder(search)
+	var err error
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
+		var snippet string
+		var read int
+		var ignore string
+		em := MessageSummary{}
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		em.Created = time.UnixMilli(created)
+		em.ID = id
+		em.MessageID = messageID
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
+		em.Read = read == 1
+		em.Snippet = snippet
+
+		allResults = append(allResults, em)
+	}); err != nil {
+		return results, nrResults, err
+	}
+
+	dbLastAction = time.Now()
+
+	nrResults = len(allResults)
+
+	if nrResults > start {
+		end := nrResults
+		if nrResults >= start+limit {
+			end = start + limit
+		}
+
+		results = allResults[start:end]
+	}
+
+	// set tags for listed messages only
+	for i, m := range results {
+		results[i].Tags = getMessageTags(m.ID)
+	}
+
+	elapsed := time.Since(tsStart)
+
+	logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
+
+	return results, nrResults, err
+}
+
+// DeleteSearch will delete all messages for search terms.
+// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
+// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
+// Negative searches also also included by prefixing the search term with a `-` or `!`
+func DeleteSearch(search string) error {
+	q := searchQueryBuilder(search)
+
+	ids := []string{}
+	deleteSize := 0
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
+		var id string
+		var messageID string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
+		// var tags string
+		var read int
+		var snippet string
+		var ignore string
+
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return
+		}
+
+		ids = append(ids, id)
+		deleteSize = deleteSize + size
+	}); err != nil {
+		return err
+	}
+
+	if len(ids) > 0 {
+		total := len(ids)
+
+		// split ids into chunks of 1000 ids
+		var chunks [][]string
+		if total > 1000 {
+			chunkSize := 1000
+			chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize)
+			for chunkSize < len(ids) {
+				ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize])
+			}
+			if len(ids) > 0 {
+				// add remaining ids <= 1000
+				chunks = append(chunks, ids)
+			}
+		} else {
+			chunks = append(chunks, ids)
+		}
+
+		// begin a transaction to ensure both the message
+		// and data are deleted successfully
+		tx, err := db.BeginTx(context.Background(), nil)
+		if err != nil {
+			return err
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		for _, ids := range chunks {
+			delIDs := make([]interface{}, len(ids))
+			for i, id := range ids {
+				delIDs[i] = id
+			}
+
+			sqlDelete1 := `DELETE FROM mailbox WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete1, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete2 := `DELETE FROM mailbox_data WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete2, delIDs...)
+			if err != nil {
+				return err
+			}
+
+			sqlDelete3 := `DELETE FROM message_tags WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec
+
+			_, err = tx.Exec(sqlDelete3, delIDs...)
+			if err != nil {
+				return err
+			}
+		}
+
+		err = tx.Commit()
+
+		if err := pruneUnusedTags(); err != nil {
+			return err
+		}
+
+		if err == nil {
+			logger.Log().Debugf("[db] deleted %d messages matching %s", total, search)
+		}
+
+		dbLastAction = time.Now()
+		addDeletedSize(int64(deleteSize))
+
+		logMessagesDeleted(total)
+
+		BroadcastMailboxStats()
+	}
+
+	return nil
+}
+
+// SearchParser returns the SQL syntax for the database search based on the search arguments
+func searchQueryBuilder(searchString string) *sqlf.Stmt {
+	// group strings with quotes as a single argument and remove quotes
+	args := tools.ArgsParser(searchString)
+
+	q := sqlf.From("mailbox m").
+		Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read,
+			m.Snippet,
+			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
+			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
+			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
+			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON,
+			IFNULL(json_extract(Metadata, '$.ReplyTo'), '{}') as ReplyToJSON
+		`).
+		OrderBy("m.Created DESC")
+
+	for _, w := range args {
+		if cleanString(w) == "" {
+			continue
+		}
+
+		// lowercase search to try match search prefixes
+		lw := strings.ToLower(w)
+
+		exclude := false
+		// search terms starting with a `-` or `!` imply an exclude
+		if len(w) > 1 && (strings.HasPrefix(w, "-") || strings.HasPrefix(w, "!")) {
+			exclude = true
+			w = w[1:]
+			lw = lw[1:]
+		}
+
+		re := regexp.MustCompile(`[a-zA-Z0-9]+`)
+		if !re.MatchString(w) {
+			continue
+		}
+
+		if strings.HasPrefix(lw, "to:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("ToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "from:") {
+			w = cleanString(w[5:])
+			if w != "" {
+				if exclude {
+					q.Where("FromJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("FromJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "cc:") {
+			w = cleanString(w[3:])
+			if w != "" {
+				if exclude {
+					q.Where("CcJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("CcJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "bcc:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where("BccJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("BccJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "reply-to:") {
+			w = cleanString(w[9:])
+			if w != "" {
+				if exclude {
+					q.Where("ReplyToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("ReplyToJSON LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "subject:") {
+			w = w[8:]
+			if w != "" {
+				if exclude {
+					q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "message-id:") {
+			w = cleanString(w[11:])
+			if w != "" {
+				if exclude {
+					q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
+		} else if strings.HasPrefix(lw, "tag:") {
+			w = cleanString(w[4:])
+			if w != "" {
+				if exclude {
+					q.Where(`m.ID NOT IN (SELECT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				} else {
+					q.Where(`m.ID IN (SELECT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID WHERE t.Name = ?)`, w)
+				}
+			}
+		} else if lw == "is:read" {
+			if exclude {
+				q.Where("Read = 0")
+			} else {
+				q.Where("Read = 1")
+			}
+		} else if lw == "is:unread" {
+			if exclude {
+				q.Where("Read = 1")
+			} else {
+				q.Where("Read = 0")
+			}
+		} else if lw == "is:tagged" {
+			if exclude {
+				q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID)`)
+			} else {
+				q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID)`)
+			}
+		} else if lw == "has:attachment" || lw == "has:attachments" {
+			if exclude {
+				q.Where("Attachments = 0")
+			} else {
+				q.Where("Attachments > 0")
+			}
+		} else {
+			// search text
+			if exclude {
+				q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			} else {
+				q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%")
+			}
+		}
+	}
+
+	return q
+}

internal/storage/search_test.go

@@ -0,0 +1,182 @@
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/jhillyerd/enmime"
+)
+
+func TestSearch(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing search")
+	for i := 0; i < testRuns; i++ {
+		msg := enmime.Builder().
+			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+			CC(fmt.Sprintf("CC %d", i), fmt.Sprintf("cc-%d@example.com", i)).
+			CC(fmt.Sprintf("CC2 %d", i), fmt.Sprintf("cc2-%d@example.com", i)).
+			Subject(fmt.Sprintf("Subject line %d end", i)).
+			Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+			To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i)).
+			To(fmt.Sprintf("To2 %d", i), fmt.Sprintf("to2-%d@example.com", i)).
+			ReplyTo(fmt.Sprintf("Reply To %d", i), fmt.Sprintf("reply-to-%d@example.com", i))
+
+		env, err := msg.Build()
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := env.Encode(buf); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		bufBytes := buf.Bytes()
+
+		if _, err := Store(&bufBytes); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	for i := 1; i < 51; i++ {
+		// search a random something that will return a single result
+		uniqueSearches := []string{
+			fmt.Sprintf("from-%d@example.com", i),
+			fmt.Sprintf("from:from-%d@example.com", i),
+			fmt.Sprintf("to-%d@example.com", i),
+			fmt.Sprintf("to:to-%d@example.com", i),
+			fmt.Sprintf("to2-%d@example.com", i),
+			fmt.Sprintf("to:to2-%d@example.com", i),
+			fmt.Sprintf("cc-%d@example.com", i),
+			fmt.Sprintf("cc:cc-%d@example.com", i),
+			fmt.Sprintf("cc2-%d@example.com", i),
+			fmt.Sprintf("cc:cc2-%d@example.com", i),
+			fmt.Sprintf("reply-to-%d@example.com", i),
+			fmt.Sprintf("reply-to:\"reply-to-%d@example.com\"", i),
+			fmt.Sprintf("\"Subject line %d end\"", i),
+			fmt.Sprintf("subject:\"Subject line %d end\"", i),
+			fmt.Sprintf("\"the email body %d jdsauk dwqmdqw\"", i),
+		}
+		searchIdx := rand.Intn(len(uniqueSearches))
+
+		search := uniqueSearches[searchIdx]
+
+		summaries, _, err := Search(search, 0, 100)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		assertEqual(t, len(summaries), 1, "search result expected")
+
+		assertEqual(t, summaries[0].From.Name, fmt.Sprintf("From %d", i), "\"From\" name does not match")
+		assertEqual(t, summaries[0].From.Address, fmt.Sprintf("from-%d@example.com", i), "\"From\" address does not match")
+		assertEqual(t, summaries[0].To[0].Name, fmt.Sprintf("To %d", i), "\"To\" name does not match")
+		assertEqual(t, summaries[0].To[0].Address, fmt.Sprintf("to-%d@example.com", i), "\"To\" address does not match")
+		assertEqual(t, summaries[0].Subject, fmt.Sprintf("Subject line %d end", i), "\"Subject\" does not match")
+	}
+
+	// search something that will return 200 results
+	summaries, _, err := Search("This is the email body", 0, testRuns)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	assertEqual(t, len(summaries), testRuns, "search results expected")
+}
+
+func TestSearchDelete100(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing search delete of 100 messages")
+	for i := 0; i < 100; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		if _, err := Store(&testMimeEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	_, total, err := Search("from:sender@example.com", 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 100, "100 search results expected")
+
+	if err := DeleteSearch("from:sender@example.com"); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	_, total, err = Search("from:sender@example.com", 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 0, "0 search results expected")
+}
+
+func TestSearchDelete1100(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing search delete of 1100 messages")
+	for i := 0; i < 1100; i++ {
+		if _, err := Store(&testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	_, total, err := Search("from:sender@example.com", 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 1100, "100 search results expected")
+
+	if err := DeleteSearch("from:sender@example.com"); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	_, total, err = Search("from:sender@example.com", 0, 100)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, total, 0, "0 search results expected")
+}
+
+func TestEscPercentChar(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a test"] = "this is a test"
+	tests["this is% a test"] = "this is%% a test"
+	tests["this is%% a test"] = "this is%%%% a test"
+	tests["this is%%% a test"] = "this is%%%%%% a test"
+	tests["%this is% a test"] = "%%this is%% a test"
+	tests["Ä"] = "Ä"
+	tests["Ä%"] = "Ä%%"
+
+	for search, expected := range tests {
+		res := escPercentChar(search)
+		assertEqual(t, res, expected, "no match")
+	}
+}

internal/storage/settings.go

@@ -0,0 +1,75 @@
+package storage
+
+import (
+	"database/sql"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/leporo/sqlf"
+)
+
+// SettingGet returns a setting string value, blank is it does not exist
+func SettingGet(k string) string {
+	var result sql.NullString
+	err := sqlf.From("settings").
+		Select("Value").To(&result).
+		Where("Key = ?", k).
+		Limit(1).
+		QueryAndClose(nil, db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return ""
+	}
+
+	return result.String
+}
+
+// SettingPut sets a setting string value, inserting if new
+func SettingPut(k, v string) error {
+	_, err := db.Exec("INSERT INTO settings (Key, Value) VALUES(?, ?) ON CONFLICT(Key) DO UPDATE SET Value = ?", k, v, v)
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return err
+}
+
+// The total deleted message size as an int64 value
+func getDeletedSize() int64 {
+	var result sql.NullInt64
+	err := sqlf.From("settings").
+		Select("Value").To(&result).
+		Where("Key = ?", "DeletedSize").
+		Limit(1).
+		QueryAndClose(nil, db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Int64
+}
+
+// The total raw non-compressed messages size in bytes of all messages in the database
+func totalMessagesSize() int64 {
+	var result sql.NullInt64
+	err := sqlf.From("mailbox").
+		Select("SUM(Size)").To(&result).
+		QueryAndClose(nil, db, func(row *sql.Rows) {})
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return 0
+	}
+
+	return result.Int64
+}
+
+// AddDeletedSize will add the value to the DeletedSize setting
+func addDeletedSize(v int64) {
+	if _, err := db.Exec("INSERT OR IGNORE INTO settings (Key, Value) VALUES(?, ?)", "DeletedSize", 0); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	if _, err := db.Exec("UPDATE settings SET Value = Value + ? WHERE Key = ?", v, "DeletedSize"); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+}

internal/storage/structs.go

@@ -0,0 +1,143 @@
+package storage
+
+import (
+	"net/mail"
+	"time"
+
+	"github.com/jhillyerd/enmime"
+)
+
+// Message data excluding physical attachments
+//
+// swagger:model Message
+type Message struct {
+	// Database ID
+	ID string
+	// Message ID
+	MessageID string
+	// From address
+	From *mail.Address
+	// To addresses
+	To []*mail.Address
+	// Cc addresses
+	Cc []*mail.Address
+	// Bcc addresses
+	Bcc []*mail.Address
+	// ReplyTo addresses
+	ReplyTo []*mail.Address
+	// Return-Path
+	ReturnPath string
+	// Message subject
+	Subject string
+	// List-Unsubscribe header information
+	// swagger:ignore
+	ListUnsubscribe ListUnsubscribe
+	// Message date if set, else date received
+	Date time.Time
+	// Message tags
+	Tags []string
+	// Message body text
+	Text string
+	// Message body HTML
+	HTML string
+	// Message size in bytes
+	Size int
+	// Inline message attachments
+	Inline []Attachment
+	// Message attachments
+	Attachments []Attachment
+}
+
+// Attachment struct for inline and attachments
+//
+// swagger:model Attachment
+type Attachment struct {
+	// Attachment part ID
+	PartID string
+	// File name
+	FileName string
+	// Content type
+	ContentType string
+	// Content ID
+	ContentID string
+	// Size in bytes
+	Size int
+}
+
+// MessageSummary struct for frontend messages
+//
+// swagger:model MessageSummary
+type MessageSummary struct {
+	// Database ID
+	ID string
+	// Message ID
+	MessageID string
+	// Read status
+	Read bool
+	// From address
+	From *mail.Address
+	// To address
+	To []*mail.Address
+	// Cc addresses
+	Cc []*mail.Address
+	// Bcc addresses
+	Bcc []*mail.Address
+	// Reply-To address
+	ReplyTo []*mail.Address
+	// Email subject
+	Subject string
+	// Created time
+	Created time.Time
+	// Message tags
+	Tags []string
+	// Message size in bytes (total)
+	Size int
+	// Whether the message has any attachments
+	Attachments int
+	// Message snippet includes up to 250 characters
+	Snippet string
+}
+
+// MailboxStats struct for quick mailbox total/read lookups
+type MailboxStats struct {
+	Total  int
+	Unread int
+	Tags   []string
+}
+
+// DBMailSummary struct for storing mail summary
+type DBMailSummary struct {
+	From    *mail.Address
+	To      []*mail.Address
+	Cc      []*mail.Address
+	Bcc     []*mail.Address
+	ReplyTo []*mail.Address
+}
+
+// AttachmentSummary returns a summary of the attachment without any binary data
+func AttachmentSummary(a *enmime.Part) Attachment {
+	o := Attachment{}
+	o.PartID = a.PartID
+	o.FileName = a.FileName
+	if o.FileName == "" {
+		o.FileName = a.ContentID
+	}
+	o.ContentType = a.ContentType
+	o.ContentID = a.ContentID
+	o.Size = len(a.Content)
+
+	return o
+}
+
+// ListUnsubscribe contains a summary of List-Unsubscribe & List-Unsubscribe-Post headers
+// including validation of the link structure
+type ListUnsubscribe struct {
+	// List-Unsubscribe header value
+	Header string
+	// Detected links, maximum one email and one HTTP(S)
+	Links []string
+	// Validation errors if any
+	Errors string
+	// List-Unsubscribe-Post value if set
+	HeaderPost string
+}

internal/storage/tags.go

@@ -0,0 +1,321 @@
+package storage
+
+import (
+	"database/sql"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/leporo/sqlf"
+)
+
+var (
+	addressPlusRe = regexp.MustCompile(`(?U)^(.*){1,}\+(.*)@`)
+)
+
+// SetMessageTags will set the tags for a given database ID
+func SetMessageTags(id string, tags []string) error {
+	applyTags := []string{}
+	for _, t := range tags {
+		t = tools.CleanTag(t)
+		if t != "" && config.ValidTagRegexp.MatchString(t) && !inArray(t, applyTags) {
+			applyTags = append(applyTags, t)
+		}
+	}
+
+	currentTags := getMessageTags(id)
+	origTagCount := len(currentTags)
+
+	for _, t := range applyTags {
+		t = tools.CleanTag(t)
+		if t == "" || !config.ValidTagRegexp.MatchString(t) || inArray(t, currentTags) {
+			continue
+		}
+
+		if err := AddMessageTag(id, t); err != nil {
+			return err
+		}
+	}
+
+	if origTagCount > 0 {
+		currentTags = getMessageTags(id)
+
+		for _, t := range currentTags {
+			if !inArray(t, applyTags) {
+				if err := DeleteMessageTag(id, t); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// AddMessageTag adds a tag to a message
+func AddMessageTag(id, name string) error {
+	var tagID int
+
+	q := sqlf.From("tags").
+		Select("ID").To(&tagID).
+		Where("Name = ?", name)
+
+	// tag exists - add tag to message
+	if err := q.QueryRowAndClose(nil, db); err == nil {
+		// check message does not already have this tag
+		var count int
+		if _, err := sqlf.From("message_tags").
+			Select("COUNT(ID)").To(&count).
+			Where("ID = ?", id).
+			Where("TagID = ?", tagID).
+			ExecAndClose(nil, db); err != nil {
+			return err
+		}
+		if count != 0 {
+			// already exists
+			return nil
+		}
+
+		logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
+
+		_, err := sqlf.InsertInto("message_tags").
+			Set("ID", id).
+			Set("TagID", tagID).
+			ExecAndClose(nil, db)
+		return err
+	}
+
+	logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
+
+	// tag dos not exist, add new one
+	if err := sqlf.InsertInto("tags").
+		Set("Name", name).
+		Returning("ID").To(&tagID).
+		QueryRowAndClose(nil, db); err != nil {
+		return err
+	}
+
+	// check message does not already have this tag
+	var count int
+	if _, err := sqlf.From("message_tags").
+		Select("COUNT(ID)").To(&count).
+		Where("ID = ?", id).
+		Where("TagID = ?", tagID).
+		ExecAndClose(nil, db); err != nil {
+		return err
+	}
+	if count != 0 {
+		return nil // already exists
+	}
+
+	// add tag to message
+	_, err := sqlf.InsertInto("message_tags").
+		Set("ID", id).
+		Set("TagID", tagID).
+		ExecAndClose(nil, db)
+	return err
+}
+
+// DeleteMessageTag deleted a tag from a message
+func DeleteMessageTag(id, name string) error {
+	if _, err := sqlf.DeleteFrom("message_tags").
+		Where("message_tags.ID = ?", id).
+		Where(`message_tags.Key IN (SELECT Key FROM message_tags LEFT JOIN tags ON TagID=tags.ID WHERE Name = ?)`, name).
+		ExecAndClose(nil, db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// DeleteAllMessageTags deleted all tags from a message
+func DeleteAllMessageTags(id string) error {
+	if _, err := sqlf.DeleteFrom("message_tags").
+		Where("message_tags.ID = ?", id).
+		ExecAndClose(nil, db); err != nil {
+		return err
+	}
+
+	return pruneUnusedTags()
+}
+
+// GetAllTags returns all used tags
+func GetAllTags() []string {
+	var tags = []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`DISTINCT Name`).
+		From("tags").To(&name).
+		OrderBy("Name").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// GetAllTagsCount returns all used tags with their total messages
+func GetAllTagsCount() map[string]int64 {
+	var tags = make(map[string]int64)
+	var name string
+	var total int64
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		Select(`COUNT(message_tags.TagID) as total`).To(&total).
+		From("tags").
+		LeftJoin("message_tags", "tags.ID = message_tags.TagID").
+		GroupBy("message_tags.TagID").
+		OrderBy("Name").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			tags[name] = total
+			// tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+	}
+
+	return tags
+}
+
+// PruneUnusedTags will delete all unused tags from the database
+func pruneUnusedTags() error {
+	q := sqlf.From("tags").
+		Select("tags.ID, tags.Name, COUNT(message_tags.ID) as COUNT").
+		LeftJoin("message_tags", "tags.ID = message_tags.TagID").
+		GroupBy("tags.ID")
+
+	toDel := []int{}
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var n string
+		var id int
+		var c int
+
+		if err := row.Scan(&id, &n, &c); err != nil {
+			logger.Log().Errorf("[tags] %s", err.Error())
+			return
+		}
+
+		if c == 0 {
+			logger.Log().Debugf("[tags] deleting unused tag \"%s\"", n)
+			toDel = append(toDel, id)
+		}
+	}); err != nil {
+		return err
+	}
+
+	if len(toDel) > 0 {
+		for _, id := range toDel {
+			if _, err := sqlf.DeleteFrom("tags").
+				Where("ID = ?", id).
+				ExecAndClose(nil, db); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Find tags set via --tags in raw message.
+// Returns a comma-separated string.
+func findTagsInRawMessage(message *[]byte) string {
+	tagStr := ""
+	if len(config.SMTPTags) == 0 {
+		return tagStr
+	}
+
+	str := strings.ToLower(string(*message))
+	for _, t := range config.SMTPTags {
+		if strings.Contains(str, t.Match) {
+			tagStr += "," + t.Tag
+		}
+	}
+
+	return tagStr
+}
+
+// Returns tags found in email plus addresses (eg: test+tagname@example.com)
+func (d DBMailSummary) tagsFromPlusAddresses() string {
+	tags := []string{}
+	for _, c := range d.To {
+		matches := addressPlusRe.FindAllStringSubmatch(c.String(), 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Cc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.String(), 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	for _, c := range d.Bcc {
+		matches := addressPlusRe.FindAllStringSubmatch(c.String(), 1)
+		if len(matches) == 1 {
+			tags = append(tags, strings.Split(matches[0][2], "+")...)
+		}
+	}
+	matches := addressPlusRe.FindAllStringSubmatch(d.From.String(), 1)
+	if len(matches) == 1 {
+		tags = append(tags, strings.Split(matches[0][2], "+")...)
+	}
+
+	return strings.Join(tags, ",")
+}
+
+// Get message tags from the database for a given database ID
+// Used when parsing a raw email.
+func getMessageTags(id string) []string {
+	tags := []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		From("Tags").
+		LeftJoin("message_tags", "Tags.ID=message_tags.TagID").
+		Where(`message_tags.ID = ?`, id).
+		OrderBy("Name").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Errorf("[tags] %s", err.Error())
+		return tags
+	}
+
+	return tags
+}
+
+// UniqueTagsFromString will split a string with commas, and extract a unique slice of formatted tags
+func uniqueTagsFromString(s string) []string {
+	tags := []string{}
+
+	if s == "" {
+		return tags
+	}
+
+	parts := strings.Split(s, ",")
+	for _, p := range parts {
+		w := tools.CleanTag(p)
+		if w == "" {
+			continue
+		}
+		if config.ValidTagRegexp.MatchString(w) {
+			if !inArray(w, tags) {
+				tags = append(tags, w)
+			}
+		} else {
+			logger.Log().Debugf("[tags] ignoring invalid tag: %s", w)
+		}
+	}
+
+	sort.Strings(tags)
+
+	return tags
+}

internal/storage/tags_test.go

@@ -0,0 +1,130 @@
+package storage
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func TestTags(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing tags")
+
+	ids := []string{}
+
+	for i := 0; i < 10; i++ {
+		id, err := Store(&testMimeEmail)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+		ids = append(ids, id)
+	}
+
+	for i := 0; i < 10; i++ {
+		if err := SetMessageTags(ids[i], []string{fmt.Sprintf("Tag-%d", i)}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		message, err := GetMessage(ids[i])
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		if len(message.Tags) != 1 || message.Tags[0] != fmt.Sprintf("Tag-%d", i) {
+			t.Fatal("Message tags do not match")
+		}
+	}
+
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	// test 20 tags
+	id, err := Store(&testMimeEmail)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	newTags := []string{}
+	for i := 0; i < 20; i++ {
+		// pad number with 0 to ensure they are returned alphabetically
+		newTags = append(newTags, fmt.Sprintf("AnotherTag %02d", i))
+	}
+	if err := SetMessageTags(id, newTags); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	returnedTags := getMessageTags(id)
+	assertEqual(t, strings.Join(newTags, "|"), strings.Join(returnedTags, "|"), "Message tags do not match")
+
+	// remove first tag
+	if err := DeleteMessageTag(id, newTags[0]); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	returnedTags = getMessageTags(id)
+	assertEqual(t, strings.Join(newTags[1:], "|"), strings.Join(returnedTags, "|"), "Message tags do not match after deleting 1")
+
+	// remove all tags
+	if err := DeleteAllMessageTags(id); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	returnedTags = getMessageTags(id)
+	assertEqual(t, "", strings.Join(returnedTags, "|"), "Message tags should be empty")
+
+	// apply the same tag twice
+	if err := SetMessageTags(id, []string{"Duplicate Tag", "Duplicate Tag"}); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	returnedTags = getMessageTags(id)
+	assertEqual(t, "Duplicate Tag", strings.Join(returnedTags, "|"), "Message tags should be duplicated")
+	if err := DeleteAllMessageTags(id); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	// apply tag with invalid characters
+	if err := SetMessageTags(id, []string{"Dirty! \"Tag\""}); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+	returnedTags = getMessageTags(id)
+	assertEqual(t, "Dirty Tag", strings.Join(returnedTags, "|"), "Dirty message tag did not clean as expected")
+	if err := DeleteAllMessageTags(id); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	// Check deleted message tags also prune the tags database
+	allTags := GetAllTags()
+	assertEqual(t, "", strings.Join(allTags, "|"), "Tags did not delete as expected")
+
+	if err := DeleteAllMessages(); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	// test 20 tags
+	id, err = Store(&testTagEmail)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	returnedTags = getMessageTags(id)
+	assertEqual(t, "BccTag|CcTag|FromFag|ToTag|X-tag1|X-tag2", strings.Join(returnedTags, "|"), "Tags not detected correctly")
+	if err := DeleteAllMessageTags(id); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+}

internal/storage/testdata/mime-attachment.eml

@@ -1,4 +1,4 @@
-Delivered-To: recipient@example.com
+Delivered-To: recipient2@example.com
 Received: by 2002:a0c:fe87:0:0:0:0:0 with SMTP id d7csp145570qvs;
         Tue, 26 Jul 2022 20:42:36 -0700 (PDT)
 X-Received: by 2002:a17:902:f788:b0:16c:f48b:905e with SMTP id q8-20020a170902f78800b0016cf48b905emr19885972pln.60.1658893355881;
@@ -23,18 +23,18 @@ ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc
          uSfA==
 ARC-Authentication-Results: i=1; mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])
         by mx.google.com with SMTPS id 11-20020aa7914b000000b0052ab192de4fsor8543241pfi.101.2022.07.26.20.42.35
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (Google Transport Security);
         Tue, 26 Jul 2022 20:42:35 -0700 (PDT)
-Received-SPF: pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
+Received-SPF: pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;
 Authentication-Results: mx.google.com;
        dkim=pass header.i=@gmail.com header.s=20210112 header.b=mywi6bMa;
-       spf=pass (google.com: domain of sender@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender@example.com;
+       spf=pass (google.com: domain of sender2@example.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=sender2@example.com;
        dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com
 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
         d=gmail.com; s=20210112;
@@ -63,10 +63,10 @@ X-Gm-Message-State: AJIora/WUqr3biShTHQBjSlCKazFbrLxeYpxmr1VF0TpBUbjnJrcLT77
 X-Google-Smtp-Source: AGRyM1tai6X1Bx130Y1yHG5w2e0r8wx6bbI+H+YppWmQoT28TV3dSoYCqmeQK5VViW8WuvdOpQzhPQ==
 X-Received: by 2002:a62:29c3:0:b0:52b:f774:7242 with SMTP id p186-20020a6229c3000000b0052bf7747242mr12504553pfp.67.1658893354675;
         Tue, 26 Jul 2022 20:42:34 -0700 (PDT)
-Return-Path: <sender@example.com>
+Return-Path: <sender2@example.com>
 Received: from [192.168.1.2] ([8.8.8.8])
         by smtp.gmail.com with ESMTPSA id oj16-20020a17090b4d9000b001f291c9d3bdsm387578pjb.48.2022.07.26.20.42.32
-        for <recipient@example.com>
+        for <recipient2@example.com>
         (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);
         Tue, 26 Jul 2022 20:42:33 -0700 (PDT)
 Content-Type: multipart/mixed; boundary="------------ae0qIOkrNQLQHe1YyfTsUXrk"
@@ -76,8 +76,8 @@ MIME-Version: 1.0
 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101
  Thunderbird/91.11.0
 Content-Language: en-NZ
-To: "Recipient Ross" <recipient@example.com>
-From: Sender Smith <sender@example.com>
+To: "Recipient Ross" <recipient2@example.com>
+From: Sender Smith <sender2@example.com>
 Subject: inline + attachment
 
 This is a multi-part message in MIME format.
@@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit
     <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   </head>
   <body>
-    Message with inline image and attachment:<br>
+    <h1>Message with inline image and attachment:</h1>
     <br>
-    <img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"
-      moz-do-not-send="false"><br>
+    <p><img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"></p>
     <br>
     <br>
   </body>

internal/storage/testdata/tags.eml

@@ -0,0 +1,49 @@
+Date: Wed, 27 Jul 2022 15:44:41 +1200
+From: Sender Smith <sender+FromFag@example.com>
+To: Recipient Ross <recipient+ToTag@example.com>
+Cc: Recipient Ross <cc+CcTag@example.com>
+Bcc: <bcc+BccTag@example.com>
+Subject: Plain text message
+X-Tags: X-tag1, X-tag2
+Message-ID: <20220727034441.7za34h6ljuzfpmj3@localhost.localhost>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras non massa lacinia, 
+fringilla ex vel, ornare nulla. Suspendisse dapibus commodo sapien, non 
+hendrerit diam feugiat sit amet. Nulla lorem quam, laoreet vitae nisl volutpat, 
+mollis bibendum felis. In eget ultricies justo. Donec vitae hendrerit tortor, at 
+posuere libero. Fusce a gravida nibh. Nulla ac odio ex.
+
+Aliquam sem turpis, cursus vitae condimentum at, scelerisque pulvinar lectus. 
+Cras tempor nisl ut arcu interdum, et luctus arcu cursus. Maecenas mollis 
+sagittis commodo. Mauris ac lorem nec ex interdum consequat. Morbi congue 
+ultrices ullamcorper. Aenean ex tortor, dapibus quis dapibus iaculis, iaculis 
+eget felis. Vestibulum purus ante, efficitur in turpis ac, tristique laoreet 
+orci. Nulla facilisi. Praesent mollis orci posuere elementum laoreet. 
+Pellentesque enim nibh, varius at ante id, consequat posuere ante.
+
+Cras maximus venenatis nulla nec cursus. Morbi convallis, enim eget viverra 
+vulputate, ipsum arcu tincidunt tortor, ut cursus dui enim commodo quam. Donec 
+et vulputate quam. Vivamus non posuere erat. Nam commodo pellentesque 
+condimentum. Vivamus condimentum eros at odio dictum feugiat. Ut imperdiet 
+tempor luctus. Aenean varius libero ac faucibus dictum. Aliquam sed finibus 
+massa. Morbi dolor lorem, feugiat quis neque et, suscipit posuere ex. Sed auctor 
+et augue at finibus. Vestibulum interdum mi ac justo porta aliquam. Curabitur 
+nec enim sit amet enim aliquet accumsan. Etiam accumsan tellus tortor, interdum 
+sodales odio finibus eu. Integer eget ante eu nisi lobortis pulvinar et vel 
+ipsum. Cras condimentum posuere vulputate.
+
+Cras nulla felis, blandit vitae egestas quis, fringilla ut dolor. Phasellus est 
+augue, feugiat eu risus quis, posuere ultrices libero. Phasellus non nunc eget 
+justo sollicitudin tincidunt. Praesent pretium dui id felis bibendum sodales. 
+Phasellus eget dictum libero, auctor tempor nibh. Suspendisse posuere libero 
+venenatis elit imperdiet porttitor. In condimentum dictum luctus. Nullam in 
+nulla vitae augue blandit posuere. Vestibulum consectetur ultricies tincidunt. 
+Vivamus dolor quam, pharetra sed eros sed, hendrerit ultrices diam. Vestibulum 
+vulputate tellus eget tellus lacinia, a pulvinar velit vulputate. Suspendisse 
+mauris odio, scelerisque eget turpis sed, tincidunt ultrices magna. Nunc arcu 
+arcu, commodo et porttitor quis, accumsan viverra purus. Fusce id libero iaculis 
+lorem tristique commodo porttitor id ipsum. Vestibulum odio dui, tincidunt eget 
+lectus vel, tristique lacinia libero. Aliquam dapibus ac felis vitae cursus.

internal/storage/testing.go

@@ -0,0 +1,63 @@
+package storage
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var (
+	testTextEmail []byte
+	testTagEmail  []byte
+	testMimeEmail []byte
+	testRuns      = 100
+)
+
+func setup() {
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.DataFile = ""
+
+	if err := InitDB(); err != nil {
+		panic(err)
+	}
+
+	var err error
+
+	testTextEmail, err = os.ReadFile("testdata/plain-text.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testTagEmail, err = os.ReadFile("testdata/tags.eml")
+	if err != nil {
+		panic(err)
+	}
+
+	testMimeEmail, err = os.ReadFile("testdata/mime-attachment.eml")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}
+
+func assertEqualStats(t *testing.T, total int, unread int) {
+	s := StatsGet()
+	if total != s.Total {
+		t.Fatalf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total)
+	}
+
+	if unread != s.Unread {
+		t.Fatalf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread)
+	}
+}

internal/storage/utils.go

@@ -0,0 +1,144 @@
+package storage
+
+import (
+	"net/mail"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/axllent/mailpit/internal/html2text"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	// for stats to prevent import cycle
+	mu sync.RWMutex
+	// StatsDeleted for counting the number of messages deleted
+	StatsDeleted int
+)
+
+// Return a header field as a []*mail.Address, or "null" is not found/empty
+func addressToSlice(env *enmime.Envelope, key string) []*mail.Address {
+	data, err := env.AddressList(key)
+	if err != nil || data == nil {
+		return []*mail.Address{}
+	}
+
+	return data
+}
+
+// Generate the search text based on some header fields (to, from, subject etc)
+// and either the stripped HTML body (if exists) or text body
+func createSearchText(env *enmime.Envelope) string {
+	var b strings.Builder
+
+	b.WriteString(env.GetHeader("From") + " ")
+	b.WriteString(env.GetHeader("Subject") + " ")
+	b.WriteString(env.GetHeader("To") + " ")
+	b.WriteString(env.GetHeader("Cc") + " ")
+	b.WriteString(env.GetHeader("Bcc") + " ")
+	b.WriteString(env.GetHeader("Reply-To") + " ")
+	b.WriteString(env.GetHeader("Return-Path") + " ")
+
+	h := html2text.Strip(env.HTML, true)
+	if h != "" {
+		b.WriteString(h + " ")
+	} else {
+		b.WriteString(env.Text + " ")
+	}
+	// add attachment filenames
+	for _, a := range env.Attachments {
+		b.WriteString(a.FileName + " ")
+	}
+
+	d := cleanString(b.String())
+
+	return d
+}
+
+// CleanString removes unwanted characters from stored search text and search queries
+func cleanString(str string) string {
+	// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+	str = strings.ReplaceAll(str, string('\uFEFF'), " ")
+
+	// remove/replace new lines
+	re := regexp.MustCompile(`(\r?\n|\t|>|<|"|\,|;|\(|\))`)
+	str = re.ReplaceAllString(str, " ")
+
+	// remove duplicate whitespace and trim
+	return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " "))
+}
+
+// LogMessagesDeleted logs the number of messages deleted
+func logMessagesDeleted(n int) {
+	mu.Lock()
+	StatsDeleted = StatsDeleted + n
+	mu.Unlock()
+}
+
+// IsFile returns whether a path is a file
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) || !info.Mode().IsRegular() {
+		return false
+	}
+
+	return true
+}
+
+// Tests if a string is within an array. It is not case sensitive.
+func inArray(k string, arr []string) bool {
+	k = strings.ToLower(k)
+	for _, v := range arr {
+		if strings.ToLower(v) == k {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Convert `%` to `%%` for SQL searches
+func escPercentChar(s string) string {
+	return strings.ReplaceAll(s, "%", "%%")
+}
+
+// Escape certain characters in search phrases
+func escSearch(str string) string {
+	dest := make([]byte, 0, 2*len(str))
+	var escape byte
+	for i := 0; i < len(str); i++ {
+		c := str[i]
+
+		escape = 0
+
+		switch c {
+		case 0: /* Must be escaped for 'mysql' */
+			escape = '0'
+			break
+		case '\n': /* Must be escaped for logs */
+			escape = 'n'
+			break
+		case '\r':
+			escape = 'r'
+			break
+		case '\\':
+			escape = '\\'
+			break
+		case '\'':
+			escape = '\''
+			break
+		case '\032': //十进制26,八进制32,十六进制1a, /* This gives problems on Win32 */
+			escape = 'Z'
+		}
+
+		if escape != 0 {
+			dest = append(dest, '\\', escape)
+		} else {
+			dest = append(dest, c)
+		}
+	}
+
+	return string(dest)
+}

internal/tools/argsparser.go

@@ -0,0 +1,32 @@
+package tools
+
+import "strings"
+
+// ArgsParser will split a string by new words and quotes phrases
+func ArgsParser(s string) []string {
+	args := []string{}
+	sb := &strings.Builder{}
+	quoted := false
+	for _, r := range s {
+		if r == '"' {
+			quoted = !quoted
+			sb.WriteRune(r) // keep '"' otherwise comment this line
+		} else if !quoted && r == ' ' {
+			v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+			if v != "" {
+				args = append(args, v)
+			}
+			sb.Reset()
+		} else {
+			sb.WriteRune(r)
+		}
+	}
+	if sb.Len() > 0 {
+		v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
+		if v != "" {
+			args = append(args, v)
+		}
+	}
+
+	return args
+}

internal/tools/html.go

@@ -0,0 +1,19 @@
+package tools
+
+import (
+	"fmt"
+
+	"golang.org/x/net/html"
+)
+
+// GetHTMLAttributeVal returns the value of an HTML Attribute, else an error.
+// Returns a blank value if the attribute is set but empty.
+func GetHTMLAttributeVal(e *html.Node, key string) (string, error) {
+	for _, a := range e.Attr {
+		if a.Key == key {
+			return a.Val, nil
+		}
+	}
+
+	return "", fmt.Errorf("%s not found", key)
+}

internal/tools/listunsubscribeparser.go

@@ -0,0 +1,99 @@
+package tools
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+// ListUnsubscribeParser will attempt to parse a `List-Unsubscribe` header and return
+// a slide of addresses (mail & URLs)
+func ListUnsubscribeParser(v string) ([]string, error) {
+	var results = []string{}
+	var re = regexp.MustCompile(`(?mU)<(.*)>`)
+	var reJoins = regexp.MustCompile(`(?imUs)>(.*)<`)
+	var reValidJoinChars = regexp.MustCompile(`(?imUs)^(\s+)?,(\s+)?$`)
+	var reWrapper = regexp.MustCompile(`(?imUs)^<(.*)>$`)
+	var reMailTo = regexp.MustCompile(`^mailto:[a-zA-Z0-9]`)
+	var reHTTP = regexp.MustCompile(`^(?i)https?://[a-zA-Z0-9]`)
+	var reSpaces = regexp.MustCompile(`\s`)
+	var reComments = regexp.MustCompile(`(?mUs)\(.*\)`)
+	var hasMailTo bool
+	var hasHTTP bool
+
+	v = strings.TrimSpace(v)
+
+	comments := reComments.FindAllStringSubmatch(v, -1)
+	for _, c := range comments {
+		// strip comments
+		v = strings.Replace(v, c[0], "", -1)
+		v = strings.TrimSpace(v)
+	}
+
+	if !re.MatchString(v) {
+		return results, fmt.Errorf("\"%s\" no valid unsubscribe links found", v)
+	}
+
+	errors := []string{}
+
+	if !reWrapper.MatchString(v) {
+		return results, fmt.Errorf("\"%s\" should be enclosed in <>", v)
+	}
+
+	matches := re.FindAllStringSubmatch(v, -1)
+
+	if len(matches) > 2 {
+		errors = append(errors, fmt.Sprintf("\"%s\" should include a maximum of one email and one HTTP link", v))
+	} else {
+		splits := reJoins.FindAllStringSubmatch(v, -1)
+		for _, g := range splits {
+			if !reValidJoinChars.MatchString(g[1]) {
+				return results, fmt.Errorf("\"%s\" <> should be split with a comma and optional spaces", v)
+			}
+		}
+
+		for _, m := range matches {
+			r := m[1]
+			if reSpaces.MatchString(r) {
+				errors = append(errors, fmt.Sprintf("\"%s\" should not contain spaces", r))
+				continue
+			}
+
+			if reMailTo.MatchString(r) {
+				if hasMailTo {
+					errors = append(errors, fmt.Sprintf("\"%s\" should only contain one mailto:", r))
+					continue
+				}
+
+				hasMailTo = true
+			} else if reHTTP.MatchString(r) {
+				if hasHTTP {
+					errors = append(errors, fmt.Sprintf("\"%s\" should only contain one HTTP link", r))
+					continue
+				}
+
+				hasHTTP = true
+
+			} else {
+				errors = append(errors, fmt.Sprintf("\"%s\" should start with either http(s):// or mailto:", r))
+				continue
+			}
+
+			_, err := url.ParseRequestURI(r)
+			if err != nil {
+				errors = append(errors, err.Error())
+				continue
+			}
+
+			results = append(results, r)
+		}
+	}
+
+	var err error
+	if len(errors) > 0 {
+		err = fmt.Errorf("%s", strings.Join(errors, ", "))
+	}
+
+	return results, err
+}

internal/tools/message.go

@@ -0,0 +1,99 @@
+// Package tools provides various methods for various things
+package tools
+
+import (
+	"bufio"
+	"bytes"
+	"net/mail"
+	"regexp"
+
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+// RemoveMessageHeaders scans a message for headers, if found them removes them.
+// It will only remove a single instance of any given message header.
+func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	reBlank := regexp.MustCompile(`^\s+`)
+
+	for _, hdr := range headers {
+		// case-insensitive
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(hdr+":"))
+
+		// header := []byte(hdr + ":")
+		if m.Header.Get(hdr) != "" {
+			scanner := bufio.NewScanner(bytes.NewReader(msg))
+			found := false
+			hdr := []byte("")
+			for scanner.Scan() {
+				line := scanner.Bytes()
+				if !found && reHdr.Match(line) {
+					// add the first line starting with <header>:
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+					found = true
+				} else if found && reBlank.Match(line) {
+					// add any following lines starting with a whitespace (tab or space)
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+				} else if found {
+					// stop scanning, we have the full <header>
+					break
+				}
+			}
+
+			if len(hdr) > 0 {
+				logger.Log().Debugf("[release] removed %s header", hdr)
+				msg = bytes.Replace(msg, hdr, []byte(""), 1)
+			}
+		}
+	}
+
+	return msg, nil
+}
+
+// UpdateMessageHeader scans a message for a header and updates its value if found.
+func UpdateMessageHeader(msg []byte, header, value string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get(header) != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(header+":"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		if len(hdr) > 0 {
+			logger.Log().Debugf("[release] replaced %s header", hdr)
+			msg = bytes.Replace(msg, hdr, []byte(header+": "+value+"\r\n"), 1)
+		}
+	}
+
+	return msg, nil
+}

internal/tools/snippets.go

@@ -0,0 +1,44 @@
+package tools
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/html2text"
+)
+
+// CreateSnippet returns a message snippet. It will use the HTML version (if it exists)
+// otherwise the text version.
+func CreateSnippet(text, html string) string {
+	text = strings.TrimSpace(text)
+	html = strings.TrimSpace(html)
+	limit := 200
+	spaceRe := regexp.MustCompile(`\s+`)
+
+	if text == "" && html == "" {
+		return ""
+	}
+
+	if html != "" {
+		data := html2text.Strip(html, false)
+
+		if len(data) <= limit {
+			return data
+		}
+
+		return data[0:limit] + "..."
+	}
+
+	if text != "" {
+		// replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184
+		text = strings.ReplaceAll(text, string('\uFEFF'), " ")
+		text = strings.TrimSpace(spaceRe.ReplaceAllString(text, " "))
+		if len(text) <= limit {
+			return text
+		}
+
+		return text[0:limit] + "..."
+	}
+
+	return ""
+}

internal/tools/tags.go

@@ -0,0 +1,36 @@
+package tools
+
+import (
+	"regexp"
+	"strings"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+var (
+	// Invalid tag characters regex
+	tagsInvalidChars = regexp.MustCompile(`[^a-zA-Z0-9\-\ \_\.]`)
+
+	// Regex to catch multiple spaces
+	multiSpaceRe = regexp.MustCompile(`(\s+)`)
+
+	// TagsTitleCase enforces TitleCase on all tags
+	TagsTitleCase bool
+)
+
+// CleanTag returns a clean tag, removing whitespace and invalid characters
+func CleanTag(s string) string {
+	s = strings.TrimSpace(
+		multiSpaceRe.ReplaceAllString(
+			tagsInvalidChars.ReplaceAllString(s, " "),
+			" ",
+		),
+	)
+
+	if TagsTitleCase {
+		return cases.Title(language.Und, cases.NoLower).String(s)
+	}
+
+	return s
+}

internal/tools/tools_test.go

@@ -0,0 +1,119 @@
+package tools
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestArgsParser(t *testing.T) {
+	tests := map[string][]string{}
+	tests["this is a test"] = []string{"this", "is", "a", "test"}
+	tests["\"this is\" a test"] = []string{"this is", "a", "test"}
+	tests["!\"this is\" a test"] = []string{"!this is", "a", "test"}
+	tests["subject:this is a test"] = []string{"subject:this", "is", "a", "test"}
+	tests["subject:\"this is\" a test"] = []string{"subject:this is", "a", "test"}
+	tests["subject:\"this is\" \"a test\""] = []string{"subject:this is", "a test"}
+	tests["subject:\"this 'is\" \"a test\""] = []string{"subject:this 'is", "a test"}
+	tests["subject:\"this 'is a test"] = []string{"subject:this 'is a test"}
+	tests["\"this is a test\"=\"this is a test\""] = []string{"this is a test=this is a test"}
+
+	for search, expected := range tests {
+		res := ArgsParser(search)
+		if !reflect.DeepEqual(res, expected) {
+			t.Log("Args parser error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestCleanTag(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test -"
+	tests["  thiS 99     IS a Test :-)"] = "thiS 99 IS a Test -"
+	tests["this_is-a test "] = "this_is-a test"
+	tests["this_is-a&^%%(*)@ test"] = "this_is-a test"
+
+	for search, expected := range tests {
+		res := CleanTag(search)
+		if res != expected {
+			t.Log("CleanTags error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestSnippets(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["this_is-a     test "] = "this_is-a test"
+	tests["this_is-a&^%%(*)@ test"] = "this_is-a&^%%(*)@ test"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading linked text."
+	// truncation to 200 chars + ...
+	tests["abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789"] = "abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmno..."
+
+	for str, expected := range tests {
+		res := CreateSnippet(str, str)
+		if res != expected {
+			t.Log("CreateSnippet error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestListUnsubscribeParser(t *testing.T) {
+	tests := map[string]bool{}
+
+	// should pass
+	tests["<mailto:unsubscribe@example.com>"] = true
+	tests["<https://example.com>"] = true
+	tests["<HTTPS://EXAMPLE.COM>"] = true
+	tests["<mailto:unsubscribe@example.com>, <http://example.com>"] = true
+	tests["<mailto:unsubscribe@example.com>, <https://example.com>"] = true
+	tests["<https://example.com>, <mailto:unsubscribe@example.com>"] = true
+	tests["<https://example.com> , 		<mailto:unsubscribe@example.com>"] = true
+	tests["<https://example.com> ,<mailto:unsubscribe@example.com>"] = true
+	tests["<mailto:unsubscribe@example.com>,<https://example.com>"] = true
+	tests[`<https://example.com> ,
+		 <mailto:unsubscribe@example.com>`] = true
+	tests["<mailto:unsubscribe@example.com?subject=unsubscribe%20me>"] = true
+	tests["(Use this command to get off the list) <mailto:unsubscribe@example.com?subject=unsubscribe%20me>"] = true
+	tests["<mailto:unsubscribe@example.com> (Use this command to get off the list)"] = true
+	tests["(Use this command to get off the list) <mailto:unsubscribe@example.com>, (Click this link to unsubscribe) <http://example.com>"] = true
+
+	// should fail
+	tests["mailto:unsubscribe@example.com"] = false                                                // no <>
+	tests["<mailto::unsubscribe@example.com>"] = false                                             // ::
+	tests["https://example.com/"] = false                                                          // no <>
+	tests["mailto:unsubscribe@example.com, <https://example.com/>"] = false                        // no <>
+	tests["<MAILTO:unsubscribe@example.com>"] = false                                              // capitals
+	tests["<mailto:unsubscribe@example.com>, <mailto:test2@example.com>"] = false                  // two emails
+	tests["<http://exampl\\e2.com>, <http://example2.com>"] = false                                // two links
+	tests["<http://example.com>, <mailto:unsubscribe@example.com>, <http://example2.com>"] = false // two links
+	tests["<mailto:unsubscribe@example.com>, <example.com>"] = false                               // no mailto || http(s)
+	tests["<mailto: unsubscribe@example.com>, <unsubscribe@lol.com>"] = false                      // space
+	tests["<mailto:unsubscribe@example.com?subject=unsubscribe me>"] = false                       // space
+	tests["<http:///example.com>"] = false                                                         // http:///
+
+	for search, expected := range tests {
+		_, err := ListUnsubscribeParser(search)
+		hasError := err != nil
+		if expected == hasError {
+			if err != nil {
+				t.Logf("ListUnsubscribeParser: %v", err)
+			} else {
+				t.Logf("ListUnsubscribeParser: \"%s\" expected: %v", search, expected)
+			}
+			t.Fail()
+		}
+	}
+}

internal/updater/targz.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -184,6 +185,10 @@ func extract(filePath string, directory string) error {
 		}
 
 		fileInfo := header.FileInfo()
+		// paths could contain a '..', is used in a file system operations
+		if strings.Contains(fileInfo.Name(), "..") {
+			continue
+		}
 		dir := filepath.Join(directory, filepath.Dir(header.Name))
 		filename := filepath.Join(dir, fileInfo.Name())
 

internal/updater/updater.go

@@ -1,3 +1,4 @@
+// package Updater checks and downloads new versions
 package updater
 
 import (
@@ -6,14 +7,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"time"
 
-	"github.com/axllent/mailpit/logger"
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/semver"
 )
 
@@ -49,13 +51,27 @@ type Release struct {
 func GithubLatest(repo, name string) (string, string, string, error) {
 	releaseURL := fmt.Sprintf("https://api.github.com/repos/%s/releases", repo)
 
-	resp, err := http.Get(releaseURL) // #nosec
+	timeout := time.Duration(5 * time.Second)
+
+	client := http.Client{
+		Timeout: timeout,
+	}
+
+	req, err := http.NewRequest("GET", releaseURL, nil)
 	if err != nil {
 		return "", "", "", err
 	}
+
+	req.Header.Set("User-Agent", "Mailpit/"+config.Version)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", "", "", err
+	}
+
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 
 	if err != nil {
 		return "", "", "", err
@@ -162,8 +178,8 @@ func GithubUpdate(repo, appName, currentVersion string) (string, error) {
 	}
 
 	if runtime.GOOS != "windows" {
-		/* #nosec G302 */
-		if err := os.Chmod(newExec, 0755); err != nil {
+		err := os.Chmod(newExec, 0755) // #nosec
+		if err != nil {
 			return "", err
 		}
 	}
@@ -178,7 +194,7 @@ func GithubUpdate(repo, appName, currentVersion string) (string, error) {
 	// get the running binary
 	oldExec, err := os.Executable()
 	if err != nil {
-		panic(err)
+		return "", err
 	}
 
 	if err = replaceFile(oldExec, newExec); err != nil {
@@ -205,7 +221,7 @@ func downloadToFile(url, fileName string) error {
 
 	defer func() {
 		if err := out.Close(); err != nil {
-			logger.Log().Errorf("Error closing file: %s\n", err)
+			logger.Log().Errorf("error closing file: %s", err.Error())
 		}
 	}()
 
@@ -289,11 +305,7 @@ func replaceFile(dst, src string) error {
 	}
 
 	// remove the src file
-	if err := os.Remove(src); err != nil {
-		return err
-	}
-
-	return nil
+	return os.Remove(src)
 }
 
 // GetTempDir will create & return a temporary directory if one has not been specified
@@ -307,7 +319,7 @@ func getTempDir() string {
 	}
 	if err := mkDirIfNotExists(tempDir); err != nil {
 		// need a better way to exit
-		logger.Log().Errorf("Error: %v", err)
+		logger.Log().Errorf("error: %s", err.Error())
 		os.Exit(2)
 	}
 

logger/logger.go

@@ -1,43 +0,0 @@
-package logger
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/axllent/mailpit/config"
-	"github.com/sirupsen/logrus"
-)
-
-var (
-	log *logrus.Logger
-)
-
-// Log returns the logger instance
-func Log() *logrus.Logger {
-	if log == nil {
-		log = logrus.New()
-		log.SetLevel(logrus.InfoLevel)
-		if config.VerboseLogging {
-			log.SetLevel(logrus.DebugLevel)
-		}
-		if config.NoLogging {
-			log.SetLevel(logrus.PanicLevel)
-		}
-
-		log.Out = os.Stdout
-		log.SetFormatter(&logrus.TextFormatter{
-			FullTimestamp:   true,
-			TimestampFormat: "15:04:05",
-			ForceColors:     true,
-		})
-	}
-
-	return log
-}
-
-// PrettyPrint for debugging
-func PrettyPrint(i interface{}) {
-	s, _ := json.MarshalIndent(i, "", "\t")
-	fmt.Println(string(s))
-}

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/axllent/mailpit/cmd"
 	sendmail "github.com/axllent/mailpit/sendmail/cmd"
@@ -15,10 +16,19 @@ func main() {
 	}
 
 	// running directly
-	if filepath.Base(exec) == filepath.Base(os.Args[0]) {
+	if normalize(filepath.Base(exec)) == normalize(filepath.Base(os.Args[0])) {
 		cmd.Execute()
 	} else {
 		// symlinked
 		sendmail.Run()
 	}
 }
+
+// Normalize returns a lowercase string stripped of the file extension (if exists).
+// Used for detecting Windows commands which ignores letter casing and `.exe`.
+// eg: "MaIlpIT.Exe" returns "mailpit"
+func normalize(s string) string {
+	s = strings.ToLower(s)
+
+	return strings.TrimSuffix(s, filepath.Ext(s))
+}

package.json

@@ -3,23 +3,32 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "build": "node esbuild.config.js",
-    "watch": "WATCH=true node esbuild.config.js",
-    "package": "MINIFY=true node esbuild.config.js"
+    "build": "MINIFY=true node esbuild.config.mjs",
+    "watch": "WATCH=true node esbuild.config.mjs",
+    "package": "MINIFY=true node esbuild.config.mjs",
+    "update-caniemail": "wget -O utils/html-check/caniemail-data.json https://www.caniemail.com/api/data.json"
   },
   "dependencies": {
-    "axios": "^0.27.2",
+    "axios": "^1.2.1",
     "bootstrap": "^5.2.0",
     "bootstrap-icons": "^1.9.1",
+    "bootstrap5-tags": "^1.6.1",
+    "color-hash": "^2.0.2",
+    "modern-screenshot": "^4.4.30",
     "moment": "^2.29.4",
-    "remove": "^0.1.5",
-    "vue": "^3.2.13"
+    "prismjs": "^1.29.0",
+    "rapidoc": "^9.3.4",
+    "vue": "^3.2.13",
+    "vue-css-donut-chart": "^2.0.0",
+    "vue-router": "^4.2.4"
   },
   "devDependencies": {
     "@popperjs/core": "^2.11.5",
+    "@types/bootstrap": "^5.2.7",
+    "@types/tinycon": "^0.6.3",
     "@vue/compiler-sfc": "^3.2.37",
-    "esbuild": "^0.14.50",
-    "esbuild-plugin-sass": "^1.0.1",
-    "esbuild-plugin-vue-next": "^0.1.4"
+    "esbuild": "^0.20.0",
+    "esbuild-plugin-vue-next": "^0.1.4",
+    "esbuild-sass-plugin": "^3.0.0"
   }
 }

sendmail/cmd/cmd.go

@@ -1,24 +1,47 @@
+// Package cmd is the sendmail cli
 package cmd
 
 /**
  * Bare bones sendmail drop-in replacement borrowed from MailHog
+ *
+ * It uses a bit of a hack for flag parsing in order to be compatible
+ * with the cobra sendmail subcommand, as sendmail uses `-bc` which
+ * is not POSIX compatible.
+ *
+ * The -bs command-line switch causes sendmail to run a single SMTP session in the
+ * foreground over its standard input and output, and then exit. The SMTP session
+ * is exactly like a network SMTP session. Usually, one or more messages are
+ * submitted to sendmail for delivery.
  */
-
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
-	"log"
+	"io"
 	"net/mail"
 	"net/smtp"
 	"os"
 	"os/user"
+	"strings"
 
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/reiver/go-telnet"
 	flag "github.com/spf13/pflag"
 )
 
-// Run the Mailpit sendmail replacement.
-func Run() {
+var (
+	// SMTPAddr address
+	SMTPAddr = "localhost:1025"
+	// FromAddr email address
+	FromAddr string
+
+	// UseB - used to set from `-bs`
+	UseB bool
+	// UseS - used to set from `-bs`
+	UseS bool
+)
+
+func init() {
 	host, err := os.Hostname()
 	if err != nil {
 		host = "localhost"
@@ -30,36 +53,76 @@ func Run() {
 		username = user.Username
 	}
 
-	fromAddr := username + "@" + host
-	smtpAddr := "localhost:1025"
-	var recip []string
+	if FromAddr == "" {
+		FromAddr = username + "@" + host
+	}
+}
+
+// Run the Mailpit sendmail replacement.
+func Run() {
+	var recipients []string
 
 	// defaults from envars if provided
 	if len(os.Getenv("MP_SENDMAIL_SMTP_ADDR")) > 0 {
-		smtpAddr = os.Getenv("MP_SENDMAIL_SMTP_ADDR")
+		SMTPAddr = os.Getenv("MP_SENDMAIL_SMTP_ADDR")
 	}
 	if len(os.Getenv("MP_SENDMAIL_FROM")) > 0 {
-		fromAddr = os.Getenv("MP_SENDMAIL_FROM")
+		FromAddr = os.Getenv("MP_SENDMAIL_FROM")
 	}
 
-	var verbose bool
+	flag.StringVarP(&FromAddr, "from", "f", FromAddr, "SMTP sender")
+	flag.StringVarP(&SMTPAddr, "smtp-addr", "S", SMTPAddr, "SMTP server address")
+	flag.BoolVarP(&UseB, "long-b", "b", false, "Handle SMTP commands on standard input (use as -bs)")
+	flag.BoolVarP(&UseS, "long-s", "s", false, "Handle SMTP commands on standard input (use as -bs)")
+	flag.BoolP("verbose", "v", false, "Ignored")
+	flag.BoolP("long-i", "i", false, "Ignored")
+	flag.BoolP("long-o", "o", false, "Ignored")
+	flag.BoolP("long-t", "t", false, "Ignored")
+
+	// set the default help
+	flag.Usage = func() {
+		fmt.Println(HelpTemplate(os.Args[0:1]))
+	}
+
+	var showHelp bool
+	// avoid 'pflag: help requested' error
+	flag.BoolVarP(&showHelp, "help", "h", false, "")
 
-	// override defaults from cli flags
-	flag.StringVar(&smtpAddr, "smtp-addr", smtpAddr, "SMTP server address")
-	flag.StringVarP(&fromAddr, "from", "f", fromAddr, "SMTP sender")
-	flag.BoolP("long-i", "i", true, "Ignored. This flag exists for sendmail compatibility.")
-	flag.BoolP("long-t", "t", true, "Ignored. This flag exists for sendmail compatibility.")
-	flag.BoolVarP(&verbose, "verbose", "v", false, "Verbose mode (sends debug output to stderr)")
 	flag.Parse()
 
-	// allow recipient to be passed as an argument
-	recip = flag.Args()
+	// allow recipients to be passed as an argument
+	recipients = flag.Args()
 
-	if verbose {
-		fmt.Fprintln(os.Stderr, smtpAddr, fromAddr)
+	// if run via `mailpit sendmail ...` then remove `sendmail` from "recipients"
+	if len(recipients) > 0 && recipients[0] == "sendmail" {
+		recipients = recipients[1:]
 	}
 
-	body, err := ioutil.ReadAll(os.Stdin)
+	if showHelp {
+		flag.Usage()
+		os.Exit(0)
+	}
+
+	// ensure -bs is set
+	if UseB && !UseS || !UseB && UseS {
+		fmt.Printf("error: use -bs")
+		os.Exit(1)
+	}
+
+	// handles `sendmail -bs`
+	if UseB && UseS {
+		var caller telnet.Caller = telnet.StandardCaller
+
+		// telnet directly to SMTP
+		if err := telnet.DialToAndCall(SMTPAddr, caller); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		return
+	}
+
+	body, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "error reading stdin")
 		os.Exit(11)
@@ -67,19 +130,61 @@ func Run() {
 
 	msg, err := mail.ReadMessage(bytes.NewReader(body))
 	if err != nil {
-		fmt.Fprintln(os.Stderr, fmt.Sprintf("error parsing message body: %s", err))
+		fmt.Fprintf(os.Stderr, "error parsing message body: %si\n", err)
 		os.Exit(11)
 	}
 
-	if len(recip) == 0 {
-		// We only need to parse the message to get a recipient if none where
-		// provided on the command line.
-		recip = append(recip, msg.Header.Get("To"))
+	addresses := []string{}
+
+	if len(recipients) > 0 {
+		addresses = recipients
+	} else {
+		// get all recipients in To, Cc and Bcc
+		if to, err := msg.Header.AddressList("To"); err == nil {
+			for _, a := range to {
+				addresses = append(addresses, a.Address)
+			}
+		}
+		if cc, err := msg.Header.AddressList("Cc"); err == nil {
+			for _, a := range cc {
+				addresses = append(addresses, a.Address)
+			}
+		}
+		if bcc, err := msg.Header.AddressList("Bcc"); err == nil {
+			for _, a := range bcc {
+				addresses = append(addresses, a.Address)
+			}
+		}
 	}
 
-	err = smtp.SendMail(smtpAddr, nil, fromAddr, recip, body)
+	from, err := mail.ParseAddress(FromAddr)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "invalid from address")
+		os.Exit(11)
+	}
+
+	err = smtp.SendMail(SMTPAddr, nil, from.Address, addresses, body)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "error sending mail")
-		log.Fatal(err)
+		logger.Log().Fatal(err)
 	}
 }
+
+// HelpTemplate returns a string of the help
+func HelpTemplate(args []string) string {
+	return fmt.Sprintf(`A sendmail command replacement for Mailpit (%s)
+
+Usage: %s [flags] [recipients] < message
+
+See: https://github.com/axllent/mailpit
+
+Flags:
+  -S  string  SMTP server address (default "localhost:1025")
+  -f  string  Set the envelope sender address (default "%s")
+  -bs         Handle SMTP commands on standard input
+  -t          Ignored
+  -i          Ignored
+  -o          Ignored
+  -v          Ignored
+`, config.Version, strings.Join(args, " "), FromAddr)
+}

server/api.go

@@ -1,224 +0,0 @@
-package server
-
-import (
-	"encoding/json"
-	"net/http"
-	"strings"
-
-	"github.com/axllent/mailpit/data"
-	"github.com/axllent/mailpit/server/websockets"
-	"github.com/axllent/mailpit/storage"
-	"github.com/gorilla/mux"
-)
-
-type messagesResult struct {
-	Total  int            `json:"total"`
-	Unread int            `json:"unread"`
-	Count  int            `json:"count"`
-	Start  int            `json:"start"`
-	Items  []data.Summary `json:"items"`
-}
-
-// Return a list of available mailboxes
-func apiListMailboxes(w http.ResponseWriter, _ *http.Request) {
-	res, err := storage.ListMailboxes()
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	bytes, _ := json.Marshal(res)
-	w.Header().Add("Content-Type", "application/json")
-	w.Write(bytes)
-}
-
-func apiListMailbox(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-
-	if !storage.MailboxExists(mailbox) {
-		fourOFour(w)
-		return
-	}
-
-	start, limit := getStartLimit(r)
-
-	messages, err := storage.List(mailbox, start, limit)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	stats := storage.StatsGet(mailbox)
-
-	var res messagesResult
-
-	res.Start = start
-	res.Items = messages
-	res.Count = len(res.Items)
-	res.Total = stats.Total
-	res.Unread = stats.Unread
-
-	bytes, _ := json.Marshal(res)
-	w.Header().Add("Content-Type", "application/json")
-	w.Write(bytes)
-}
-
-func apiSearchMailbox(w http.ResponseWriter, r *http.Request) {
-	search := strings.TrimSpace(r.URL.Query().Get("query"))
-	if search == "" {
-		fourOFour(w)
-		return
-	}
-
-	vars := mux.Vars(r)
-	mailbox := vars["mailbox"]
-
-	if !storage.MailboxExists(mailbox) {
-		fourOFour(w)
-		return
-	}
-
-	// we will only return up to 200 results
-	start := 0
-	limit := 200
-
-	messages, err := storage.Search(mailbox, search, start, limit)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	stats := storage.StatsGet(mailbox)
-
-	var res messagesResult
-
-	res.Start = start
-	res.Items = messages
-	res.Count = len(messages)
-	res.Total = stats.Total
-	res.Unread = stats.Unread
-
-	bytes, _ := json.Marshal(res)
-	w.Header().Add("Content-Type", "application/json")
-	w.Write(bytes)
-}
-
-// Open a message
-func apiOpenMessage(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-	id := vars["id"]
-
-	msg, err := storage.GetMessage(mailbox, id)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	bytes, _ := json.Marshal(msg)
-	w.Header().Add("Content-Type", "application/json")
-	w.Write(bytes)
-}
-
-// Download/view an attachment
-func apiDownloadAttachment(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-	id := vars["id"]
-	partID := vars["partID"]
-
-	a, err := storage.GetAttachmentPart(mailbox, id, partID)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-	fileName := a.FileName
-	if fileName == "" {
-		fileName = a.ContentID
-	}
-
-	w.Header().Add("Content-Type", a.ContentType)
-	w.Header().Set("Content-Disposition", "filename=\""+fileName+"\"")
-	w.Write(a.Content)
-}
-
-// View the full email source as plain text
-func apiDownloadSource(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-	id := vars["id"]
-
-	dl := r.FormValue("dl")
-
-	data, err := storage.GetMessageRaw(mailbox, id)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	w.Header().Set("Content-Type", "text/plain")
-	if dl == "1" {
-		w.Header().Set("Content-Disposition", "attachment; filename=\""+id+".eml\"")
-	}
-	w.Write(data)
-}
-
-// Delete all messages in the mailbox
-func apiDeleteAll(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-
-	err := storage.DeleteAllMessages(mailbox)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	w.Header().Add("Content-Type", "text/plain")
-	w.Write([]byte("ok"))
-}
-
-// Delete a single message
-func apiDeleteOne(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-	id := vars["id"]
-
-	err := storage.DeleteOneMessage(mailbox, id)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	w.Header().Add("Content-Type", "text/plain")
-	w.Write([]byte("ok"))
-}
-
-// Mark single message as unread
-func apiUnreadOne(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	mailbox := vars["mailbox"]
-	id := vars["id"]
-
-	err := storage.UnreadMessage(mailbox, id)
-	if err != nil {
-		httpError(w, err.Error())
-		return
-	}
-
-	w.Header().Add("Content-Type", "text/plain")
-	w.Write([]byte("ok"))
-}
-
-// Websocket to broadcast changes
-func apiWebsocket(w http.ResponseWriter, r *http.Request) {
-	websockets.ServeWs(websockets.MessageHub, w, r)
-}

server/apiv1/api.go

@@ -0,0 +1,916 @@
+// Package apiv1 handles all the API responses
+package apiv1
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/mail"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/htmlcheck"
+	"github.com/axllent/mailpit/internal/linkcheck"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/spamassassin"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/smtpd"
+	"github.com/gorilla/mux"
+	"github.com/lithammer/shortuuid/v4"
+)
+
+// GetMessages returns a paginated list of messages as JSON
+func GetMessages(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/messages messages GetMessages
+	//
+	// # List messages
+	//
+	// Returns messages from the mailbox ordered from newest to oldest.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: start
+	//	    in: query
+	//	    description: Pagination offset
+	//	    required: false
+	//	    type: integer
+	//	    default: 0
+	//	  + name: limit
+	//	    in: query
+	//	    description: Limit results
+	//	    required: false
+	//	    type: integer
+	//	    default: 50
+	//
+	//	Responses:
+	//		200: MessagesSummaryResponse
+	//		default: ErrorResponse
+	start, limit := getStartLimit(r)
+
+	messages, err := storage.List(start, limit)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	stats := storage.StatsGet()
+
+	var res MessagesSummary
+
+	res.Start = start
+	res.Messages = messages
+	res.Count = len(messages) // legacy - now undocumented in API specs
+	res.Total = stats.Total
+	res.Unread = stats.Unread
+	res.Tags = stats.Tags
+	res.MessagesCount = stats.Total
+
+	bytes, _ := json.Marshal(res)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// Search returns the latest messages as JSON
+func Search(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/search messages MessagesSummary
+	//
+	// # Search messages
+	//
+	// Returns the latest messages matching a search.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: query
+	//	    in: query
+	//	    description: Search query
+	//	    required: true
+	//	    type: string
+	//	  + name: start
+	//	    in: query
+	//	    description: Pagination offset
+	//	    required: false
+	//	    type: integer
+	//	    default: 0
+	//	  + name: limit
+	//	    in: query
+	//	    description: Limit results
+	//	    required: false
+	//	    type: integer
+	//	    default: 50
+	//
+	//	Responses:
+	//		200: MessagesSummaryResponse
+	//		default: ErrorResponse
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search == "" {
+		httpError(w, "Error: no search query")
+		return
+	}
+
+	start, limit := getStartLimit(r)
+
+	messages, results, err := storage.Search(search, start, limit)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	stats := storage.StatsGet()
+
+	var res MessagesSummary
+
+	res.Start = start
+	res.Messages = messages
+	res.Count = len(messages) // legacy - now undocumented in API specs
+	res.Total = stats.Total   // total messages in mailbox
+	res.MessagesCount = results
+	res.Unread = stats.Unread
+	res.Tags = stats.Tags
+
+	bytes, _ := json.Marshal(res)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// DeleteSearch will delete all messages matching a search
+func DeleteSearch(w http.ResponseWriter, r *http.Request) {
+	// swagger:route DELETE /api/v1/search messages DeleteSearch
+	//
+	// # Delete messages by search
+	//
+	// Delete all messages matching a search.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: query
+	//	    in: query
+	//	    description: Search query
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search == "" {
+		httpError(w, "Error: no search query")
+		return
+	}
+
+	if err := storage.DeleteSearch(search); err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
+// GetMessage (method: GET) returns the Message as JSON
+func GetMessage(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID} message Message
+	//
+	// # Get message summary
+	//
+	// Returns the summary of a message, marking the message as read.
+	//
+	// The ID can be set to `latest` to return the latest message.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Message database ID or "latest"
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: Message
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	bytes, _ := json.Marshal(msg)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// DownloadAttachment (method: GET) returns the attachment data
+func DownloadAttachment(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/part/{PartID} message Attachment
+	//
+	// # Get message attachment
+	//
+	// This will return the attachment part using the appropriate Content-Type.
+	//
+	//	Produces:
+	//	- application/*
+	//	- image/*
+	//	- text/*
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Message database ID
+	//	    required: true
+	//	    type: string
+	//	  + name: PartID
+	//	    in: path
+	//	    description: Attachment part ID
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: BinaryResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+	partID := vars["partID"]
+
+	a, err := storage.GetAttachmentPart(id, partID)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+	fileName := a.FileName
+	if fileName == "" {
+		fileName = a.ContentID
+	}
+
+	w.Header().Add("Content-Type", a.ContentType)
+	w.Header().Set("Content-Disposition", "filename=\""+fileName+"\"")
+	_, _ = w.Write(a.Content)
+}
+
+// GetHeaders (method: GET) returns the message headers as JSON
+func GetHeaders(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/headers message Headers
+	//
+	// # Get message headers
+	//
+	// Returns the message headers as an array.
+	//
+	// The ID can be set to `latest` to return the latest message headers.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Message database ID or "latest"
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//	  200: MessageHeaders
+	//	  default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	data, err := storage.GetMessageRaw(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	reader := bytes.NewReader(data)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	bytes, _ := json.Marshal(m.Header)
+
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// DownloadRaw (method: GET) returns the full email source as plain text
+func DownloadRaw(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/raw message Raw
+	//
+	// # Get message source
+	//
+	// Returns the full email source as plain text.
+	//
+	// The ID can be set to `latest` to return the latest message source.
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Message database ID or "latest"
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: TextResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+	dl := r.FormValue("dl")
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	data, err := storage.GetMessageRaw(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if dl == "1" {
+		w.Header().Set("Content-Disposition", "attachment; filename=\""+id+".eml\"")
+	}
+	_, _ = w.Write(data)
+}
+
+// DeleteMessages (method: DELETE) deletes all messages matching IDS.
+func DeleteMessages(w http.ResponseWriter, r *http.Request) {
+	// swagger:route DELETE /api/v1/messages messages DeleteMessages
+	//
+	// # Delete messages
+	//
+	// Delete individual or all messages. If no IDs are provided then all messages are deleted.
+	//
+	//	Consumes:
+	//	- application/json
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+
+	decoder := json.NewDecoder(r.Body)
+	var data struct {
+		IDs []string
+	}
+	err := decoder.Decode(&data)
+	if err != nil || len(data.IDs) == 0 {
+		if err := storage.DeleteAllMessages(); err != nil {
+			httpError(w, err.Error())
+			return
+		}
+	} else {
+		for _, id := range data.IDs {
+			if err := storage.DeleteOneMessage(id); err != nil {
+				httpError(w, err.Error())
+				return
+			}
+		}
+	}
+
+	w.Header().Add("Content-Type", "application/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
+// SetReadStatus (method: PUT) will update the status to Read/Unread for all provided IDs
+// If no IDs are provided then all messages are updated.
+func SetReadStatus(w http.ResponseWriter, r *http.Request) {
+	// swagger:route PUT /api/v1/messages messages SetReadStatus
+	//
+	// # Set read status
+	//
+	// If no IDs are provided then all messages are updated.
+	//
+	//	Consumes:
+	//	- application/json
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+
+	decoder := json.NewDecoder(r.Body)
+
+	var data struct {
+		Read bool
+		IDs  []string
+	}
+
+	err := decoder.Decode(&data)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	ids := data.IDs
+
+	if len(ids) == 0 {
+		if data.Read {
+			err := storage.MarkAllRead()
+			if err != nil {
+				httpError(w, err.Error())
+				return
+			}
+		} else {
+			err := storage.MarkAllUnread()
+			if err != nil {
+				httpError(w, err.Error())
+				return
+			}
+		}
+	} else {
+		if data.Read {
+			for _, id := range ids {
+				if err := storage.MarkRead(id); err != nil {
+					httpError(w, err.Error())
+					return
+				}
+			}
+		} else {
+			for _, id := range ids {
+				if err := storage.MarkUnread(id); err != nil {
+					httpError(w, err.Error())
+					return
+				}
+			}
+		}
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
+// GetAllTags (method: GET) will get all tags currently in use
+func GetAllTags(w http.ResponseWriter, _ *http.Request) {
+	// swagger:route GET /api/v1/tags tags GetAllTags
+	//
+	// # Get all current tags
+	//
+	// Returns a JSON array of all unique message tags.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: ArrayResponse
+	//		default: ErrorResponse
+
+	tags := storage.GetAllTags()
+
+	data, err := json.Marshal(tags)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(data)
+}
+
+// SetMessageTags (method: PUT) will set the tags for all provided IDs
+func SetMessageTags(w http.ResponseWriter, r *http.Request) {
+	// swagger:route PUT /api/v1/tags tags SetTags
+	//
+	// # Set message tags
+	//
+	// This will overwrite any existing tags for selected message database IDs. To remove all tags from a message, pass an empty tags array.
+	//
+	//	Consumes:
+	//	- application/json
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+
+	decoder := json.NewDecoder(r.Body)
+
+	var data struct {
+		Tags []string
+		IDs  []string
+	}
+
+	err := decoder.Decode(&data)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	ids := data.IDs
+
+	if len(ids) > 0 {
+		for _, id := range ids {
+			if err := storage.SetMessageTags(id, data.Tags); err != nil {
+				httpError(w, err.Error())
+				return
+			}
+		}
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
+// ReleaseMessage (method: POST) will release a message via a pre-configured external SMTP server.
+func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
+	// swagger:route POST /api/v1/message/{ID}/release message ReleaseMessage
+	//
+	// # Release message
+	//
+	// Release a message via a pre-configured external SMTP server. This is only enabled if message relaying has been configured.
+	//
+	//	Consumes:
+	//	- application/json
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	msg, err := storage.GetMessageRaw(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	decoder := json.NewDecoder(r.Body)
+
+	data := releaseMessageRequestBody{}
+
+	if err := decoder.Decode(&data); err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	tos := data.To
+	if len(tos) == 0 {
+		httpError(w, "No valid addresses found")
+		return
+	}
+
+	for _, to := range tos {
+		address, err := mail.ParseAddress(to)
+
+		if err != nil {
+			httpError(w, "Invalid email address: "+to)
+			return
+		}
+
+		if config.SMTPRelayConfig.AllowedRecipientsRegexp != nil && !config.SMTPRelayConfig.AllowedRecipientsRegexp.MatchString(address.Address) {
+			httpError(w, "Mail address does not match allowlist: "+to)
+			return
+		}
+	}
+
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	froms, err := m.Header.AddressList("From")
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	from := froms[0].Address
+
+	// if sender is used, then change from to the sender
+	if senders, err := m.Header.AddressList("Sender"); err == nil {
+		from = senders[0].Address
+	}
+
+	msg, err = tools.RemoveMessageHeaders(msg, []string{"Bcc"})
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	// set the Return-Path and SMTP mfrom
+	if config.SMTPRelayConfig.ReturnPath != "" {
+		if m.Header.Get("Return-Path") != "<"+config.SMTPRelayConfig.ReturnPath+">" {
+			msg, err = tools.RemoveMessageHeaders(msg, []string{"Return-Path"})
+			if err != nil {
+				httpError(w, err.Error())
+				return
+			}
+			msg = append([]byte("Return-Path: <"+config.SMTPRelayConfig.ReturnPath+">\r\n"), msg...)
+		}
+
+		from = config.SMTPRelayConfig.ReturnPath
+	}
+
+	// update message date
+	msg, err = tools.UpdateMessageHeader(msg, "Date", time.Now().Format(time.RFC1123Z))
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	// generate unique ID
+	uid := shortuuid.New() + "@mailpit"
+	// update Message-Id with unique ID
+	msg, err = tools.UpdateMessageHeader(msg, "Message-Id", "<"+uid+">")
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	if err := smtpd.Send(from, tos, msg); err != nil {
+		logger.Log().Errorf("[smtp] error sending message: %s", err.Error())
+		httpError(w, "SMTP error: "+err.Error())
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
+// HTMLCheck returns a summary of the HTML client support
+func HTMLCheck(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/html-check Other HTMLCheck
+	//
+	// # HTML check (beta)
+	//
+	// Returns the summary of the message HTML checker.
+	//
+	// NOTE: This feature is currently in beta and is documented for reference only.
+	// Please do not integrate with it (yet) as there may be changes.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: HTMLCheckResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	if msg.HTML == "" {
+		httpError(w, "message does not contain HTML")
+		return
+	}
+
+	checks, err := htmlcheck.RunTests(msg.HTML)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	bytes, _ := json.Marshal(checks)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// LinkCheck returns a summary of links in the email
+func LinkCheck(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/link-check Other LinkCheck
+	//
+	// # Link check (beta)
+	//
+	// Returns the summary of the message Link checker.
+	//
+	// NOTE: This feature is currently in beta and is documented for reference only.
+	// Please do not integrate with it (yet) as there may be changes.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: LinkCheckResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	f := r.URL.Query().Get("follow")
+	followRedirects := f == "true" || f == "1"
+
+	summary, err := linkcheck.RunTests(msg, followRedirects)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	bytes, _ := json.Marshal(summary)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// SpamAssassinCheck returns a summary of SpamAssassin results (if enabled)
+func SpamAssassinCheck(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/sa-check Other SpamAssassinCheck
+	//
+	// # SpamAssassin check (beta)
+	//
+	// Returns the SpamAssassin (if enabled) summary of the message.
+	//
+	// NOTE: This feature is currently in beta and is documented for reference only.
+	// Please do not integrate with it (yet) as there may be changes.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: SpamAssassinResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessageRaw(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	summary, err := spamassassin.Check(msg)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	bytes, _ := json.Marshal(summary)
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}
+
+// FourOFour returns a basic 404 message
+func fourOFour(w http.ResponseWriter) {
+	w.Header().Set("Referrer-Policy", "no-referrer")
+	w.Header().Set("Content-Security-Policy", config.ContentSecurityPolicy)
+	w.WriteHeader(http.StatusNotFound)
+	w.Header().Set("Content-Type", "text/plain")
+	fmt.Fprint(w, "404 page not found")
+}
+
+// HTTPError returns a basic error message (400 response)
+func httpError(w http.ResponseWriter, msg string) {
+	w.Header().Set("Referrer-Policy", "no-referrer")
+	w.Header().Set("Content-Security-Policy", config.ContentSecurityPolicy)
+	w.WriteHeader(http.StatusBadRequest)
+	w.Header().Set("Content-Type", "text/plain")
+	fmt.Fprint(w, msg)
+}
+
+// Get the start and limit based on query params. Defaults to 0, 50
+func getStartLimit(req *http.Request) (start int, limit int) {
+	start = 0
+	limit = 50
+
+	s := req.URL.Query().Get("start")
+	if n, err := strconv.Atoi(s); err == nil && n > 0 {
+		start = n
+	}
+
+	l := req.URL.Query().Get("limit")
+	if n, err := strconv.Atoi(l); err == nil && n > 0 {
+		limit = n
+	}
+
+	return start, limit
+}
+
+// GetOptions returns a blank response
+func GetOptions(w http.ResponseWriter, _ *http.Request) {
+
+	w.Header().Set("Content-Type", "text/plain")
+	_, _ = w.Write([]byte(""))
+}

server/apiv1/info.go

@@ -0,0 +1,33 @@
+package apiv1
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/axllent/mailpit/internal/stats"
+)
+
+// AppInfo returns some basic details about the running app, and latest release.
+func AppInfo(w http.ResponseWriter, _ *http.Request) {
+	// swagger:route GET /api/v1/info application AppInformation
+	//
+	// # Get application information
+	//
+	// Returns basic runtime information, message totals and latest release version.
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: InfoResponse
+	//		default: ErrorResponse
+
+	info := stats.Load()
+
+	bytes, _ := json.Marshal(info)
+
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}

server/apiv1/structs.go

@@ -0,0 +1,56 @@
+package apiv1
+
+import (
+	"github.com/axllent/mailpit/internal/htmlcheck"
+	"github.com/axllent/mailpit/internal/linkcheck"
+	"github.com/axllent/mailpit/internal/spamassassin"
+	"github.com/axllent/mailpit/internal/storage"
+)
+
+// MessagesSummary is a summary of a list of messages
+type MessagesSummary struct {
+	// Total number of messages in mailbox
+	Total int `json:"total"`
+
+	// Total number of unread messages in mailbox
+	Unread int `json:"unread"`
+
+	// Legacy - now undocumented in API specs but left for backwards compatibility.
+	// Removed from API documentation 2023-07-12
+	// swagger:ignore
+	Count int `json:"count"`
+
+	// Total number of messages matching current query
+	MessagesCount int `json:"messages_count"`
+
+	// Pagination offset
+	Start int `json:"start"`
+
+	// All current tags
+	Tags []string `json:"tags"`
+
+	// Messages summary
+	// in: body
+	Messages []storage.MessageSummary `json:"messages"`
+}
+
+// The following structs & aliases are provided for easy import
+// and understanding of the JSON structure.
+
+// MessageSummary - summary of a single message
+type MessageSummary = storage.MessageSummary
+
+// Message data
+type Message = storage.Message
+
+// Attachment summary
+type Attachment = storage.Attachment
+
+// HTMLCheckResponse summary
+type HTMLCheckResponse = htmlcheck.Response
+
+// LinkCheckResponse summary
+type LinkCheckResponse = linkcheck.Response
+
+// SpamAssassinResponse summary
+type SpamAssassinResponse = spamassassin.Result

server/apiv1/swagger-config.yml

@@ -0,0 +1,19 @@
+consumes:
+  - application/json
+info:
+  description:  |-
+    OpenAPI 2.0 documentation for [Mailpit](https://github.com/axllent/mailpit).
+  title: Mailpit API
+  contact:
+    name: GitHub
+    url: https://github.com/axllent/mailpit
+  license:
+    name: MIT license
+    url: https://github.com/axllent/mailpit/blob/develop/LICENSE
+  version: "v1"
+paths: {}
+produces:
+  - application/json
+schemes:
+  - http
+swagger: "2.0"

server/apiv1/swagger.go

@@ -0,0 +1,181 @@
+package apiv1
+
+import "github.com/axllent/mailpit/internal/stats"
+
+// These structs are for the purpose of defining swagger HTTP parameters & responses
+
+// Application information
+// swagger:response InfoResponse
+type infoResponse struct {
+	// Application information
+	//
+	// in: body
+	Body stats.AppInformation
+}
+
+// Web UI configuration
+// swagger:response WebUIConfigurationResponse
+type webUIConfigurationResponse struct {
+	// Web UI configuration settings
+	//
+	// in: body
+	Body webUIConfiguration
+}
+
+// Message summary
+// swagger:response MessagesSummaryResponse
+type messagesSummaryResponse struct {
+	// The message summary
+	// in: body
+	Body MessagesSummary
+}
+
+// Message headers
+// swagger:model MessageHeaders
+type messageHeaders map[string][]string
+
+// swagger:parameters DeleteMessages
+type deleteMessagesParams struct {
+	// in: body
+	Body *deleteMessagesRequestBody
+}
+
+// Delete request
+// swagger:model DeleteRequest
+type deleteMessagesRequestBody struct {
+	// Array of message database IDs
+	//
+	// required: false
+	// example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"]
+	IDs []string `json:"ids"`
+}
+
+// swagger:parameters SetReadStatus
+type setReadStatusParams struct {
+	// in: body
+	Body *setReadStatusRequestBody
+}
+
+// Set read status request
+// swagger:model setReadStatusRequestBody
+type setReadStatusRequestBody struct {
+	// Read status
+	//
+	// required: false
+	// default: false
+	// example: true
+	Read bool `json:"read"`
+
+	// Array of message database IDs
+	//
+	// required: false
+	// example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"]
+	IDs []string `json:"ids"`
+}
+
+// swagger:parameters SetTags
+type setTagsParams struct {
+	// in: body
+	Body *setTagsRequestBody
+}
+
+// Set tags request
+// swagger:model setTagsRequestBody
+type setTagsRequestBody struct {
+	// Array of tag names to set
+	//
+	// required: true
+	// example: ["Tag 1", "Tag 2"]
+	Tags []string `json:"tags"`
+
+	// Array of message database IDs
+	//
+	// required: true
+	// example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"]
+	IDs []string `json:"ids"`
+}
+
+// swagger:parameters ReleaseMessage
+type releaseMessageParams struct {
+	// Message database ID
+	//
+	// in: path
+	// description: Message database ID
+	// required: true
+	ID string
+
+	// in: body
+	Body *releaseMessageRequestBody
+}
+
+// Release request
+// swagger:model releaseMessageRequestBody
+type releaseMessageRequestBody struct {
+	// Array of email addresses to relay the message to
+	//
+	// required: true
+	// example: ["user1@example.com", "user2@example.com"]
+	To []string `json:"to"`
+}
+
+// swagger:parameters HTMLCheck
+type htmlCheckParams struct {
+	// Message database ID or "latest"
+	//
+	// in: path
+	// description: Message database ID or "latest"
+	// required: true
+	ID string
+}
+
+// swagger:parameters LinkCheck
+type linkCheckParams struct {
+	// Message database ID or "latest"
+	//
+	// in: path
+	// description: Message database ID or "latest"
+	// required: true
+	ID string
+
+	// Follow redirects
+	//
+	// in: query
+	// description: Follow redirects
+	// required: false
+	// default: false
+	Follow string `json:"follow"`
+}
+
+// swagger:parameters SpamAssassinCheck
+type spamAssassinCheckParams struct {
+	// Message database ID or "latest"
+	//
+	// in: path
+	// description: Message database ID or "latest"
+	// required: true
+	ID string
+}
+
+// Binary data response inherits the attachment's content type
+// swagger:response BinaryResponse
+type binaryResponse string
+
+// Plain text response
+// swagger:response TextResponse
+type textResponse string
+
+// HTML response
+// swagger:response HTMLResponse
+type htmlResponse string
+
+// HTTP error response will return with a >= 400 response code
+// swagger:response ErrorResponse
+type errorResponse string
+
+// Plain text "ok" response
+// swagger:response OKResponse
+type okResponse string
+
+// Plain JSON array response
+// swagger:response ArrayResponse
+type arrayResponse []string

server/apiv1/thumbnails.go

@@ -0,0 +1,134 @@
+package apiv1
+
+import (
+	"bufio"
+	"bytes"
+	"image"
+	"image/color"
+	"image/draw"
+	"image/jpeg"
+	"net/http"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/disintegration/imaging"
+	"github.com/gorilla/mux"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	thumbWidth  = 180
+	thumbHeight = 120
+)
+
+// Thumbnail returns a thumbnail image for an attachment (images only)
+func Thumbnail(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /api/v1/message/{ID}/part/{PartID}/thumb message Thumbnail
+	//
+	// # Get an attachment image thumbnail
+	//
+	// This will return a cropped 180x120 JPEG thumbnail of an image attachment.
+	// If the image is smaller than 180x120 then the image is padded. If the attachment is not an image then a blank image is returned.
+	//
+	//	Produces:
+	//	- image/jpeg
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Database ID
+	//	    required: true
+	//	    type: string
+	//	  + name: PartID
+	//	    in: path
+	//	    description: Attachment part ID
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: BinaryResponse
+	//		default: ErrorResponse
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+	partID := vars["partID"]
+
+	a, err := storage.GetAttachmentPart(id, partID)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	fileName := a.FileName
+	if fileName == "" {
+		fileName = a.ContentID
+	}
+
+	if !strings.HasPrefix(a.ContentType, "image/") {
+		blankImage(a, w)
+		return
+	}
+
+	buf := bytes.NewBuffer(a.Content)
+
+	img, err := imaging.Decode(buf)
+	if err != nil {
+		// it's not an image, return default
+		logger.Log().Warnf("[image] %s", err.Error())
+		blankImage(a, w)
+		return
+	}
+
+	var b bytes.Buffer
+	foo := bufio.NewWriter(&b)
+
+	var dstImageFill *image.NRGBA
+
+	if img.Bounds().Dx() < thumbWidth || img.Bounds().Dy() < thumbHeight {
+		dstImageFill = imaging.Fit(img, thumbWidth, thumbHeight, imaging.Lanczos)
+	} else {
+		dstImageFill = imaging.Fill(img, thumbWidth, thumbHeight, imaging.Center, imaging.Lanczos)
+	}
+	// create white image and paste image over the top
+	// preventing black backgrounds for transparent GIF/PNG images
+	dst := imaging.New(thumbWidth, thumbHeight, color.White)
+	// paste the original over the top
+	dst = imaging.OverlayCenter(dst, dstImageFill, 1.0)
+
+	if err := jpeg.Encode(foo, dst, &jpeg.Options{Quality: 70}); err != nil {
+		logger.Log().Warnf("[image] %s", err.Error())
+		blankImage(a, w)
+		return
+	}
+
+	w.Header().Add("Content-Type", "image/jpeg")
+	w.Header().Set("Content-Disposition", "filename=\""+fileName+"\"")
+	_, _ = w.Write(b.Bytes())
+}
+
+// Return a blank image instead of an error when file or image not supported
+func blankImage(a *enmime.Part, w http.ResponseWriter) {
+	rect := image.Rect(0, 0, thumbWidth, thumbHeight)
+	img := image.NewRGBA(rect)
+	background := color.RGBA{255, 255, 255, 255}
+	draw.Draw(img, img.Bounds(), &image.Uniform{background}, image.ZP, draw.Src)
+	var b bytes.Buffer
+	foo := bufio.NewWriter(&b)
+	dstImageFill := imaging.Fill(img, thumbWidth, thumbHeight, imaging.Center, imaging.Lanczos)
+
+	if err := jpeg.Encode(foo, dstImageFill, &jpeg.Options{Quality: 70}); err != nil {
+		logger.Log().Warnf("[image] %s", err.Error())
+	}
+
+	fileName := a.FileName
+	if fileName == "" {
+		fileName = a.ContentID
+	}
+
+	w.Header().Add("Content-Type", "image/jpeg")
+	w.Header().Set("Content-Disposition", "filename=\""+fileName+"\"")
+	_, _ = w.Write(b.Bytes())
+}

server/apiv1/webui.go

@@ -0,0 +1,76 @@
+package apiv1
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/axllent/mailpit/config"
+)
+
+// Response includes global web UI settings
+//
+// swagger:model WebUIConfiguration
+type webUIConfiguration struct {
+	// Message Relay information
+	MessageRelay struct {
+		// Whether message relaying (release) is enabled
+		Enabled bool
+		// The configured SMTP server address
+		SMTPServer string
+		// Enforced Return-Path (if set) for relay bounces
+		ReturnPath string
+		// Only allow relaying to these recipients (regex)
+		AllowedRecipients string
+		// DEPRECATED 2024/03/12
+		// swagger:ignore
+		RecipientAllowlist string
+	}
+
+	// Whether the HTML check has been globally disabled
+	DisableHTMLCheck bool
+
+	// Whether SpamAssassin is enabled
+	SpamAssassin bool
+
+	// Whether messages with duplicate IDs are ignored
+	DuplicatesIgnored bool
+}
+
+// WebUIConfig returns configuration settings for the web UI.
+func WebUIConfig(w http.ResponseWriter, _ *http.Request) {
+	// swagger:route GET /api/v1/webui application WebUIConfiguration
+	//
+	// # Get web UI configuration
+	//
+	// Returns configuration settings for the web UI.
+	// Intended for web UI only!
+	//
+	//	Produces:
+	//	- application/json
+	//
+	//	Schemes: http, https
+	//
+	//	Responses:
+	//		200: WebUIConfigurationResponse
+	//		default: ErrorResponse
+	conf := webUIConfiguration{}
+
+	conf.MessageRelay.Enabled = config.ReleaseEnabled
+	if config.ReleaseEnabled {
+		conf.MessageRelay.SMTPServer = fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		conf.MessageRelay.ReturnPath = config.SMTPRelayConfig.ReturnPath
+		conf.MessageRelay.AllowedRecipients = config.SMTPRelayConfig.AllowedRecipients
+		// DEPRECATED 2024/03/12
+		conf.MessageRelay.RecipientAllowlist = config.SMTPRelayConfig.AllowedRecipients
+	}
+
+	conf.DisableHTMLCheck = config.DisableHTMLCheck
+	conf.SpamAssassin = config.EnableSpamAssassin != ""
+	conf.DuplicatesIgnored = config.IgnoreDuplicateIDs
+
+	bytes, _ := json.Marshal(conf)
+
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(bytes)
+}

server/handlers/k8healthz.go

@@ -0,0 +1,8 @@
+package handlers
+
+import "net/http"
+
+// HealthzHandler is a liveness probe
+func HealthzHandler(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}

server/handlers/k8sready.go

@@ -0,0 +1,17 @@
+package handlers
+
+import (
+	"net/http"
+	"sync/atomic"
+)
+
+// ReadyzHandler is a ready probe that signals k8s to be able to retrieve traffic
+func ReadyzHandler(isReady *atomic.Value) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		if isReady == nil || !isReady.Load().(bool) {
+			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}

server/handlers/messages.go

@@ -0,0 +1,185 @@
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/gorilla/mux"
+)
+
+// RedirectToLatestMessage (method: GET) redirects the web UI to the latest message
+func RedirectToLatestMessage(w http.ResponseWriter, r *http.Request) {
+	var messages []storage.MessageSummary
+	var err error
+
+	search := strings.TrimSpace(r.URL.Query().Get("query"))
+	if search != "" {
+		messages, _, err = storage.Search(search, 0, 1)
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+	} else {
+		messages, err = storage.List(0, 1)
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+	}
+
+	uri := config.Webroot
+
+	if len(messages) == 1 {
+		uri, err = url.JoinPath(uri, "/view/"+messages[0].ID)
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+	}
+
+	http.Redirect(w, r, uri, 302)
+}
+
+// GetMessageHTML (method: GET) returns a rendered version of a message's HTML part
+func GetMessageHTML(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /view/{ID}.html testing GetMessageHTML
+	//
+	// # Render message HTML part
+	//
+	// Renders just the message's HTML part which can be used for UI integration testing.
+	// Attached inline images are modified to link to the API provided they exist.
+	// Note that is the message does not contain a HTML part then an 404 error is returned.
+	//
+	// The ID can be set to `latest` to return the latest message.
+	//
+	//	Produces:
+	//	- text/html
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Database ID or latest
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: HTMLResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "Message not found")
+		return
+	}
+	if msg.HTML == "" {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "This message does not contain a HTML part")
+		return
+	}
+
+	html := linkInlineImages(msg)
+	w.Header().Add("Content-Type", "text/html; charset=utf-8")
+	_, _ = w.Write([]byte(html))
+}
+
+// GetMessageText (method: GET) returns a message's text part
+func GetMessageText(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /view/{ID}.txt testing GetMessageText
+	//
+	// # Render message text part
+	//
+	// Renders just the message's text part which can be used for UI integration testing.
+	//
+	// The ID can be set to `latest` to return the latest message.
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Database ID or latest
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: TextResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		var err error
+		id, err = storage.LatestID(r)
+		if err != nil {
+			w.WriteHeader(404)
+			fmt.Fprint(w, err.Error())
+			return
+		}
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "Message not found")
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain; charset=utf-8")
+	_, _ = w.Write([]byte(msg.Text))
+}
+
+// This will rewrite all inline image paths to API URLs
+func linkInlineImages(msg *storage.Message) string {
+	html := msg.HTML
+
+	for _, a := range msg.Inline {
+		if a.ContentID != "" {
+			re := regexp.MustCompile(`(?i)(=["\']?)(cid:` + regexp.QuoteMeta(a.ContentID) + `)(["|\'|\\s|\\/|>|;])`)
+			u := config.Webroot + "api/v1/message/" + msg.ID + "/part/" + a.PartID
+			matches := re.FindAllStringSubmatch(html, -1)
+			for _, m := range matches {
+				html = strings.ReplaceAll(html, m[0], m[1]+u+m[3])
+			}
+		}
+	}
+
+	for _, a := range msg.Attachments {
+		if a.ContentID != "" {
+			re := regexp.MustCompile(`(?i)(=["\']?)(cid:` + regexp.QuoteMeta(a.ContentID) + `)(["|\'|\\s|\\/|>|;])`)
+			u := config.Webroot + "api/v1/message/" + msg.ID + "/part/" + a.PartID
+			matches := re.FindAllStringSubmatch(html, -1)
+			for _, m := range matches {
+				html = strings.ReplaceAll(html, m[0], m[1]+u+m[3])
+			}
+		}
+	}
+
+	return html
+}

server/handlers/proxy.go

@@ -0,0 +1,157 @@
+// Package handlers contains a specific handlers
+package handlers
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+)
+
+var linkRe = regexp.MustCompile(`(?i)^https?:\/\/`)
+
+// ProxyHandler is used to proxy assets for printing
+func ProxyHandler(w http.ResponseWriter, r *http.Request) {
+	uri := strings.TrimSpace(r.URL.Query().Get("url"))
+	if uri == "" {
+		logger.Log().Warn("[proxy] URL missing")
+		httpError(w, "Error: URL missing")
+		return
+	}
+
+	if !linkRe.MatchString(uri) {
+		logger.Log().Warnf("[proxy] invalid URL %s", uri)
+		httpError(w, "Error: invalid URL")
+		return
+	}
+
+	tr := &http.Transport{}
+
+	if config.AllowUntrustedTLS {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec
+	}
+
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   10 * time.Second,
+	}
+
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		logger.Log().Warnf("[proxy] %s", err.Error())
+		httpError(w, err.Error())
+		return
+	}
+
+	// use requesting useragent
+	req.Header.Set("User-Agent", r.UserAgent())
+
+	resp, err := client.Do(req)
+	if err != nil {
+		logger.Log().Warnf("[proxy] %s", err.Error())
+		httpError(w, err.Error())
+		return
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logger.Log().Warnf("[proxy] %s", err.Error())
+		httpError(w, err.Error())
+		return
+	}
+
+	// relay common headers
+	if resp.Header.Get("content-type") != "" {
+		w.Header().Set("content-type", resp.Header.Get("content-type"))
+	}
+	if resp.Header.Get("last-modified") != "" {
+		w.Header().Set("last-modified", resp.Header.Get("last-modified"))
+	}
+	if resp.Header.Get("content-disposition") != "" {
+		w.Header().Set("content-disposition", resp.Header.Get("content-disposition"))
+	}
+	if resp.Header.Get("cache-control") != "" {
+		w.Header().Set("cache-control", resp.Header.Get("cache-control"))
+	}
+
+	// replace url() values with proxy address, eg: fonts & images
+	if strings.HasPrefix(resp.Header.Get("content-type"), "text/css") {
+		var re = regexp.MustCompile(`(?mi)(url\((\'|\")?([^\)\'\"]+)(\'|\")?\))`)
+		body = re.ReplaceAllFunc(body, func(s []byte) []byte {
+			parts := re.FindStringSubmatch(string(s))
+
+			// don't resolve inline `data:..`
+			if strings.HasPrefix(parts[3], "data:") {
+				return []byte(parts[3])
+			}
+
+			address, err := absoluteURL(parts[3], uri)
+			if err != nil {
+				logger.Log().Errorf("[proxy] %s", err.Error())
+				return []byte(parts[3])
+			}
+
+			return []byte("url(" + parts[2] + config.Webroot + "proxy?url=" + url.QueryEscape(address) + parts[4] + ")")
+		})
+	}
+
+	logger.Log().Debugf("[proxy] %s (%d)", uri, resp.StatusCode)
+
+	// relay status code - WriteHeader must come after Header.Set()
+	w.WriteHeader(resp.StatusCode)
+
+	if _, err := w.Write(body); err != nil {
+		logger.Log().Warnf("[proxy] %s", err.Error())
+	}
+}
+
+// AbsoluteURL will return a full URL regardless whether it is relative or absolute
+func absoluteURL(link, baseURL string) (string, error) {
+	// scheme relative links, eg <script src="//example.com/script.js">
+	if len(link) > 1 && link[0:2] == "//" {
+		base, err := url.Parse(baseURL)
+		if err != nil {
+			return link, err
+		}
+		link = base.Scheme + ":" + link
+	}
+
+	u, err := url.Parse(link)
+	if err != nil {
+		return link, err
+	}
+
+	// remove hashes
+	u.Fragment = ""
+
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return link, err
+	}
+
+	result := base.ResolveReference(u)
+
+	// ensure link is HTTP(S)
+	if result.Scheme != "http" && result.Scheme != "https" {
+		return link, fmt.Errorf("Invalid URL: %s", result.String())
+	}
+
+	return result.String(), nil
+}
+
+// HTTPError returns a basic error message (400 response)
+func httpError(w http.ResponseWriter, msg string) {
+	w.Header().Set("Referrer-Policy", "no-referrer")
+	w.Header().Set("Content-Security-Policy", config.ContentSecurityPolicy)
+	w.WriteHeader(http.StatusBadRequest)
+	w.Header().Set("Content-Type", "text/plain")
+	fmt.Fprint(w, msg)
+}

server/pop3/functions.go

@@ -0,0 +1,76 @@
+package pop3
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+)
+
+func authUser(username, password string) bool {
+	return auth.POP3Credentials.Match(username, password)
+}
+
+// Send a response with debug logging
+func sendResponse(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+	logger.Log().Debugf("[pop3] response: %s", m)
+}
+
+// Send a response without debug logging (for data)
+func sendData(c net.Conn, m string) {
+	fmt.Fprintf(c, "%s\r\n", m)
+}
+
+func getMessages() ([]message, error) {
+	messages := []message{}
+	list, err := storage.List(0, 100)
+	if err != nil {
+		return messages, err
+	}
+
+	for _, m := range list {
+		msg := message{}
+		msg.ID = m.ID
+		msg.Size = m.Size
+		messages = append(messages, msg)
+	}
+
+	return messages, nil
+}
+
+// POP3 TOP command returns the headers, followed by the next x lines
+func getTop(id string, nr int) (string, string, error) {
+	var header, body string
+	raw, err := storage.GetMessageRaw(id)
+	if err != nil {
+		return header, body, errors.New("-ERR no such message")
+	}
+
+	parts := strings.SplitN(string(raw), "\r\n\r\n", 2)
+	header = parts[0]
+	lines := []string{}
+	if nr > 0 && len(parts) == 2 {
+		lines = strings.SplitN(parts[1], "\r\n", nr)
+	}
+
+	return header, strings.Join(lines, "\r\n"), nil
+}
+
+// cuts the line into command and arguments
+func getCommand(line string) (string, []string) {
+	line = strings.Trim(line, "\r \n")
+	cmd := strings.Split(line, " ")
+	return cmd[0], cmd[1:]
+}
+
+func getSafeArg(args []string, nr int) (string, error) {
+	if nr < len(args) {
+		return args[nr], nil
+	}
+	return "", errors.New("Out of range")
+}

server/pop3/pop3.go

@@ -0,0 +1,314 @@
+// Package pop3 is a simple POP3 server for Mailpit.
+// By default it is disabled unless password credentials have been loaded.
+//
+// References: https://github.com/r0stig/golang-pop3 | https://github.com/inbucket/inbucket
+// See RFC: https://datatracker.ietf.org/doc/html/rfc1939
+package pop3
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/websockets"
+)
+
+const (
+	// UNAUTHORIZED state
+	UNAUTHORIZED = 1
+	// TRANSACTION state
+	TRANSACTION = 2
+	// UPDATE state
+	UPDATE = 3
+)
+
+// Run will start the pop3 server if enabled
+func Run() {
+	if auth.POP3Credentials == nil || config.POP3Listen == "" {
+		// POP3 server is disabled without authentication
+		return
+	}
+
+	var listener net.Listener
+	var err error
+
+	if config.POP3TLSCert != "" {
+		cer, err := tls.LoadX509KeyPair(config.POP3TLSCert, config.POP3TLSKey)
+		if err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+			return
+		}
+
+		tlsConfig := &tls.Config{
+			Certificates: []tls.Certificate{cer},
+			MinVersion:   tls.VersionTLS12,
+		}
+
+		listener, err = tls.Listen("tcp", config.POP3Listen, tlsConfig)
+	} else {
+		// unencrypted
+		listener, err = net.Listen("tcp", config.POP3Listen)
+	}
+
+	if err != nil {
+		logger.Log().Errorf("[pop3] %s", err.Error())
+		return
+	}
+
+	logger.Log().Infof("[pop3] starting on %s", config.POP3Listen)
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			continue
+		}
+
+		// run as goroutine
+		go handleClient(conn)
+	}
+}
+
+type message struct {
+	ID   string
+	Size int
+}
+
+func handleClient(conn net.Conn) {
+
+	var (
+		user     = ""
+		state    = 1
+		toDelete = []string{}
+	)
+
+	defer func() {
+		if state == UPDATE {
+			for _, id := range toDelete {
+				_ = storage.DeleteOneMessage(id)
+			}
+			if len(toDelete) > 0 {
+				// update web UI to remove deleted messages
+				websockets.Broadcast("prune", nil)
+			}
+		}
+
+		if err := conn.Close(); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+		}
+	}()
+
+	reader := bufio.NewReader(conn)
+
+	messages := []message{}
+
+	// State
+	// 1 = Unauthorized
+	// 2 = Transaction mode
+	// 3 = update mode
+
+	logger.Log().Debugf("[pop3] connection opened by %s", conn.RemoteAddr().String())
+
+	// First welcome the new connection
+	sendResponse(conn, "+OK Mailpit POP3 server")
+
+	timeoutDuration := 30 * time.Second
+
+	for {
+		// POP3 server enforced a timeout of 30 seconds
+		if err := conn.SetDeadline(time.Now().Add(timeoutDuration)); err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+			return
+		}
+
+		// Reads a line from the client
+		rawLine, err := reader.ReadString('\n')
+		if err != nil {
+			logger.Log().Errorf("[pop3] %s", err.Error())
+			return
+		}
+
+		// Parses the command
+		cmd, args := getCommand(rawLine)
+
+		logger.Log().Debugf("[pop3] received: %s (%s)", strings.TrimSpace(rawLine), conn.RemoteAddr().String())
+
+		if cmd == "CAPA" {
+			// List our capabilities per RFC2449
+			sendResponse(conn, "+OK Capability list follows")
+			sendResponse(conn, "TOP")
+			sendResponse(conn, "USER")
+			sendResponse(conn, "UIDL")
+			sendResponse(conn, "IMPLEMENTATION Mailpit")
+			sendResponse(conn, ".")
+			continue
+		} else if cmd == "USER" && state == UNAUTHORIZED {
+			if len(args) != 1 {
+				sendResponse(conn, "-ERR must supply a user")
+				return
+			}
+			// always true - stash for PASS
+			sendResponse(conn, "+OK")
+			user = args[0]
+
+		} else if cmd == "PASS" && state == UNAUTHORIZED {
+			if len(args) != 1 {
+				sendResponse(conn, "-ERR must supply a password")
+				return
+			}
+
+			pass := args[0]
+			if authUser(user, pass) {
+				sendResponse(conn, "+OK signed in")
+				messages, err = getMessages()
+				if err != nil {
+					logger.Log().Errorf("[pop3] %s", err.Error())
+				}
+				state = 2
+			} else {
+				sendResponse(conn, "-ERR invalid password")
+				logger.Log().Warnf("[pop3] failed login: %s", user)
+			}
+
+		} else if cmd == "STAT" && state == TRANSACTION {
+			totalSize := 0
+			for _, m := range messages {
+				totalSize = totalSize + m.Size
+			}
+
+			sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), totalSize))
+
+		} else if cmd == "LIST" && state == TRANSACTION {
+			totalSize := 0
+			for _, m := range messages {
+				totalSize = totalSize + m.Size
+			}
+			sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), totalSize))
+
+			// print all sizes
+			for row, m := range messages {
+				sendData(conn, fmt.Sprintf("%d %d", row+1, m.Size))
+			}
+			// end
+			sendData(conn, ".")
+
+		} else if cmd == "UIDL" && state == TRANSACTION {
+			totalSize := 0
+			for _, m := range messages {
+				totalSize = totalSize + m.Size
+			}
+
+			sendData(conn, "+OK unique-id listing follows")
+
+			// print all message IDS
+			for row, m := range messages {
+				sendData(conn, fmt.Sprintf("%d %s", row+1, m.ID))
+			}
+			// end
+			sendData(conn, ".")
+
+		} else if cmd == "RETR" && state == TRANSACTION {
+			if len(args) != 1 {
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+
+			nr, err := strconv.Atoi(args[0])
+			if err != nil {
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+
+			if nr < 1 || nr > len(messages) {
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+
+			m := messages[nr-1]
+			raw, err := storage.GetMessageRaw(m.ID)
+			if err != nil {
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+
+			size := len(raw)
+			sendData(conn, fmt.Sprintf("+OK %d octets", size))
+			sendData(conn, string(raw))
+			sendData(conn, ".")
+
+		} else if cmd == "TOP" && state == TRANSACTION {
+			arg, err := getSafeArg(args, 0)
+			if err != nil {
+				sendResponse(conn, "-ERR TOP requires two arguments")
+				return
+			}
+			nr, err := strconv.Atoi(arg)
+			if err != nil {
+				sendResponse(conn, "-ERR TOP requires two arguments")
+				return
+			}
+
+			if nr < 1 || nr > len(messages) {
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+			arg2, err := getSafeArg(args, 1)
+			if err != nil {
+				sendResponse(conn, "-ERR TOP requires two arguments")
+				return
+			}
+
+			lines, err := strconv.Atoi(arg2)
+			if err != nil {
+				sendResponse(conn, "-ERR TOP requires two arguments")
+				return
+			}
+
+			m := messages[nr-1]
+			headers, body, err := getTop(m.ID, lines)
+
+			sendData(conn, "+OK Top of message follows")
+			sendData(conn, headers+"\r\n")
+			sendData(conn, body)
+			sendData(conn, ".")
+
+		} else if cmd == "NOOP" && state == TRANSACTION {
+			sendData(conn, "+OK")
+		} else if cmd == "DELE" && state == TRANSACTION {
+			arg, _ := getSafeArg(args, 0)
+			nr, err := strconv.Atoi(arg)
+			if err != nil {
+				logger.Log().Warnf("[pop3] -ERR invalid DELETE integer: %s", arg)
+				sendResponse(conn, "-ERR invalid integer")
+				return
+			}
+
+			if nr < 1 || nr > len(messages) {
+				logger.Log().Warnf("[pop3] -ERR no such message")
+				sendResponse(conn, "-ERR no such message")
+				return
+			}
+			toDelete = append(toDelete, messages[nr-1].ID)
+
+			sendResponse(conn, "+OK")
+
+		} else if cmd == "RSET" && state == TRANSACTION {
+			toDelete = []string{}
+			sendData(conn, "+OK")
+
+		} else if cmd == "QUIT" {
+			state = UPDATE
+			return
+		} else {
+			logger.Log().Warnf("[pop3] -ERR %s not implemented", cmd)
+			sendResponse(conn, fmt.Sprintf("-ERR %s not implemented", cmd))
+		}
+	}
+}

server/server.go

@@ -1,19 +1,28 @@
+// Package server is the HTTP daemon
 package server
 
 import (
+	"bytes"
 	"compress/gzip"
 	"embed"
 	"fmt"
 	"io"
 	"io/fs"
-	"log"
 	"net/http"
 	"os"
-	"strconv"
 	"strings"
+	"sync/atomic"
+	"text/template"
+	"time"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/logger"
+	"github.com/axllent/mailpit/internal/auth"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/stats"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/apiv1"
+	"github.com/axllent/mailpit/server/handlers"
+	"github.com/axllent/mailpit/server/pop3"
 	"github.com/axllent/mailpit/server/websockets"
 	"github.com/gorilla/mux"
 )
@@ -21,11 +30,18 @@ import (
 //go:embed ui
 var embeddedFS embed.FS
 
+// AccessControlAllowOrigin CORS policy
+var AccessControlAllowOrigin string
+
 // Listen will start the httpd
 func Listen() {
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	stats.Track()
+
 	serverRoot, err := fs.Sub(embeddedFS, "ui")
 	if err != nil {
-		logger.Log().Errorf("[http] %s", err)
+		logger.Log().Errorf("[http] %s", err.Error())
 		os.Exit(1)
 	}
 
@@ -33,29 +49,112 @@ func Listen() {
 
 	go websockets.MessageHub.Run()
 
-	r := mux.NewRouter()
-	r.HandleFunc("/api/mailboxes", gzipHandlerFunc(apiListMailboxes))
-	r.HandleFunc("/api/{mailbox}/messages", gzipHandlerFunc(apiListMailbox))
-	r.HandleFunc("/api/{mailbox}/search", gzipHandlerFunc(apiSearchMailbox))
-	r.HandleFunc("/api/{mailbox}/delete", gzipHandlerFunc(apiDeleteAll))
-	r.HandleFunc("/api/{mailbox}/events", apiWebsocket)
-	r.HandleFunc("/api/{mailbox}/{id}/source", gzipHandlerFunc(apiDownloadSource))
-	r.HandleFunc("/api/{mailbox}/{id}/part/{partID}", gzipHandlerFunc(apiDownloadAttachment))
-	r.HandleFunc("/api/{mailbox}/{id}/delete", gzipHandlerFunc(apiDeleteOne))
-	r.HandleFunc("/api/{mailbox}/{id}/unread", gzipHandlerFunc(apiUnreadOne))
-	r.HandleFunc("/api/{mailbox}/{id}", gzipHandlerFunc(apiOpenMessage))
-	r.HandleFunc("/api/{mailbox}/search", gzipHandlerFunc(apiSearchMailbox))
-	r.PathPrefix("/").Handler(gzipHandler(http.FileServer(http.FS(serverRoot))))
-	http.Handle("/", r)
+	go pop3.Run()
 
-	if config.SSLCert != "" && config.SSLKey != "" {
-		logger.Log().Infof("[http] starting secure server on https://%s", config.HTTPListen)
-		log.Fatal(http.ListenAndServeTLS(config.HTTPListen, config.SSLCert, config.SSLKey, nil))
-	} else {
-		logger.Log().Infof("[http] starting server on http://%s", config.HTTPListen)
-		log.Fatal(http.ListenAndServe(config.HTTPListen, nil))
+	r := apiRoutes()
+
+	// kubernetes probes
+	r.HandleFunc(config.Webroot+"livez", handlers.HealthzHandler)
+	r.HandleFunc(config.Webroot+"readyz", handlers.ReadyzHandler(isReady))
+
+	// proxy handler for screenshots
+	r.HandleFunc(config.Webroot+"proxy", middleWareFunc(handlers.ProxyHandler)).Methods("GET")
+
+	// virtual filesystem for /dist/ & some individual files
+	r.PathPrefix(config.Webroot + "dist/").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+	r.PathPrefix(config.Webroot + "api/").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+	r.Path(config.Webroot + "favicon.ico").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+	r.Path(config.Webroot + "favicon.svg").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+	r.Path(config.Webroot + "mailpit.svg").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+	r.Path(config.Webroot + "notification.png").Handler(middlewareHandler(http.StripPrefix(config.Webroot, http.FileServer(http.FS(serverRoot)))))
+
+	// redirect to webroot if no trailing slash
+	if config.Webroot != "/" {
+		redirect := strings.TrimRight(config.Webroot, "/")
+		r.HandleFunc(redirect, middleWareFunc(addSlashToWebroot)).Methods("GET")
 	}
 
+	// UI shortcut
+	r.HandleFunc(config.Webroot+"view/latest", handlers.RedirectToLatestMessage).Methods("GET")
+
+	// frontend testing
+	r.HandleFunc(config.Webroot+"view/{id}.html", handlers.GetMessageHTML).Methods("GET")
+	r.HandleFunc(config.Webroot+"view/{id}.txt", handlers.GetMessageText).Methods("GET")
+
+	// web UI via virtual index.html
+	r.PathPrefix(config.Webroot + "view/").Handler(middleWareFunc(index)).Methods("GET")
+	r.Path(config.Webroot + "search").Handler(middleWareFunc(index)).Methods("GET")
+	r.Path(config.Webroot).Handler(middleWareFunc(index)).Methods("GET")
+
+	// put it all together
+	http.Handle("/", r)
+
+	if auth.UICredentials != nil {
+		logger.Log().Info("[http] enabling basic authentication")
+	}
+
+	// Mark the application here as ready
+	isReady.Store(true)
+
+	logger.Log().Infof("[http] starting on %s", config.HTTPListen)
+
+	server := &http.Server{
+		Addr:         config.HTTPListen,
+		ReadTimeout:  30 * time.Second,
+		WriteTimeout: 30 * time.Second,
+	}
+
+	if config.UITLSCert != "" && config.UITLSKey != "" {
+		logger.Log().Infof("[http] accessible via https://%s%s", logger.CleanHTTPIP(config.HTTPListen), config.Webroot)
+		logger.Log().Fatal(server.ListenAndServeTLS(config.UITLSCert, config.UITLSKey))
+	} else {
+		logger.Log().Infof("[http] accessible via http://%s%s", logger.CleanHTTPIP(config.HTTPListen), config.Webroot)
+		logger.Log().Fatal(server.ListenAndServe())
+	}
+}
+
+func apiRoutes() *mux.Router {
+	r := mux.NewRouter()
+
+	// API V1
+	r.HandleFunc(config.Webroot+"api/v1/messages", middleWareFunc(apiv1.GetMessages)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/messages", middleWareFunc(apiv1.SetReadStatus)).Methods("PUT")
+	r.HandleFunc(config.Webroot+"api/v1/messages", middleWareFunc(apiv1.DeleteMessages)).Methods("DELETE")
+	r.HandleFunc(config.Webroot+"api/v1/tags", middleWareFunc(apiv1.GetAllTags)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/tags", middleWareFunc(apiv1.SetMessageTags)).Methods("PUT")
+	r.HandleFunc(config.Webroot+"api/v1/search", middleWareFunc(apiv1.Search)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/search", middleWareFunc(apiv1.DeleteSearch)).Methods("DELETE")
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/part/{partID}", middleWareFunc(apiv1.DownloadAttachment)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/part/{partID}/thumb", middleWareFunc(apiv1.Thumbnail)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/headers", middleWareFunc(apiv1.GetHeaders)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/raw", middleWareFunc(apiv1.DownloadRaw)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/release", middleWareFunc(apiv1.ReleaseMessage)).Methods("POST")
+	if !config.DisableHTMLCheck {
+		r.HandleFunc(config.Webroot+"api/v1/message/{id}/html-check", middleWareFunc(apiv1.HTMLCheck)).Methods("GET")
+	}
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}/link-check", middleWareFunc(apiv1.LinkCheck)).Methods("GET")
+	if config.EnableSpamAssassin != "" {
+		r.HandleFunc(config.Webroot+"api/v1/message/{id}/sa-check", middleWareFunc(apiv1.SpamAssassinCheck)).Methods("GET")
+	}
+	r.HandleFunc(config.Webroot+"api/v1/message/{id}", middleWareFunc(apiv1.GetMessage)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/info", middleWareFunc(apiv1.AppInfo)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/webui", middleWareFunc(apiv1.WebUIConfig)).Methods("GET")
+	r.HandleFunc(config.Webroot+"api/v1/swagger.json", middleWareFunc(swaggerBasePath)).Methods("GET")
+
+	// web UI websocket
+	r.HandleFunc(config.Webroot+"api/events", apiWebsocket).Methods("GET")
+
+	// return blank 200 response for OPTIONS requests for CORS
+	r.PathPrefix(config.Webroot + "api/v1/").Handler(middleWareFunc(apiv1.GetOptions)).Methods("OPTIONS")
+
+	return r
+}
+
+// BasicAuthResponse returns an basic auth response to the browser
+func basicAuthResponse(w http.ResponseWriter) {
+	w.Header().Set("WWW-Authenticate", `Basic realm="Login"`)
+	w.WriteHeader(http.StatusUnauthorized)
+	_, _ = w.Write([]byte("Unauthorised.\n"))
 }
 
 type gzipResponseWriter struct {
@@ -67,9 +166,47 @@ func (w gzipResponseWriter) Write(b []byte) (int, error) {
 	return w.Writer.Write(b)
 }
 
-// GzipHandlerFunc http middleware
-func gzipHandlerFunc(fn http.HandlerFunc) http.HandlerFunc {
+// MiddleWareFunc http middleware adds optional basic authentication
+// and gzip compression.
+func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("Content-Security-Policy", config.ContentSecurityPolicy)
+
+		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
+			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
+			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT, OPTIONS")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
+		}
+
+		if auth.UICredentials != nil {
+			user, pass, ok := r.BasicAuth()
+
+			if !ok {
+				basicAuthResponse(w)
+				return
+			}
+
+			if !auth.UICredentials.Match(user, pass) {
+				basicAuthResponse(w)
+				return
+			}
+		}
+
+		if auth.UICredentials != nil {
+			user, pass, ok := r.BasicAuth()
+
+			if !ok {
+				basicAuthResponse(w)
+				return
+			}
+
+			if !auth.UICredentials.Match(user, pass) {
+				basicAuthResponse(w)
+				return
+			}
+		}
+
 		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
 			fn(w, r)
 			return
@@ -82,8 +219,33 @@ func gzipHandlerFunc(fn http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-func gzipHandler(h http.Handler) http.Handler {
+// MiddlewareHandler http middleware adds optional basic authentication
+// and gzip compression
+func middlewareHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("Content-Security-Policy", config.ContentSecurityPolicy)
+
+		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
+			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
+			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT, OPTIONS")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
+		}
+
+		if auth.UICredentials != nil {
+			user, pass, ok := r.BasicAuth()
+
+			if !ok {
+				basicAuthResponse(w)
+				return
+			}
+
+			if !auth.UICredentials.Match(user, pass) {
+				basicAuthResponse(w)
+				return
+			}
+		}
+
 		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
 			h.ServeHTTP(w, r)
 			return
@@ -95,37 +257,83 @@ func gzipHandler(h http.Handler) http.Handler {
 	})
 }
 
-// FourOFour returns a standard 404 meesage
-func fourOFour(w http.ResponseWriter) {
-	w.WriteHeader(http.StatusNotFound)
-	w.Header().Set("Content-Type", "text/plain")
-	fmt.Fprint(w, "404 page not found")
+// Redirect to webroot
+func addSlashToWebroot(w http.ResponseWriter, r *http.Request) {
+	http.Redirect(w, r, config.Webroot, http.StatusFound)
 }
 
-// HTTPError returns a standard 404 meesage
-func httpError(w http.ResponseWriter, msg string) {
-	w.WriteHeader(http.StatusBadRequest)
-	w.Header().Set("Content-Type", "text/plain")
-	fmt.Fprint(w, msg)
+// Websocket to broadcast changes
+func apiWebsocket(w http.ResponseWriter, r *http.Request) {
+	websockets.ServeWs(websockets.MessageHub, w, r)
+	storage.BroadcastMailboxStats()
 }
 
-// Get the start and limit based on query params. Defaults to 0, 50
-func getStartLimit(req *http.Request) (start int, limit int) {
-	start = 0
-	limit = 50
-
-	s := req.URL.Query().Get("start")
-	if n, e := strconv.ParseInt(s, 10, 64); e == nil && n > 0 {
-		start = int(n)
+// Wrapper to artificially inject a basePath to the swagger.json if a webroot has been specified
+func swaggerBasePath(w http.ResponseWriter, _ *http.Request) {
+	f, err := embeddedFS.ReadFile("ui/api/v1/swagger.json")
+	if err != nil {
+		panic(err)
 	}
 
-	l := req.URL.Query().Get("limit")
-	if n, e := strconv.ParseInt(l, 10, 64); e == nil && n > 0 {
-		if n > 500 {
-			n = 500
-		}
-		limit = int(n)
+	if config.Webroot != "/" {
+		// artificially inject a path at the start
+		replacement := fmt.Sprintf("{\n  \"basePath\": \"%s\",", strings.TrimRight(config.Webroot, "/"))
+
+		f = bytes.Replace(f, []byte("{"), []byte(replacement), 1)
 	}
 
-	return start, limit
+	w.Header().Add("Content-Type", "application/json")
+	_, _ = w.Write(f)
+}
+
+// Just returns the default HTML template
+func index(w http.ResponseWriter, _ *http.Request) {
+
+	var h = `<!DOCTYPE html>
+<html lang="en" class="h-100">
+
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width,initial-scale=1.0">
+	<meta name="referrer" content="no-referrer">
+	<meta name="robots" content="noindex, nofollow, noarchive">
+	<link rel="icon" href="{{ .Webroot }}favicon.svg">
+	<title>Mailpit</title>
+	<link rel=stylesheet href="{{ .Webroot }}dist/app.css?{{ .Version }}">
+</head>
+
+<body class="h-100">
+	<div class="container-fluid h-100 d-flex flex-column" id="app" data-webroot="{{ .Webroot }}" data-version="{{ .Version }}">
+		<noscript>You require JavaScript to use this app.</noscript>
+	</div>
+
+	<script src="{{ .Webroot }}dist/app.js?{{ .Version }}"></script>
+</body>
+
+</html>`
+
+	t, err := template.New("index").Parse(h)
+	if err != nil {
+		panic(err)
+	}
+
+	data := struct {
+		Webroot string
+		Version string
+	}{
+		Webroot: config.Webroot,
+		Version: config.Version,
+	}
+
+	buff := new(bytes.Buffer)
+
+	err = t.Execute(buff, data)
+	if err != nil {
+		panic(err)
+	}
+
+	buff.Bytes()
+
+	w.Header().Add("Content-Type", "text/html")
+	_, _ = w.Write(buff.Bytes())
 }

server/server_test.go

@@ -0,0 +1,382 @@
+package server
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/server/apiv1"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	putDataStruct struct {
+		Read bool     `json:"read"`
+		IDs  []string `json:"ids"`
+	}
+)
+
+func TestAPIv1Messages(t *testing.T) {
+	setup()
+	defer storage.Close()
+
+	r := apiRoutes()
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	m, err := fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// check count of empty database
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+
+	// insert 100
+	t.Log("Insert 100 messages")
+	insertEmailData(t)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	// store this for later tests
+
+	m, err = fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// read first 10 messages
+	t.Log("Read first 10 messages including raw & headers")
+	putIDS := []string{}
+	for idx, msg := range m.Messages {
+		if idx == 10 {
+			break
+		}
+
+		if _, err := clientGet(ts.URL + "/api/v1/message/" + msg.ID); err != nil {
+			t.Errorf(err.Error())
+		}
+
+		// get RAW
+		if _, err := clientGet(ts.URL + "/api/v1/message/" + msg.ID + "/raw"); err != nil {
+			t.Errorf(err.Error())
+		}
+
+		// het headers
+		if _, err := clientGet(ts.URL + "/api/v1/message/" + msg.ID + "/headers"); err != nil {
+			t.Errorf(err.Error())
+		}
+
+		// store for later
+		putIDS = append(putIDS, msg.ID)
+	}
+
+	// 10 should be marked as read
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 100)
+
+	// delete all
+	t.Log("Delete all messages")
+	_, err = clientDelete(ts.URL+"/api/v1/messages", "{}")
+	if err != nil {
+		t.Errorf("Expected nil, received %s", err.Error())
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+}
+
+func TestAPIv1ToggleReadStatus(t *testing.T) {
+	setup()
+	defer storage.Close()
+
+	r := apiRoutes()
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	m, err := fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// check count of empty database
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+
+	// insert 100
+	t.Log("Insert 100 messages")
+	insertEmailData(t)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	m, err = fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// read first 10 IDs
+	t.Log("Get first 10 IDs")
+	putIDS := []string{}
+	for idx, msg := range m.Messages {
+		if idx == 10 {
+			break
+		}
+
+		// store for later
+		putIDS = append(putIDS, msg.ID)
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	// mark first 10 as unread
+	t.Log("Mark first 10 as read")
+	putData := putDataStruct
+	putData.Read = true
+	putData.IDs = putIDS
+	j, err := json.Marshal(putData)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	_, err = clientPut(ts.URL+"/api/v1/messages", string(j))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 100)
+
+	// mark first 10 as read
+	t.Log("Mark first 10 as unread")
+	putData.Read = false
+	j, err = json.Marshal(putData)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	_, err = clientPut(ts.URL+"/api/v1/messages", string(j))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	// mark all as read
+	putData.Read = true
+	putData.IDs = []string{}
+	j, err = json.Marshal(putData)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	t.Log("Mark all read")
+	_, err = clientPut(ts.URL+"/api/v1/messages", string(j))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 100)
+}
+
+func TestAPIv1Search(t *testing.T) {
+	setup()
+	defer storage.Close()
+
+	r := apiRoutes()
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	// insert 100
+	t.Log("Insert 100 messages & tag")
+	insertEmailData(t)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	// search
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from-1@example.com", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from:from-1@example.com", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "-from:from-1@example.com", 99)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "-FROM:FROM-1@EXAMPLE.COM", 99)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "to:from-1@example.com", 0)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from:@example.com", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"Subject line\"", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"SUBJECT LINE 17 END\"", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "!thisdoesnotexist", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "-ThisDoesNotExist", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "thisdoesnotexist", 0)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "tag:\"Test tag 065\"", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "tag:\"TEST TAG 065\"", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "!tag:\"Test tag 023\"", 99)
+}
+
+func setup() {
+	logger.NoLogging = true
+	config.MaxMessages = 0
+	config.DataFile = ""
+
+	if err := storage.InitDB(); err != nil {
+		panic(err)
+	}
+}
+
+func assertStatsEqual(t *testing.T, uri string, unread, total int) {
+	m := apiv1.MessagesSummary{}
+
+	data, err := clientGet(uri)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	if err := json.Unmarshal(data, &m); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, unread, m.Unread, "wrong unread count")
+	assertEqual(t, total, m.Total, "wrong total count")
+}
+
+func assertSearchEqual(t *testing.T, uri, query string, count int) {
+	t.Logf("Test search: %s", query)
+	m := apiv1.MessagesSummary{}
+
+	limit := fmt.Sprintf("%d", count)
+
+	data, err := clientGet(uri + "?query=" + url.QueryEscape(query) + "&limit=" + limit)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	if err := json.Unmarshal(data, &m); err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	assertEqual(t, count, m.MessagesCount, "wrong search results count")
+}
+
+func insertEmailData(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		msg := enmime.Builder().
+			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
+			Subject(fmt.Sprintf("Subject line %d end", i)).
+			Text([]byte(fmt.Sprintf("This is the email body %d <jdsauk;dwqmdqw;>.", i))).
+			To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i))
+
+		env, err := msg.Build()
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		buf := new(bytes.Buffer)
+
+		if err := env.Encode(buf); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		bufBytes := buf.Bytes()
+
+		id, err := storage.Store(&bufBytes)
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+
+		if err := storage.SetMessageTags(id, []string{fmt.Sprintf("Test tag %03d", i)}); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+}
+
+func fetchMessages(url string) (apiv1.MessagesSummary, error) {
+	m := apiv1.MessagesSummary{}
+
+	data, err := clientGet(url)
+	if err != nil {
+		return m, err
+	}
+
+	if err := json.Unmarshal(data, &m); err != nil {
+		return m, err
+	}
+
+	return m, nil
+}
+
+func clientGet(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s returned status %d", url, resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+
+	return data, err
+}
+
+func clientDelete(url, body string) ([]byte, error) {
+	client := new(http.Client)
+
+	b := strings.NewReader(body)
+	req, err := http.NewRequest("DELETE", url, b)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s returned status %d", url, resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+
+	return data, err
+}
+
+func clientPut(url, body string) ([]byte, error) {
+	client := new(http.Client)
+
+	b := strings.NewReader(body)
+	req, err := http.NewRequest("PUT", url, b)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s returned status %d", url, resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+
+	return data, err
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
+	t.Fatal(message)
+}