Merge tag 'v1.30.3' into develop

Release v1.30.3

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.30.3]
+
+### Feature
+- Add link check rate limiting and caching mechanism
+
+### Chore
+- Update Go dependencies
+- Update node dependencies
+
+### Fix
+- Correctly parse after/before datetimes with timestamp in search query ([#704](https://github.com/axllent/mailpit/issues/704))
+- Update Swagger response definitions for MessageHeadersResponse ([#703](https://github.com/axllent/mailpit/issues/703))
+- Refactor Web UI configuration definitions in Swagger documentation
+- Handle MaxBytesError in SendMessageHandler and return JSON error response
+
+
 ## [v1.30.2]
 
 ### Security

cmd/root.go

@@ -108,6 +108,7 @@ func init() {
 	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set CORS origin(s) for the API, comma-separated (eg: example.com,foo.com)")
 	rootCmd.Flags().BoolVar(&config.BlockRemoteCSSAndFonts, "block-remote-css-and-fonts", config.BlockRemoteCSSAndFonts, "Block access to remote CSS & fonts")
 	rootCmd.Flags().BoolVar(&config.AllowInternalHTTPRequests, "allow-internal-http-requests", config.AllowInternalHTTPRequests, "Allow link checker, HTML checker & screenshots to access internal IP addresses")
+	rootCmd.Flags().BoolVar(&config.DisableLinkCheckRateLimit, "disable-link-check-rate-limit", config.DisableLinkCheckRateLimit, "Disable the per-domain rate limiter and result cache used by the link checker")
 	rootCmd.Flags().StringVar(&config.EnableSpamAssassin, "enable-spamassassin", config.EnableSpamAssassin, "Enable integration with SpamAssassin")
 	rootCmd.Flags().BoolVar(&config.AllowUntrustedTLS, "allow-untrusted-tls", config.AllowUntrustedTLS, "Do not verify HTTPS certificates (link checker & screenshots)")
 	rootCmd.Flags().BoolVar(&config.DisableHTTPCompression, "disable-http-compression", config.DisableHTTPCompression, "Disable HTTP compression support (web UI & API)")
@@ -261,6 +262,9 @@ func initConfigFromEnv() {
 	if getEnabledFromEnv("MP_ALLOW_INTERNAL_HTTP_REQUESTS") {
 		config.AllowInternalHTTPRequests = true
 	}
+	if getEnabledFromEnv("MP_DISABLE_LINK_CHECK_RATE_LIMIT") {
+		config.DisableLinkCheckRateLimit = true
+	}
 	if len(os.Getenv("MP_ENABLE_SPAMASSASSIN")) > 0 {
 		config.EnableSpamAssassin = os.Getenv("MP_ENABLE_SPAMASSASSIN")
 	}

config/config.go

@@ -140,6 +140,11 @@ var (
 	// This policy applies to both link checking and screenshot generation (proxy) features and is disabled by default for security reasons.
 	AllowInternalHTTPRequests = false
 
+	// DisableLinkCheckRateLimit disables the per-domain rate limiter, concurrency
+	// cap, and result cache used by the link checker. Off by default; set when
+	// running in a trusted environment where the limiter's pacing is unwanted.
+	DisableLinkCheckRateLimit = false
+
 	// CLITagsArg is used to map the CLI args
 	CLITagsArg string
 

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/jhillyerd/enmime/v2 v2.4.1
 	github.com/klauspost/compress v1.18.6
-	github.com/kovidgoyal/imaging v1.8.21
+	github.com/kovidgoyal/imaging v1.8.22
 	github.com/leporo/sqlf v1.4.0
 	github.com/pkg/errors v0.9.1
 	github.com/rqlite/gorqlite v0.0.0-20260504155303-50d445fd0ab9
@@ -25,7 +25,7 @@ require (
 	golang.org/x/net v0.56.0
 	golang.org/x/text v0.38.0
 	golang.org/x/time v0.15.0
-	modernc.org/sqlite v1.52.0
+	modernc.org/sqlite v1.53.0
 )
 
 require (
@@ -57,10 +57,10 @@ require (
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vanng822/css v1.0.1 // indirect
-	golang.org/x/image v0.42.0 // indirect
+	golang.org/x/image v0.43.0 // indirect
 	golang.org/x/mod v0.37.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
-	modernc.org/libc v1.73.4 // indirect
+	modernc.org/libc v1.73.5 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -60,8 +60,8 @@ github.com/kovidgoyal/go-parallel v1.1.1 h1:1OzpNjtrUkBPq3UaqrnvOoB2F9RttSt811ui
 github.com/kovidgoyal/go-parallel v1.1.1/go.mod h1:BJNIbe6+hxyFWv7n6oEDPj3PA5qSw5OCtf0hcVxWJiw=
 github.com/kovidgoyal/go-shm v1.0.0 h1:HJEel9D1F9YhULvClEHJLawoRSj/1u/EDV7MJbBPgQo=
 github.com/kovidgoyal/go-shm v1.0.0/go.mod h1:Yzb80Xf9L3kaoB2RGok9hHwMIt7Oif61kT6t3+VnZds=
-github.com/kovidgoyal/imaging v1.8.21 h1:95S2+dowTeKJJHNpf6lnScvIennTr2H0zQotu+ptNQw=
-github.com/kovidgoyal/imaging v1.8.21/go.mod h1:976F+zjiQeZ7sd87Pxlm0a64S/w9bImSIWg3sSk1rdQ=
+github.com/kovidgoyal/imaging v1.8.22 h1:CtpoRXQpS79xxJsKu8+LUJJE/0i4FLquJZy0QH+QNlM=
+github.com/kovidgoyal/imaging v1.8.22/go.mod h1:y8wo4JTv4D+skbtQf6fHg8nA1qtagvCcn8J2Nu5k2Jg=
 github.com/leporo/sqlf v1.4.0 h1:SyWnX/8GSGOzVmanG0Ub1c04mR9nNl6Tq3IeFKX2/4c=
 github.com/leporo/sqlf v1.4.0/go.mod h1:pgN9yKsAnQ+2ewhbZogr98RcasUjPsHF3oXwPPhHvBw=
 github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
@@ -123,8 +123,8 @@ github.com/vanng822/go-premailer v1.34.0/go.mod h1:LGYI7ym6FQ7KcHN16LiQRF+tlan7q
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
 golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
-golang.org/x/image v0.42.0 h1:1gSs6ehNWXLbkHBIPcWztk3D/6aIA/8hauiAYtlodVY=
-golang.org/x/image v0.42.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
+golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
+golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
 golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
 golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
@@ -137,26 +137,26 @@ golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
-golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
-golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
+golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
-modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
-modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
-modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.5 h1:hcwnthv2/LBl+mRLOYwnQA/LuW44Oln1NQlWppNaS1Q=
+modernc.org/ccgo/v4 v4.34.5/go.mod h1:aow0HNkO30OSA/2NrtDXkis92ff8ZFiDOmDOPhqhF8U=
 modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
 modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
 modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
 modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
-modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
-modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
 modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
 modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
-modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
-modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
+modernc.org/libc v1.73.5 h1:G34rN/cRqL+zOUnrbz9uPq/+OxJ8/vzQ2CQwTJ42Wmw=
+modernc.org/libc v1.73.5/go.mod h1:+Aoyx4M0etg6GikzCrip1VtvAtUlMlo2Aq+GHwQSqOA=
 modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
 modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
 modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
@@ -165,8 +165,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
 modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-modernc.org/sqlite v1.52.0 h1:p4dhYh2tXZCiyaqHwRVJDjIGKWyXayiQpThxgDzJaxo=
-modernc.org/sqlite v1.52.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM=
+modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
+modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
 modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
 modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

internal/linkcheck/linkcheck_test.go

@@ -1,7 +1,9 @@
 package linkcheck
 
 import (
+	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/axllent/mailpit/internal/storage"
@@ -72,15 +74,47 @@ func TestLinkDetection(t *testing.T) {
 	m.Text = testTextLinks
 	m.HTML = testHTML
 
-	textLinks := extractTextLinks(&m)
+	textC := &linkCollector{seen: make(map[string]bool)}
+	extractTextLinks(&m, textC)
 
-	if !reflect.DeepEqual(textLinks, expectedTextLinks) {
+	if !reflect.DeepEqual(textC.links, expectedTextLinks) {
 		t.Fatalf("Failed to detect text links correctly")
 	}
 
-	htmlLinks := extractHTMLLinks(&m)
+	htmlC := &linkCollector{seen: make(map[string]bool)}
+	extractHTMLLinks(&m, htmlC)
 
-	if !reflect.DeepEqual(htmlLinks, expectedHTMLLinks) {
+	if !reflect.DeepEqual(htmlC.links, expectedHTMLLinks) {
 		t.Fatalf("Failed to detect HTML links correctly")
 	}
 }
+
+func TestLinkLimit(t *testing.T) {
+	var html strings.Builder
+	html.WriteString("<html><body>")
+	for i := range maxUniqueLinks + 50 {
+		fmt.Fprintf(&html, `<a href="http://example.com/%d">link</a>`, i)
+	}
+	html.WriteString("</body></html>")
+
+	var text strings.Builder
+	for i := range 100 {
+		fmt.Fprintf(&text, " http://text-example.com/%d ", i)
+	}
+
+	m := storage.Message{HTML: html.String(), Text: text.String()}
+
+	c := &linkCollector{seen: make(map[string]bool)}
+	extractHTMLLinks(&m, c)
+	extractTextLinks(&m, c)
+
+	if len(c.links) != maxUniqueLinks {
+		t.Fatalf("expected %d links, got %d", maxUniqueLinks, len(c.links))
+	}
+
+	for _, l := range c.links {
+		if strings.HasPrefix(l, "http://text-example.com/") {
+			t.Fatalf("text extractor should not have run once HTML filled the collector, got %q", l)
+		}
+	}
+}

internal/linkcheck/main.go

@@ -2,6 +2,7 @@
 package linkcheck
 
 import (
+	"context"
 	"regexp"
 	"strings"
 
@@ -12,13 +13,17 @@ import (
 
 var linkRe = regexp.MustCompile(`(?im)\b(http|https):\/\/([\-\w@:%_\+'!.~#?,&\/\/=;]+)`)
 
+// maxUniqueLinks caps how many unique links will be tested per message.
+const maxUniqueLinks = 100
+
 // RunTests will run all tests on an HTML string
-func RunTests(msg *storage.Message, followRedirects bool) (Response, error) {
+func RunTests(ctx context.Context, msg *storage.Message, followRedirects bool) (Response, error) {
 	s := Response{}
 
-	allLinks := extractHTMLLinks(msg)
-	allLinks = strUnique(append(allLinks, extractTextLinks(msg)...))
-	s.Links = getHTTPStatuses(allLinks, followRedirects)
+	c := &linkCollector{seen: make(map[string]bool)}
+	extractHTMLLinks(msg, c)
+	extractTextLinks(msg, c)
+	s.Links = getHTTPStatuses(ctx, c.links, followRedirects)
 
 	for _, l := range s.Links {
 		if l.StatusCode >= 400 || l.StatusCode == 0 {
@@ -29,81 +34,91 @@ func RunTests(msg *storage.Message, followRedirects bool) (Response, error) {
 	return s, nil
 }
 
-func extractTextLinks(msg *storage.Message) []string {
+// linkCollector accumulates unique links up to maxUniqueLinks.
+type linkCollector struct {
+	seen  map[string]bool
+	links []string
+}
+
+// full reports whether the collector has reached maxUniqueLinks.
+func (c *linkCollector) full() bool {
+	return len(c.links) >= maxUniqueLinks
+}
+
+// add appends link if new and within capacity, returning false when the
+// collector is full and the caller should stop producing more links.
+func (c *linkCollector) add(link string) bool {
+	if c.full() {
+		return false
+	}
+	if !c.seen[link] {
+		c.seen[link] = true
+		c.links = append(c.links, link)
+	}
+	return !c.full()
+}
+
+func extractTextLinks(msg *storage.Message, c *linkCollector) {
+	if c.full() {
+		return
+	}
+
 	testLinkRe := regexp.MustCompile(`(?im)([^<]\b)((http|https):\/\/([\-\w@:%_\+'!.~#?,&\/\/=;]+))`)
 	// RFC2396 appendix E states angle brackets are recommended for text/plain emails to
 	// recognize potential spaces in between the URL
 	// @see https://www.rfc-editor.org/rfc/rfc2396#appendix-E
 	bracketLinkRe := regexp.MustCompile(`(?im)<((http|https):\/\/([\-\w@:%_\+'!.~#?,&\/\/=;][^>]+))>`)
 
-	links := []string{}
+	// Cap the regex match count to bound work on very large bodies; the
+	// 3x multiplier leaves headroom for duplicates the collector will drop.
+	matchLimit := maxUniqueLinks * 3
 
-	matches := testLinkRe.FindAllStringSubmatch(msg.Text, -1)
+	matches := testLinkRe.FindAllStringSubmatch(msg.Text, matchLimit)
 	for _, match := range matches {
 		if len(match) > 0 {
-			links = append(links, match[2])
+			if !c.add(match[2]) {
+				return
+			}
 		}
 	}
 
-	angleMatches := bracketLinkRe.FindAllStringSubmatch(msg.Text, -1)
+	angleMatches := bracketLinkRe.FindAllStringSubmatch(msg.Text, matchLimit)
 	for _, match := range angleMatches {
 		if len(match) > 0 {
 			link := strings.ReplaceAll(match[1], "\n", "")
-			links = append(links, link)
+			if !c.add(link) {
+				return
+			}
 		}
 	}
-
-	return links
 }
 
-func extractHTMLLinks(msg *storage.Message) []string {
-	links := []string{}
+func extractHTMLLinks(msg *storage.Message, c *linkCollector) {
+	if c.full() {
+		return
+	}
 
 	reader := strings.NewReader(msg.HTML)
 
 	// Load the HTML document
 	doc, err := goquery.NewDocumentFromReader(reader)
 	if err != nil {
-		return links
+		return
 	}
 
-	aLinks := doc.Find("a[href]").Nodes
-	for _, link := range aLinks {
-		l, err := tools.GetHTMLAttributeVal(link, "href")
-		if err == nil && linkRe.MatchString(l) {
-			links = append(links, l)
+	for _, sel := range []struct{ selector, attr string }{
+		{"a[href]", "href"},
+		{`link[rel="stylesheet"]`, "href"},
+		{"img[src]", "src"},
+	} {
+		for _, node := range doc.Find(sel.selector).Nodes {
+			l, err := tools.GetHTMLAttributeVal(node, sel.attr)
+			if err != nil || !linkRe.MatchString(l) {
+				continue
+			}
+			if !c.add(l) {
+				return
+			}
 		}
 	}
-
-	cssLinks := doc.Find("link[rel=\"stylesheet\"]").Nodes
-	for _, link := range cssLinks {
-		l, err := tools.GetHTMLAttributeVal(link, "href")
-		if err == nil && linkRe.MatchString(l) {
-			links = append(links, l)
-		}
-	}
-
-	imgLinks := doc.Find("img[src]").Nodes
-	for _, link := range imgLinks {
-		l, err := tools.GetHTMLAttributeVal(link, "src")
-		if err == nil && linkRe.MatchString(l) {
-			links = append(links, l)
-		}
-	}
-
-	return links
-}
-
-// strUnique return a slice of unique strings from a slice
-func strUnique(strSlice []string) []string {
-	keys := make(map[string]bool)
-	list := []string{}
-	for _, entry := range strSlice {
-		if _, value := keys[entry]; !value {
-			keys[entry] = true
-			list = append(list, entry)
-		}
-	}
-
-	return list
 }

internal/linkcheck/status.go

@@ -17,26 +17,31 @@ import (
 	"github.com/axllent/mailpit/internal/tools"
 )
 
-func getHTTPStatuses(links []string, followRedirects bool) []Link {
-	// allow 5 threads
-	threads := make(chan int, 5)
-
-	results := make(map[string]Link, len(links))
-	resultsMutex := sync.RWMutex{}
-
-	output := []Link{}
-
+func getHTTPStatuses(ctx context.Context, links []string, followRedirects bool) []Link {
+	results := make([]Link, len(links))
 	var wg sync.WaitGroup
+	var warnedDomains sync.Map
+
+	for i, l := range links {
+		if cached, ok := cachedLink(l); ok {
+			results[i] = cached
+			continue
+		}
 
-	for _, l := range links {
 		wg.Add(1)
-		go func(link string, w *sync.WaitGroup) {
-			threads <- 1 // will block if MAX threads
-			defer w.Done()
+		go func(idx int, link string) {
+			defer wg.Done()
 
-			code, err := doHead(link, followRedirects)
-			l := Link{}
-			l.URL = link
+			domain := registeredDomain(link)
+			release, err := acquireDomainSlot(ctx, domain, &warnedDomains)
+			if err != nil {
+				results[idx] = Link{URL: link, StatusCode: 0, Status: httpErrorSummary(err)}
+				return
+			}
+			defer release()
+
+			code, err := doHead(ctx, link, followRedirects)
+			l := Link{URL: link}
 			if err != nil {
 				l.StatusCode = 0
 				l.Status = httpErrorSummary(err)
@@ -48,25 +53,17 @@ func getHTTPStatuses(links []string, followRedirects bool) []Link {
 				l.StatusCode = code
 				l.Status = http.StatusText(code)
 			}
-			resultsMutex.Lock()
-			results[link] = l
-			resultsMutex.Unlock()
-
-			<-threads // remove from threads
-		}(l, &wg)
+			results[idx] = l
+			storeLink(link, l)
+		}(i, l)
 	}
 
 	wg.Wait()
-
-	for _, l := range results {
-		output = append(output, l)
-	}
-
-	return output
+	return results
 }
 
 // Do a HEAD request to return HTTP status code
-func doHead(link string, followRedirects bool) (int, error) {
+func doHead(ctx context.Context, link string, followRedirects bool) (int, error) {
 	if !tools.IsValidLinkURL(link) {
 		return 0, fmt.Errorf("invalid URL: %s", link)
 	}
@@ -102,7 +99,7 @@ func doHead(link string, followRedirects bool) (int, error) {
 		},
 	}
 
-	req, err := http.NewRequest("HEAD", link, nil)
+	req, err := http.NewRequestWithContext(ctx, "HEAD", link, nil)
 	if err != nil {
 		logger.Log().Errorf("[link-check] %s", err.Error())
 		return 0, err

internal/linkcheck/throttle.go

@@ -0,0 +1,219 @@
+package linkcheck
+
+import (
+	"container/list"
+	"context"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"golang.org/x/net/publicsuffix"
+	"golang.org/x/time/rate"
+)
+
+// Per-domain rate-limiter parameters. The bucket starts full so a single
+// fresh check of a 100-link newsletter completes without waiting; refill
+// caps sustained traffic to any one registered domain at 1 req/s across
+// all concurrent API calls.
+const (
+	perDomainBurst       = 100
+	perDomainRefill      = rate.Limit(1)
+	perDomainConcurrency = 2
+
+	// limiterRegistryCap bounds memory regardless of attacker effort.
+	// Eviction prefers buckets at full capacity (safe to drop).
+	limiterRegistryCap = 10000
+
+	// resultCacheTTL deduplicates repeated checks of the same URL so a
+	// user retesting the same email doesn't drain the rate limiter twice
+	// and an attacker can't multiply outbound load by looping the API.
+	resultCacheTTL = 60 * time.Second
+)
+
+type domainState struct {
+	limiter *rate.Limiter
+	sem     chan struct{}
+	lruElem *list.Element
+}
+
+type registry struct {
+	mu      sync.Mutex
+	entries map[string]*domainState
+	lru     *list.List // front = most recently used
+}
+
+func newRegistry() *registry {
+	return &registry{
+		entries: make(map[string]*domainState),
+		lru:     list.New(),
+	}
+}
+
+// get returns the state for a registered domain, creating it on demand.
+// When the registry is at capacity, prefers to evict entries whose bucket
+// is at full capacity (no security cost — recreating yields identical state).
+func (r *registry) get(domain string) *domainState {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if st, ok := r.entries[domain]; ok {
+		r.lru.MoveToFront(st.lruElem)
+		return st
+	}
+
+	if len(r.entries) >= limiterRegistryCap {
+		r.evictLocked()
+	}
+
+	st := &domainState{
+		limiter: rate.NewLimiter(perDomainRefill, perDomainBurst),
+		sem:     make(chan struct{}, perDomainConcurrency),
+	}
+	st.lruElem = r.lru.PushFront(domainKey{domain: domain, state: st})
+	r.entries[domain] = st
+
+	return st
+}
+
+type domainKey struct {
+	domain string
+	state  *domainState
+}
+
+// evictLocked drops one entry. Caller must hold r.mu.
+// Walks the LRU from the back looking for a full bucket; if none, drops the LRU.
+func (r *registry) evictLocked() {
+	for e := r.lru.Back(); e != nil; e = e.Prev() {
+		k := e.Value.(domainKey)
+		if k.state.limiter.Tokens() >= float64(perDomainBurst) {
+			r.lru.Remove(e)
+			delete(r.entries, k.domain)
+			return
+		}
+	}
+
+	e := r.lru.Back()
+	if e == nil {
+		return
+	}
+
+	k := e.Value.(domainKey)
+	r.lru.Remove(e)
+	delete(r.entries, k.domain)
+}
+
+type cachedResult struct {
+	link    Link
+	expires time.Time
+}
+
+type resultCache struct {
+	mu      sync.Mutex
+	entries map[string]cachedResult
+}
+
+func newResultCache() *resultCache {
+	return &resultCache{entries: make(map[string]cachedResult)}
+}
+
+func (c *resultCache) get(u string) (Link, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.entries[u]
+	if !ok {
+		return Link{}, false
+	}
+	if time.Now().After(e.expires) {
+		delete(c.entries, u)
+		return Link{}, false
+	}
+
+	return e.link, true
+}
+
+func (c *resultCache) put(u string, l Link) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.entries[u] = cachedResult{link: l, expires: time.Now().Add(resultCacheTTL)}
+	// Opportunistic sweep: when the cache grows past a threshold,
+	// drop expired entries. Avoids unbounded growth without a goroutine.
+	if len(c.entries) > 2*limiterRegistryCap {
+		now := time.Now()
+		for k, v := range c.entries {
+			if now.After(v.expires) {
+				delete(c.entries, k)
+			}
+		}
+	}
+}
+
+var (
+	domainRegistry = newRegistry()
+	linkCache      = newResultCache()
+)
+
+// registeredDomain returns the eTLD+1 for a URL's host, or the lowercased
+// host if no registered domain can be determined (e.g. IP literals).
+// Subdomains share the same key so wildcard-DNS bypass is closed.
+func registeredDomain(rawurl string) string {
+	u, err := url.Parse(rawurl)
+	if err != nil {
+		return ""
+	}
+	host := strings.ToLower(u.Hostname())
+	if host == "" {
+		return ""
+	}
+	d, err := publicsuffix.EffectiveTLDPlusOne(host)
+	if err != nil {
+		return host
+	}
+
+	return d
+}
+
+// acquireDomainSlot blocks until both a rate-limit token and a per-domain
+// concurrency slot are available, or ctx is cancelled. Returns a release
+// function that must be called when the request completes.
+func acquireDomainSlot(ctx context.Context, domain string, warned *sync.Map) (release func(), err error) {
+	if config.DisableLinkCheckRateLimit {
+		return func() {}, nil
+	}
+	st := domainRegistry.get(domain)
+	if st.limiter.Tokens() < 1 {
+		if _, alreadyWarned := warned.LoadOrStore(domain, struct{}{}); !alreadyWarned {
+			logger.Log().Warnf("[link-check] rate limiting active for %s - use --disable-link-check-rate-limit to disable", domain)
+		}
+	}
+	if err := st.limiter.Wait(ctx); err != nil {
+		return nil, err
+	}
+	select {
+	case st.sem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	return func() { <-st.sem }, nil
+}
+
+// cachedLink returns a previously-checked result if still fresh.
+func cachedLink(u string) (Link, bool) {
+	if config.DisableLinkCheckRateLimit {
+		return Link{}, false
+	}
+	return linkCache.get(u)
+}
+
+// storeLink caches a result so repeat checks of the same URL skip the
+// rate limiter and the outbound HEAD.
+func storeLink(u string, l Link) {
+	if config.DisableLinkCheckRateLimit {
+		return
+	}
+
+	linkCache.put(u, l)
+}

internal/storage/search.go

@@ -465,7 +465,7 @@ func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt {
 				q.Where("Attachments > 0")
 			}
 		} else if strings.HasPrefix(lw, "after:") {
-			w = cleanString(w[6:])
+			w = strings.ToUpper(cleanString(w[6:]))
 			if w != "" {
 				t, err := dateparse.ParseIn(w, loc)
 				if err != nil {
@@ -480,7 +480,7 @@ func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt {
 				}
 			}
 		} else if strings.HasPrefix(lw, "before:") {
-			w = cleanString(w[7:])
+			w = strings.ToUpper(cleanString(w[7:]))
 			if w != "" {
 				t, err := dateparse.ParseIn(w, loc)
 				if err != nil {

server/apiv1/other.go

@@ -123,7 +123,7 @@ func LinkCheck(w http.ResponseWriter, r *http.Request) {
 	f := r.URL.Query().Get("follow")
 	followRedirects := f == "true" || f == "1"
 
-	summary, err := linkcheck.RunTests(msg, followRedirects)
+	summary, err := linkcheck.RunTests(r.Context(), msg, followRedirects)
 	if err != nil {
 		httpError(w, err.Error())
 		return

server/apiv1/send.go

@@ -54,7 +54,10 @@ func SendMessageHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		var maxErr *http.MaxBytesError
 		if errors.As(err, &maxErr) {
+			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(http.StatusRequestEntityTooLarge)
+			_ = json.NewEncoder(w).Encode(struct{ Error string }{Error: err.Error()})
+			return
 		}
 		httpJSONError(w, err.Error())
 		return

server/apiv1/swaggerResponses.go

@@ -53,49 +53,53 @@ type jsonErrorResponse struct {
 	}
 }
 
+// Web UI configuration settings
+// swagger:model WebUIConfiguration
+type WebUIConfiguration struct {
+	// Optional label to identify this Mailpit instance
+	Label string
+	// Message Relay information
+	MessageRelay struct {
+		// Whether message relaying (release) is enabled
+		Enabled bool
+		// The configured SMTP server address
+		SMTPServer string
+		// Enforced Return-Path (if set) for relay bounces
+		ReturnPath string
+		// Only allow relaying to these recipients (regex)
+		AllowedRecipients string
+		// Block relaying to these recipients (regex)
+		BlockedRecipients string
+		// Overrides the "From" address for all relayed messages
+		OverrideFrom string
+		// Preserve the original Message-IDs when relaying messages
+		PreserveMessageIDs bool
+
+		// DEPRECATED 2024/03/12
+		// swagger:ignore
+		RecipientAllowlist string
+	}
+
+	// Whether SpamAssassin is enabled
+	SpamAssassin bool
+
+	// Whether Chaos support is enabled at runtime
+	ChaosEnabled bool
+
+	// Whether messages with duplicate IDs are ignored
+	DuplicatesIgnored bool
+
+	// Whether the delete button should be hidden
+	HideDeleteAllButton bool
+}
+
 // Web UI configuration response
 // swagger:response WebUIConfigurationResponse
 type webUIConfigurationResponse struct {
 	// Web UI configuration settings
 	//
 	// in: body
-	Body struct {
-		// Optional label to identify this Mailpit instance
-		Label string
-		// Message Relay information
-		MessageRelay struct {
-			// Whether message relaying (release) is enabled
-			Enabled bool
-			// The configured SMTP server address
-			SMTPServer string
-			// Enforced Return-Path (if set) for relay bounces
-			ReturnPath string
-			// Only allow relaying to these recipients (regex)
-			AllowedRecipients string
-			// Block relaying to these recipients (regex)
-			BlockedRecipients string
-			// Overrides the "From" address for all relayed messages
-			OverrideFrom string
-			// Preserve the original Message-IDs when relaying messages
-			PreserveMessageIDs bool
-
-			// DEPRECATED 2024/03/12
-			// swagger:ignore
-			RecipientAllowlist string
-		}
-
-		// Whether SpamAssassin is enabled
-		SpamAssassin bool
-
-		// Whether Chaos support is enabled at runtime
-		ChaosEnabled bool
-
-		// Whether messages with duplicate IDs are ignored
-		DuplicatesIgnored bool
-
-		// Whether the delete button should be hidden
-		HideDeleteAllButton bool
-	}
+	Body WebUIConfiguration
 }
 
 // Application information
@@ -117,7 +121,7 @@ type chaosResponse struct {
 }
 
 // Message headers
-// swagger:model MessageHeadersResponse
+// swagger:response MessageHeadersResponse
 type messageHeadersResponse map[string][]string
 
 // Summary of messages

server/ui/api/v1/swagger.json

@@ -174,10 +174,7 @@
         ],
         "responses": {
           "200": {
-            "description": "MessageHeadersResponse",
-            "schema": {
-              "$ref": "#/definitions/MessageHeadersResponse"
-            }
+            "$ref": "#/responses/MessageHeadersResponse"
           },
           "400": {
             "$ref": "#/responses/ErrorResponse"
@@ -1774,18 +1771,6 @@
       },
       "x-go-package": "github.com/axllent/mailpit/internal/storage"
     },
-    "MessageHeadersResponse": {
-      "description": "Message headers",
-      "type": "object",
-      "additionalProperties": {
-        "type": "array",
-        "items": {
-          "type": "string"
-        }
-      },
-      "x-go-name": "messageHeadersResponse",
-      "x-go-package": "github.com/axllent/mailpit/server/apiv1"
-    },
     "MessageSummary": {
       "description": "MessageSummary struct for frontend messages",
       "type": "object",
@@ -1875,7 +1860,7 @@
       "type": "object",
       "properties": {
         "messages": {
-          "description": "Messages summary\nin: body",
+          "description": "Messages summary",
           "type": "array",
           "items": {
             "$ref": "#/definitions/MessageSummary"
@@ -1970,6 +1955,67 @@
       },
       "x-go-name": "Result",
       "x-go-package": "github.com/axllent/mailpit/internal/spamassassin"
+    },
+    "WebUIConfiguration": {
+      "description": "Web UI configuration settings",
+      "type": "object",
+      "properties": {
+        "ChaosEnabled": {
+          "description": "Whether Chaos support is enabled at runtime",
+          "type": "boolean"
+        },
+        "DuplicatesIgnored": {
+          "description": "Whether messages with duplicate IDs are ignored",
+          "type": "boolean"
+        },
+        "HideDeleteAllButton": {
+          "description": "Whether the delete button should be hidden",
+          "type": "boolean"
+        },
+        "Label": {
+          "description": "Optional label to identify this Mailpit instance",
+          "type": "string"
+        },
+        "MessageRelay": {
+          "description": "Message Relay information",
+          "type": "object",
+          "properties": {
+            "AllowedRecipients": {
+              "description": "Only allow relaying to these recipients (regex)",
+              "type": "string"
+            },
+            "BlockedRecipients": {
+              "description": "Block relaying to these recipients (regex)",
+              "type": "string"
+            },
+            "Enabled": {
+              "description": "Whether message relaying (release) is enabled",
+              "type": "boolean"
+            },
+            "OverrideFrom": {
+              "description": "Overrides the \"From\" address for all relayed messages",
+              "type": "string"
+            },
+            "PreserveMessageIDs": {
+              "description": "Preserve the original Message-IDs when relaying messages",
+              "type": "boolean"
+            },
+            "ReturnPath": {
+              "description": "Enforced Return-Path (if set) for relay bounces",
+              "type": "string"
+            },
+            "SMTPServer": {
+              "description": "The configured SMTP server address",
+              "type": "string"
+            }
+          }
+        },
+        "SpamAssassin": {
+          "description": "Whether SpamAssassin is enabled",
+          "type": "boolean"
+        }
+      },
+      "x-go-package": "github.com/axllent/mailpit/server/apiv1"
     }
   },
   "responses": {
@@ -2025,6 +2071,18 @@
         }
       }
     },
+    "MessageHeadersResponse": {
+      "description": "Message headers",
+      "schema": {
+        "type": "object",
+        "additionalProperties": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "MessagesSummaryResponse": {
       "description": "Summary of messages",
       "schema": {
@@ -2065,63 +2123,7 @@
     "WebUIConfigurationResponse": {
       "description": "Web UI configuration response",
       "schema": {
-        "type": "object",
-        "properties": {
-          "ChaosEnabled": {
-            "description": "Whether Chaos support is enabled at runtime",
-            "type": "boolean"
-          },
-          "DuplicatesIgnored": {
-            "description": "Whether messages with duplicate IDs are ignored",
-            "type": "boolean"
-          },
-          "HideDeleteAllButton": {
-            "description": "Whether the delete button should be hidden",
-            "type": "boolean"
-          },
-          "Label": {
-            "description": "Optional label to identify this Mailpit instance",
-            "type": "string"
-          },
-          "MessageRelay": {
-            "description": "Message Relay information",
-            "type": "object",
-            "properties": {
-              "AllowedRecipients": {
-                "description": "Only allow relaying to these recipients (regex)",
-                "type": "string"
-              },
-              "BlockedRecipients": {
-                "description": "Block relaying to these recipients (regex)",
-                "type": "string"
-              },
-              "Enabled": {
-                "description": "Whether message relaying (release) is enabled",
-                "type": "boolean"
-              },
-              "OverrideFrom": {
-                "description": "Overrides the \"From\" address for all relayed messages",
-                "type": "string"
-              },
-              "PreserveMessageIDs": {
-                "description": "Preserve the original Message-IDs when relaying messages",
-                "type": "boolean"
-              },
-              "ReturnPath": {
-                "description": "Enforced Return-Path (if set) for relay bounces",
-                "type": "string"
-              },
-              "SMTPServer": {
-                "description": "The configured SMTP server address",
-                "type": "string"
-              }
-            }
-          },
-          "SpamAssassin": {
-            "description": "Whether SpamAssassin is enabled",
-            "type": "boolean"
-          }
-        }
+        "$ref": "#/definitions/WebUIConfiguration"
       }
     }
   }