diff --git a/README.md b/README.md index 1a54f60..496537d 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ ![CodeQL](https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/axllent/mailpit)](https://goreportcard.com/report/github.com/axllent/mailpit) -Mailpit is a multi-platform email testing tool & API for developers. +Mailpit is a small, fast, low memory, zero-dependency, multi-platform email testing tool & API for developers. -It acts as both an SMTP server, and provides a web interface to view all captured emails. It also contains an API for automated integration testing. +It acts as an SMTP server, provides a modern web interface to view & test captured emails, and contains an API for automated integration testing. -Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but modern and much, much faster. +Mailpit was originally **inspired** by MailHog which is now [no longer maintained](https://github.com/mailhog/MailHog/issues/442#issuecomment-1493415258) and hasn't seen active development for a few years now. ![Mailpit](https://raw.githubusercontent.com/axllent/mailpit/develop/docs/screenshot.png) @@ -21,6 +21,7 @@ Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but modern and much, muc - SMTP server (default `0.0.0.0:1025`) - Web UI to view emails (formatted HTML, highlighted HTML source, text, headers, raw source and MIME attachments including image thumbnails) - HTML check to test & score mail client compatibility with HTML emails +- Link check to test message links (HTML & text) & linked images - Light & dark web UI theme with auto-detect - Mobile and tablet HTML preview toggle in desktop mode - Advanced mail search ([see wiki](https://github.com/axllent/mailpit/wiki/Mail-search)) @@ -88,12 +89,3 @@ Please refer to [the documentation](https://github.com/axllent/mailpit/wiki/Test ### Configuring sendmail Mailpit's SMTP server (by default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. A common MTA (Mail Transfer Agent) that delivers system emails to a SMTP server is `sendmail`, used by many applications including PHP. Mailpit can also act as substitute for sendmail. For instructions of how to set this up, please refer to the [sendmail documentation](https://github.com/axllent/mailpit/wiki/Configuring-sendmail). - - -## Why rewrite MailHog? - -I had been using MailHog for a few years to intercept and test emails, but experienced a number of severe performance issues. Many of the frontend and Go libraries are very out of date, and the project [is no longer maintained](https://github.com/mailhog/MailHog/issues/442#issuecomment-1493415258). - -Initially I tried to upgrade a fork of MailHog (the UI, the HTTP server and the API), but discovered that it is (with all due respect to its authors) far too complex. I found it over-engineered (split over 9 separate projects), and performs very poorly when dealing with large amounts of emails or emails with attachments (a single email with a 3MB attachment can take over a minute to ingest). Finally the API transmits a lot of duplicate & irrelevant data on every browser request, all without any HTTP compression. - -In order to improve it I felt it needed to be completely rewritten, and so Mailpit was born. diff --git a/cmd/root.go b/cmd/root.go index 338efaf..7483e4c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -100,6 +100,7 @@ func init() { rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key") rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert") rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication") + rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain ") rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages") rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)") @@ -185,6 +186,9 @@ func initConfigFromEnv() { if getEnabledFromEnv("MP_SMTP_AUTH_ALLOW_INSECURE") { config.SMTPAuthAllowInsecure = true } + if getEnabledFromEnv("MP_STRICT_RFC_HEADERS") { + config.SMTPStrictRFCHeaders = true + } // Relay server config if len(os.Getenv("MP_SMTP_RELAY_CONFIG")) > 0 { diff --git a/config/config.go b/config/config.go index 072b0f9..fe18ea9 100644 --- a/config/config.go +++ b/config/config.go @@ -90,6 +90,10 @@ var ( // SMTPRelayConfig to parse a yaml file and store config of relay SMTP server SMTPRelayConfig smtpRelayConfigStruct + // SMTPStrictRFCHeaders will return an error if the email headers contain (\r\r\n) + // @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153 + SMTPStrictRFCHeaders bool + // ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile ReleaseEnabled = false diff --git a/server/apiv1/api.go b/server/apiv1/api.go index d5fd63d..59d564e 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -14,6 +14,7 @@ import ( "github.com/axllent/mailpit/server/smtpd" "github.com/axllent/mailpit/storage" "github.com/axllent/mailpit/utils/htmlcheck" + "github.com/axllent/mailpit/utils/linkcheck" "github.com/axllent/mailpit/utils/logger" "github.com/axllent/mailpit/utils/tools" "github.com/gorilla/mux" @@ -638,7 +639,7 @@ func HTMLCheck(w http.ResponseWriter, r *http.Request) { // // # HTML check (beta) // - // Returns the summary of HTML check. + // Returns the summary of the message HTML checker. // // NOTE: This feature is currently in beta and is documented for reference only. // Please do not integrate with it (yet) as there may be changes. @@ -684,6 +685,62 @@ func HTMLCheck(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(bytes) } +// LinkCheck returns a summary of links in the email +func LinkCheck(w http.ResponseWriter, r *http.Request) { + // swagger:route GET /api/v1/message/{ID}/link-check Other LinkCheckResponse + // + // # Link check (beta) + // + // Returns the summary of the message Link checker. + // + // NOTE: This feature is currently in beta and is documented for reference only. + // Please do not integrate with it (yet) as there may be changes. + // + // Produces: + // - application/json + // + // Schemes: http, https + // + // Parameters: + // + name: ID + // in: path + // description: Database ID + // required: true + // type: string + // + name: follow + // in: query + // description: Follow redirects + // required: false + // type: boolean + // default: false + // + // Responses: + // 200: LinkCheckResponse + // default: ErrorResponse + + vars := mux.Vars(r) + id := vars["id"] + + msg, err := storage.GetMessage(id) + if err != nil { + fourOFour(w) + return + } + + f := r.URL.Query().Get("follow") + followRedirects := f == "true" || f == "1" + + summary, err := linkcheck.RunTests(msg, followRedirects) + if err != nil { + httpError(w, err.Error()) + return + } + + bytes, _ := json.Marshal(summary) + w.Header().Add("Content-Type", "application/json") + _, _ = w.Write(bytes) +} + // FourOFour returns a basic 404 message func fourOFour(w http.ResponseWriter) { w.Header().Set("Referrer-Policy", "no-referrer") diff --git a/server/apiv1/structs.go b/server/apiv1/structs.go index 95f0bd3..7bdb5a0 100644 --- a/server/apiv1/structs.go +++ b/server/apiv1/structs.go @@ -3,6 +3,7 @@ package apiv1 import ( "github.com/axllent/mailpit/storage" "github.com/axllent/mailpit/utils/htmlcheck" + "github.com/axllent/mailpit/utils/linkcheck" ) // MessagesSummary is a summary of a list of messages @@ -49,3 +50,6 @@ type Attachment = storage.Attachment // HTMLCheckResponse summary type HTMLCheckResponse = htmlcheck.Response + +// LinkCheckResponse summary +type LinkCheckResponse = linkcheck.Response diff --git a/server/server.go b/server/server.go index b90c06f..b1c6946 100644 --- a/server/server.go +++ b/server/server.go @@ -95,6 +95,7 @@ func defaultRoutes() *mux.Router { if !config.DisableHTMLCheck { r.HandleFunc(config.Webroot+"api/v1/message/{id}/html-check", middleWareFunc(apiv1.HTMLCheck)).Methods("GET") } + r.HandleFunc(config.Webroot+"api/v1/message/{id}/link-check", middleWareFunc(apiv1.LinkCheck)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/message/{id}", middleWareFunc(apiv1.GetMessage)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/info", middleWareFunc(apiv1.AppInfo)).Methods("GET") r.HandleFunc(config.Webroot+"api/v1/webui", middleWareFunc(apiv1.WebUIConfig)).Methods("GET") diff --git a/server/smtpd/smtpd.go b/server/smtpd/smtpd.go index 67b2aa2..d78f4a5 100644 --- a/server/smtpd/smtpd.go +++ b/server/smtpd/smtpd.go @@ -17,6 +17,12 @@ import ( ) func mailHandler(origin net.Addr, from string, to []string, data []byte) error { + if !config.SMTPStrictRFCHeaders { + // replace all (\r\r\n) with (\r\n) + // @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153 + data = bytes.ReplaceAll(data, []byte("\r\r\n"), []byte("\r\n")) + } + msg, err := mail.ReadMessage(bytes.NewReader(data)) if err != nil { logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error()) diff --git a/server/ui-src/App.vue b/server/ui-src/App.vue index c998064..9749b4b 100644 --- a/server/ui-src/App.vue +++ b/server/ui-src/App.vue @@ -88,6 +88,10 @@ export default { }, mounted() { + + let title = document.title + ' - ' + location.hostname + document.title = title + this.currentPath = window.location.hash.slice(1) window.addEventListener('hashchange', () => { this.currentPath = window.location.hash.slice(1) @@ -1006,7 +1010,7 @@ export default { - diff --git a/server/ui-src/assets/styles.scss b/server/ui-src/assets/styles.scss index 7695651..0cf6341 100644 --- a/server/ui-src/assets/styles.scss +++ b/server/ui-src/assets/styles.scss @@ -39,14 +39,13 @@ } .nav-tabs .nav-link { - @include media-breakpoint-down(sm) { - // font-size: 14px; + @include media-breakpoint-down(xl) { padding-left: 10px; padding-right: 10px; } } -:not(.text-view) > a { +:not(.text-view) > a:not(.no-icon) { &[href^="http://"], &[href^="https://"] { diff --git a/server/ui-src/templates/Message.vue b/server/ui-src/templates/Message.vue index 5795c62..224a5cf 100644 --- a/server/ui-src/templates/Message.vue +++ b/server/ui-src/templates/Message.vue @@ -6,6 +6,7 @@ import Tags from "bootstrap5-tags" import Attachments from './Attachments.vue' import Headers from './Headers.vue' import HTMLCheck from './MessageHTMLCheck.vue' +import LinkCheck from './MessageLinkCheck.vue' export default { props: { @@ -18,6 +19,7 @@ export default { Attachments, Headers, HTMLCheck, + LinkCheck, }, mixins: [commonMixins], @@ -33,6 +35,7 @@ export default { loadHeaders: false, htmlScore: false, htmlScoreColor: false, + linkCheckErrors: false, showMobileButtons: false, scaleHTMLPreview: 'display', // keys names match bootstrap icon names @@ -219,7 +222,7 @@ export default { let html = s // full links with http(s) - let re = /(\b(https?|ftp):\/\/[\-\w@:%_\+.~#?,&\/\/=;]+)\b/gim + let re = /(\b(https?|ftp):\/\/[\-\w@:%_\+'!.~#?,&\/\/=;]+)/gim html = html.replace(re, '˱˱˱a href=ˠˠˠ$&ˠˠˠ target=_blank rel=noopener˲˲˲$&˱˱˱/a˲˲˲') // plain www links without https?:// prefix @@ -366,15 +369,51 @@ export default { role="tab" aria-controls="nav-raw" aria-selected="false"> Raw - +
    +
  • + +
    • +
  • + +
    • +
+ + +
diff --git a/server/ui-src/templates/MessageHTMLCheck.vue b/server/ui-src/templates/MessageHTMLCheck.vue index 32ab463..376f67c 100644 --- a/server/ui-src/templates/MessageHTMLCheck.vue +++ b/server/ui-src/templates/MessageHTMLCheck.vue @@ -640,7 +640,7 @@ export default {
-

About HTML check

+

HTML check options

diff --git a/server/ui-src/templates/MessageLinkCheck.vue b/server/ui-src/templates/MessageLinkCheck.vue new file mode 100644 index 0000000..608b277 --- /dev/null +++ b/server/ui-src/templates/MessageLinkCheck.vue @@ -0,0 +1,398 @@ + + + diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index e74165f..f6b0ff0 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -124,7 +124,7 @@ }, "/api/v1/message/{ID}/html-check": { "get": { - "description": "Returns the summary of HTML check.\n\nNOTE: This feature is currently in beta and is documented for reference only.\nPlease do not integrate with it (yet) as there may be changes.", + "description": "Returns the summary of the message HTML checker.\n\nNOTE: This feature is currently in beta and is documented for reference only.\nPlease do not integrate with it (yet) as there may be changes.", "produces": [ "application/json" ], @@ -159,6 +159,50 @@ } } }, + "/api/v1/message/{ID}/link-check": { + "get": { + "description": "Returns the summary of the message Link checker.\n\nNOTE: This feature is currently in beta and is documented for reference only.\nPlease do not integrate with it (yet) as there may be changes.", + "produces": [ + "application/json" + ], + "schemes": [ + "http", + "https" + ], + "tags": [ + "Other" + ], + "summary": "Link check (beta)", + "operationId": "LinkCheckResponse", + "parameters": [ + { + "type": "string", + "description": "Database ID", + "name": "ID", + "in": "path", + "required": true + }, + { + "type": "boolean", + "default": false, + "description": "Follow redirects", + "name": "follow", + "in": "query" + } + ], + "responses": { + "200": { + "description": "LinkCheckResponse", + "schema": { + "$ref": "#/definitions/LinkCheckResponse" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + } + } + }, "/api/v1/message/{ID}/part/{PartID}": { "get": { "description": "This will return the attachment part using the appropriate Content-Type.", @@ -828,6 +872,46 @@ "x-go-name": "Warning", "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck" }, + "Link": { + "description": "Link struct", + "type": "object", + "properties": { + "Status": { + "description": "HTTP status definition", + "type": "string" + }, + "StatusCode": { + "description": "HTTP status code", + "type": "integer", + "format": "int64" + }, + "URL": { + "description": "Link URL", + "type": "string" + } + }, + "x-go-package": "github.com/axllent/mailpit/utils/linkcheck" + }, + "LinkCheckResponse": { + "description": "Response represents the Link check response", + "type": "object", + "properties": { + "Errors": { + "description": "Total number of errors", + "type": "integer", + "format": "int64" + }, + "Links": { + "description": "Tested links", + "type": "array", + "items": { + "$ref": "#/definitions/Link" + } + } + }, + "x-go-name": "Response", + "x-go-package": "github.com/axllent/mailpit/utils/linkcheck" + }, "Message": { "description": "Message data excluding physical attachments", "type": "object", diff --git a/utils/htmlcheck/css.go b/utils/htmlcheck/css.go index 0621ae2..16120bd 100644 --- a/utils/htmlcheck/css.go +++ b/utils/htmlcheck/css.go @@ -11,6 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/utils/logger" + "github.com/axllent/mailpit/utils/tools" "github.com/vanng822/go-premailer/premailer" "golang.org/x/net/html" "golang.org/x/net/html/atom" @@ -59,7 +60,7 @@ func runCSSTests(html string) ([]Warning, int, error) { // get a list of all generated styles from all nodes allNodeStyles := []string{} for _, n := range doc.Find("*[style]").Nodes { - style, err := getHTMLAttributeVal(n, "style") + style, err := tools.GetHTMLAttributeVal(n, "style") if err == nil { allNodeStyles = append(allNodeStyles, style) } diff --git a/utils/htmlcheck/html.go b/utils/htmlcheck/html.go index 7416a12..8429e27 100644 --- a/utils/htmlcheck/html.go +++ b/utils/htmlcheck/html.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/PuerkitoBio/goquery" - "golang.org/x/net/html" + "github.com/axllent/mailpit/utils/tools" ) // HTML tests @@ -72,7 +72,7 @@ func runHTMLTests(html string) ([]Warning, int, error) { totalTests = totalTests + len(imageRegexpTests) for _, image := range images { - src, err := getHTMLAttributeVal(image, "src") + src, err := tools.GetHTMLAttributeVal(image, "src") if err != nil { continue } @@ -100,13 +100,3 @@ func runHTMLTests(html string) ([]Warning, int, error) { return results, totalTests, nil } - -func getHTMLAttributeVal(e *html.Node, key string) (string, error) { - for _, a := range e.Attr { - if a.Key == key { - return a.Val, nil - } - } - - return "", nil -} diff --git a/utils/linkcheck/main.go b/utils/linkcheck/main.go new file mode 100644 index 0000000..50440de --- /dev/null +++ b/utils/linkcheck/main.go @@ -0,0 +1,92 @@ +// Package linkcheck handles message links checking +package linkcheck + +import ( + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/axllent/mailpit/storage" + "github.com/axllent/mailpit/utils/tools" +) + +var linkRe = regexp.MustCompile(`(?m)\b(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:'!\/~+#-]*[\w@?^=%&\/~+#-])`) + +// RunTests will run all tests on an HTML string +func RunTests(msg *storage.Message, followRedirects bool) (Response, error) { + s := Response{} + + allLinks := extractHTMLLinks(msg) + allLinks = strUnique(append(allLinks, extractTextLinks(msg)...)) + s.Links = getHTTPStatuses(allLinks, followRedirects) + + for _, l := range s.Links { + if l.StatusCode >= 400 || l.StatusCode == 0 { + s.Errors++ + } + } + + return s, nil +} + +func extractTextLinks(msg *storage.Message) []string { + links := []string{} + + for _, match := range linkRe.FindAllString(msg.Text, -1) { + links = append(links, match) + } + + return links +} + +func extractHTMLLinks(msg *storage.Message) []string { + links := []string{} + + reader := strings.NewReader(msg.HTML) + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(reader) + if err != nil { + return links + } + + aLinks := doc.Find("a[href]").Nodes + for _, link := range aLinks { + l, err := tools.GetHTMLAttributeVal(link, "href") + if err == nil && linkRe.MatchString(l) { + links = append(links, l) + } + } + + cssLinks := doc.Find("link[rel=\"stylesheet\"]").Nodes + for _, link := range cssLinks { + l, err := tools.GetHTMLAttributeVal(link, "href") + if err == nil && linkRe.MatchString(l) { + links = append(links, l) + } + } + + imgLinks := doc.Find("img[src]").Nodes + for _, link := range imgLinks { + l, err := tools.GetHTMLAttributeVal(link, "src") + if err == nil && linkRe.MatchString(l) { + links = append(links, l) + } + } + + return links +} + +// strUnique return a slice of unique strings from a slice +func strUnique(strSlice []string) []string { + keys := make(map[string]bool) + list := []string{} + for _, entry := range strSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + + return list +} diff --git a/utils/linkcheck/status.go b/utils/linkcheck/status.go new file mode 100644 index 0000000..1b9acfc --- /dev/null +++ b/utils/linkcheck/status.go @@ -0,0 +1,106 @@ +package linkcheck + +import ( + "net/http" + "regexp" + "sync" + "time" + + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/utils/logger" +) + +func getHTTPStatuses(links []string, followRedirects bool) []Link { + // allow 5 threads + threads := make(chan int, 5) + + results := make(map[string]Link, len(links)) + resultsMutex := sync.RWMutex{} + + output := []Link{} + + var wg sync.WaitGroup + + for _, l := range links { + wg.Add(1) + go func(link string, w *sync.WaitGroup) { + threads <- 1 // will block if MAX threads + defer w.Done() + + code, err := doHead(link, followRedirects) + l := Link{} + l.URL = link + if err != nil { + l.StatusCode = 0 + l.Status = httpErrorSummary(err) + } else { + l.StatusCode = code + l.Status = http.StatusText(code) + } + resultsMutex.Lock() + results[link] = l + resultsMutex.Unlock() + + <-threads // remove from threads + }(l, &wg) + } + + wg.Wait() + + for _, l := range results { + output = append(output, l) + } + + return output +} + +// Do a HEAD request to return HTTP status code +func doHead(link string, followRedirects bool) (int, error) { + + timeout := time.Duration(10 * time.Second) + + client := http.Client{ + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if followRedirects { + return nil + } + return http.ErrUseLastResponse + }, + } + + req, err := http.NewRequest("HEAD", link, nil) + if err != nil { + logger.Log().Error(err) + return 0, err + } + + req.Header.Set("User-Agent", "Mailpit/"+config.Version) + + res, err := client.Do(req) + if err != nil { + if res != nil { + return res.StatusCode, err + } + + return 0, err + + } + + return res.StatusCode, nil +} + +// HTTP errors include a lot more info that just the actual error, so this +// tries to take the final part of it, eg: `no such host` +func httpErrorSummary(err error) string { + var re = regexp.MustCompile(`.*: (.*)$`) + + e := err.Error() + if !re.MatchString(e) { + return e + } + + parts := re.FindAllStringSubmatch(e, -1) + + return parts[0][len(parts[0])-1] +} diff --git a/utils/linkcheck/structs.go b/utils/linkcheck/structs.go new file mode 100644 index 0000000..210dce0 --- /dev/null +++ b/utils/linkcheck/structs.go @@ -0,0 +1,21 @@ +package linkcheck + +// Response represents the Link check response +// +// swagger:model LinkCheckResponse +type Response struct { + // Total number of errors + Errors int `json:"Errors"` + // Tested links + Links []Link `json:"Links"` +} + +// Link struct +type Link struct { + // Link URL + URL string `json:"URL"` + // HTTP status code + StatusCode int `json:"StatusCode"` + // HTTP status definition + Status string `json:"Status"` +} diff --git a/utils/tools/html.go b/utils/tools/html.go new file mode 100644 index 0000000..6e5e883 --- /dev/null +++ b/utils/tools/html.go @@ -0,0 +1,19 @@ +package tools + +import ( + "fmt" + + "golang.org/x/net/html" +) + +// GetHTMLAttributeVal returns the value of an HTML Attribute, else an error. +// Returns a blank value if the attribute is set but empty. +func GetHTMLAttributeVal(e *html.Node, key string) (string, error) { + for _, a := range e.Attr { + if a.Key == key { + return a.Val, nil + } + } + + return "", fmt.Errorf("%s not found", key) +}