diff --git a/go.mod b/go.mod index f38cd5e..59b13dd 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/klauspost/compress v1.17.0 github.com/leporo/sqlf v1.4.0 github.com/mhale/smtpd v0.8.0 + github.com/microcosm-cc/bluemonday v1.0.25 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.9.3 @@ -32,6 +33,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cznic/ql v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 16b2c2b..df39836 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E= github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -97,6 +99,8 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/storage/database.go b/internal/storage/database.go index f7cbbee..c6d55ed 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -18,9 +18,9 @@ import ( "syscall" "time" - "github.com/GuiaBolso/darwin" "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" "github.com/axllent/mailpit/server/websockets" "github.com/jhillyerd/enmime" "github.com/klauspost/compress/zstd" @@ -42,81 +42,8 @@ var ( // zstd compression encoder & decoder dbEncoder, _ = zstd.NewWriter(nil) dbDecoder, _ = zstd.NewReader(nil) - - dbMigrations = []darwin.Migration{ - { - Version: 1.0, - Description: "Creating tables", - Script: `CREATE TABLE IF NOT EXISTS mailbox ( - Sort INTEGER PRIMARY KEY AUTOINCREMENT, - ID TEXT NOT NULL, - Data BLOB, - Search TEXT, - Read INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - - CREATE TABLE IF NOT EXISTS mailbox_data ( - ID TEXT KEY NOT NULL, - Email BLOB - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`, - }, - { - Version: 1.1, - Description: "Create tags column", - Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - { - Version: 1.2, - Description: "Creating new mailbox format", - Script: `CREATE TABLE IF NOT EXISTS mailboxtmp ( - Created INTEGER NOT NULL, - ID TEXT NOT NULL, - MessageID TEXT NOT NULL, - Subject TEXT NOT NULL, - Metadata TEXT, - Size INTEGER NOT NULL, - Inline INTEGER NOT NULL, - Attachments INTEGER NOT NULL, - Read INTEGER, - Tags TEXT, - SearchText TEXT - ); - INSERT INTO mailboxtmp - (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) - SELECT - Sort, ID, '', json_extract(Data, '$.Subject'),Data, - json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), - Search, Read, Tags - FROM mailbox; - - DROP TABLE IF EXISTS mailbox; - ALTER TABLE mailboxtmp RENAME TO mailbox; - CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID); - CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject); - CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size); - CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline); - CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - } ) -// DBMailSummary struct for storing mail summary -type DBMailSummary struct { - From *mail.Address - To []*mail.Address - Cc []*mail.Address - Bcc []*mail.Address -} - // InitDB will initialise the database func InitDB() error { p := config.DataFile @@ -178,15 +105,6 @@ func InitDB() error { return nil } -// Create tables and apply migrations if required -func dbApplyMigrations() error { - driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{}) - - d := darwin.New(driver, dbMigrations, nil) - - return d.Migrate() -} - // Close will close the database, and delete if a temporary table func Close() { if db != nil { @@ -281,10 +199,11 @@ func Store(body []byte) (string, error) { size := len(body) inline := len(env.Inlines) attachments := len(env.Attachments) + snippet := tools.CreateSnippet(env.Text, env.HTML) // insert mail summary data - _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)", - created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON)) + _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read, Snippet) values(?,?,?,?,?,?,?,?,?,?,0, ?)", + created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON), snippet) if err != nil { return "", err } @@ -312,6 +231,7 @@ func Store(body []byte) (string, error) { c.Subject = subject c.Size = size c.Tags = tagData + c.Snippet = snippet websockets.Broadcast("new", c) @@ -328,7 +248,7 @@ func List(start, limit int) ([]MessageSummary, error) { results := []MessageSummary{} q := sqlf.From("mailbox"). - Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags`). + Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet`). OrderBy("Created DESC"). Limit(limit). Offset(start) @@ -343,9 +263,10 @@ func List(start, limit int) ([]MessageSummary, error) { var attachments int var tags string var read int + var snippet string em := MessageSummary{} - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet); err != nil { logger.Log().Error(err) return } @@ -367,6 +288,7 @@ func List(start, limit int) ([]MessageSummary, error) { em.Size = size em.Attachments = attachments em.Read = read == 1 + em.Snippet = snippet results = append(results, em) }); err != nil { diff --git a/internal/storage/migrations.go b/internal/storage/migrations.go new file mode 100644 index 0000000..ac5937b --- /dev/null +++ b/internal/storage/migrations.go @@ -0,0 +1,84 @@ +package storage + +import "github.com/GuiaBolso/darwin" + +var ( + dbMigrations = []darwin.Migration{ + { + Version: 1.0, + Description: "Creating tables", + Script: `CREATE TABLE IF NOT EXISTS mailbox ( + Sort INTEGER PRIMARY KEY AUTOINCREMENT, + ID TEXT NOT NULL, + Data BLOB, + Search TEXT, + Read INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort); + CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); + CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); + + CREATE TABLE IF NOT EXISTS mailbox_data ( + ID TEXT KEY NOT NULL, + Email BLOB + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`, + }, + { + Version: 1.1, + Description: "Create tags column", + Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; + CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, + }, + { + Version: 1.2, + Description: "Creating new mailbox format", + Script: `CREATE TABLE IF NOT EXISTS mailboxtmp ( + Created INTEGER NOT NULL, + ID TEXT NOT NULL, + MessageID TEXT NOT NULL, + Subject TEXT NOT NULL, + Metadata TEXT, + Size INTEGER NOT NULL, + Inline INTEGER NOT NULL, + Attachments INTEGER NOT NULL, + Read INTEGER, + Tags TEXT, + SearchText TEXT + ); + INSERT INTO mailboxtmp + (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) + SELECT + Sort, ID, '', json_extract(Data, '$.Subject'),Data, + json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), + Search, Read, Tags + FROM mailbox; + + DROP TABLE IF EXISTS mailbox; + ALTER TABLE mailboxtmp RENAME TO mailbox; + CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created); + CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); + CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID); + CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject); + CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size); + CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline); + CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments); + CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); + CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, + }, + { + Version: 1.3, + Description: "Create snippet column", + Script: `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`, + }, + } +) + +// Create tables and apply migrations if required +func dbApplyMigrations() error { + driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{}) + + d := darwin.New(driver, dbMigrations, nil) + + return d.Migrate() +} diff --git a/internal/storage/search.go b/internal/storage/search.go index 9bd844d..abb8a6a 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -38,11 +38,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { var size int var attachments int var tags string + var snippet string var read int var ignore string em := MessageSummary{} - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Error(err) return } @@ -64,6 +65,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { em.Size = size em.Attachments = attachments em.Read = read == 1 + em.Snippet = snippet allResults = append(allResults, em) }); err != nil { @@ -109,9 +111,10 @@ func DeleteSearch(search string) error { var attachments int var tags string var read int + var snippet string var ignore string - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Error(err) return } @@ -193,7 +196,7 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt { args := tools.ArgsParser(searchString) q := sqlf.From("mailbox"). - Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, + Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet, IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON, IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON, IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON, diff --git a/internal/storage/structs.go b/internal/storage/structs.go index e3e05df..2ad9721 100644 --- a/internal/storage/structs.go +++ b/internal/storage/structs.go @@ -89,6 +89,8 @@ type MessageSummary struct { Size int // Whether the message has any attachments Attachments int + // Message snippet includes up to 250 characters + Snippet string } // MailboxStats struct for quick mailbox total/read lookups @@ -98,6 +100,14 @@ type MailboxStats struct { Tags []string } +// DBMailSummary struct for storing mail summary +type DBMailSummary struct { + From *mail.Address + To []*mail.Address + Cc []*mail.Address + Bcc []*mail.Address +} + // AttachmentSummary returns a summary of the attachment without any binary data func AttachmentSummary(a *enmime.Part) Attachment { o := Attachment{} diff --git a/internal/storage/test_shared.go b/internal/storage/testing.go similarity index 100% rename from internal/storage/test_shared.go rename to internal/storage/testing.go diff --git a/internal/tools/html.go b/internal/tools/html.go index 6e5e883..06de636 100644 --- a/internal/tools/html.go +++ b/internal/tools/html.go @@ -2,7 +2,9 @@ package tools import ( "fmt" + "strings" + "github.com/microcosm-cc/bluemonday" "golang.org/x/net/html" ) @@ -17,3 +19,12 @@ func GetHTMLAttributeVal(e *html.Node, key string) (string, error) { return "", fmt.Errorf("%s not found", key) } + +// StripHTML returns text from an HTML string +func stripHTML(h string) string { + p := bluemonday.StrictPolicy() + // // ensure joining html elements are spaced apart, eg table cells etc + h = strings.ReplaceAll(h, "><", "> <") + // return p.Sanitize(h) + return html.UnescapeString(p.Sanitize(h)) +} diff --git a/internal/tools/snippets.go b/internal/tools/snippets.go new file mode 100644 index 0000000..6dee2ad --- /dev/null +++ b/internal/tools/snippets.go @@ -0,0 +1,42 @@ +package tools + +import ( + "regexp" + "strings" +) + +// CreateSnippet returns a message snippet. It will use the HTML version (if it exists) +// and fall back to the text version. +func CreateSnippet(text, html string) string { + text = strings.TrimSpace(text) + html = strings.TrimSpace(html) + characters := 200 + spaceRe := regexp.MustCompile(`\s+`) + nlRe := regexp.MustCompile(`\r?\n`) + + if text == "" && html == "" { + return "" + } + + if html != "" { + data := nlRe.ReplaceAllString(stripHTML(html), " ") + data = strings.TrimSpace(spaceRe.ReplaceAllString(data, " ")) + + if len(data) <= characters { + return data + } + + return data[0:characters] + "..." + } + + if text != "" { + text = spaceRe.ReplaceAllString(text, " ") + if len(text) <= characters { + return text + } + + return text[0:characters] + "..." + } + + return "" +} diff --git a/server/ui-src/assets/styles.scss b/server/ui-src/assets/styles.scss index cadff15..bbfadea 100644 --- a/server/ui-src/assets/styles.scss +++ b/server/ui-src/assets/styles.scss @@ -94,11 +94,33 @@ } .message { + .subject { + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 1; + overflow: hidden; + color: $text-muted; + + b { + color: $list-group-color; + } + + small { + opacity: 0.5; + } + } + &.read { color: $text-muted; b { + opacity: 0.7; font-weight: normal; + color: $list-group-color; + } + + small { + opacity: 0.5; } } &.selected { diff --git a/server/ui-src/components/ListMessages.vue b/server/ui-src/components/ListMessages.vue index 1d8c8e1..2028f18 100644 --- a/server/ui-src/components/ListMessages.vue +++ b/server/ui-src/components/ListMessages.vue @@ -141,7 +141,10 @@ export default {
-
{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}
+
+ {{ message.Subject != "" ? message.Subject : "[ no subject ]" }} +   {{ message.Snippet }} +