From 6a410a28b69fe2faf971ca159f74db3a98cd8d4d Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Tue, 9 Apr 2024 21:30:56 +1200 Subject: [PATCH] Feature: Add optional tenant ID to isolate data in shared databases (#254) --- cmd/readyz.go | 3 +- cmd/root.go | 14 +- config/config.go | 14 ++ go.mod | 3 - go.sum | 27 ---- internal/storage/cron.go | 8 +- internal/storage/database.go | 18 ++- internal/storage/messages.go | 74 +++++----- internal/storage/migrations.go | 189 ------------------------ internal/storage/reindex.go | 5 +- internal/storage/schemas.go | 222 +++++++++++++++++++++++++++++ internal/storage/schemas/1.0.0.sql | 19 +++ internal/storage/schemas/1.1.0.sql | 3 + internal/storage/schemas/1.2.0.sql | 36 +++++ internal/storage/schemas/1.3.0.sql | 2 + internal/storage/schemas/1.4.0.sql | 16 +++ internal/storage/schemas/1.5.0.sql | 7 + internal/storage/schemas/README.md | 5 + internal/storage/search.go | 16 +-- internal/storage/settings.go | 12 +- internal/storage/tags.go | 44 +++--- 21 files changed, 427 insertions(+), 310 deletions(-) delete mode 100644 internal/storage/migrations.go create mode 100644 internal/storage/schemas.go create mode 100644 internal/storage/schemas/1.0.0.sql create mode 100644 internal/storage/schemas/1.1.0.sql create mode 100644 internal/storage/schemas/1.2.0.sql create mode 100644 internal/storage/schemas/1.3.0.sql create mode 100644 internal/storage/schemas/1.4.0.sql create mode 100644 internal/storage/schemas/1.5.0.sql create mode 100644 internal/storage/schemas/README.md diff --git a/cmd/readyz.go b/cmd/readyz.go index d65a7a2..c680782 100644 --- a/cmd/readyz.go +++ b/cmd/readyz.go @@ -41,7 +41,8 @@ settings to determine the HTTP bind interface & port. IdleConnTimeout: time.Second * 5, ExpectContinueTimeout: time.Second * 5, TLSHandshakeTimeout: time.Second * 5, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + // do not verify TLS in case this instance is using HTTPS + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec } client := &http.Client{Transport: conf} diff --git a/cmd/root.go b/cmd/root.go index 792ce3b..d88fc54 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -81,6 +81,7 @@ func init() { initConfigFromEnv() rootCmd.Flags().StringVarP(&config.DataFile, "db-file", "d", config.DataFile, "Database file to store persistent data") + rootCmd.Flags().StringVar(&config.TenantID, "db-tenant-id", config.TenantID, "Database tenant ID to isolate data") rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store") rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates") rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)") @@ -155,7 +156,12 @@ func init() { // Load settings from environment func initConfigFromEnv() { // General - config.DataFile = os.Getenv("MP_DATA_FILE") + if len(os.Getenv("MP_DB_FILE")) > 0 { + config.DataFile = os.Getenv("MP_DB_FILE") + } + + config.TenantID = os.Getenv("MP_DB_TENANT") + if len(os.Getenv("MP_MAX_MESSAGES")) > 0 { config.MaxMessages, _ = strconv.Atoi(os.Getenv("MP_MAX_MESSAGES")) } @@ -289,6 +295,12 @@ func initConfigFromEnv() { // load deprecated settings from environment and warn func initDeprecatedConfigFromEnv() { + // deprecated 2024/04/08 + if len(os.Getenv("MP_DATA_FILE")) > 0 { + // do not warn - this will remain for quite some time + // logger.Log().Warn("ENV MP_DATA_FILE has been deprecated, use MP_DB_FILE") + config.DataFile = os.Getenv("MP_DATA_FILE") + } // deprecated 2023/03/12 if len(os.Getenv("MP_UI_SSL_CERT")) > 0 { logger.Log().Warn("ENV MP_UI_SSL_CERT has been deprecated, use MP_UI_TLS_CERT") diff --git a/config/config.go b/config/config.go index 5cdf58e..cfee030 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,10 @@ var ( // DataFile for mail (optional) DataFile string + // TenantID is an optional prefix to be applied to all database tables, + // allowing multiple isolated instances of Mailpit to share a database. + TenantID = "" + // MaxMessages is the maximum number of messages a mailbox can have (auto-pruned every minute) MaxMessages = 500 @@ -189,6 +193,16 @@ func VerifyConfig() error { DataFile = filepath.Join(DataFile, "mailpit.db") } + TenantID = strings.TrimSpace(TenantID) + if TenantID != "" { + logger.Log().Infof("[db] using tenant \"%s\"", TenantID) + re := regexp.MustCompile(`[^a-zA-Z0-9\_]`) + TenantID = re.ReplaceAllString(TenantID, "_") + if !strings.HasSuffix(TenantID, "_") { + TenantID = TenantID + "_" + } + } + re := regexp.MustCompile(`.*:\d+$`) if !re.MatchString(SMTPListen) { return errors.New("[smtp] bind should be in the format of :") diff --git a/go.mod b/go.mod index dbd8f41..d2dfa8b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/axllent/mailpit go 1.20 require ( - github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 github.com/PuerkitoBio/goquery v1.9.1 github.com/axllent/semver v0.0.1 github.com/disintegration/imaging v1.6.2 @@ -30,11 +29,9 @@ require ( ) require ( - github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect - github.com/cznic/ql v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index dfd85eb..9dde7a4 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= -github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 h1:dqzm54OhCqY8RinR/cx+Ppb0y56Ds5I3wwWhx4XybDg= -github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244/go.mod h1:3sqgkckuISJ5rs1EpOp6vCvwOUKe/z9vPmyuIlq8Q/A= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= @@ -16,26 +12,6 @@ github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4 h1:CVAqftqbj+exlab+8KJQrE+kNIVlQfJt58j4GxCMF1s= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00 h1:FHpbUtp2K8X53/b4aFNj4my5n+i3x+CQCZWNuHWH/+E= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= -github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs= -github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= -github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk= -github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/ql v1.2.0 h1:lcKp95ZtdF0XkWhGnVIXGF8dVD2X+ClS08tglKtf+ak= -github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc h1:YKKpTb2BrXN2GYyGaygIdis1vXbE7SSAG9axGWIMClg= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -43,13 +19,10 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8= -github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47 h1:k4Tw0nt6lwro3Uin8eqoET7MDA4JnT8YgbCjc/g5E3k= github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= diff --git a/internal/storage/cron.go b/internal/storage/cron.go index c0f170c..30d1c09 100644 --- a/internal/storage/cron.go +++ b/internal/storage/cron.go @@ -55,7 +55,7 @@ func pruneMessages() { start := time.Now() q := sqlf.Select("ID, Size"). - From("mailbox"). + From(tenant("mailbox")). OrderBy("Created DESC"). Limit(5000). Offset(config.MaxMessages) @@ -93,19 +93,19 @@ func pruneMessages() { args[i] = id } - _, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM `+tenant("mailbox_data")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } - _, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM `+tenant("message_tags")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } - _, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM `+tenant("mailbox")+` WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return diff --git a/internal/storage/database.go b/internal/storage/database.go index b04c891..863faa4 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -48,6 +48,7 @@ func InitDB() error { p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano()) dbIsTemp = true sqlDriver = "sqlite" + dsn = p logger.Log().Debugf("[db] using temporary database: %s", p) } else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") { sqlDriver = "rqlite" @@ -92,7 +93,7 @@ func InitDB() error { } // create tables if necessary & apply migrations - if err := dbApplyMigrations(); err != nil { + if err := dbApplySchemas(); err != nil { return err } @@ -121,6 +122,11 @@ func InitDB() error { return nil } +// Tenant applies an optional prefix to the table name +func tenant(table string) string { + return fmt.Sprintf("%s%s", config.TenantID, table) +} + // Close will close the database, and delete if a temporary table func Close() { if db != nil { @@ -163,7 +169,7 @@ func StatsGet() MailboxStats { func CountTotal() float64 { var total float64 - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&total). QueryRowAndClose(context.TODO(), db) @@ -174,7 +180,7 @@ func CountTotal() float64 { func CountUnread() float64 { var total float64 - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&total). Where("Read = ?", 0). QueryRowAndClose(context.TODO(), db) @@ -186,7 +192,7 @@ func CountUnread() float64 { func CountRead() float64 { var total float64 - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&total). Where("Read = ?", 1). QueryRowAndClose(context.TODO(), db) @@ -212,7 +218,7 @@ func DbSize() float64 { func IsUnread(id string) bool { var unread int - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&unread). Where("Read = ?", 0). Where("ID = ?", id). @@ -225,7 +231,7 @@ func IsUnread(id string) bool { func MessageIDExists(id string) bool { var total int - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&total). Where("MessageID = ?", id). QueryRowAndClose(context.TODO(), db) diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 5310a84..7fef1f7 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -95,9 +95,14 @@ func Store(body *[]byte) (string, error) { attachments := len(env.Attachments) snippet := tools.CreateSnippet(env.Text, env.HTML) + sql := fmt.Sprintf(`INSERT INTO %s + (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) + VALUES(?,?,?,?,?,?,?,?,?,0,?)`, + tenant("mailbox"), + ) // #nosec + // insert mail summary data - _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) VALUES(?,?,?,?,?,?,?,?,?,0,?)", - created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet) + _, err = tx.Exec(sql, created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet) if err != nil { return "", err } @@ -105,7 +110,7 @@ func Store(body *[]byte) (string, error) { // insert compressed raw message encoded := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size))) hexStr := hex.EncodeToString(encoded) - _, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) VALUES(?, x'"+hexStr+"')", id) + _, err = tx.Exec(fmt.Sprintf(`INSERT INTO %s (ID, Email) VALUES(?, x'%s')`, tenant("mailbox_data"), hexStr), id) // #nosec if err != nil { return "", err } @@ -151,7 +156,7 @@ func List(start, limit int) ([]MessageSummary, error) { results := []MessageSummary{} tsStart := time.Now() - q := sqlf.From("mailbox m"). + q := sqlf.From(tenant("mailbox") + " m"). Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet`). OrderBy("m.Created DESC"). Limit(limit). @@ -244,7 +249,7 @@ func GetMessage(id string) (*Message, error) { date, err := env.Date() if err != nil { // return received datetime when message does not contain a date header - q := sqlf.From("mailbox"). + q := sqlf.From(tenant("mailbox")). Select(`Created`). Where(`ID = ?`, id) @@ -330,7 +335,7 @@ func GetMessage(id string) (*Message, error) { func GetMessageRaw(id string) ([]byte, error) { var i string var msg string - q := sqlf.From("mailbox_data"). + q := sqlf.From(tenant("mailbox_data")). Select(`ID`).To(&i). Select(`Email`).To(&msg). Where(`ID = ?`, id) @@ -433,7 +438,7 @@ func MarkRead(id string) error { return nil } - _, err := sqlf.Update("mailbox"). + _, err := sqlf.Update(tenant("mailbox")). Set("Read", 1). Where("ID = ?", id). ExecAndClose(context.Background(), db) @@ -454,7 +459,7 @@ func MarkAllRead() error { total = CountUnread() ) - _, err := sqlf.Update("mailbox"). + _, err := sqlf.Update(tenant("mailbox")). Set("Read", 1). Where("Read = ?", 0). ExecAndClose(context.Background(), db) @@ -479,7 +484,7 @@ func MarkAllUnread() error { total = CountRead() ) - _, err := sqlf.Update("mailbox"). + _, err := sqlf.Update(tenant("mailbox")). Set("Read", 0). Where("Read = ?", 1). ExecAndClose(context.Background(), db) @@ -503,7 +508,7 @@ func MarkUnread(id string) error { return nil } - _, err := sqlf.Update("mailbox"). + _, err := sqlf.Update(tenant("mailbox")). Set("Read", 0). Where("ID = ?", id). ExecAndClose(context.Background(), db) @@ -532,7 +537,8 @@ func DeleteMessages(ids []string) error { args[i] = id } - rows, err := db.Query(`SELECT ID, Size FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(args)-1)+`)`, args...) + sql := fmt.Sprintf(`SELECT ID, Size FROM %s WHERE ID IN (?%s)`, tenant("mailbox"), strings.Repeat(",?", len(args)-1)) // #nosec + rows, err := db.Query(sql, args...) if err != nil { return err } @@ -569,19 +575,15 @@ func DeleteMessages(ids []string) error { args[i] = id } - _, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec - if err != nil { - return err - } + tables := []string{"mailbox", "mailbox_data", "message_tags"} - _, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec - if err != nil { - return err - } + for _, t := range tables { + sql = fmt.Sprintf(`DELETE FROM %s WHERE ID IN (?%s)`, tenant(t), strings.Repeat(",?", len(ids)-1)) - _, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec - if err != nil { - return err + _, err = tx.Exec(sql, args...) // #nosec + if err != nil { + return err + } } err = tx.Commit() @@ -591,7 +593,7 @@ func DeleteMessages(ids []string) error { logMessagesDeleted(len(toDelete)) - pruneUnusedTags() + _ = pruneUnusedTags() elapsed := time.Since(start) @@ -614,7 +616,7 @@ func DeleteAllMessages() error { total int ) - _ = sqlf.From("mailbox"). + _ = sqlf.From(tenant("mailbox")). Select("COUNT(*)").To(&total). QueryRowAndClose(context.TODO(), db) @@ -628,24 +630,14 @@ func DeleteAllMessages() error { // roll back if it fails defer tx.Rollback() - _, err = tx.Exec("DELETE FROM mailbox") - if err != nil { - return err - } + tables := []string{"mailbox", "mailbox_data", "tags", "message_tags"} - _, err = tx.Exec("DELETE FROM mailbox_data") - if err != nil { - return err - } - - _, err = tx.Exec("DELETE FROM tags") - if err != nil { - return err - } - - _, err = tx.Exec("DELETE FROM message_tags") - if err != nil { - return err + for _, t := range tables { + sql := fmt.Sprintf(`DELETE FROM %s`, tenant(t)) // #nosec + _, err := tx.Exec(sql) + if err != nil { + return err + } } if err := tx.Commit(); err != nil { diff --git a/internal/storage/migrations.go b/internal/storage/migrations.go deleted file mode 100644 index a6e6087..0000000 --- a/internal/storage/migrations.go +++ /dev/null @@ -1,189 +0,0 @@ -package storage - -import ( - "context" - "database/sql" - "encoding/json" - - "github.com/GuiaBolso/darwin" - "github.com/axllent/mailpit/internal/logger" - "github.com/leporo/sqlf" -) - -var ( - dbMigrations = []darwin.Migration{ - { - Version: 1.0, - Description: "Creating tables", - Script: `CREATE TABLE IF NOT EXISTS mailbox ( - Sort INTEGER PRIMARY KEY AUTOINCREMENT, - ID TEXT NOT NULL, - Data BLOB, - Search TEXT, - Read INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - - CREATE TABLE IF NOT EXISTS mailbox_data ( - ID TEXT KEY NOT NULL, - Email BLOB - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`, - }, - { - Version: 1.1, - Description: "Create tags column", - Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - { - Version: 1.2, - Description: "Creating new mailbox format", - Script: `CREATE TABLE IF NOT EXISTS mailboxtmp ( - Created INTEGER NOT NULL, - ID TEXT NOT NULL, - MessageID TEXT NOT NULL, - Subject TEXT NOT NULL, - Metadata TEXT, - Size INTEGER NOT NULL, - Inline INTEGER NOT NULL, - Attachments INTEGER NOT NULL, - Read INTEGER, - Tags TEXT, - SearchText TEXT - ); - INSERT INTO mailboxtmp - (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) - SELECT - Sort, ID, '', json_extract(Data, '$.Subject'),Data, - json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), - Search, Read, Tags - FROM mailbox; - - DROP TABLE IF EXISTS mailbox; - ALTER TABLE mailboxtmp RENAME TO mailbox; - CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID); - CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject); - CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size); - CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline); - CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - { - Version: 1.3, - Description: "Create snippet column", - Script: `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`, - }, - { - Version: 1.4, - Description: "Create tag tables", - Script: `CREATE TABLE IF NOT EXISTS tags ( - ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - Name TEXT COLLATE NOCASE - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_tag_name ON tags (Name); - - CREATE TABLE IF NOT EXISTS message_tags( - Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - ID TEXT REFERENCES mailbox(ID), - TagID INT REFERENCES tags(ID) - ); - CREATE INDEX IF NOT EXISTS idx_message_tag_id ON message_tags (ID); - CREATE INDEX IF NOT EXISTS idx_message_tag_tagid ON message_tags (TagID);`, - }, - { - // assume deleted messages account for 50% of storage - // to handle previously-deleted messages - Version: 1.5, - Description: "Create settings table", - Script: `CREATE TABLE IF NOT EXISTS settings ( - Key TEXT, - Value TEXT - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_settings_key ON settings (Key); - INSERT INTO settings (Key, Value) VALUES("DeletedSize", (SELECT SUM(Size)/2 FROM mailbox));`, - }, - } -) - -// Create tables and apply migrations if required -func dbApplyMigrations() error { - driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{}) - - d := darwin.New(driver, dbMigrations, nil) - - return d.Migrate() -} - -// These functions are used to migrate data formats/structure on startup. -func dataMigrations() { - // ensure DeletedSize has a value if empty - if SettingGet("DeletedSize") == "" { - _ = SettingPut("DeletedSize", "0") - } - - migrateTagsToManyMany() -} - -// Migrate tags to ManyMany structure -// Migration task implemented 12/2023 -// Can be removed end 06/2024 and Tags column & index dropped from mailbox -func migrateTagsToManyMany() { - toConvert := make(map[string][]string) - q := sqlf. - Select("ID, Tags"). - From("mailbox"). - Where("Tags != ?", "[]"). - Where("Tags IS NOT NULL") - - if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { - var id string - var jsonTags string - if err := row.Scan(&id, &jsonTags); err != nil { - logger.Log().Errorf("[migration] %s", err.Error()) - return - } - - tags := []string{} - - if err := json.Unmarshal([]byte(jsonTags), &tags); err != nil { - logger.Log().Errorf("[json] %s", err.Error()) - return - } - - toConvert[id] = tags - }); err != nil { - logger.Log().Errorf("[migration] %s", err.Error()) - } - - if len(toConvert) > 0 { - logger.Log().Infof("[migration] converting %d message tags", len(toConvert)) - for id, tags := range toConvert { - if err := SetMessageTags(id, tags); err != nil { - logger.Log().Errorf("[migration] %s", err.Error()) - } else { - if _, err := sqlf.Update("mailbox"). - Set("Tags", nil). - Where("ID = ?", id). - ExecAndClose(context.TODO(), db); err != nil { - logger.Log().Errorf("[migration] %s", err.Error()) - } - } - } - - logger.Log().Info("[migration] tags conversion complete") - } - - // set all legacy `[]` tags to NULL - if _, err := sqlf.Update("mailbox"). - Set("Tags", nil). - Where("Tags = ?", "[]"). - ExecAndClose(context.TODO(), db); err != nil { - logger.Log().Errorf("[migration] %s", err.Error()) - } -} diff --git a/internal/storage/reindex.go b/internal/storage/reindex.go index 8f1a3c4..8d040d2 100644 --- a/internal/storage/reindex.go +++ b/internal/storage/reindex.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "net/mail" "os" @@ -24,7 +25,7 @@ func ReindexAll() { finished := 0 err := sqlf.Select("ID").To(&i). - From("mailbox"). + From(tenant("mailbox")). OrderBy("Created DESC"). QueryAndClose(context.TODO(), db, func(row *sql.Rows) { ids = append(ids, i) @@ -112,7 +113,7 @@ func ReindexAll() { // insert mail summary data for _, u := range updates { - _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ?, Metadata = ? WHERE ID = ?", u.SearchText, u.Snippet, u.Metadata, u.ID) + _, err = tx.Exec(fmt.Sprintf(`UPDATE %s SET SearchText = ?, Snippet = ?, Metadata = ? WHERE ID = ?`, tenant("mailbox")), u.SearchText, u.Snippet, u.Metadata, u.ID) if err != nil { logger.Log().Errorf("[db] %s", err.Error()) continue diff --git a/internal/storage/schemas.go b/internal/storage/schemas.go new file mode 100644 index 0000000..a7c6149 --- /dev/null +++ b/internal/storage/schemas.go @@ -0,0 +1,222 @@ +package storage + +import ( + "bytes" + "context" + "database/sql" + "embed" + "encoding/json" + "log" + "path/filepath" + "sort" + "strings" + "text/template" + "time" + + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/semver" + "github.com/leporo/sqlf" +) + +//go:embed schemas/* +var schemaScripts embed.FS + +// Create tables and apply schemas if required +func dbApplySchemas() error { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS ` + tenant("schemas") + ` (Version TEXT PRIMARY KEY NOT NULL)`); err != nil { + return err + } + + var legacyMigrationTable int + err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)`, tenant("darwin_migrations")).Scan(&legacyMigrationTable) + if err != nil { + return err + } + + if legacyMigrationTable == 1 { + rows, err := db.Query(`SELECT version FROM ` + tenant("darwin_migrations")) + if err != nil { + return err + } + + legacySchemas := []string{} + + for rows.Next() { + var oldID string + if err := rows.Scan(&oldID); err == nil { + legacySchemas = append(legacySchemas, semver.MajorMinor(oldID)+"."+semver.Patch(oldID)) + } + } + + legacySchemas = semver.SortMin(legacySchemas) + + for _, v := range legacySchemas { + var migrated int + err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, v).Scan(&migrated) + if err != nil { + return err + } + if migrated == 0 { + // copy to tenant("schemas") + if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, v); err != nil { + return err + } + } + } + + // delete legacy migration database after 01/10/2024 + if time.Now().After(time.Date(2024, 10, 1, 0, 0, 0, 0, time.Local)) { + if _, err := db.Exec(`DROP TABLE IF EXISTS ` + tenant("darwin_migrations")); err != nil { + return err + } + } + } + + schemaFiles, err := schemaScripts.ReadDir("schemas") + if err != nil { + log.Fatal(err) + } + + temp := template.New("") + temp.Funcs( + template.FuncMap{ + "tenant": tenant, + }, + ) + + type schema struct { + Name string + Semver string + } + + scripts := []schema{} + + for _, s := range schemaFiles { + if !s.Type().IsRegular() || !strings.HasSuffix(s.Name(), ".sql") { + continue + } + + schemaID := strings.TrimRight(s.Name(), ".sql") + + if !semver.IsValid(schemaID) { + logger.Log().Warnf("[db] invalid schema name: %s", s.Name()) + continue + } + + s := schema{s.Name(), semver.MajorMinor(schemaID) + "." + semver.Patch(schemaID)} + scripts = append(scripts, s) + } + + // sort schemas by semver, low to high + sort.Slice(scripts, func(i, j int) bool { + return semver.Compare(scripts[j].Semver, scripts[i].Semver) == 1 + }) + + for _, s := range scripts { + var complete int + err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM `+tenant("schemas")+` WHERE Version = ?)`, s.Semver).Scan(&complete) + if err != nil { + return err + } + + if complete == 1 { + // already completed, ignore + continue + } + + b, err := schemaScripts.ReadFile(filepath.Join("schemas", s.Name)) + if err != nil { + return err + } + + // parse import script + t1, err := temp.Parse(string(b)) + if err != nil { + return err + } + + buf := new(bytes.Buffer) + + err = t1.Execute(buf, nil) + + if _, err := db.Exec(buf.String()); err != nil { + return err + } + + if _, err := db.Exec(`INSERT INTO `+tenant("schemas")+` (Version) VALUES (?)`, s.Semver); err != nil { + return err + } + + logger.Log().Debugf("[db] applied schema: %s", s.Name) + } + + return nil +} + +// These functions are used to migrate data formats/structure on startup. +func dataMigrations() { + // ensure DeletedSize has a value if empty + if SettingGet("DeletedSize") == "" { + _ = SettingPut("DeletedSize", "0") + } + + migrateTagsToManyMany() +} + +// Migrate tags to ManyMany structure +// Migration task implemented 12/2023 +// TODO: Can be removed end 06/2024 and Tags column & index dropped from mailbox +func migrateTagsToManyMany() { + toConvert := make(map[string][]string) + q := sqlf. + Select("ID, Tags"). + From(tenant("mailbox")). + Where("Tags != ?", "[]"). + Where("Tags IS NOT NULL") + + if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { + var id string + var jsonTags string + if err := row.Scan(&id, &jsonTags); err != nil { + logger.Log().Errorf("[migration] %s", err.Error()) + return + } + + tags := []string{} + + if err := json.Unmarshal([]byte(jsonTags), &tags); err != nil { + logger.Log().Errorf("[json] %s", err.Error()) + return + } + + toConvert[id] = tags + }); err != nil { + logger.Log().Errorf("[migration] %s", err.Error()) + } + + if len(toConvert) > 0 { + logger.Log().Infof("[migration] converting %d message tags", len(toConvert)) + for id, tags := range toConvert { + if err := SetMessageTags(id, tags); err != nil { + logger.Log().Errorf("[migration] %s", err.Error()) + } else { + if _, err := sqlf.Update(tenant("mailbox")). + Set("Tags", nil). + Where("ID = ?", id). + ExecAndClose(context.TODO(), db); err != nil { + logger.Log().Errorf("[migration] %s", err.Error()) + } + } + } + + logger.Log().Info("[migration] tags conversion complete") + } + + // set all legacy `[]` tags to NULL + if _, err := sqlf.Update(tenant("mailbox")). + Set("Tags", nil). + Where("Tags = ?", "[]"). + ExecAndClose(context.TODO(), db); err != nil { + logger.Log().Errorf("[migration] %s", err.Error()) + } +} diff --git a/internal/storage/schemas/1.0.0.sql b/internal/storage/schemas/1.0.0.sql new file mode 100644 index 0000000..07f69fd --- /dev/null +++ b/internal/storage/schemas/1.0.0.sql @@ -0,0 +1,19 @@ +-- CREATE TABLES +CREATE TABLE IF NOT EXISTS {{ tenant "mailbox" }} ( + Sort INTEGER PRIMARY KEY AUTOINCREMENT, + ID TEXT NOT NULL, + Data BLOB, + Search TEXT, + Read INTEGER +); + +CREATE INDEX IF NOT EXISTS {{ tenant "idx_sort" }} ON {{ tenant "mailbox" }} (Sort); +CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read); + +CREATE TABLE IF NOT EXISTS {{ tenant "mailbox_data" }} ( + ID TEXT KEY NOT NULL, + Email BLOB +); + +CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_data_id" }} ON {{ tenant "mailbox_data" }} (ID); diff --git a/internal/storage/schemas/1.1.0.sql b/internal/storage/schemas/1.1.0.sql new file mode 100644 index 0000000..8b2134d --- /dev/null +++ b/internal/storage/schemas/1.1.0.sql @@ -0,0 +1,3 @@ +-- CREATE TAGS COLUMN +ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; +CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags); diff --git a/internal/storage/schemas/1.2.0.sql b/internal/storage/schemas/1.2.0.sql new file mode 100644 index 0000000..a989546 --- /dev/null +++ b/internal/storage/schemas/1.2.0.sql @@ -0,0 +1,36 @@ +-- CREATING NEW MAILBOX FORMAT +CREATE TABLE IF NOT EXISTS {{ tenant "mailboxtmp" }} ( + Created INTEGER NOT NULL, + ID TEXT NOT NULL, + MessageID TEXT NOT NULL, + Subject TEXT NOT NULL, + Metadata TEXT, + Size INTEGER NOT NULL, + Inline INTEGER NOT NULL, + Attachments INTEGER NOT NULL, + Read INTEGER, + Tags TEXT, + SearchText TEXT +); + +INSERT INTO {{ tenant "mailboxtmp" }} + (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) + SELECT + Sort, ID, '', json_extract(Data, '$.Subject'),Data, + json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), + Search, Read, Tags + FROM {{ tenant "mailbox" }}; + +DROP TABLE IF EXISTS {{ tenant "mailbox" }}; + +ALTER TABLE {{ tenant "mailboxtmp" }} RENAME TO {{ tenant "mailbox" }}; + +CREATE INDEX IF NOT EXISTS {{ tenant "idx_created" }} ON {{ tenant "mailbox" }} (Created); +CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_id" }} ON {{ tenant "mailbox" }} (ID); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_id" }} ON {{ tenant "mailbox" }} (MessageID); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_subject" }} ON {{ tenant "mailbox" }} (Subject); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_size" }} ON {{ tenant "mailbox" }} (Size); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_inline" }} ON {{ tenant "mailbox" }} (Inline); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_attachments" }} ON {{ tenant "mailbox" }} (Attachments); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_read" }} ON {{ tenant "mailbox" }} (Read); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_tags" }} ON {{ tenant "mailbox" }} (Tags); diff --git a/internal/storage/schemas/1.3.0.sql b/internal/storage/schemas/1.3.0.sql new file mode 100644 index 0000000..b0064d2 --- /dev/null +++ b/internal/storage/schemas/1.3.0.sql @@ -0,0 +1,2 @@ +-- CREATE SNIPPET COLUMN +ALTER TABLE {{ tenant "mailbox" }} ADD COLUMN Snippet TEXT NOT NULL DEFAULT ''; diff --git a/internal/storage/schemas/1.4.0.sql b/internal/storage/schemas/1.4.0.sql new file mode 100644 index 0000000..423269a --- /dev/null +++ b/internal/storage/schemas/1.4.0.sql @@ -0,0 +1,16 @@ +-- CREATE TAG TABLES +CREATE TABLE IF NOT EXISTS {{ tenant "tags" }} ( + ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + Name TEXT COLLATE NOCASE +); + +CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_tag_name" }} ON {{ tenant "tags" }} (Name); + +CREATE TABLE IF NOT EXISTS {{ tenant "message_tags" }} ( + Key INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + ID TEXT REFERENCES {{ tenant "mailbox" }} (ID), + TagID INT REFERENCES {{ tenant "tags" }} (ID) +); + +CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_id" }} ON {{ tenant "message_tags" }} (ID); +CREATE INDEX IF NOT EXISTS {{ tenant "idx_message_tag_tagid" }} ON {{ tenant "message_tags" }} (TagID); diff --git a/internal/storage/schemas/1.5.0.sql b/internal/storage/schemas/1.5.0.sql new file mode 100644 index 0000000..d338eb5 --- /dev/null +++ b/internal/storage/schemas/1.5.0.sql @@ -0,0 +1,7 @@ +-- CREATE SETTINGS TABLE +CREATE TABLE IF NOT EXISTS {{ tenant "settings" }} ( + Key TEXT, + Value TEXT +); +CREATE UNIQUE INDEX IF NOT EXISTS {{ tenant "idx_settings_key" }} ON {{ tenant "settings" }} (Key); +INSERT INTO {{ tenant "settings" }} (Key, Value) VALUES ("DeletedSize", (SELECT SUM(Size)/2 FROM {{ tenant "mailbox" }})); diff --git a/internal/storage/schemas/README.md b/internal/storage/schemas/README.md new file mode 100644 index 0000000..d926069 --- /dev/null +++ b/internal/storage/schemas/README.md @@ -0,0 +1,5 @@ +# Migration scripts + +- Scripts should be named using semver and have the `.sql` extension. +- Inline comments should be prefixed with a `--` +- All references to tables and indexes should be wrapped with a `{{ tenant "" }}` diff --git a/internal/storage/search.go b/internal/storage/search.go index 30e67ff..b9ce2cc 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -159,21 +159,21 @@ func DeleteSearch(search string) error { delIDs[i] = id } - sqlDelete1 := `DELETE FROM mailbox WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec + sqlDelete1 := `DELETE FROM ` + tenant("mailbox") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete1, delIDs...) if err != nil { return err } - sqlDelete2 := `DELETE FROM mailbox_data WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec + sqlDelete2 := `DELETE FROM ` + tenant("mailbox_data") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete2, delIDs...) if err != nil { return err } - sqlDelete3 := `DELETE FROM message_tags WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec + sqlDelete3 := `DELETE FROM ` + tenant("message_tags") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete3, delIDs...) if err != nil { @@ -207,7 +207,7 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt { // group strings with quotes as a single argument and remove quotes args := tools.ArgsParser(searchString) - q := sqlf.From("mailbox m"). + q := sqlf.From(tenant("mailbox") + " m"). Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet, IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON, @@ -306,9 +306,9 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt { w = cleanString(w[4:]) if w != "" { if exclude { - q.Where(`m.ID NOT IN (SELECT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) + q.Where(`m.ID NOT IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) } else { - q.Where(`m.ID IN (SELECT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) + q.Where(`m.ID IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) } } } else if lw == "is:read" { @@ -325,9 +325,9 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt { } } else if lw == "is:tagged" { if exclude { - q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID)`) + q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`) } else { - q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM message_tags mt JOIN tags t ON mt.TagID = t.ID)`) + q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`) } } else if lw == "has:attachment" || lw == "has:attachments" { if exclude { diff --git a/internal/storage/settings.go b/internal/storage/settings.go index 02f91a9..6f1754f 100644 --- a/internal/storage/settings.go +++ b/internal/storage/settings.go @@ -11,7 +11,7 @@ import ( // SettingGet returns a setting string value, blank is it does not exist func SettingGet(k string) string { var result sql.NullString - err := sqlf.From("settings"). + err := sqlf.From(tenant("settings")). Select("Value").To(&result). Where("Key = ?", k). Limit(1). @@ -26,7 +26,7 @@ func SettingGet(k string) string { // SettingPut sets a setting string value, inserting if new func SettingPut(k, v string) error { - _, err := db.Exec("INSERT INTO settings (Key, Value) VALUES(?, ?) ON CONFLICT(Key) DO UPDATE SET Value = ?", k, v, v) + _, err := db.Exec(`INSERT INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?) ON CONFLICT(Key) DO UPDATE SET Value = ?`, k, v, v) if err != nil { logger.Log().Errorf("[db] %s", err.Error()) } @@ -37,7 +37,7 @@ func SettingPut(k, v string) error { // The total deleted message size as an int64 value func getDeletedSize() float64 { var result sql.NullFloat64 - err := sqlf.From("settings"). + err := sqlf.From(tenant("settings")). Select("Value").To(&result). Where("Key = ?", "DeletedSize"). Limit(1). @@ -53,7 +53,7 @@ func getDeletedSize() float64 { // The total raw non-compressed messages size in bytes of all messages in the database func totalMessagesSize() float64 { var result sql.NullFloat64 - err := sqlf.From("mailbox"). + err := sqlf.From(tenant("mailbox")). Select("SUM(Size)").To(&result). QueryAndClose(context.TODO(), db, func(row *sql.Rows) {}) if err != nil { @@ -66,11 +66,11 @@ func totalMessagesSize() float64 { // AddDeletedSize will add the value to the DeletedSize setting func addDeletedSize(v int64) { - if _, err := db.Exec("INSERT OR IGNORE INTO settings (Key, Value) VALUES(?, ?)", "DeletedSize", 0); err != nil { + if _, err := db.Exec(`INSERT OR IGNORE INTO `+tenant("settings")+` (Key, Value) VALUES(?, ?)`, "DeletedSize", 0); err != nil { logger.Log().Errorf("[db] %s", err.Error()) } - if _, err := db.Exec("UPDATE settings SET Value = Value + ? WHERE Key = ?", v, "DeletedSize"); err != nil { + if _, err := db.Exec(`UPDATE `+tenant("settings")+` SET Value = Value + ? WHERE Key = ?`, v, "DeletedSize"); err != nil { logger.Log().Errorf("[db] %s", err.Error()) } } diff --git a/internal/storage/tags.go b/internal/storage/tags.go index bf03a7f..fbafbe5 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -60,7 +60,7 @@ func SetMessageTags(id string, tags []string) error { func AddMessageTag(id, name string) error { var tagID int - q := sqlf.From("tags"). + q := sqlf.From(tenant("tags")). Select("ID").To(&tagID). Where("Name = ?", name) @@ -68,7 +68,7 @@ func AddMessageTag(id, name string) error { if err := q.QueryRowAndClose(context.TODO(), db); err == nil { // check message does not already have this tag var count int - if _, err := sqlf.From("message_tags"). + if _, err := sqlf.From(tenant("message_tags")). Select("COUNT(ID)").To(&count). Where("ID = ?", id). Where("TagID = ?", tagID). @@ -82,7 +82,7 @@ func AddMessageTag(id, name string) error { logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id) - _, err := sqlf.InsertInto("message_tags"). + _, err := sqlf.InsertInto(tenant("message_tags")). Set("ID", id). Set("TagID", tagID). ExecAndClose(context.TODO(), db) @@ -92,7 +92,7 @@ func AddMessageTag(id, name string) error { logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id) // tag dos not exist, add new one - if _, err := sqlf.InsertInto("tags"). + if _, err := sqlf.InsertInto(tenant("tags")). Set("Name", name). ExecAndClose(context.TODO(), db); err != nil { return err @@ -103,9 +103,9 @@ func AddMessageTag(id, name string) error { // DeleteMessageTag deleted a tag from a message func DeleteMessageTag(id, name string) error { - if _, err := sqlf.DeleteFrom("message_tags"). - Where("message_tags.ID = ?", id). - Where(`message_tags.Key IN (SELECT Key FROM message_tags LEFT JOIN tags ON TagID=tags.ID WHERE Name = ?)`, name). + if _, err := sqlf.DeleteFrom(tenant("message_tags")). + Where(tenant("message_tags.ID")+" = ?", id). + Where(tenant("message_tags.Key")+` IN (SELECT Key FROM `+tenant("message_tags")+` LEFT JOIN tags ON `+tenant("TagID")+"="+tenant("tags.ID")+` WHERE Name = ?)`, name). ExecAndClose(context.TODO(), db); err != nil { return err } @@ -115,8 +115,8 @@ func DeleteMessageTag(id, name string) error { // DeleteAllMessageTags deleted all tags from a message func DeleteAllMessageTags(id string) error { - if _, err := sqlf.DeleteFrom("message_tags"). - Where("message_tags.ID = ?", id). + if _, err := sqlf.DeleteFrom(tenant("message_tags")). + Where(tenant("message_tags.ID")+" = ?", id). ExecAndClose(context.TODO(), db); err != nil { return err } @@ -131,7 +131,7 @@ func GetAllTags() []string { if err := sqlf. Select(`DISTINCT Name`). - From("tags").To(&name). + From(tenant("tags")).To(&name). OrderBy("Name"). QueryAndClose(context.TODO(), db, func(row *sql.Rows) { tags = append(tags, name) @@ -150,10 +150,10 @@ func GetAllTagsCount() map[string]int64 { if err := sqlf. Select(`Name`).To(&name). - Select(`COUNT(message_tags.TagID) as total`).To(&total). - From("tags"). - LeftJoin("message_tags", "tags.ID = message_tags.TagID"). - GroupBy("message_tags.TagID"). + Select(`COUNT(`+tenant("message_tags.TagID")+`) as total`).To(&total). + From(tenant("tags")). + LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")). + GroupBy(tenant("message_tags.TagID")). OrderBy("Name"). QueryAndClose(context.TODO(), db, func(row *sql.Rows) { tags[name] = total @@ -167,10 +167,10 @@ func GetAllTagsCount() map[string]int64 { // PruneUnusedTags will delete all unused tags from the database func pruneUnusedTags() error { - q := sqlf.From("tags"). - Select("tags.ID, tags.Name, COUNT(message_tags.ID) as COUNT"). - LeftJoin("message_tags", "tags.ID = message_tags.TagID"). - GroupBy("tags.ID") + q := sqlf.From(tenant("tags")). + Select(tenant("tags.ID")+", "+tenant("tags.Name")+", COUNT("+tenant("message_tags.ID")+") as COUNT"). + LeftJoin(tenant("message_tags"), tenant("tags.ID")+" = "+tenant("message_tags.TagID")). + GroupBy(tenant("tags.ID")) toDel := []int{} @@ -194,7 +194,7 @@ func pruneUnusedTags() error { if len(toDel) > 0 { for _, id := range toDel { - if _, err := sqlf.DeleteFrom("tags"). + if _, err := sqlf.DeleteFrom(tenant("tags")). Where("ID = ?", id). ExecAndClose(context.TODO(), db); err != nil { return err @@ -260,9 +260,9 @@ func getMessageTags(id string) []string { if err := sqlf. Select(`Name`).To(&name). - From("Tags"). - LeftJoin("message_tags", "Tags.ID=message_tags.TagID"). - Where(`message_tags.ID = ?`, id). + From(tenant("Tags")). + LeftJoin(tenant("message_tags"), tenant("Tags.ID")+"="+tenant("message_tags.TagID")). + Where(tenant("message_tags.ID")+` = ?`, id). OrderBy("Name"). QueryAndClose(context.TODO(), db, func(row *sql.Rows) { tags = append(tags, name)