From 0af11fcb288df878800b052d11a2fe0670abd51f Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Tue, 2 Jan 2024 13:23:16 +1300 Subject: [PATCH] Chore: Include runtime statistics in API (info) & UI (About) Resolves #218 --- internal/stats/stats.go | 139 ++++++++++++++++++++++ internal/storage/database.go | 22 +--- internal/storage/tags.go | 41 +++++++ internal/storage/utils.go | 15 +++ server/apiv1/info.go | 45 +------ server/apiv1/swagger.go | 4 +- server/server.go | 2 + server/smtpd/smtpd.go | 12 +- server/ui-src/assets/_bootstrap.scss | 2 +- server/ui-src/components/AboutMailpit.vue | 128 +++++++++++++++----- server/ui-src/mixins/CommonMixins.js | 7 ++ server/ui/api/v1/swagger.json | 64 ++++++++-- 12 files changed, 377 insertions(+), 104 deletions(-) create mode 100644 internal/stats/stats.go diff --git a/internal/stats/stats.go b/internal/stats/stats.go new file mode 100644 index 0000000..e31ed07 --- /dev/null +++ b/internal/stats/stats.go @@ -0,0 +1,139 @@ +// Package stats stores and returns Mailpit statistics +package stats + +import ( + "os" + "runtime" + "sync" + "time" + + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/internal/storage" + "github.com/axllent/mailpit/internal/updater" +) + +var ( + // to prevent hammering Github for latest version + latestVersionCache string + + // StartedAt is set to the current ime when Mailpit starts + startedAt time.Time + + mu sync.RWMutex + + smtpReceived int + smtpReceivedSize int + smtpErrors int + smtpIgnored int +) + +// AppInformation struct +// swagger:model AppInformation +type AppInformation struct { + // Current Mailpit version + Version string + // Latest Mailpit version + LatestVersion string + // Database path + Database string + // Database size in bytes + DatabaseSize int64 + // Total number of messages in the database + Messages int + // Total number of messages in the database + Unread int + // Tags and message totals per tag + Tags map[string]int64 + // Runtime statistics + RuntimeStats struct { + // Mailpit server uptime in seconds + Uptime int + // Current memory usage in bytes + Memory uint64 + // Messages deleted + MessagesDeleted int + // SMTP messages received via since run + SMTPReceived int + // Total size in bytes of received messages since run + SMTPReceivedSize int + // SMTP errors since run + SMTPErrors int + // SMTP messages ignored since run (duplicate IDs) + SMTPIgnored int + } +} + +// Load the current statistics +func Load() AppInformation { + info := AppInformation{} + info.Version = config.Version + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + info.RuntimeStats.Memory = m.Sys - m.HeapReleased + + info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds()) + info.RuntimeStats.MessagesDeleted = storage.StatsDeleted + info.RuntimeStats.SMTPReceived = smtpReceived + info.RuntimeStats.SMTPReceivedSize = smtpReceivedSize + info.RuntimeStats.SMTPErrors = smtpErrors + info.RuntimeStats.SMTPIgnored = smtpIgnored + + if latestVersionCache != "" { + info.LatestVersion = latestVersionCache + } else { + latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName) + if err == nil { + info.LatestVersion = latest + latestVersionCache = latest + + // clear latest version cache after 5 minutes + go func() { + time.Sleep(5 * time.Minute) + latestVersionCache = "" + }() + } + } + + info.Database = config.DataFile + + db, err := os.Stat(info.Database) + if err == nil { + info.DatabaseSize = db.Size() + } + + info.Messages = storage.CountTotal() + info.Unread = storage.CountUnread() + + info.Tags = storage.GetAllTagsCount() + + return info +} + +// Track will start the statistics logging in memory +func Track() { + startedAt = time.Now() +} + +// LogSMTPReceived logs a successfully SMTP transaction +func LogSMTPReceived(size int) { + mu.Lock() + smtpReceived = smtpReceived + 1 + smtpReceivedSize = smtpReceivedSize + size + mu.Unlock() +} + +// LogSMTPError logs a failed SMTP transaction +func LogSMTPError() { + mu.Lock() + smtpErrors = smtpErrors + 1 + mu.Unlock() +} + +// LogSMTPIgnored logs an ignored SMTP transaction +func LogSMTPIgnored() { + mu.Lock() + smtpIgnored = smtpIgnored + 1 + mu.Unlock() +} diff --git a/internal/storage/database.go b/internal/storage/database.go index b3e3478..db469c0 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -628,6 +628,8 @@ func DeleteOneMessage(id string) error { dbLastAction = time.Now() dbDataDeleted = true + logMessagesDeleted(1) + BroadcastMailboxStats() return err @@ -684,6 +686,8 @@ func DeleteAllMessages() error { logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed) } + logMessagesDeleted(total) + dbLastAction = time.Now() dbDataDeleted = false @@ -693,24 +697,6 @@ func DeleteAllMessages() error { return err } -// GetAllTags returns all used tags -func GetAllTags() []string { - var tags = []string{} - var name string - - if err := sqlf. - Select(`DISTINCT Name`). - From("tags").To(&name). - OrderBy("Name"). - QueryAndClose(nil, db, func(row *sql.Rows) { - tags = append(tags, name) - }); err != nil { - logger.Log().Error(err) - } - - return tags -} - // StatsGet returns the total/unread statistics for a mailbox func StatsGet() MailboxStats { var ( diff --git a/internal/storage/tags.go b/internal/storage/tags.go index 0ccc2a9..8d3ef6b 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -137,6 +137,47 @@ func DeleteAllMessageTags(id string) error { return pruneUnusedTags() } +// GetAllTags returns all used tags +func GetAllTags() []string { + var tags = []string{} + var name string + + if err := sqlf. + Select(`DISTINCT Name`). + From("tags").To(&name). + OrderBy("Name"). + QueryAndClose(nil, db, func(row *sql.Rows) { + tags = append(tags, name) + }); err != nil { + logger.Log().Error(err) + } + + return tags +} + +// GetAllTagsCount returns all used tags with their total messages +func GetAllTagsCount() map[string]int64 { + var tags = make(map[string]int64) + var name string + var total int64 + + if err := sqlf. + Select(`Name`).To(&name). + Select(`COUNT(message_tags.TagID) as total`).To(&total). + From("tags"). + LeftJoin("message_tags", "tags.ID = message_tags.TagID"). + GroupBy("message_tags.TagID"). + OrderBy("Name"). + QueryAndClose(nil, db, func(row *sql.Rows) { + tags[name] = total + // tags = append(tags, name) + }); err != nil { + logger.Log().Error(err) + } + + return tags +} + // PruneUnusedTags will delete all unused tags from the database func pruneUnusedTags() error { q := sqlf.From("tags"). diff --git a/internal/storage/utils.go b/internal/storage/utils.go index a5ff20f..25d7aec 100644 --- a/internal/storage/utils.go +++ b/internal/storage/utils.go @@ -7,6 +7,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/axllent/mailpit/config" @@ -17,6 +18,13 @@ import ( "github.com/leporo/sqlf" ) +var ( + // for stats to prevent import cycle + mu sync.RWMutex + // StatsDeleted for counting the number of messages deleted + StatsDeleted int +) + // Return a header field as a []*mail.Address, or "null" is not found/empty func addressToSlice(env *enmime.Envelope, key string) []*mail.Address { data, err := env.AddressList(key) @@ -168,6 +176,13 @@ func dbCron() { } } +// LogMessagesDeleted logs the number of messages deleted +func logMessagesDeleted(n int) { + mu.Lock() + StatsDeleted = StatsDeleted + n + mu.Unlock() +} + // IsFile returns whether a path is a file func isFile(path string) bool { info, err := os.Stat(path) diff --git a/server/apiv1/info.go b/server/apiv1/info.go index 85afe9e..06bd9d9 100644 --- a/server/apiv1/info.go +++ b/server/apiv1/info.go @@ -3,32 +3,10 @@ package apiv1 import ( "encoding/json" "net/http" - "os" - "runtime" - "github.com/axllent/mailpit/config" - "github.com/axllent/mailpit/internal/storage" - "github.com/axllent/mailpit/internal/updater" + "github.com/axllent/mailpit/internal/stats" ) -// Response includes the current and latest Mailpit version, database info, and memory usage -// -// swagger:model AppInformation -type appInformation struct { - // Current Mailpit version - Version string - // Latest Mailpit version - LatestVersion string - // Database path - Database string - // Database size in bytes - DatabaseSize int64 - // Total number of messages in the database - Messages int - // Current memory usage in bytes - Memory uint64 -} - // AppInfo returns some basic details about the running app, and latest release. func AppInfo(w http.ResponseWriter, _ *http.Request) { // swagger:route GET /api/v1/info application AppInformation @@ -45,27 +23,8 @@ func AppInfo(w http.ResponseWriter, _ *http.Request) { // Responses: // 200: InfoResponse // default: ErrorResponse - info := appInformation{} - info.Version = config.Version - var m runtime.MemStats - runtime.ReadMemStats(&m) - - info.Memory = m.Sys - m.HeapReleased - - latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName) - if err == nil { - info.LatestVersion = latest - } - - info.Database = config.DataFile - - db, err := os.Stat(info.Database) - if err == nil { - info.DatabaseSize = db.Size() - } - - info.Messages = storage.CountTotal() + info := stats.Load() bytes, _ := json.Marshal(info) diff --git a/server/apiv1/swagger.go b/server/apiv1/swagger.go index 4e1df61..7bdd36a 100644 --- a/server/apiv1/swagger.go +++ b/server/apiv1/swagger.go @@ -1,5 +1,7 @@ package apiv1 +import "github.com/axllent/mailpit/internal/stats" + // These structs are for the purpose of defining swagger HTTP parameters & responses // Application information @@ -8,7 +10,7 @@ type infoResponse struct { // Application information // // in: body - Body appInformation + Body stats.AppInformation } // Web UI configuration diff --git a/server/server.go b/server/server.go index 85101ca..ac954f9 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/auth" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/stats" "github.com/axllent/mailpit/internal/storage" "github.com/axllent/mailpit/server/apiv1" "github.com/axllent/mailpit/server/handlers" @@ -34,6 +35,7 @@ var AccessControlAllowOrigin string func Listen() { isReady := &atomic.Value{} isReady.Store(false) + stats.Track() serverRoot, err := fs.Sub(embeddedFS, "ui") if err != nil { diff --git a/server/smtpd/smtpd.go b/server/smtpd/smtpd.go index 2a37582..ba09058 100644 --- a/server/smtpd/smtpd.go +++ b/server/smtpd/smtpd.go @@ -12,6 +12,7 @@ import ( "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/auth" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/stats" "github.com/axllent/mailpit/internal/storage" "github.com/google/uuid" "github.com/mhale/smtpd" @@ -27,7 +28,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { msg, err := mail.ReadMessage(bytes.NewReader(data)) if err != nil { logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error()) - + stats.LogSMTPError() return err } @@ -63,6 +64,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { } else if config.IgnoreDuplicateIDs { if storage.MessageIDExists(messageID) { logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID) + stats.LogSMTPIgnored() return nil } } @@ -116,13 +118,17 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", ")) } - _, err = storage.Store(data) + _, err = storage.Store(&data) if err != nil { logger.Log().Errorf("[db] error storing message: %s", err.Error()) - + stats.LogSMTPError() return err } + stats.LogSMTPReceived(len(data)) + + data = nil // avoid memory leaks + subject := msg.Header.Get("Subject") logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject) diff --git a/server/ui-src/assets/_bootstrap.scss b/server/ui-src/assets/_bootstrap.scss index 91c2bfa..47d6c73 100644 --- a/server/ui-src/assets/_bootstrap.scss +++ b/server/ui-src/assets/_bootstrap.scss @@ -16,7 +16,7 @@ @import "bootstrap/scss/images"; @import "bootstrap/scss/containers"; @import "bootstrap/scss/grid"; -// @import "bootstrap/scss/tables"; +@import "bootstrap/scss/tables"; @import "bootstrap/scss/forms"; @import "bootstrap/scss/buttons"; @import "bootstrap/scss/transitions"; diff --git a/server/ui-src/components/AboutMailpit.vue b/server/ui-src/components/AboutMailpit.vue index 4c6cdfc..06991ed 100644 --- a/server/ui-src/components/AboutMailpit.vue +++ b/server/ui-src/components/AboutMailpit.vue @@ -148,10 +148,11 @@ export default {