diff --git a/cmd/reindex.go b/cmd/reindex.go new file mode 100644 index 0000000..97ad990 --- /dev/null +++ b/cmd/reindex.go @@ -0,0 +1,40 @@ +/* +Copyright © 2022-Now() Ralph Slooten +This file is part of a CLI application. +*/ +package cmd + +import ( + "os" + + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/storage" + "github.com/spf13/cobra" +) + +// reindexCmd represents the reindex command +var reindexCmd = &cobra.Command{ + Use: "reindex ", + Short: "Reindex the database", + Long: `This will reindex all messages in the entire database. + +If you have several thousand messages in your mailbox, then it is advised to shut down +Mailpit while you reindex as this process will likely result in database locking issues.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + config.DataFile = args[0] + config.MaxMessages = 0 + + if err := storage.InitDB(); err != nil { + logger.Log().Error(err) + os.Exit(1) + } + + storage.ReindexAll() + }, +} + +func init() { + rootCmd.AddCommand(reindexCmd) +} diff --git a/internal/storage/reindex.go b/internal/storage/reindex.go new file mode 100644 index 0000000..7b14324 --- /dev/null +++ b/internal/storage/reindex.go @@ -0,0 +1,184 @@ +package storage + +import ( + "bytes" + "context" + "database/sql" + "os" + + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" + "github.com/jhillyerd/enmime" + "github.com/leporo/sqlf" +) + +// ReindexAll will regenerate the search text and snippet for a message +// and update the database. +func ReindexAll() { + ids := []string{} + var i string + chunkSize := 1000 + + finished := 0 + + err := sqlf.Select("ID").To(&i). + From("mailbox"). + OrderBy("Created DESC"). + QueryAndClose(nil, db, func(row *sql.Rows) { + ids = append(ids, i) + }) + + if err != nil { + logger.Log().Error(err) + os.Exit(1) + } + + total := len(ids) + + chunks := chunkBy(ids, chunkSize) + + logger.Log().Infof("Reindexing %d messages", total) + + // fmt.Println(len(ids), " = ", len(chunks), "chunks") + + type updateStruct struct { + ID string + SearchText string + Snippet string + } + + for _, ids := range chunks { + updates := []updateStruct{} + + for _, id := range ids { + raw, err := GetMessageRaw(id) + if err != nil { + logger.Log().Error(err) + continue + } + + r := bytes.NewReader(raw) + + env, err := enmime.ReadEnvelope(r) + if err != nil { + logger.Log().Error(err) + continue + } + + searchText := createSearchText(env) + snippet := tools.CreateSnippet(env.Text, env.HTML) + + u := updateStruct{} + u.ID = id + u.SearchText = searchText + u.Snippet = snippet + + updates = append(updates, u) + } + + ctx := context.Background() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + logger.Log().Error(err) + continue + } + + // roll back if it fails + defer tx.Rollback() + + // insert mail summary data + for _, u := range updates { + _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID) + if err != nil { + logger.Log().Error(err) + continue + } + } + + if err := tx.Commit(); err != nil { + logger.Log().Error(err) + continue + } + + finished += len(updates) + + logger.Log().Printf("Reindexed: %d / %d (%d%%)", finished, total, finished*100/total) + } +} + +// Reindex will regenerate the search text and snippet for a message +// and update the database. +func Reindex(id string) error { + // ids := []string{} + // var i string + // // chunkSize := 100 + + // err := sqlf.Select("ID").To(&i).From("mailbox_data").QueryAndClose(nil, db, func(row *sql.Rows) { + // ids = append(ids, id) + // }) + + // if err != nil { + // return err + // } + + // chunks := chunkBy(ids, 100) + + // fmt.Println(len(ids), " = ", len(chunks), "chunks") + + // return nil + + raw, err := GetMessageRaw(id) + if err != nil { + return err + } + + r := bytes.NewReader(raw) + + env, err := enmime.ReadEnvelope(r) + if err != nil { + return err + } + + searchText := createSearchText(env) + snippet := tools.CreateSnippet(env.Text, env.HTML) + + // return nil + + // ctx := context.Background() + // tx, err := db.BeginTx(ctx, nil) + // if err != nil { + // return err + // } + + // // roll back if it fails + // defer tx.Rollback() + + // // insert mail summary data + // _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", searchText, snippet, id) + // if err != nil { + // return err + // } + + // return tx.Commit() + + _, err = sqlf.Update("mailbox"). + Set("SearchText", searchText). + Set("Snippet", snippet). + Where("ID = ?", id). + ExecAndClose(context.Background(), db) + + return err +} + +// ctx := context.Background() +// tx, err := db.BeginTx(ctx, nil) +// if err != nil { +// return "", err +// } + +func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) { + for chunkSize < len(items) { + items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize]) + } + return append(chunks, items) +}