// Package dump is used to export all messages from mailpit into a directory package dump import ( "encoding/json" "errors" "io" "net/http" "os" "path/filepath" "regexp" "strconv" "strings" "time" "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/logger" "github.com/axllent/mailpit/internal/storage" "github.com/axllent/mailpit/internal/tools" "github.com/axllent/mailpit/server/apiv1" ) // httpClient bounds each remote request so a slow or hostile --http endpoint // cannot hang the dump indefinitely. var httpClient = &http.Client{Timeout: time.Minute} // maxSummarySize caps the bytes read from the remote messages-summary endpoint // to prevent a hostile server from exhausting memory via an unbounded response. const maxSummarySize = 20 * 1024 * 1024 // 20 MiB // pageSize is the per-request limit when paging through the remote messages // summary endpoint. const pageSize = 10000 var ( linkRe = regexp.MustCompile(`(?i)^https?:\/\/`) // idRe matches a valid Mailpit message ID (alphanumeric or dash, 8–60 chars). idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{8,60}$`) outDir string // Base URL of mailpit instance base string // URL is the base URL of a remove Mailpit instance URL string dumpIDs = make(map[string]struct { Timestamp time.Time }) ) // Sync will sync all messages from the specified database or API to the specified output directory func Sync(d string) error { outDir = filepath.Clean(d) if URL != "" { if !linkRe.MatchString(URL) { return errors.New("invalid URL") } base = strings.TrimRight(URL, "/") + "/" } if base == "" && config.Database == "" { return errors.New("no database or API URL specified") } if !tools.IsDir(outDir) { if err := os.MkdirAll(outDir, 0755); /* #nosec */ err != nil { return err } } if err := loadIDs(); err != nil { return err } if err := saveMessages(); err != nil { return err } return nil } // LoadIDs will load all message IDs from the specified database or API func loadIDs() error { if base != "" { // remote logger.Log().Debugf("Fetching messages summary from %s", base) start := 0 var total uint64 for { data, err := fetchSummaryPage(start) if err != nil { return err } if start == 0 { total = data.Total } for _, m := range data.Messages { dumpIDs[m.ID] = struct { Timestamp time.Time }{Timestamp: m.Created} } logger.Log().Debugf("Fetched messages summary page start=%d size=%d (%d/%d)", start, len(data.Messages), len(dumpIDs), total) // stop on empty page to guard against stale/inconsistent Total if len(data.Messages) == 0 { break } if uint64(len(dumpIDs)) >= total { break } start += pageSize } } else { // make sure the database isn't pruned while open config.MaxMessages = 0 // local database if err := storage.InitDB(); err != nil { return err } logger.Log().Debugf("Fetching messages summary from %s", config.Database) start := 0 for { page, err := storage.List(start, 0, pageSize) if err != nil { return err } for _, m := range page { dumpIDs[m.ID] = struct { Timestamp time.Time }{Timestamp: m.Created} } if len(page) < pageSize { break } start += pageSize } } if len(dumpIDs) == 0 { return errors.New("no messages found") } return nil } // fetchSummaryPage fetches a single page of the remote messages summary, // starting at the given offset. func fetchSummaryPage(start int) (*apiv1.MessagesSummary, error) { url := base + "api/v1/messages?limit=" + strconv.Itoa(pageSize) + "&start=" + strconv.Itoa(start) res, err := httpClient.Get(url) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return nil, errors.New("error fetching messages summary: HTTP " + res.Status) } body, err := io.ReadAll(io.LimitReader(res.Body, maxSummarySize+1)) if err != nil { return nil, err } if int64(len(body)) > maxSummarySize { return nil, errors.New("messages summary exceeds size cap") } var data apiv1.MessagesSummary if err := json.Unmarshal(body, &data); err != nil { return nil, err } return &data, nil } func saveMessages() error { for id, m := range dumpIDs { if !idRe.MatchString(id) { logger.Log().Errorf("skipping message with invalid ID: %q", id) continue } out := filepath.Join(outDir, id+".eml") // skip if message exists if tools.IsFile(out) { continue } var b []byte limit := int64(config.MaxMessageSize) * 1024 * 1024 if base != "" { res, err := httpClient.Get(base + "api/v1/message/" + id + "/raw") if err != nil { logger.Log().Errorf("error fetching message %s: %s", id, err.Error()) continue } if res.StatusCode != http.StatusOK { res.Body.Close() logger.Log().Errorf("error fetching message %s: HTTP %d", id, res.StatusCode) continue } if config.MaxMessageSize > 0 { b, err = io.ReadAll(io.LimitReader(res.Body, limit+1)) res.Body.Close() if err != nil { logger.Log().Errorf("error fetching message %s: %s", id, err.Error()) continue } if int64(len(b)) > limit { logger.Log().Warnf("message %s exceeds %d MiB size cap, skipping", id, config.MaxMessageSize) continue } } else { b, err = io.ReadAll(res.Body) res.Body.Close() if err != nil { logger.Log().Errorf("error fetching message %s: %s", id, err.Error()) continue } } } else { var err error b, err = storage.GetMessageRaw(id) if err != nil { logger.Log().Errorf("error fetching message %s: %s", id, err.Error()) continue } if config.MaxMessageSize > 0 && int64(len(b)) > limit { logger.Log().Warnf("message %s exceeds %d MiB size cap, skipping", id, config.MaxMessageSize) continue } } if err := os.WriteFile(out, b, 0644); /* #nosec */ err != nil { logger.Log().Errorf("error writing message %s: %s", id, err.Error()) continue } _ = os.Chtimes(out, m.Timestamp, m.Timestamp) logger.Log().Debugf("Saved message %s to %s", id, out) } return nil }