diff --git a/.gitignore b/.gitignore
index 8148291..7ad41a5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -62,6 +62,7 @@ backup/
htmlcov/
.tox/
docker-compose-dev.yml
+Dev/
# Documentation
docs/_build/
@@ -76,4 +77,5 @@ Thumbs.db
.DS_Store
# Maxmind
-data
\ No newline at end of file
+data
+old-data
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f96f32e..980a6ed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,205 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.1.0] - 2026-01-20
+
+### Added
+
+#### Mailbox Statistics Page
+- **Complete Mailbox Statistics Feature**: New page showing per-mailbox message statistics
+ - Summary cards: Total Sent, Received, Failed, and Failure Rate
+ - Accordion-style mailbox list with expandable details
+ - Message counts aggregated from MessageCorrelation table (not Mailcow API)
+ - Per-alias message statistics with sent/received/failed counts
+ - Combined totals (mailbox + all its aliases)
+
+- **Clickable Statistics Links**: All stat cards and alias table cells are clickable
+ - Click on any stat (Sent, Received, Internal, Delivered, Deferred, Bounced, Rejected) to navigate to Messages page
+ - Search automatically pre-filled with the mailbox/alias email address
+
+- **Mailbox Details Display (Domains-style)**:
+ - Quota usage with percentage
+ - Messages in mailbox count
+ - Last IMAP/SMTP/POP3 login times
+ - Created and Modified dates
+ - Rate limiting settings (value/frame)
+ - Access permissions indicators: IMAP, POP3, SMTP, Sieve, SOGo, TLS Enforce
+ - Color-coded status dots (green=enabled, red=disabled)
+
+- **Filtering & Search**:
+ - **Date Range Picker with Presets**:
+ - Quick select preset buttons: Today, 7 Days, 30 Days, 90 Days
+ - Custom date range with From/To date inputs
+ - Domain filter dropdown
+ - Search by mailbox username, name, or alias address
+ - "Active Only" checkbox (default: checked)
+ - "Hide Zero Activity" checkbox (default: checked) - filters mailboxes and aliases with no messages
+ - Sort by: Sent, Received, Failure Rate, Quota Used, Username
+
+- **Pagination**: 50 mailboxes per page with navigation controls
+
+#### Background Jobs
+- **Mailbox Statistics Job**: Fetches mailbox data from Mailcow API every 5 minutes
+ - Syncs quota, messages, login times, rate limits, and attributes
+ - Marks deleted mailboxes as inactive (preserves historical data)
+
+- **Alias Statistics Job**: Fetches alias data from Mailcow API every 5 minutes
+ - Links aliases to their target mailboxes
+ - Marks deleted aliases as inactive (preserves historical data)
+
+#### SMTP Relay Mode
+- **No-Authentication SMTP Support**: New option for sending emails through local relay servers without credentials
+ - Enable via `SMTP_RELAY_MODE=true` in environment variables
+ - When enabled, `SMTP_USER` and `SMTP_PASSWORD` are not required
+ - Useful for local Postfix relay servers, internal mail gateways, or trusted SMTP relays
+ - Connection test in Settings page properly handles relay mode authentication bypass
+
+#### Clean URL Routing (History API)
+- **Shareable URLs for All Pages**: Implemented History API-based routing for the SPA
+ - Direct navigation to any tab via clean URLs (e.g., `/dashboard`, `/messages`, `/dmarc`, `/settings`)
+ - Browser Back/Forward buttons now work correctly between pages
+ - URLs can be bookmarked and shared
+
+- **DMARC Nested Routes**: Deep linking support for all DMARC views
+ - `/dmarc` - Domains list
+ - `/dmarc/{domain}` - Domain overview
+ - `/dmarc/{domain}/reports` - Daily Reports tab
+ - `/dmarc/{domain}/sources` - Source IPs tab
+ - `/dmarc/{domain}/tls` - TLS Reports tab
+ - `/dmarc/{domain}/report/{date}` - Specific daily report details
+ - `/dmarc/{domain}/source/{ip}` - Specific source IP details
+
+- **Removed Internal Back Button**: DMARC section no longer uses custom back button
+ - Users now use browser's native Back button
+ - Cleaner UI without duplicate navigation controls
+
+#### TLS-RPT (TLS Reporting) Support
+- **Complete TLS-RPT Implementation**: Full support for TLS aggregate reports (RFC 8460)
+ - TLS-RPT parser for JSON reports (gzip compressed)
+ - Database models for TLS reports and policies
+ - IMAP auto-import support for TLS-RPT emails
+ - Manual upload support for TLS-RPT files
+
+- **TLS Reports Tab in DMARC Page**:
+ - New "TLS Reports" sub-tab alongside Daily Reports and Source IPs
+ - Daily aggregated view showing reports grouped by date
+ - Success rate with color-coded progress bars (green ≥95%, yellow ≥80%, red <80%)
+ - Provider breakdown with session counts
+
+- **TLS Report Details View**:
+ - Click any daily report to see detailed breakdown
+ - Stats cards: Sessions, Success Rate, Successful, Failed
+ - Providers table with per-provider success rates
+
+- **TLS in Domain List**:
+ - TLS Success Rate column in DMARC domains table
+ - TLS report count displayed per domain
+ - Domains with only TLS reports (no DMARC) now included in list
+
+#### DMARC Navigation Improvements
+- **Breadcrumb Navigation**: Clear path indicator for all DMARC views
+ - Shows current location: `domain.com > Daily Reports > Jan 14, 2026`
+ - Clickable links to navigate back to any level
+ - Displayed below page description
+
+#### Mobile Navigation Hamburger Menu
+- **Hamburger Menu for Mobile**:
+ - Replaced horizontal scrolling tabs with a proper hamburger menu on mobile devices
+
+### Fixed
+
+#### DMARC Source IPs - Broken Flag Images
+- **Fixed broken flag images when MAXMIND is not configured**: When GeoIP integration is not set up, the Source IPs tab was showing broken images
+ - Now displays a generic server icon instead of a broken image when country data is unavailable
+ - Flag is completely hidden in source details view when no GeoIP data exists
+ - Added `onerror` fallback handlers to gracefully handle missing flag files
+ - Improves UX for users who haven't configured MAXMIND integration
+
+#### DMARC Parser
+- **DMARC 2.0 XML Namespace Support**: Fixed parsing error for DMARC reports using XML namespaces
+ - Reports from GMX and other providers using the new format now parse correctly
+ - Parser now tries both namespaced and non-namespaced element lookups
+
+### Improved
+
+#### Backend API Performance
+- **In-Memory Caching for Statistics API**: Added 5-minute TTL cache for `/api/mailbox-stats/all` endpoint
+ - Cache key generated from all query parameters
+ - First request fetches from database, subsequent requests return from cache
+ - Cache automatically expires after 5 minutes for fresh data
+ - Significantly reduces database load and improves response times
+
+- **In-Memory Caching for DMARC API**: Added 5-minute TTL cache for `/api/dmarc/domains` endpoint
+ - Reduces heavy database queries for domain statistics
+ - Cache cleared on new report imports
+
+#### DMARC IMAP Auto-Import
+- **Batch Processing**: Emails are now processed in configurable batches to prevent memory issues
+ - New `DMARC_IMAP_BATCH_SIZE` environment variable (default: 10)
+ - Processes emails in chunks, re-searching after each batch
+ - Progress logging shows batch number and completion status
+ - Prevents application crashes when syncing mailboxes with lots of emails
+
+- **UID-Based Email Handling**: Fixed "Invalid messageset" IMAP errors
+ - Changed from sequence numbers to UIDs for all IMAP operations
+ - UIDs remain stable even after deleting emails during sync
+ - Affects SEARCH, FETCH, STORE operations
+
+- **Flexible DMARC Email Detection**: Now supports more email providers
+ - Yahoo and other providers that don't include "Report-ID:" now detected correctly
+ - Primary validation is now attachment-based (.xml.gz or .zip files)
+ - Accepts: "Report Domain:" only, "Report Domain:" + "Submitter:", or "DMARC" keyword
+
+- **Improved Error Reporting in Notifications**:
+ - Error notification emails now show actual error messages
+ - Parse failures show: "Failed to parse: filename.xml.gz"
+ - Processing errors show: "Error processing filename: exception details"
+ - Duplicate reports no longer counted as failures
+
+- **Infinite Loop Prevention**: Fixed sync running endlessly when emails fail validation
+ - Added `UNSEEN` filter to search criteria
+ - Failed or processed emails are marked as Seen and excluded from next search
+ - Prevents re-processing the same emails repeatedly
+
+- **Microsoft Outlook Support**: Fixed DMARC reports from Microsoft not being recognized
+ - Now detects DMARC reports by filename pattern (contains `!` separator)
+ - Supports filenames like: `enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz`
+
+- **Enhanced Attachment Extraction**: More robust attachment detection
+ - Now supports plain `.xml` files in addition to `.xml.gz` and `.zip`
+ - Falls back to Content-Type `name` parameter when filename header is missing
+ - Recognizes attachments by Content-Type: `application/gzip`, `application/zip`, `text/xml`, etc.
+ - Added debug logging to help troubleshoot attachment detection issues
+
+#### Domains Page - DKIM View Record
+- **DKIM Record Viewer**: Added "View Record" functionality for DKIM, similar to SPF
+ - Displays the full DNS record name including selector (e.g., `dkim._domainkey.example.com`)
+ - Shows the DKIM public key record value
+ - Helps users identify exactly which DNS record to configure
+
+### Technical
+
+#### New API Endpoints
+```
+GET /api/mailbox-stats/summary
+GET /api/mailbox-stats/all
+GET /api/mailbox-stats/domains
+```
+
+#### API Parameters
+- `date_range`: today, 7days, 30days, 90days, custom
+- `start_date`: Custom start date (YYYY-MM-DD) - required when date_range is 'custom'
+- `end_date`: Custom end date (YYYY-MM-DD) - required when date_range is 'custom'
+- `domain`: Filter by specific domain
+- `active_only`: true/false
+- `hide_zero`: true/false (filter zero-activity mailboxes)
+- `search`: Search mailbox/alias addresses
+- `sort_by`: sent_total, received_total, failure_rate, quota_used, username
+- `sort_order`: asc, desc
+- `page`, `page_size`: Pagination
+
+---
+
## [2.0.4] - 2026-01-15
### Fixed
diff --git a/README.md b/README.md
index b3ed1bf..77f70ff 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,31 @@ A modern, self-hosted dashboard for viewing and analyzing Mailcow mail server lo
- Quarantined emails
- Search and filter
+### 🌐 Domains Overview
+- Complete domains overview with DNS validation
+- SPF validation with DNS lookup counter and server IP authorization check
+- DKIM validation with Mailcow configuration comparison
+- DMARC policy checking and recommendations
+- Automated DNS checks every 6 hours
+- Domain info: mailboxes, aliases, storage, relay settings
+- Color-coded status indicators (✓ green, ⚠ amber, ✗ red)
+
+### 📊 Mailbox Statistics
+- Per-mailbox message statistics (sent, received, failed, failure rate)
+- Accordion-style mailbox list with expandable details
+- Per-alias statistics with message counts
+- Quota usage, login times, rate limits
+- Filtering by date range, domain, search, active only, hide zero activity
+- Sorting by sent, received, failure rate, quota, username
+
+### 📧 DMARC Reports
+- DMARC report viewing and analysis
+- GeoIP enrichment with MaxMind (City + ASN)
+- Daily aggregated reports with compliance rates
+- Manual upload (XML, GZ, ZIP formats)
+- IMAP auto-import with configurable sync interval
+- Sync history tracking with error notifications
+
### 📈 Status
- All container states (running/stopped count)
- Storage usage with percentage bar
diff --git a/VERSION b/VERSION
index 26e3379..50aea0e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0.4
\ No newline at end of file
+2.1.0
\ No newline at end of file
diff --git a/backend/app/auth.py b/backend/app/auth.py
index 5fb46a9..c8030ce 100644
--- a/backend/app/auth.py
+++ b/backend/app/auth.py
@@ -50,10 +50,11 @@ class BasicAuthMiddleware(BaseHTTPMiddleware):
if not settings.auth_enabled:
return await call_next(request)
- # Allow access to login page, static files, and health check without authentication
+ # Allow access to login page, static files, health check, and info endpoint without authentication
# Health check endpoint must be accessible for Docker health monitoring
+ # Info endpoint is used to check if authentication is enabled
path = request.url.path
- if path == "/login" or path.startswith("/static/") or path == "/api/health":
+ if path == "/login" or path.startswith("/static/") or path == "/api/health" or path == "/api/info":
return await call_next(request)
# Check if password is configured
@@ -67,9 +68,10 @@ class BasicAuthMiddleware(BaseHTTPMiddleware):
# Extract credentials from Authorization header
authorization = request.headers.get("Authorization", "")
- # For root page, allow access without Authorization header
+ # For frontend routes (not API), allow access without Authorization header
# The frontend JavaScript will handle authentication and redirect if needed
- if path == "/":
+ # This enables clean URLs like /dashboard, /messages, /dmarc etc.
+ if not path.startswith("/api/"):
return await call_next(request)
# For all other paths (API endpoints), require authentication
diff --git a/backend/app/config.py b/backend/app/config.py
index 026ab2f..32ca7ca 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -162,6 +162,12 @@ class Settings(BaseSettings):
description='Run IMAP sync once on application startup'
)
+ dmarc_imap_batch_size: int = Field(
+ default=10,
+ env='DMARC_IMAP_BATCH_SIZE',
+ description='Number of emails to process per batch (prevents memory issues with large mailboxes)'
+ )
+
dmarc_error_email: Optional[str] = Field(
default=None,
env='DMARC_ERROR_EMAIL',
@@ -188,11 +194,17 @@ class Settings(BaseSettings):
)
smtp_use_tls: bool = Field(
- default=True,
+ default=False,
env='SMTP_USE_TLS',
description='Use STARTTLS for SMTP connection'
)
+ smtp_use_ssl: bool = Field(
+ default=False,
+ env='SMTP_USE_SSL',
+ description='Use Implicit SSL/TLS for SMTP connection (usually port 465)'
+ )
+
smtp_user: Optional[str] = Field(
default=None,
env='SMTP_USER',
@@ -211,6 +223,12 @@ class Settings(BaseSettings):
description='From address for emails (defaults to SMTP user if not set)'
)
+ smtp_relay_mode: bool = Field(
+ default=False,
+ env='SMTP_RELAY_MODE',
+ description='Relay mode - send emails without authentication (for local relay servers)'
+ )
+
# Global Admin Email
admin_email: Optional[str] = Field(
default=None,
@@ -260,12 +278,21 @@ class Settings(BaseSettings):
@property
def notification_smtp_configured(self) -> bool:
"""Check if SMTP is properly configured for notifications"""
- return (
- self.smtp_enabled and
- self.smtp_host is not None and
- self.smtp_user is not None and
- self.smtp_password is not None
- )
+ if self.smtp_relay_mode:
+ # Relay mode - only need host and from address
+ return (
+ self.smtp_enabled and
+ self.smtp_host is not None and
+ self.smtp_from is not None
+ )
+ else:
+ # Standard mode - need authentication
+ return (
+ self.smtp_enabled and
+ self.smtp_host is not None and
+ self.smtp_user is not None and
+ self.smtp_password is not None
+ )
@property
def database_url(self) -> str:
diff --git a/backend/app/main.py b/backend/app/main.py
index 08fd274..2970abf 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -20,6 +20,7 @@ from .routers import logs, stats
from .routers import export as export_router
from .routers import domains as domains_router
from .routers import dmarc as dmarc_router
+from .routers import mailbox_stats as mailbox_stats_router
from .routers import documentation
from .migrations import run_migrations
from .auth import BasicAuthMiddleware
@@ -172,6 +173,7 @@ app.include_router(messages_router.router, prefix="/api", tags=["Messages"])
app.include_router(settings_router.router, prefix="/api", tags=["Settings"])
app.include_router(domains_router.router, prefix="/api", tags=["Domains"])
app.include_router(dmarc_router.router, prefix="/api", tags=["DMARC"])
+app.include_router(mailbox_stats_router.router, prefix="/api", tags=["Mailbox Stats"])
app.include_router(documentation.router, prefix="/api", tags=["Documentation"])
# Mount static files (frontend)
@@ -256,6 +258,23 @@ async def global_exception_handler(request: Request, exc: Exception):
)
+# SPA catch-all route - must be AFTER all other routes and exception handlers
+# Returns index.html for all frontend routes (e.g., /dashboard, /messages, /dmarc)
+@app.get("/{full_path:path}", response_class=HTMLResponse)
+async def spa_catch_all(full_path: str):
+ """Serve the SPA for all frontend routes - enables clean URLs"""
+ # API and static routes are handled by their respective routers/mounts
+ # This catch-all only receives unmatched routes
+ try:
+ with open("/app/frontend/index.html", "r") as f:
+ return HTMLResponse(content=f.read())
+ except FileNotFoundError:
+ return HTMLResponse(
+ content="
Mailcow Logs Viewer
Frontend not found. Please check installation.
",
+ status_code=500
+ )
+
+
if __name__ == "__main__":
import uvicorn
uvicorn.run(
diff --git a/backend/app/migrations.py b/backend/app/migrations.py
index bd7af7a..50b8850 100644
--- a/backend/app/migrations.py
+++ b/backend/app/migrations.py
@@ -795,6 +795,59 @@ def create_dmarc_sync_table(db: Session):
db.rollback()
raise
+def ensure_mailbox_statistics_table(db: Session):
+ """Ensure mailbox_statistics table exists with proper structure"""
+ try:
+ # Check if table exists
+ result = db.execute(text("""
+ SELECT EXISTS (
+ SELECT FROM information_schema.tables
+ WHERE table_name = 'mailbox_statistics'
+ )
+ """)).scalar()
+
+ if not result:
+ logger.info("Creating mailbox_statistics table...")
+ db.execute(text("""
+ CREATE TABLE mailbox_statistics (
+ id SERIAL PRIMARY KEY,
+ username VARCHAR(255) UNIQUE NOT NULL,
+ domain VARCHAR(255) NOT NULL,
+ name VARCHAR(255),
+ quota BIGINT DEFAULT 0,
+ quota_used BIGINT DEFAULT 0,
+ percent_in_use FLOAT DEFAULT 0.0,
+ messages INTEGER DEFAULT 0,
+ active BOOLEAN DEFAULT TRUE,
+ last_imap_login BIGINT,
+ last_pop3_login BIGINT,
+ last_smtp_login BIGINT,
+ spam_aliases INTEGER DEFAULT 0,
+ rl_value INTEGER,
+ rl_frame VARCHAR(20),
+ attributes JSONB,
+ created_at TIMESTAMP DEFAULT NOW(),
+ updated_at TIMESTAMP DEFAULT NOW()
+ )
+ """))
+
+ # Create indexes
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_domain ON mailbox_statistics(domain)"))
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_active ON mailbox_statistics(active)"))
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_quota_used ON mailbox_statistics(quota_used)"))
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_username ON mailbox_statistics(username)"))
+
+ db.commit()
+ logger.info("mailbox_statistics table created successfully")
+ else:
+ logger.debug("mailbox_statistics table already exists")
+
+ except Exception as e:
+ logger.error(f"Error ensuring mailbox_statistics table: {e}")
+ db.rollback()
+ raise
+
+
def run_migrations():
"""
Run all database migrations and maintenance tasks
@@ -825,6 +878,17 @@ def run_migrations():
create_dmarc_sync_table(db)
# GeoIP fields
+ add_geoip_fields_to_dmarc(db)
+
+ # Mailbox statistics table
+ ensure_mailbox_statistics_table(db)
+
+ # Alias statistics table
+ ensure_alias_statistics_table(db)
+
+ # System settings (for cache signaling)
+ ensure_system_settings_table(db)
+
add_geoip_fields_to_dmarc(db)
add_geoip_fields_to_rspamd(db)
@@ -837,4 +901,82 @@ def run_migrations():
logger.error(f"Migration failed: {e}")
raise
finally:
- db.close()
\ No newline at end of file
+ db.close()
+
+
+def ensure_alias_statistics_table(db: Session):
+ """Ensure alias_statistics table exists with all required fields"""
+ try:
+ # Check if table exists
+ result = db.execute(text("""
+ SELECT EXISTS (
+ SELECT FROM information_schema.tables
+ WHERE table_name = 'alias_statistics'
+ )
+ """))
+ table_exists = result.scalar()
+
+ if not table_exists:
+ logger.info("Creating alias_statistics table...")
+ db.execute(text("""
+ CREATE TABLE alias_statistics (
+ id SERIAL PRIMARY KEY,
+ alias_address VARCHAR(255) NOT NULL UNIQUE,
+ goto TEXT,
+ domain VARCHAR(255) NOT NULL,
+ active BOOLEAN DEFAULT TRUE,
+ is_catch_all BOOLEAN DEFAULT FALSE,
+ primary_mailbox VARCHAR(255),
+ created_at TIMESTAMP DEFAULT NOW(),
+ updated_at TIMESTAMP DEFAULT NOW()
+ )
+ """))
+
+ # Create indexes
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_domain ON alias_statistics(domain)"))
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_active ON alias_statistics(active)"))
+ db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_primary_mailbox ON alias_statistics(primary_mailbox)"))
+ db.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_alias_address ON alias_statistics(alias_address)"))
+
+ db.commit()
+ logger.info("alias_statistics table created successfully")
+ else:
+ logger.debug("alias_statistics table already exists")
+
+ except Exception as e:
+ logger.error(f"Error ensuring alias_statistics table: {e}")
+ db.rollback()
+ raise
+
+
+def ensure_system_settings_table(db: Session):
+ """Ensure system_settings table exists"""
+ try:
+ # Check if table exists
+ result = db.execute(text("""
+ SELECT EXISTS (
+ SELECT FROM information_schema.tables
+ WHERE table_name = 'system_settings'
+ )
+ """))
+ table_exists = result.scalar()
+
+ if not table_exists:
+ logger.info("Creating system_settings table...")
+ db.execute(text("""
+ CREATE TABLE system_settings (
+ key VARCHAR(255) PRIMARY KEY,
+ value TEXT,
+ updated_at TIMESTAMP DEFAULT NOW()
+ )
+ """))
+
+ db.commit()
+ logger.info("system_settings table created successfully")
+ else:
+ logger.debug("system_settings table already exists")
+
+ except Exception as e:
+ logger.error(f"Error ensuring system_settings table: {e}")
+ db.rollback()
+ raise
diff --git a/backend/app/models.py b/backend/app/models.py
index c7c3376..41a5be6 100644
--- a/backend/app/models.py
+++ b/backend/app/models.py
@@ -6,7 +6,7 @@ SIMPLIFIED VERSION:
- Removed old generate_correlation_key function
- Correlation key is now SHA256 of Message-ID
"""
-from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint
+from sqlalchemy import Column, Integer, BigInteger, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB
from datetime import datetime
@@ -292,4 +292,174 @@ class DMARCSync(Base):
)
def __repr__(self):
- return f""
\ No newline at end of file
+ return f""
+
+
+class MailboxStatistics(Base):
+ """
+ Mailbox statistics fetched from Mailcow API
+ Tracks quota usage, message counts, and last access times for each mailbox
+ """
+ __tablename__ = "mailbox_statistics"
+
+ id = Column(Integer, primary_key=True, index=True)
+
+ # Mailbox identification
+ username = Column(String(255), unique=True, index=True, nullable=False) # email address
+ domain = Column(String(255), index=True, nullable=False)
+ name = Column(String(255)) # Display name
+
+ # Quota information (in bytes)
+ quota = Column(BigInteger, default=0) # Allocated quota
+ quota_used = Column(BigInteger, default=0) # Used quota
+ percent_in_use = Column(Float, default=0.0) # Percentage used
+
+ # Message counts
+ messages = Column(Integer, default=0) # Total messages in mailbox
+
+ # Status
+ active = Column(Boolean, default=True, index=True)
+
+ # Access times (Unix timestamps from API, stored as integers)
+ last_imap_login = Column(BigInteger, nullable=True)
+ last_pop3_login = Column(BigInteger, nullable=True)
+ last_smtp_login = Column(BigInteger, nullable=True)
+
+ # Spam filter settings
+ spam_aliases = Column(Integer, default=0)
+
+ # Rate limits
+ rl_value = Column(Integer, nullable=True) # Rate limit value
+ rl_frame = Column(String(20), nullable=True) # Rate limit time frame (e.g., "s", "m", "h")
+
+ # Attributes from API
+ attributes = Column(JSONB) # Store full attributes for reference
+
+ # Timestamps
+ created_at = Column(DateTime, default=datetime.utcnow)
+ updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ __table_args__ = (
+ Index('idx_mailbox_domain', 'domain'),
+ Index('idx_mailbox_active', 'active'),
+ Index('idx_mailbox_quota_used', 'quota_used'),
+ )
+
+ def __repr__(self):
+ return f""
+
+
+class AliasStatistics(Base):
+ """
+ Alias statistics for tracking message counts per alias
+ Links aliases to their target mailboxes
+ """
+ __tablename__ = "alias_statistics"
+
+ id = Column(Integer, primary_key=True, index=True)
+
+ # Alias identification
+ alias_address = Column(String(255), unique=True, index=True, nullable=False) # The alias email
+ goto = Column(Text) # Target mailbox(es), comma-separated
+ domain = Column(String(255), index=True, nullable=False)
+
+ # Status
+ active = Column(Boolean, default=True, index=True)
+ is_catch_all = Column(Boolean, default=False) # Is this a catch-all alias
+
+ # Link to primary mailbox (if applicable)
+ primary_mailbox = Column(String(255), index=True, nullable=True) # Main target mailbox
+
+ # Timestamps
+ created_at = Column(DateTime, default=datetime.utcnow)
+ updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ __table_args__ = (
+ Index('idx_alias_domain', 'domain'),
+ Index('idx_alias_active', 'active'),
+ Index('idx_alias_primary_mailbox', 'primary_mailbox'),
+ )
+
+ def __repr__(self):
+ return f""
+
+
+class TLSReport(Base):
+ """TLS-RPT (SMTP TLS Reporting) reports received from email providers"""
+ __tablename__ = "tls_reports"
+
+ id = Column(Integer, primary_key=True, index=True)
+
+ # Report identification
+ report_id = Column(String(255), unique=True, index=True, nullable=False)
+
+ # Organization that sent the report
+ organization_name = Column(String(255), index=True)
+ contact_info = Column(String(255))
+
+ # Domain being reported on
+ policy_domain = Column(String(255), index=True, nullable=False)
+
+ # Date range of the report
+ start_datetime = Column(DateTime, nullable=False)
+ end_datetime = Column(DateTime, nullable=False)
+
+ # Raw JSON for reference
+ raw_json = Column(Text)
+
+ created_at = Column(DateTime, default=datetime.utcnow, index=True)
+ updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ __table_args__ = (
+ Index('idx_tls_report_domain_date', 'policy_domain', 'start_datetime'),
+ Index('idx_tls_report_org', 'organization_name'),
+ )
+
+ def __repr__(self):
+ return f""
+
+
+class TLSReportPolicy(Base):
+ """Individual policy records within a TLS-RPT report"""
+ __tablename__ = "tls_report_policies"
+
+ id = Column(Integer, primary_key=True, index=True)
+ tls_report_id = Column(Integer, index=True, nullable=False)
+
+ # Policy information
+ policy_type = Column(String(50)) # "sts", "no-policy-found", etc.
+ policy_domain = Column(String(255))
+ policy_string = Column(JSONB) # The policy string array
+ mx_host = Column(JSONB) # List of MX hosts
+
+ # Session counts
+ successful_session_count = Column(Integer, default=0)
+ failed_session_count = Column(Integer, default=0)
+
+ # Failure details if any
+ failure_details = Column(JSONB)
+
+ created_at = Column(DateTime, default=datetime.utcnow)
+
+ __table_args__ = (
+ Index('idx_tls_policy_report', 'tls_report_id'),
+ Index('idx_tls_policy_type', 'policy_type'),
+ )
+
+ def __repr__(self):
+ return f""
+
+
+class SystemSetting(Base):
+ """
+ Global system settings and state
+ Used for inter-process signaling (e.g., cache invalidation)
+ """
+ __tablename__ = "system_settings"
+
+ key = Column(String(255), primary_key=True, index=True)
+ value = Column(Text) # JSON string or simple text
+ updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ def __repr__(self):
+ return f""
diff --git a/backend/app/routers/dmarc.py b/backend/app/routers/dmarc.py
index a15cbbf..16ac34a 100644
--- a/backend/app/routers/dmarc.py
+++ b/backend/app/routers/dmarc.py
@@ -2,6 +2,8 @@
DMARC Router - Domain-centric view (Cloudflare style)
"""
import logging
+import hashlib
+import json
from typing import List, Optional
from datetime import datetime, timedelta, timezone
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, BackgroundTasks
@@ -9,10 +11,16 @@ from sqlalchemy.orm import Session
from sqlalchemy import func, and_, or_, case
from ..database import get_db
-from ..models import DMARCReport, DMARCRecord, DMARCSync
+from ..models import DMARCReport, DMARCRecord, DMARCSync, TLSReport, TLSReportPolicy
from ..services.dmarc_parser import parse_dmarc_file
from ..services.geoip_service import enrich_dmarc_record
from ..services.dmarc_imap_service import sync_dmarc_reports_from_imap
+from ..services.dmarc_cache import (
+ get_dmarc_cache_key,
+ get_dmarc_cached,
+ set_dmarc_cache,
+ clear_dmarc_cache
+)
from ..config import settings
from ..scheduler import update_job_status
@@ -20,21 +28,52 @@ logger = logging.getLogger(__name__)
router = APIRouter()
+# =============================================================================
+# CACHING SYSTEM
+# =============================================================================
+
+# =============================================================================
+# CACHING SYSTEM (Delegated to services.dmarc_cache)
+# =============================================================================
+
+# Cache functions imported from ..services.dmarc_cache
+
# =============================================================================
# DOMAINS LIST
# =============================================================================
+@router.post("/dmarc/cache/clear")
+async def clear_cache(
+ db: Session = Depends(get_db)
+):
+ """
+ Clear all DMARC related cache
+ """
+ try:
+ clear_dmarc_cache(db)
+ return {"status": "success", "message": "Cache cleared"}
+ except Exception as e:
+ logger.error(f"Error clearing DMARC cache: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+
@router.get("/dmarc/domains")
async def get_domains_list(
db: Session = Depends(get_db)
):
"""
- Get list of all domains with DMARC reports and their statistics
- Similar to Cloudflare's domain list
+ Get list of all domains with DMARC and/or TLS-RPT reports and their statistics
"""
try:
- domains_query = db.query(
+ # Check cache first
+ cache_key = get_dmarc_cache_key("domains_list")
+ cached_result = get_dmarc_cached(cache_key, db)
+ if cached_result is not None:
+ return cached_result
+
+ # Get domains from DMARC reports
+ dmarc_domains = db.query(
DMARCReport.domain,
func.count(DMARCReport.id).label('report_count'),
func.min(DMARCReport.begin_date).label('first_report'),
@@ -43,55 +82,139 @@ async def get_domains_list(
DMARCReport.domain
).all()
+ # Get domains from TLS reports
+ tls_domains = db.query(
+ TLSReport.policy_domain
+ ).distinct().all()
+ tls_domain_set = {d[0] for d in tls_domains}
+
+ # Combine domains
+ all_domains = set()
+ dmarc_domain_data = {}
+
+ for domain, report_count, first_report, last_report in dmarc_domains:
+ all_domains.add(domain)
+ dmarc_domain_data[domain] = {
+ 'report_count': report_count,
+ 'first_report': first_report,
+ 'last_report': last_report
+ }
+
+ # Add TLS-only domains
+ all_domains.update(tls_domain_set)
+
domains_list = []
- for domain, report_count, first_report, last_report in domains_query:
+ for domain in all_domains:
thirty_days_ago = int((datetime.now() - timedelta(days=30)).timestamp())
+ thirty_days_ago_datetime = datetime.now() - timedelta(days=30)
- stats = db.query(
- func.sum(DMARCRecord.count).label('total_messages'),
- func.count(func.distinct(DMARCRecord.source_ip)).label('unique_ips'),
- func.sum(
- case(
- (and_(DMARCRecord.spf_result == 'pass', DMARCRecord.dkim_result == 'pass'), DMARCRecord.count),
- else_=0
- )
- ).label('dmarc_pass_count')
+ # Get TLS stats for this domain
+ tls_stats = db.query(
+ func.count(TLSReport.id).label('tls_report_count'),
+ func.sum(TLSReportPolicy.successful_session_count).label('tls_success'),
+ func.sum(TLSReportPolicy.failed_session_count).label('tls_fail')
).join(
- DMARCReport,
- DMARCRecord.dmarc_report_id == DMARCReport.id
+ TLSReportPolicy,
+ TLSReportPolicy.tls_report_id == TLSReport.id
).filter(
and_(
- DMARCReport.domain == domain,
- DMARCReport.begin_date >= thirty_days_ago
+ TLSReport.policy_domain == domain,
+ TLSReport.start_datetime >= thirty_days_ago_datetime
)
).first()
- total_msgs = stats.total_messages or 0
- dmarc_pass = stats.dmarc_pass_count or 0
+ tls_report_count = tls_stats.tls_report_count or 0
+ tls_success = tls_stats.tls_success or 0
+ tls_fail = tls_stats.tls_fail or 0
+ tls_total = tls_success + tls_fail
+ tls_success_pct = round((tls_success / tls_total * 100) if tls_total > 0 else 100, 2)
- domains_list.append({
- 'domain': domain,
- 'report_count': report_count,
- 'first_report': first_report,
- 'last_report': last_report,
- 'stats_30d': {
- 'total_messages': total_msgs,
- 'unique_ips': stats.unique_ips or 0,
- 'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2)
- }
- })
+ # Get DMARC stats
+ dmarc_data = dmarc_domain_data.get(domain)
+ if dmarc_data:
+ stats = db.query(
+ func.sum(DMARCRecord.count).label('total_messages'),
+ func.count(func.distinct(DMARCRecord.source_ip)).label('unique_ips'),
+ func.sum(
+ case(
+ (and_(DMARCRecord.spf_result == 'pass', DMARCRecord.dkim_result == 'pass'), DMARCRecord.count),
+ else_=0
+ )
+ ).label('dmarc_pass_count')
+ ).join(
+ DMARCReport,
+ DMARCRecord.dmarc_report_id == DMARCReport.id
+ ).filter(
+ and_(
+ DMARCReport.domain == domain,
+ DMARCReport.begin_date >= thirty_days_ago
+ )
+ ).first()
+
+ total_msgs = stats.total_messages or 0
+ dmarc_pass = stats.dmarc_pass_count or 0
+
+ domains_list.append({
+ 'domain': domain,
+ 'report_count': dmarc_data['report_count'],
+ 'tls_report_count': tls_report_count,
+ 'first_report': dmarc_data['first_report'],
+ 'last_report': dmarc_data['last_report'],
+ 'has_dmarc': True,
+ 'has_tls': domain in tls_domain_set,
+ 'stats_30d': {
+ 'total_messages': total_msgs,
+ 'unique_ips': stats.unique_ips or 0,
+ 'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2),
+ 'tls_success_pct': tls_success_pct
+ }
+ })
+ else:
+ # TLS-only domain
+ tls_report = db.query(TLSReport).filter(
+ TLSReport.policy_domain == domain
+ ).order_by(TLSReport.end_datetime.desc()).first()
+
+ first_tls = db.query(func.min(TLSReport.start_datetime)).filter(
+ TLSReport.policy_domain == domain
+ ).scalar()
+
+ domains_list.append({
+ 'domain': domain,
+ 'report_count': 0,
+ 'tls_report_count': tls_report_count,
+ 'first_report': int(first_tls.timestamp()) if first_tls else None,
+ 'last_report': int(tls_report.end_datetime.timestamp()) if tls_report and tls_report.end_datetime else None,
+ 'has_dmarc': False,
+ 'has_tls': True,
+ 'stats_30d': {
+ 'total_messages': 0,
+ 'unique_ips': 0,
+ 'dmarc_pass_pct': 0,
+ 'tls_success_pct': tls_success_pct
+ }
+ })
- return {
- 'domains': sorted(domains_list, key=lambda x: x['last_report'], reverse=True),
+ # Sort by last_report, handling None values
+ domains_list.sort(key=lambda x: x['last_report'] or 0, reverse=True)
+
+ response = {
+ 'domains': domains_list,
'total': len(domains_list)
}
+ # Cache the result
+ set_dmarc_cache(cache_key, response)
+
+ return response
+
except Exception as e:
logger.error(f"Error fetching domains list: {e}")
raise HTTPException(status_code=500, detail=str(e))
+
# =============================================================================
# DOMAIN OVERVIEW
# =============================================================================
@@ -624,6 +747,297 @@ async def get_source_details(
except Exception as e:
logger.error(f"Error fetching source details: {e}")
raise HTTPException(status_code=500, detail=str(e))
+# =============================================================================
+# TLS-RPT REPORTS
+# =============================================================================
+
+@router.get("/dmarc/domains/{domain}/tls-reports")
+async def get_domain_tls_reports(
+ domain: str,
+ days: int = 30,
+ page: int = 1,
+ limit: int = 50,
+ db: Session = Depends(get_db)
+):
+ """
+ Get TLS-RPT reports for a domain
+ Returns list of TLS reports with organization, date range, success/fail counts
+ """
+ try:
+ cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
+
+ # Query TLS reports for domain
+ reports_query = db.query(TLSReport).filter(
+ and_(
+ TLSReport.policy_domain == domain,
+ TLSReport.start_datetime >= cutoff_date
+ )
+ ).order_by(TLSReport.start_datetime.desc())
+
+ total = reports_query.count()
+ reports = reports_query.offset((page - 1) * limit).limit(limit).all()
+
+ # Get aggregated totals
+ total_success = 0
+ total_fail = 0
+ organizations = set()
+
+ reports_list = []
+ for report in reports:
+ # Get policies for this report
+ policies = db.query(TLSReportPolicy).filter(
+ TLSReportPolicy.tls_report_id == report.id
+ ).all()
+
+ report_success = sum(p.successful_session_count for p in policies)
+ report_fail = sum(p.failed_session_count for p in policies)
+
+ total_success += report_success
+ total_fail += report_fail
+ organizations.add(report.organization_name)
+
+ policies_list = [{
+ 'policy_type': p.policy_type,
+ 'policy_domain': p.policy_domain,
+ 'mx_host': p.mx_host,
+ 'successful_sessions': p.successful_session_count,
+ 'failed_sessions': p.failed_session_count,
+ 'failure_details': p.failure_details
+ } for p in policies]
+
+ reports_list.append({
+ 'id': report.id,
+ 'report_id': report.report_id,
+ 'organization_name': report.organization_name,
+ 'contact_info': report.contact_info,
+ 'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None,
+ 'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None,
+ 'successful_sessions': report_success,
+ 'failed_sessions': report_fail,
+ 'total_sessions': report_success + report_fail,
+ 'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2),
+ 'policies': policies_list
+ })
+
+ return {
+ 'domain': domain,
+ 'total': total,
+ 'page': page,
+ 'limit': limit,
+ 'pages': (total + limit - 1) // limit if total > 0 else 0,
+ 'totals': {
+ 'total_reports': total,
+ 'total_successful_sessions': total_success,
+ 'total_failed_sessions': total_fail,
+ 'overall_success_rate': round((total_success / (total_success + total_fail) * 100) if (total_success + total_fail) > 0 else 100, 2),
+ 'unique_organizations': len(organizations)
+ },
+ 'data': reports_list
+ }
+
+ except Exception as e:
+ logger.error(f"Error fetching TLS reports for domain {domain}: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/dmarc/domains/{domain}/tls-reports/daily")
+async def get_domain_tls_daily_reports(
+ domain: str,
+ days: int = 30,
+ page: int = 1,
+ limit: int = 30,
+ db: Session = Depends(get_db)
+):
+ """
+ Get TLS-RPT reports aggregated by date
+ Groups multiple reports from same day together, like DMARC daily reports
+ """
+ try:
+ cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
+
+ # Query all TLS reports for domain
+ reports = db.query(TLSReport).filter(
+ and_(
+ TLSReport.policy_domain == domain,
+ TLSReport.start_datetime >= cutoff_date
+ )
+ ).order_by(TLSReport.start_datetime.desc()).all()
+
+ # Group by date
+ daily_data = {}
+ for report in reports:
+ if report.start_datetime:
+ date_key = report.start_datetime.strftime('%Y-%m-%d')
+ if date_key not in daily_data:
+ daily_data[date_key] = {
+ 'date': date_key,
+ 'report_count': 0,
+ 'organizations': set(),
+ 'total_success': 0,
+ 'total_fail': 0,
+ 'reports': []
+ }
+
+ # Get policies for this report
+ policies = db.query(TLSReportPolicy).filter(
+ TLSReportPolicy.tls_report_id == report.id
+ ).all()
+
+ report_success = sum(p.successful_session_count for p in policies)
+ report_fail = sum(p.failed_session_count for p in policies)
+
+ daily_data[date_key]['report_count'] += 1
+ daily_data[date_key]['organizations'].add(report.organization_name)
+ daily_data[date_key]['total_success'] += report_success
+ daily_data[date_key]['total_fail'] += report_fail
+ daily_data[date_key]['reports'].append({
+ 'id': report.id,
+ 'organization_name': report.organization_name,
+ 'successful_sessions': report_success,
+ 'failed_sessions': report_fail
+ })
+
+ # Convert to list and add success rate
+ daily_list = []
+ for date_key in sorted(daily_data.keys(), reverse=True):
+ data = daily_data[date_key]
+ total = data['total_success'] + data['total_fail']
+ success_rate = round((data['total_success'] / total * 100) if total > 0 else 100, 2)
+
+ daily_list.append({
+ 'date': date_key,
+ 'report_count': data['report_count'],
+ 'organizations': list(data['organizations']),
+ 'organization_count': len(data['organizations']),
+ 'total_success': data['total_success'],
+ 'total_fail': data['total_fail'],
+ 'total_sessions': total,
+ 'success_rate': success_rate,
+ 'reports': data['reports']
+ })
+
+ # Pagination
+ total = len(daily_list)
+ start_idx = (page - 1) * limit
+ end_idx = start_idx + limit
+ paginated = daily_list[start_idx:end_idx]
+
+ # Calculate overall totals
+ overall_success = sum(d['total_success'] for d in daily_list)
+ overall_fail = sum(d['total_fail'] for d in daily_list)
+ overall_total = overall_success + overall_fail
+
+ return {
+ 'domain': domain,
+ 'total': total,
+ 'page': page,
+ 'limit': limit,
+ 'pages': (total + limit - 1) // limit if total > 0 else 0,
+ 'totals': {
+ 'total_days': total,
+ 'total_reports': sum(d['report_count'] for d in daily_list),
+ 'total_successful_sessions': overall_success,
+ 'total_failed_sessions': overall_fail,
+ 'overall_success_rate': round((overall_success / overall_total * 100) if overall_total > 0 else 100, 2)
+ },
+ 'data': paginated
+ }
+
+ except Exception as e:
+ logger.error(f"Error fetching TLS daily reports for domain {domain}: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/dmarc/domains/{domain}/tls-reports/{report_date}/details")
+async def get_tls_report_details(
+ domain: str,
+ report_date: str,
+ db: Session = Depends(get_db)
+):
+ """
+ Get detailed TLS-RPT reports for a specific date
+ Shows all reports and policies from that day with breakdown by provider
+ """
+ try:
+ date_obj = datetime.strptime(report_date, '%Y-%m-%d').date()
+ start_dt = datetime.combine(date_obj, datetime.min.time(), tzinfo=timezone.utc)
+ end_dt = datetime.combine(date_obj, datetime.max.time(), tzinfo=timezone.utc)
+
+ reports = db.query(TLSReport).filter(
+ and_(
+ TLSReport.policy_domain == domain,
+ TLSReport.start_datetime >= start_dt,
+ TLSReport.start_datetime <= end_dt
+ )
+ ).all()
+
+ if not reports:
+ raise HTTPException(status_code=404, detail="No TLS reports found for this date")
+
+ # Aggregate stats
+ total_success = 0
+ total_fail = 0
+ providers = []
+
+ for report in reports:
+ policies = db.query(TLSReportPolicy).filter(
+ TLSReportPolicy.tls_report_id == report.id
+ ).all()
+
+ report_success = sum(p.successful_session_count for p in policies)
+ report_fail = sum(p.failed_session_count for p in policies)
+
+ total_success += report_success
+ total_fail += report_fail
+
+ # Add provider details
+ policies_list = []
+ for p in policies:
+ policies_list.append({
+ 'policy_type': p.policy_type,
+ 'policy_domain': p.policy_domain,
+ 'mx_host': p.mx_host,
+ 'successful_sessions': p.successful_session_count,
+ 'failed_sessions': p.failed_session_count,
+ 'total_sessions': p.successful_session_count + p.failed_session_count,
+ 'success_rate': round((p.successful_session_count / (p.successful_session_count + p.failed_session_count) * 100) if (p.successful_session_count + p.failed_session_count) > 0 else 100, 2),
+ 'failure_details': p.failure_details
+ })
+
+ providers.append({
+ 'report_id': report.report_id,
+ 'organization_name': report.organization_name,
+ 'contact_info': report.contact_info,
+ 'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None,
+ 'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None,
+ 'successful_sessions': report_success,
+ 'failed_sessions': report_fail,
+ 'total_sessions': report_success + report_fail,
+ 'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2),
+ 'policies': policies_list
+ })
+
+ total = total_success + total_fail
+
+ return {
+ 'domain': domain,
+ 'date': report_date,
+ 'stats': {
+ 'total_reports': len(reports),
+ 'total_providers': len(set(r.organization_name for r in reports)),
+ 'total_success': total_success,
+ 'total_fail': total_fail,
+ 'total_sessions': total,
+ 'success_rate': round((total_success / total * 100) if total > 0 else 100, 2)
+ },
+ 'providers': providers
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error fetching TLS report details for {domain}/{report_date}: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
# =============================================================================
@@ -796,66 +1210,156 @@ async def get_sync_history(
logger.error(f"Error fetching sync history: {e}")
raise HTTPException(status_code=500, detail=str(e))
-
# =============================================================================
# UPLOAD
# =============================================================================
@router.post("/dmarc/upload")
-async def upload_dmarc_report(
+async def upload_report(
file: UploadFile = File(...),
db: Session = Depends(get_db)
):
+ """
+ Upload and parse DMARC or TLS-RPT report file
+
+ Supported formats:
+ - DMARC: .xml, .xml.gz, .zip (XML content)
+ - TLS-RPT: .json, .json.gz
+ """
if not settings.dmarc_manual_upload_enabled:
raise HTTPException(
status_code=403,
- detail="Manual DMARC report upload is disabled"
+ detail="Manual report upload is disabled"
)
- """Upload and parse DMARC report file (GZ or ZIP)"""
try:
file_content = await file.read()
+ filename = file.filename.lower()
- parsed_data = parse_dmarc_file(file_content, file.filename)
+ # Detect file type based on extension
+ # TLS-RPT files: .json, .json.gz, .json.zip
+ is_tls_rpt = filename.endswith('.json') or filename.endswith('.json.gz') or filename.endswith('.json.zip')
- if not parsed_data:
- raise HTTPException(status_code=400, detail="Failed to parse DMARC report")
-
- records_data = parsed_data.pop('records', [])
- report_data = parsed_data
-
- existing = db.query(DMARCReport).filter(
- DMARCReport.report_id == report_data['report_id']
- ).first()
-
- if existing:
- return {
- 'status': 'duplicate',
- 'message': f'Report {report_data["report_id"]} already exists'
- }
-
- report = DMARCReport(**report_data)
- db.add(report)
- db.flush()
-
- for record_data in records_data:
- record_data['dmarc_report_id'] = report.id
- enriched = enrich_dmarc_record(record_data)
- record = DMARCRecord(**enriched)
- db.add(record)
-
- db.commit()
-
- return {
- 'status': 'success',
- 'message': f'Uploaded report for {report.domain} from {report.org_name}',
- 'report_id': report.id,
- 'records_count': len(records_data)
- }
+ if is_tls_rpt:
+ # Process TLS-RPT report
+ return await _upload_tls_rpt_report(file_content, file.filename, db)
+ else:
+ # Process DMARC report (default)
+ return await _upload_dmarc_report(file_content, file.filename, db)
except HTTPException:
raise
except Exception as e:
db.rollback()
- logger.error(f"Error uploading DMARC report: {e}", exc_info=True)
- raise HTTPException(status_code=500, detail=str(e))
\ No newline at end of file
+ logger.error(f"Error uploading report: {e}", exc_info=True)
+ raise HTTPException(status_code=500, detail=str(e))
+
+
+async def _upload_dmarc_report(file_content: bytes, filename: str, db: Session):
+ """Handle DMARC report upload"""
+ parsed_data = parse_dmarc_file(file_content, filename)
+
+ if not parsed_data:
+ raise HTTPException(status_code=400, detail="Failed to parse DMARC report")
+
+ records_data = parsed_data.pop('records', [])
+ report_data = parsed_data
+
+ existing = db.query(DMARCReport).filter(
+ DMARCReport.report_id == report_data['report_id']
+ ).first()
+
+ if existing:
+ return {
+ 'status': 'duplicate',
+ 'report_type': 'dmarc',
+ 'message': f'Report {report_data["report_id"]} already exists'
+ }
+
+ report = DMARCReport(**report_data)
+ db.add(report)
+ db.flush()
+
+ for record_data in records_data:
+ record_data['dmarc_report_id'] = report.id
+ enriched = enrich_dmarc_record(record_data)
+ record = DMARCRecord(**enriched)
+ db.add(record)
+
+ db.commit()
+
+ # Clear cache
+ clear_dmarc_cache(db)
+
+ return {
+ 'status': 'success',
+ 'report_type': 'dmarc',
+ 'message': f'Uploaded DMARC report for {report.domain} from {report.org_name}',
+ 'report_id': report.id,
+ 'records_count': len(records_data)
+ }
+
+
+async def _upload_tls_rpt_report(file_content: bytes, filename: str, db: Session):
+ """Handle TLS-RPT report upload"""
+ from ..services.tls_rpt_parser import parse_tls_rpt_file
+
+ parsed_data = parse_tls_rpt_file(file_content, filename)
+
+ if not parsed_data:
+ raise HTTPException(status_code=400, detail="Failed to parse TLS-RPT report")
+
+ # Extract policies
+ policies_data = parsed_data.pop('policies', [])
+
+ # Check for duplicate
+ existing = db.query(TLSReport).filter(
+ TLSReport.report_id == parsed_data['report_id']
+ ).first()
+
+ if existing:
+ return {
+ 'status': 'duplicate',
+ 'report_type': 'tls-rpt',
+ 'message': f'TLS-RPT report {parsed_data["report_id"]} already exists'
+ }
+
+ # Create TLS report
+ tls_report = TLSReport(
+ report_id=parsed_data['report_id'],
+ organization_name=parsed_data.get('organization_name', 'Unknown'),
+ contact_info=parsed_data.get('contact_info', ''),
+ policy_domain=parsed_data['policy_domain'],
+ start_datetime=parsed_data['start_datetime'],
+ end_datetime=parsed_data['end_datetime'],
+ raw_json=parsed_data.get('raw_json', '')
+ )
+ db.add(tls_report)
+ db.flush()
+
+ # Create policy records
+ for policy_data in policies_data:
+ policy = TLSReportPolicy(
+ tls_report_id=tls_report.id,
+ policy_type=policy_data.get('policy_type', 'unknown'),
+ policy_domain=policy_data.get('policy_domain', ''),
+ policy_string=policy_data.get('policy_string', []),
+ mx_host=policy_data.get('mx_host', []),
+ successful_session_count=policy_data.get('successful_session_count', 0),
+ failed_session_count=policy_data.get('failed_session_count', 0),
+ failure_details=policy_data.get('failure_details', [])
+ )
+ db.add(policy)
+
+ db.commit()
+
+ # Clear cache
+ clear_dmarc_cache(db)
+
+ return {
+ 'status': 'success',
+ 'report_type': 'tls-rpt',
+ 'message': f'Uploaded TLS-RPT report for {tls_report.policy_domain} from {tls_report.organization_name}',
+ 'report_id': tls_report.id,
+ 'policies_count': len(policies_data)
+ }
\ No newline at end of file
diff --git a/backend/app/routers/domains.py b/backend/app/routers/domains.py
index da01008..bb1f04f 100644
--- a/backend/app/routers/domains.py
+++ b/backend/app/routers/domains.py
@@ -404,88 +404,6 @@ async def count_spf_dns_lookups(domain: str, spf_record: str, resolver, visited_
return lookup_count
-async def check_ip_in_spf(domain: str, ip_to_check: str, spf_record: str, resolver, visited_domains: set = None, depth: int = 0) -> tuple:
- """
- Check if IP is authorized in SPF record recursively
- Returns: (authorized: bool, method: str or None)
- """
- if depth > 10:
- return False, None
-
- if visited_domains is None:
- visited_domains = set()
-
- if domain in visited_domains:
- return False, None
-
- visited_domains.add(domain)
-
- parts = spf_record.split()
-
- for part in parts:
- clean_part = part.lstrip('+-~?')
-
- if clean_part.startswith('ip4:'):
- ip_spec = clean_part.replace('ip4:', '')
- try:
- if '/' in ip_spec:
- network = ipaddress.ip_network(ip_spec, strict=False)
- if ipaddress.ip_address(ip_to_check) in network:
- return True, f'ip4:{ip_spec}'
- else:
- if ip_to_check == ip_spec:
- return True, f'ip4:{ip_spec}'
- except:
- pass
-
- elif clean_part in ['a'] or clean_part.startswith('a:'):
- check_domain = domain if clean_part == 'a' else clean_part.split(':', 1)[1]
- try:
- a_records = await resolver.resolve(check_domain, 'A')
- for rdata in a_records:
- if str(rdata) == ip_to_check:
- return True, f'a:{check_domain}' if clean_part.startswith('a:') else 'a'
- except:
- pass
-
- elif clean_part in ['mx'] or clean_part.startswith('mx:'):
- check_domain = domain if clean_part == 'mx' else clean_part.split(':', 1)[1]
- try:
- mx_records = await resolver.resolve(check_domain, 'MX')
- for mx in mx_records:
- try:
- mx_a_records = await resolver.resolve(str(mx.exchange), 'A')
- for rdata in mx_a_records:
- if str(rdata) == ip_to_check:
- return True, f'mx:{check_domain}' if clean_part.startswith('mx:') else 'mx'
- except:
- pass
- except:
- pass
-
- elif clean_part.startswith('include:'):
- include_domain = clean_part.replace('include:', '')
- try:
- include_answers = await resolver.resolve(include_domain, 'TXT')
- for rdata in include_answers:
- include_spf = b''.join(rdata.strings).decode('utf-8')
- if include_spf.startswith('v=spf1'):
- authorized, method = await check_ip_in_spf(
- include_domain,
- ip_to_check,
- include_spf,
- resolver,
- visited_domains.copy(),
- depth + 1
- )
- if authorized:
- return True, f'include:{include_domain} ({method})'
- except:
- pass
-
- return False, None
-
-
def parse_dkim_parameters(dkim_record: str) -> Dict[str, Any]:
"""
Parse and validate DKIM record parameters
diff --git a/backend/app/routers/mailbox_stats.py b/backend/app/routers/mailbox_stats.py
new file mode 100644
index 0000000..0bf424a
--- /dev/null
+++ b/backend/app/routers/mailbox_stats.py
@@ -0,0 +1,553 @@
+"""
+API endpoints for mailbox statistics with message counts
+Shows per-mailbox/per-alias message statistics from MessageCorrelation table
+"""
+import logging
+import hashlib
+import json
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.orm import Session
+from sqlalchemy import func, case, and_, or_
+from datetime import datetime, timezone, timedelta
+from typing import Optional, List
+
+from ..database import get_db
+from ..models import MailboxStatistics, AliasStatistics, MessageCorrelation
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+# =============================================================================
+# CACHING SYSTEM
+# =============================================================================
+
+# In-memory cache for mailbox stats
+_stats_cache = {}
+_cache_ttl_seconds = 300 # 5 minutes cache TTL
+
+
+def _get_cache_key(prefix: str, **params) -> str:
+ """Generate a cache key from parameters"""
+ param_str = json.dumps(params, sort_keys=True, default=str)
+ hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16]
+ return f"{prefix}:{hash_val}"
+
+
+def _get_cached(key: str):
+ """Get cached value if not expired"""
+ if key in _stats_cache:
+ cached_data, cached_time = _stats_cache[key]
+ if datetime.now(timezone.utc) - cached_time < timedelta(seconds=_cache_ttl_seconds):
+ logger.debug(f"Cache hit for key: {key}")
+ return cached_data
+ else:
+ # Cache expired, remove it
+ del _stats_cache[key]
+ return None
+
+
+def _set_cache(key: str, data):
+ """Set cached value with current timestamp"""
+ _stats_cache[key] = (data, datetime.now(timezone.utc))
+ logger.debug(f"Cache set for key: {key}")
+
+
+def clear_stats_cache():
+ """Clear all stats cache - call after data changes"""
+ global _stats_cache
+ _stats_cache = {}
+ logger.info("Stats cache cleared")
+
+
+def format_datetime_utc(dt: Optional[datetime]) -> Optional[str]:
+ """Format datetime for API response with proper UTC timezone"""
+ if dt is None:
+ return None
+
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+
+ dt_utc = dt.astimezone(timezone.utc)
+ return dt_utc.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
+
+
+def format_bytes(bytes_value) -> str:
+ """Format bytes into human-readable format"""
+ if bytes_value is None:
+ return "0 B"
+
+ bytes_value = float(bytes_value)
+
+ for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+ if abs(bytes_value) < 1024.0:
+ return f"{bytes_value:.1f} {unit}"
+ bytes_value /= 1024.0
+ return f"{bytes_value:.1f} PB"
+
+
+def format_unix_timestamp(timestamp: Optional[int]) -> Optional[str]:
+ """Convert Unix timestamp to ISO format string"""
+ if timestamp is None or timestamp == 0:
+ return None
+ try:
+ dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+ return dt.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
+ except (ValueError, OSError):
+ return None
+
+
+def parse_date_range(date_range: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> tuple[datetime, datetime]:
+ """Parse date range string into start and end datetimes
+
+ For custom date ranges, start_date and end_date should be ISO format strings (YYYY-MM-DD)
+ """
+ now = datetime.now(timezone.utc)
+
+ if date_range == "custom" and start_date and end_date:
+ # Parse custom date range
+ try:
+ start = datetime.fromisoformat(start_date.replace('Z', '+00:00'))
+ if start.tzinfo is None:
+ start = start.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc)
+
+ end = datetime.fromisoformat(end_date.replace('Z', '+00:00'))
+ if end.tzinfo is None:
+ # Set end time to end of day
+ end = end.replace(hour=23, minute=59, second=59, microsecond=999999, tzinfo=timezone.utc)
+
+ return start, end
+ except ValueError:
+ # If parsing fails, fall back to 30 days
+ logger.warning(f"Failed to parse custom date range: {start_date} - {end_date}, falling back to 30 days")
+ start = now - timedelta(days=30)
+ end = now
+ elif date_range == "today":
+ start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+ end = now
+ elif date_range == "7days":
+ start = now - timedelta(days=7)
+ end = now
+ elif date_range == "30days":
+ start = now - timedelta(days=30)
+ end = now
+ elif date_range == "90days":
+ start = now - timedelta(days=90)
+ end = now
+ else:
+ # Default to 30 days
+ start = now - timedelta(days=30)
+ end = now
+
+ return start, end
+
+
+def get_message_counts_for_email(db: Session, email: str, start_date: datetime, end_date: datetime) -> dict:
+ """
+ Get message counts for a specific email address (mailbox or alias)
+ Counts sent (as sender) and received (as recipient) with status breakdown
+ Also counts by direction (inbound, outbound, internal)
+ """
+ # Count sent messages (this email as sender) by status
+ sent_query = db.query(
+ MessageCorrelation.final_status,
+ func.count(MessageCorrelation.id).label('count')
+ ).filter(
+ MessageCorrelation.sender == email,
+ MessageCorrelation.first_seen >= start_date,
+ MessageCorrelation.first_seen <= end_date
+ ).group_by(MessageCorrelation.final_status).all()
+
+ # Count received messages (this email as recipient) by status
+ received_query = db.query(
+ MessageCorrelation.final_status,
+ func.count(MessageCorrelation.id).label('count')
+ ).filter(
+ MessageCorrelation.recipient == email,
+ MessageCorrelation.first_seen >= start_date,
+ MessageCorrelation.first_seen <= end_date
+ ).group_by(MessageCorrelation.final_status).all()
+
+ # Count by direction (inbound, outbound, internal)
+ direction_query = db.query(
+ MessageCorrelation.direction,
+ func.count(MessageCorrelation.id).label('count')
+ ).filter(
+ or_(
+ MessageCorrelation.sender == email,
+ MessageCorrelation.recipient == email
+ ),
+ MessageCorrelation.first_seen >= start_date,
+ MessageCorrelation.first_seen <= end_date
+ ).group_by(MessageCorrelation.direction).all()
+
+ # Process results
+ sent_by_status = {row.final_status or 'unknown': row.count for row in sent_query}
+ received_by_status = {row.final_status or 'unknown': row.count for row in received_query}
+ direction_counts = {row.direction or 'unknown': row.count for row in direction_query}
+
+ sent_total = sum(sent_by_status.values())
+ received_total = sum(received_by_status.values())
+
+ # Calculate failures (only for sent messages - bounces and rejections are outbound failures)
+ sent_failed = sent_by_status.get('bounced', 0) + sent_by_status.get('rejected', 0)
+
+ total = sent_total + received_total
+ failure_rate = round((sent_failed / sent_total * 100) if sent_total > 0 else 0, 1)
+
+ return {
+ "sent_total": sent_total,
+ "sent_delivered": sent_by_status.get('delivered', 0) + sent_by_status.get('sent', 0),
+ "sent_bounced": sent_by_status.get('bounced', 0),
+ "sent_rejected": sent_by_status.get('rejected', 0),
+ "sent_deferred": sent_by_status.get('deferred', 0),
+ "sent_expired": sent_by_status.get('expired', 0),
+ "sent_failed": sent_failed,
+ "received_total": received_total,
+ "total_messages": total,
+ "failure_rate": failure_rate,
+ # Direction counts
+ "direction_inbound": direction_counts.get('inbound', 0),
+ "direction_outbound": direction_counts.get('outbound', 0),
+ "direction_internal": direction_counts.get('internal', 0)
+ }
+
+
+@router.get("/mailbox-stats/summary")
+async def get_mailbox_stats_summary(
+ date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"),
+ start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"),
+ end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"),
+ db: Session = Depends(get_db)
+):
+ """
+ Get summary statistics for all mailboxes
+ """
+ try:
+ parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date)
+
+ # Total mailboxes
+ total_mailboxes = db.query(func.count(MailboxStatistics.id)).scalar() or 0
+ active_mailboxes = db.query(func.count(MailboxStatistics.id)).filter(
+ MailboxStatistics.active == True
+ ).scalar() or 0
+
+ # Total aliases
+ total_aliases = db.query(func.count(AliasStatistics.id)).scalar() or 0
+ active_aliases = db.query(func.count(AliasStatistics.id)).filter(
+ AliasStatistics.active == True
+ ).scalar() or 0
+
+ # Unique domains
+ unique_domains = db.query(func.count(func.distinct(MailboxStatistics.domain))).scalar() or 0
+
+ # Get total storage used
+ total_quota_used = db.query(func.sum(MailboxStatistics.quota_used)).scalar() or 0
+
+ # Get last update time
+ last_update = db.query(func.max(MailboxStatistics.updated_at)).scalar()
+
+ # Get all local mailbox emails and alias emails
+ mailbox_emails = [m.username for m in db.query(MailboxStatistics.username).all()]
+ alias_emails = [a.alias_address for a in db.query(AliasStatistics.alias_address).all()]
+ all_local_emails = set(mailbox_emails + alias_emails)
+
+ # Count total messages for all local emails
+ total_sent = 0
+ total_received = 0
+ total_failed = 0
+
+ if all_local_emails:
+ # Sent messages
+ sent_result = db.query(func.count(MessageCorrelation.id)).filter(
+ MessageCorrelation.sender.in_(all_local_emails),
+ MessageCorrelation.first_seen >= parsed_start,
+ MessageCorrelation.first_seen <= parsed_end
+ ).scalar() or 0
+ total_sent = sent_result
+
+ # Received messages
+ received_result = db.query(func.count(MessageCorrelation.id)).filter(
+ MessageCorrelation.recipient.in_(all_local_emails),
+ MessageCorrelation.first_seen >= parsed_start,
+ MessageCorrelation.first_seen <= parsed_end
+ ).scalar() or 0
+ total_received = received_result
+
+ # Failed messages (only sent that bounced/rejected - failures are outbound)
+ failed_result = db.query(func.count(MessageCorrelation.id)).filter(
+ MessageCorrelation.sender.in_(all_local_emails),
+ MessageCorrelation.first_seen >= parsed_start,
+ MessageCorrelation.first_seen <= parsed_end,
+ MessageCorrelation.final_status.in_(['bounced', 'rejected'])
+ ).scalar() or 0
+ total_sent_failed = failed_result
+
+ total_messages = total_sent + total_received
+ failure_rate = round((total_sent_failed / total_sent * 100) if total_sent > 0 else 0, 1)
+
+ return {
+ "total_sent": total_sent,
+ "total_received": total_received,
+ "total_messages": total_messages,
+ "sent_failed": total_sent_failed,
+ "failure_rate": failure_rate,
+ "date_range": date_range,
+ "start_date": format_datetime_utc(parsed_start),
+ "end_date": format_datetime_utc(parsed_end),
+ "last_update": format_datetime_utc(last_update)
+ }
+ except Exception as e:
+ logger.error(f"Error fetching mailbox stats summary: {e}")
+ return {"error": str(e), "total_mailboxes": 0}
+
+
+@router.get("/mailbox-stats/all")
+async def get_all_mailbox_stats(
+ domain: Optional[str] = None,
+ active_only: bool = True, # Changed default to True
+ hide_zero: bool = False, # Filter out mailboxes with zero activity
+ search: Optional[str] = None,
+ date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"),
+ start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"),
+ end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"),
+ sort_by: str = "sent_total",
+ sort_order: str = "desc",
+ page: int = Query(1, ge=1, description="Page number"),
+ page_size: int = Query(50, ge=10, le=100, description="Items per page"),
+ db: Session = Depends(get_db)
+):
+ """
+ Get all mailbox statistics with message counts and aliases (paginated)
+ """
+ try:
+ # Check cache first
+ cache_key = _get_cache_key(
+ "mailbox_stats_all",
+ domain=domain,
+ active_only=active_only,
+ hide_zero=hide_zero,
+ search=search,
+ date_range=date_range,
+ start_date=start_date,
+ end_date=end_date,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ page=page,
+ page_size=page_size
+ )
+
+ cached_result = _get_cached(cache_key)
+ if cached_result is not None:
+ return cached_result
+
+ parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date)
+
+ query = db.query(MailboxStatistics)
+
+ # Apply domain filter
+ if domain:
+ query = query.filter(MailboxStatistics.domain == domain)
+
+ # Apply active filter
+ if active_only:
+ query = query.filter(MailboxStatistics.active == True)
+
+ # Apply search filter on mailbox username/name OR mailboxes that have matching aliases
+ if search:
+ mailbox_search_term = f"%{search}%"
+
+ # Find mailboxes that have matching aliases
+ alias_matched_usernames = db.query(AliasStatistics.primary_mailbox).filter(
+ AliasStatistics.alias_address.ilike(mailbox_search_term)
+ ).distinct().subquery()
+
+ query = query.filter(
+ or_(
+ MailboxStatistics.username.ilike(mailbox_search_term),
+ MailboxStatistics.name.ilike(mailbox_search_term),
+ MailboxStatistics.username.in_(alias_matched_usernames)
+ )
+ )
+
+ # Get total count before pagination
+ total_count = query.count()
+
+ # Get all for sorting (we need to calculate counts before pagination)
+ mailboxes = query.all()
+
+ # Build result with message counts for each mailbox
+ result = []
+ for mb in mailboxes:
+ # Get message counts for this mailbox
+ counts = get_message_counts_for_email(db, mb.username, parsed_start, parsed_end)
+
+ # Get aliases for this mailbox
+ aliases = db.query(AliasStatistics).filter(
+ AliasStatistics.primary_mailbox == mb.username
+ ).all()
+
+ # Get message counts for each alias
+ alias_list = []
+ alias_sent_total = 0
+ alias_received_total = 0
+ alias_failed_total = 0
+ alias_internal_total = 0
+ alias_delivered_total = 0
+
+ for alias in aliases:
+ alias_counts = get_message_counts_for_email(db, alias.alias_address, parsed_start, parsed_end)
+ alias_sent_total += alias_counts['sent_total']
+ alias_received_total += alias_counts['received_total']
+ alias_failed_total += alias_counts['sent_failed']
+ alias_internal_total += alias_counts['direction_internal']
+ alias_delivered_total += alias_counts['sent_delivered']
+
+ alias_list.append({
+ "alias_address": alias.alias_address,
+ "active": alias.active,
+ "is_catch_all": alias.is_catch_all,
+ **alias_counts
+ })
+
+ # Calculate combined totals (mailbox + all aliases)
+ combined_sent = counts['sent_total'] + alias_sent_total
+ combined_received = counts['received_total'] + alias_received_total
+ combined_total = combined_sent + combined_received
+ combined_failed = counts['sent_failed'] + alias_failed_total
+ combined_failure_rate = round((combined_failed / combined_sent * 100) if combined_sent > 0 else 0, 1)
+ combined_internal = counts['direction_internal'] + alias_internal_total
+ combined_delivered = counts['sent_delivered'] + alias_delivered_total
+ combined_inbound = counts['direction_inbound']
+ combined_outbound = counts['direction_outbound']
+
+ result.append({
+ "id": mb.id,
+ "username": mb.username,
+ "domain": mb.domain,
+ "name": mb.name,
+ "active": mb.active,
+ # Quota info
+ "quota": float(mb.quota or 0),
+ "quota_formatted": format_bytes(mb.quota),
+ "quota_used": float(mb.quota_used or 0),
+ "quota_used_formatted": format_bytes(mb.quota_used),
+ "percent_in_use": round(float(mb.percent_in_use or 0), 1),
+ "messages_in_mailbox": mb.messages or 0,
+ # Last login times
+ "last_imap_login": format_unix_timestamp(mb.last_imap_login),
+ "last_pop3_login": format_unix_timestamp(mb.last_pop3_login),
+ "last_smtp_login": format_unix_timestamp(mb.last_smtp_login),
+ # Rate limiting
+ "rl_value": mb.rl_value,
+ "rl_frame": mb.rl_frame,
+ # Attributes (access permissions)
+ "attributes": mb.attributes or {},
+ # Message counts for mailbox only
+ "mailbox_counts": counts,
+ # Aliases
+ "aliases": alias_list,
+ "alias_count": len(alias_list),
+ # Combined totals (mailbox + aliases)
+ "combined_sent": combined_sent,
+ "combined_received": combined_received,
+ "combined_total": combined_total,
+ "combined_failed": combined_failed,
+ "combined_failure_rate": combined_failure_rate,
+ # Direction and status combined counts
+ "combined_internal": combined_internal,
+ "combined_delivered": combined_delivered,
+ "combined_inbound": combined_inbound,
+ "combined_outbound": combined_outbound,
+ # Metadata
+ "created": format_datetime_utc(mb.created_at),
+ "modified": format_datetime_utc(mb.updated_at)
+ })
+
+ # Sort results
+ reverse = sort_order.lower() == "desc"
+ if sort_by == "sent_total":
+ result.sort(key=lambda x: x['combined_sent'], reverse=reverse)
+ elif sort_by == "received_total":
+ result.sort(key=lambda x: x['combined_received'], reverse=reverse)
+ elif sort_by == "failure_rate":
+ result.sort(key=lambda x: x['combined_failure_rate'], reverse=reverse)
+ elif sort_by == "username":
+ result.sort(key=lambda x: x['username'].lower(), reverse=reverse)
+ elif sort_by == "quota_used":
+ result.sort(key=lambda x: x['quota_used'], reverse=reverse)
+ else:
+ result.sort(key=lambda x: x['combined_total'], reverse=reverse)
+
+ # Apply hide_zero filter - remove mailboxes with no activity
+ if hide_zero:
+ result = [r for r in result if r['combined_total'] > 0]
+
+ # Apply pagination
+ total_pages = (len(result) + page_size - 1) // page_size
+ start_index = (page - 1) * page_size
+ end_index = start_index + page_size
+ paginated_result = result[start_index:end_index]
+
+ response = {
+ "total": len(result),
+ "page": page,
+ "page_size": page_size,
+ "total_pages": total_pages,
+ "date_range": date_range,
+ "start_date": format_datetime_utc(parsed_start),
+ "end_date": format_datetime_utc(parsed_end),
+ "mailboxes": paginated_result
+ }
+
+ # Cache the result
+ _set_cache(cache_key, response)
+
+ return response
+ except Exception as e:
+ logger.error(f"Error fetching all mailbox stats: {e}")
+ return {"error": str(e), "total": 0, "mailboxes": []}
+
+
+@router.get("/mailbox-stats/domains")
+async def get_mailbox_domains(db: Session = Depends(get_db)):
+ """
+ Get list of all domains for filtering
+ """
+ try:
+ domains = db.query(
+ MailboxStatistics.domain,
+ func.count(MailboxStatistics.id).label('count')
+ ).group_by(
+ MailboxStatistics.domain
+ ).order_by(
+ MailboxStatistics.domain
+ ).all()
+
+ return {
+ "domains": [
+ {"domain": d.domain, "mailbox_count": d.count}
+ for d in domains
+ ]
+ }
+ except Exception as e:
+ logger.error(f"Error fetching mailbox domains: {e}")
+ return {"error": str(e), "domains": []}
+
+
+@router.get("/mailbox-stats/refresh")
+async def refresh_mailbox_stats(db: Session = Depends(get_db)):
+ """
+ Get last update time for mailbox statistics
+ """
+ try:
+ last_mailbox_update = db.query(func.max(MailboxStatistics.updated_at)).scalar()
+ last_alias_update = db.query(func.max(AliasStatistics.updated_at)).scalar()
+
+ return {
+ "last_mailbox_update": format_datetime_utc(last_mailbox_update),
+ "last_alias_update": format_datetime_utc(last_alias_update)
+ }
+ except Exception as e:
+ logger.error(f"Error getting refresh info: {e}")
+ return {"error": str(e)}
diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py
index d540bf8..e6f463e 100644
--- a/backend/app/routers/settings.py
+++ b/backend/app/routers/settings.py
@@ -224,6 +224,20 @@ async def get_settings_info(db: Session = Depends(get_db)):
"status": jobs_status.get('update_geoip', {}).get('status', 'idle') if is_license_configured() else 'disabled',
"last_run": format_datetime_utc(jobs_status.get('update_geoip', {}).get('last_run')) if is_license_configured() else None,
"error": jobs_status.get('update_geoip', {}).get('error') if is_license_configured() else None
+ },
+ "mailbox_stats": {
+ "interval": "5 minutes",
+ "description": "Fetches mailbox statistics from Mailcow API",
+ "status": jobs_status.get('mailbox_stats', {}).get('status', 'unknown'),
+ "last_run": format_datetime_utc(jobs_status.get('mailbox_stats', {}).get('last_run')),
+ "error": jobs_status.get('mailbox_stats', {}).get('error')
+ },
+ "alias_stats": {
+ "interval": "5 minutes",
+ "description": "Syncs alias data from Mailcow API",
+ "status": jobs_status.get('alias_stats', {}).get('status', 'unknown'),
+ "last_run": format_datetime_utc(jobs_status.get('alias_stats', {}).get('last_run')),
+ "error": jobs_status.get('alias_stats', {}).get('error')
}
},
"smtp_configuration": {
diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py
index b00cd9d..bd2abde 100644
--- a/backend/app/scheduler.py
+++ b/backend/app/scheduler.py
@@ -18,7 +18,7 @@ from sqlalchemy.exc import IntegrityError
from .config import settings, set_cached_active_domains
from .database import get_db_context, SessionLocal
from .mailcow_api import mailcow_api
-from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck
+from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck, MailboxStatistics, AliasStatistics
from .correlation import detect_direction, parse_postfix_message
from .routers.domains import check_domain_dns, save_dns_check_to_db
from .services.dmarc_imap_service import sync_dmarc_reports_from_imap
@@ -44,7 +44,9 @@ job_status = {
'check_app_version': {'last_run': None, 'status': 'idle', 'error': None},
'dns_check': {'last_run': None, 'status': 'idle', 'error': None},
'update_geoip': {'last_run': None, 'status': 'idle', 'error': None},
- 'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None}
+ 'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None},
+ 'mailbox_stats': {'last_run': None, 'status': 'idle', 'error': None},
+ 'alias_stats': {'last_run': None, 'status': 'idle', 'error': None}
}
def update_job_status(job_name: str, status: str, error: str = None):
@@ -1404,6 +1406,266 @@ async def sync_local_domains():
update_job_status('sync_local_domains', 'failed', str(e))
return False
+# =============================================================================
+# MAILBOX STATISTICS
+# =============================================================================
+
+def safe_int(value, default=0):
+ """Safely convert a value to int, handling '- ', None, and other invalid values"""
+ if value is None:
+ return default
+ if isinstance(value, int):
+ return value
+ if isinstance(value, str):
+ value = value.strip()
+ if value in ('', '-', '- '):
+ return default
+ try:
+ return int(value)
+ except (ValueError, TypeError):
+ return default
+ try:
+ return int(value)
+ except (ValueError, TypeError):
+ return default
+
+def safe_float(value, default=0.0):
+ """Safely convert a value to float, handling '- ', None, and other invalid values"""
+ if value is None:
+ return default
+ if isinstance(value, (int, float)):
+ return float(value)
+ if isinstance(value, str):
+ value = value.strip()
+ if value in ('', '-', '- '):
+ return default
+ try:
+ return float(value)
+ except (ValueError, TypeError):
+ return default
+ try:
+ return float(value)
+ except (ValueError, TypeError):
+ return default
+
+async def update_mailbox_statistics():
+ """
+ Fetch mailbox statistics from Mailcow API and update the database.
+ Runs every 5 minutes.
+ Also removes mailboxes that no longer exist in Mailcow.
+ """
+ update_job_status('mailbox_stats', 'running')
+ logger.info("Starting mailbox statistics update...")
+
+ try:
+ # Fetch mailboxes from Mailcow API
+ mailboxes = await mailcow_api.get_mailboxes()
+
+ if not mailboxes:
+ logger.warning("No mailboxes retrieved from Mailcow API")
+ update_job_status('mailbox_stats', 'success')
+ return
+
+ # Get set of current mailbox usernames from API
+ api_mailbox_usernames = {mb.get('username') for mb in mailboxes if mb.get('username')}
+
+ with get_db_context() as db:
+ updated = 0
+ created = 0
+ deleted = 0
+
+ # First, mark mailboxes that no longer exist in Mailcow as inactive
+ db_mailboxes = db.query(MailboxStatistics).all()
+ for db_mb in db_mailboxes:
+ if db_mb.username not in api_mailbox_usernames:
+ if db_mb.active: # Only log and count if it was previously active
+ logger.info(f"Marking deleted mailbox as inactive: {db_mb.username}")
+ db_mb.active = False
+ db_mb.updated_at = datetime.now(timezone.utc)
+ deleted += 1
+
+ for mb in mailboxes:
+ try:
+ username = mb.get('username')
+ if not username:
+ continue
+
+ # Extract domain from username
+ domain = username.split('@')[-1] if '@' in username else ''
+
+ # Check if mailbox exists
+ existing = db.query(MailboxStatistics).filter(
+ MailboxStatistics.username == username
+ ).first()
+
+ # Prepare data - safely convert values
+ attributes = mb.get('attributes', {})
+ rl_value_raw = mb.get('rl_value')
+ rl_value = safe_int(rl_value_raw) if rl_value_raw not in (None, '', '-', '- ') else None
+ rl_frame = mb.get('rl_frame')
+ if rl_frame in ('', '-', '- '):
+ rl_frame = None
+
+ if existing:
+ # Update existing record
+ existing.domain = domain
+ existing.name = mb.get('name', '') or ''
+ existing.quota = safe_int(mb.get('quota'), 0)
+ existing.quota_used = safe_int(mb.get('quota_used'), 0)
+ existing.percent_in_use = safe_float(mb.get('percent_in_use'), 0.0)
+ existing.messages = safe_int(mb.get('messages'), 0)
+ existing.active = mb.get('active', 1) == 1
+ existing.last_imap_login = safe_int(mb.get('last_imap_login'), 0) or None
+ existing.last_pop3_login = safe_int(mb.get('last_pop3_login'), 0) or None
+ existing.last_smtp_login = safe_int(mb.get('last_smtp_login'), 0) or None
+ existing.spam_aliases = safe_int(mb.get('spam_aliases'), 0)
+ existing.rl_value = rl_value
+ existing.rl_frame = rl_frame
+ existing.attributes = attributes
+ existing.updated_at = datetime.now(timezone.utc)
+ updated += 1
+ else:
+ # Create new record
+ new_mailbox = MailboxStatistics(
+ username=username,
+ domain=domain,
+ name=mb.get('name', '') or '',
+ quota=safe_int(mb.get('quota'), 0),
+ quota_used=safe_int(mb.get('quota_used'), 0),
+ percent_in_use=safe_float(mb.get('percent_in_use'), 0.0),
+ messages=safe_int(mb.get('messages'), 0),
+ active=mb.get('active', 1) == 1,
+ last_imap_login=safe_int(mb.get('last_imap_login'), 0) or None,
+ last_pop3_login=safe_int(mb.get('last_pop3_login'), 0) or None,
+ last_smtp_login=safe_int(mb.get('last_smtp_login'), 0) or None,
+ spam_aliases=safe_int(mb.get('spam_aliases'), 0),
+ rl_value=rl_value,
+ rl_frame=rl_frame,
+ attributes=attributes
+ )
+ db.add(new_mailbox)
+ created += 1
+
+ except Exception as e:
+ logger.error(f"Error processing mailbox {mb.get('username', 'unknown')}: {e}")
+ continue
+
+ db.commit()
+ logger.info(f"✓ Mailbox statistics updated: {updated} updated, {created} created, {deleted} deactivated")
+ update_job_status('mailbox_stats', 'success')
+
+ except Exception as e:
+ logger.error(f"✗ Failed to update mailbox statistics: {e}")
+ update_job_status('mailbox_stats', 'failed', str(e))
+
+
+# =============================================================================
+# ALIAS STATISTICS
+# =============================================================================
+
+async def update_alias_statistics():
+ """
+ Fetch aliases from Mailcow API and update the database.
+ Links aliases to their target mailboxes.
+ Runs every 5 minutes.
+ Also removes aliases that no longer exist in Mailcow.
+ """
+ update_job_status('alias_stats', 'running')
+ logger.info("Starting alias statistics update...")
+
+ try:
+ # Fetch aliases from Mailcow API
+ aliases = await mailcow_api.get_aliases()
+
+ if not aliases:
+ logger.warning("No aliases retrieved from Mailcow API")
+ update_job_status('alias_stats', 'success')
+ return
+
+ # Get set of current alias addresses from API
+ api_alias_addresses = {alias.get('address') for alias in aliases if alias.get('address')}
+
+ with get_db_context() as db:
+ updated = 0
+ created = 0
+ deleted = 0
+
+ # First, mark aliases that no longer exist in Mailcow as inactive
+ db_aliases = db.query(AliasStatistics).all()
+ for db_alias in db_aliases:
+ if db_alias.alias_address not in api_alias_addresses:
+ if db_alias.active: # Only log and count if it was previously active
+ logger.info(f"Marking deleted alias as inactive: {db_alias.alias_address}")
+ db_alias.active = False
+ db_alias.updated_at = datetime.now(timezone.utc)
+ deleted += 1
+
+ for alias in aliases:
+ try:
+ alias_address = alias.get('address')
+ if not alias_address:
+ continue
+
+ # Skip if this is a mailbox address (not an alias)
+ if alias.get('is_catch_all') is None and not alias.get('goto'):
+ continue
+
+ # Extract domain from alias address
+ domain = alias_address.split('@')[-1] if '@' in alias_address else ''
+
+ # Get the target mailbox(es)
+ goto = alias.get('goto', '')
+
+ # Determine primary mailbox (first in goto list)
+ primary_mailbox = None
+ if goto:
+ goto_list = [g.strip() for g in goto.split(',') if g.strip()]
+ if goto_list:
+ primary_mailbox = goto_list[0]
+
+ # Check if alias exists
+ existing = db.query(AliasStatistics).filter(
+ AliasStatistics.alias_address == alias_address
+ ).first()
+
+ is_catch_all = alias.get('is_catch_all', 0) == 1
+ is_active = alias.get('active', 1) == 1
+
+ if existing:
+ # Update existing record
+ existing.goto = goto
+ existing.domain = domain
+ existing.active = is_active
+ existing.is_catch_all = is_catch_all
+ existing.primary_mailbox = primary_mailbox
+ existing.updated_at = datetime.now(timezone.utc)
+ updated += 1
+ else:
+ # Create new record
+ new_alias = AliasStatistics(
+ alias_address=alias_address,
+ goto=goto,
+ domain=domain,
+ active=is_active,
+ is_catch_all=is_catch_all,
+ primary_mailbox=primary_mailbox
+ )
+ db.add(new_alias)
+ created += 1
+
+ except Exception as e:
+ logger.error(f"Error processing alias {alias.get('address', 'unknown')}: {e}")
+ continue
+
+ db.commit()
+ logger.info(f"✓ Alias statistics updated: {updated} updated, {created} created, {deleted} deactivated")
+ update_job_status('alias_stats', 'success')
+
+ except Exception as e:
+ logger.error(f"✗ Failed to update alias statistics: {e}")
+ update_job_status('alias_stats', 'failed', str(e))
+
+
# =============================================================================
# SCHEDULER SETUP
# =============================================================================
@@ -1562,6 +1824,46 @@ def start_scheduler():
)
logger.info("Scheduled initial DMARC IMAP sync on startup")
+ # Job 13: Mailbox Statistics (every 5 minutes)
+ scheduler.add_job(
+ update_mailbox_statistics,
+ IntervalTrigger(minutes=5),
+ id='mailbox_stats',
+ name='Update Mailbox Statistics',
+ replace_existing=True,
+ max_instances=1
+ )
+
+ # Run once on startup (after 45 seconds)
+ scheduler.add_job(
+ update_mailbox_statistics,
+ 'date',
+ run_date=datetime.now(timezone.utc) + timedelta(seconds=45),
+ id='mailbox_stats_startup',
+ name='Mailbox Statistics (Startup)'
+ )
+ logger.info("Scheduled mailbox statistics job (interval: 5 minutes)")
+
+ # Job 14: Alias Statistics (every 5 minutes)
+ scheduler.add_job(
+ update_alias_statistics,
+ IntervalTrigger(minutes=5),
+ id='alias_stats',
+ name='Update Alias Statistics',
+ replace_existing=True,
+ max_instances=1
+ )
+
+ # Run once on startup (after 50 seconds)
+ scheduler.add_job(
+ update_alias_statistics,
+ 'date',
+ run_date=datetime.now(timezone.utc) + timedelta(seconds=50),
+ id='alias_stats_startup',
+ name='Alias Statistics (Startup)'
+ )
+ logger.info("Scheduled alias statistics job (interval: 5 minutes)")
+
scheduler.start()
logger.info("[OK] Scheduler started")
diff --git a/backend/app/services/connection_test.py b/backend/app/services/connection_test.py
index 222e10a..6e900e1 100644
--- a/backend/app/services/connection_test.py
+++ b/backend/app/services/connection_test.py
@@ -20,11 +20,18 @@ def test_smtp_connection() -> Dict:
logs.append(f"Host: {settings.smtp_host}")
logs.append(f"Port: {settings.smtp_port}")
logs.append(f"Use TLS: {settings.smtp_use_tls}")
+ logs.append(f"Relay Mode: {settings.smtp_relay_mode}")
logs.append(f"User: {settings.smtp_user}")
- if not settings.smtp_host or not settings.smtp_user or not settings.smtp_password:
- logs.append("ERROR: SMTP not fully configured")
- return {"success": False, "logs": logs}
+ # Different validation for relay mode
+ if settings.smtp_relay_mode:
+ if not settings.smtp_host or not settings.smtp_from:
+ logs.append("ERROR: SMTP relay mode requires host and from address")
+ return {"success": False, "logs": logs}
+ else:
+ if not settings.smtp_host or not settings.smtp_user or not settings.smtp_password:
+ logs.append("ERROR: SMTP not fully configured")
+ return {"success": False, "logs": logs}
logs.append("Connecting to SMTP server...")
@@ -40,9 +47,13 @@ def test_smtp_connection() -> Dict:
server.starttls()
logs.append("TLS established")
- logs.append("Logging in...")
- server.login(settings.smtp_user, settings.smtp_password)
- logs.append("Login successful")
+ # Skip login in relay mode
+ if not settings.smtp_relay_mode:
+ logs.append("Logging in...")
+ server.login(settings.smtp_user, settings.smtp_password)
+ logs.append("Login successful")
+ else:
+ logs.append("Relay mode - skipping authentication")
logs.append("Sending test email...")
msg = MIMEMultipart()
diff --git a/backend/app/services/dmarc_cache.py b/backend/app/services/dmarc_cache.py
new file mode 100644
index 0000000..e62d2c4
--- /dev/null
+++ b/backend/app/services/dmarc_cache.py
@@ -0,0 +1,95 @@
+"""
+Shared DMARC Caching Service
+Used to share cache state between API router and background services
+"""
+import logging
+import json
+import hashlib
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Any
+from sqlalchemy.orm import Session
+from ..models import SystemSetting
+
+logger = logging.getLogger(__name__)
+
+# In-memory cache for DMARC stats
+_dmarc_cache = {}
+_dmarc_cache_ttl_seconds = 300 # 5 minutes cache TTL
+
+# Global signal tracking
+_cache_valid_since = datetime.now(timezone.utc)
+_last_db_check = datetime.min.replace(tzinfo=timezone.utc)
+_db_check_interval_seconds = 5
+
+
+def get_dmarc_cache_key(prefix: str, **params) -> str:
+ """Generate a cache key from parameters"""
+ param_str = json.dumps(params, sort_keys=True, default=str)
+ hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16]
+ return f"dmarc:{prefix}:{hash_val}"
+
+
+def get_dmarc_cached(key: str, db: Session) -> Optional[Any]:
+ """Get cached value if not expired and not invalidated globally"""
+ global _dmarc_cache, _last_db_check, _cache_valid_since
+
+ now = datetime.now(timezone.utc)
+
+ # Periodically check DB for invalidation signal
+ if (now - _last_db_check).total_seconds() > _db_check_interval_seconds:
+ _last_db_check = now
+ try:
+ setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first()
+ if setting and setting.updated_at:
+ # Ensure timezone awareness
+ db_updated_at = setting.updated_at
+ if db_updated_at.tzinfo is None:
+ db_updated_at = db_updated_at.replace(tzinfo=timezone.utc)
+
+ # If DB signal is newer than our local validity, clear cache
+ if db_updated_at > _cache_valid_since:
+ logger.info("DMARC cache invalidated by another process")
+ _dmarc_cache = {}
+ _cache_valid_since = now
+ return None
+ except Exception as e:
+ logger.error(f"Error checking cache signal: {e}")
+
+ if key in _dmarc_cache:
+ cached_data, cached_time = _dmarc_cache[key]
+ if now - cached_time < timedelta(seconds=_dmarc_cache_ttl_seconds):
+ logger.debug(f"DMARC cache hit for key: {key}")
+ return cached_data
+ else:
+ # Cache expired, remove it
+ del _dmarc_cache[key]
+ return None
+
+
+def set_dmarc_cache(key: str, data: Any) -> None:
+ """Set cached value with current timestamp"""
+ _dmarc_cache[key] = (data, datetime.now(timezone.utc))
+ logger.debug(f"DMARC cache set for key: {key}")
+
+
+def clear_dmarc_cache(db: Session) -> None:
+ """Clear all DMARC cache locally and signal other processes via DB"""
+ global _dmarc_cache, _cache_valid_since
+
+ # local clear
+ _dmarc_cache = {}
+ _cache_valid_since = datetime.now(timezone.utc)
+
+ try:
+ # DB signal
+ setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first()
+ if not setting:
+ setting = SystemSetting(key="dmarc_last_update", value="signal")
+ db.add(setting)
+
+ setting.updated_at = datetime.utcnow()
+ db.commit()
+ logger.info("DMARC cache cleared and signaled to DB")
+ except Exception as e:
+ logger.error(f"Error clearing cache signal: {e}")
+ db.rollback()
diff --git a/backend/app/services/dmarc_imap_service.py b/backend/app/services/dmarc_imap_service.py
index ea7a596..5d3363c 100644
--- a/backend/app/services/dmarc_imap_service.py
+++ b/backend/app/services/dmarc_imap_service.py
@@ -1,6 +1,6 @@
"""
-DMARC IMAP Service
-Automatically fetches and processes DMARC reports from email inbox
+DMARC & TLS-RPT IMAP Service
+Automatically fetches and processes DMARC and TLS-RPT reports from email inbox
"""
import logging
import imaplib
@@ -8,16 +8,19 @@ import email
import gzip
import zipfile
import io
+import json
from datetime import datetime, timezone
from typing import List, Dict, Optional, Tuple
from email.message import EmailMessage
from ..config import settings
from ..database import SessionLocal
-from ..models import DMARCSync, DMARCReport, DMARCRecord
+from ..models import DMARCSync, DMARCReport, DMARCRecord, TLSReport, TLSReportPolicy
from ..services.dmarc_parser import parse_dmarc_file
+from ..services.tls_rpt_parser import parse_tls_rpt_file, is_tls_rpt_json
from ..services.geoip_service import enrich_dmarc_record
from ..services.dmarc_notifications import send_dmarc_error_notification
+from ..services.dmarc_cache import clear_dmarc_cache
logger = logging.getLogger(__name__)
@@ -75,30 +78,33 @@ class DMARCImapService:
logger.error(f"Error selecting folder: {e}")
return False
- def search_dmarc_emails(self) -> List[bytes]:
+ def search_report_emails(self) -> List[bytes]:
"""
- Search for DMARC report emails
+ Search for DMARC and TLS-RPT report emails
Looking for emails with subject containing:
- - "Report Domain:"
- - "Submitter:"
- - "Report-ID:"
+ - "Report Domain:" (DMARC)
+ - "DMARC" (DMARC)
+ - "Report-ID:" (DMARC)
+ - "TLS-RPT" (TLS-RPT)
+ - "TLS Report" (TLS-RPT)
Returns list of email IDs
"""
try:
- # Search for emails with DMARC-related subject
+ # Search for emails with DMARC or TLS-RPT related subjects
# Using OR to be more flexible
- search_criteria = '(OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (SUBJECT "Report-ID:")))'
+ # UNSEEN ensures we don't re-process emails that were already handled (marked as Seen)
+ search_criteria = '(UNSEEN (OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (OR (SUBJECT "Report-ID:") (OR (SUBJECT "TLS-RPT") (SUBJECT "TLS Report"))))))'
- status, messages = self.connection.search(None, search_criteria)
+ status, messages = self.connection.uid('SEARCH', None, search_criteria)
if status != 'OK':
- logger.error("Failed to search for DMARC emails")
+ logger.error("Failed to search for report emails")
return []
email_ids = messages[0].split()
- logger.info(f"Found {len(email_ids)} potential DMARC emails")
+ logger.info(f"Found {len(email_ids)} potential DMARC/TLS-RPT emails")
return email_ids
@@ -106,83 +112,238 @@ class DMARCImapService:
logger.error(f"Error searching for emails: {e}")
return []
+ def search_dmarc_emails(self) -> List[bytes]:
+ """Alias for backward compatibility"""
+ return self.search_report_emails()
+
def is_valid_dmarc_email(self, msg: EmailMessage) -> bool:
"""
Validate that this is a genuine DMARC report email
- Checks:
- 1. Subject contains "Report Domain:" AND ("Submitter:" OR "Report-ID:")
- 2. Has at least one compressed attachment (.xml.gz or .zip)
+ Accepts multiple DMARC email formats:
+ - Standard: "Report Domain: X Submitter: Y Report-ID: Z"
+ - Yahoo format: "Report Domain: X Submitter: Y" (no Report-ID)
+ - Alternative: Contains "DMARC" in subject
+ - Microsoft Outlook: DMARC-like attachment filename pattern
+
+ Primary validation is the attachment (.xml.gz or .zip with DMARC content)
"""
try:
subject = msg.get('subject', '').lower()
- # Check subject format
- has_report_domain = 'report domain:' in subject
- has_submitter = 'submitter:' in subject
- has_report_id = 'report-id:' in subject
+ # Check for compressed DMARC attachments FIRST (most reliable indicator)
+ has_dmarc_attachment = False
+ has_dmarc_filename = False
- if not (has_report_domain and (has_submitter or has_report_id)):
- logger.debug(f"Email does not match DMARC subject pattern: {subject}")
- return False
-
- # Check for compressed attachments
- has_attachment = False
for part in msg.walk():
filename = part.get_filename()
if filename:
filename_lower = filename.lower()
- if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip'):
- has_attachment = True
+ # DMARC reports come as .xml.gz, .xml, or .zip files
+ if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip') or filename_lower.endswith('.xml'):
+ has_dmarc_attachment = True
+ # Check if filename looks like a DMARC report
+ # Microsoft format: enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz
+ # Standard format: domain!report-domain!timestamp!timestamp.xml.gz
+ if '!' in filename and (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml') or filename_lower.endswith('.zip')):
+ has_dmarc_filename = True
break
- if not has_attachment:
+ if not has_dmarc_attachment:
logger.debug(f"Email has no compressed DMARC attachment: {subject}")
return False
+ # Check subject format - be flexible to support different providers
+ has_report_domain = 'report domain:' in subject
+ has_submitter = 'submitter:' in subject
+ has_report_id = 'report-id:' in subject
+ has_dmarc_keyword = 'dmarc' in subject
+
+ # Accept if:
+ # 1. Has "Report Domain:" and ("Submitter:" or "Report-ID:") - standard format
+ # 2. Has "Report Domain:" only (Yahoo and others) - we have verified attachment
+ # 3. Has "DMARC" keyword in subject with valid attachment
+ # 4. Has DMARC-like filename pattern (Microsoft Outlook and others) - attachment name contains '!'
+ is_valid_subject = (
+ (has_report_domain and (has_submitter or has_report_id)) or # Standard format
+ (has_report_domain) or # Yahoo/minimal format (attachment already verified)
+ (has_dmarc_keyword) or # DMARC keyword with attachment
+ (has_dmarc_filename) # Microsoft Outlook format - DMARC filename pattern
+ )
+
+ if not is_valid_subject:
+ logger.debug(f"Email does not match DMARC subject/filename pattern: {subject}")
+ return False
+
return True
except Exception as e:
logger.error(f"Error validating DMARC email: {e}")
return False
- def extract_attachments(self, msg: EmailMessage) -> List[Tuple[str, bytes]]:
+ def is_valid_tls_rpt_email(self, msg: EmailMessage) -> bool:
+ """
+ Validate that this is a TLS-RPT report email
+
+ TLS-RPT emails typically have:
+ - Subject containing "TLS-RPT" or "TLS Report"
+ - JSON or JSON.GZ attachment
+ - Some providers send with generic subjects like "Report Domain: ..."
+ """
+ try:
+ subject = msg.get('subject', '').lower()
+
+ # Check for JSON/ZIP attachments
+ has_json_attachment = False
+
+ for part in msg.walk():
+ filename = part.get_filename()
+ if filename:
+ filename_lower = filename.lower()
+ if filename_lower.endswith('.json') or filename_lower.endswith('.json.gz') or filename_lower.endswith('.zip'):
+ has_json_attachment = True
+ break
+
+ if not has_json_attachment:
+ return False
+
+ # Trust the attachment if it looks like a TLS report
+ # If it has a json/gz/zip attachment, we should try to process it as potential TLS-RPT
+ # The parser will validate the content anyway
+ return True
+
+ except Exception as e:
+ logger.error(f"Error validating TLS-RPT email: {e}")
+ return False
+
+ def detect_email_type(self, msg: EmailMessage) -> str:
+ """
+ Detect if email is DMARC or TLS-RPT by inspecting attachments
+
+ Returns: 'dmarc', 'tls-rpt', or 'unknown'
+ """
+ try:
+ # Check attachments FIRST - content is king
+ for part in msg.walk():
+ filename = part.get_filename()
+ if not filename:
+ continue
+
+ filename_lower = filename.lower()
+ content = None
+
+ # Check explicit extensions
+ if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml'):
+ return 'dmarc'
+
+ if filename_lower.endswith('.json.gz') or filename_lower.endswith('.json'):
+ return 'tls-rpt'
+
+ # Check ZIP content
+ if filename_lower.endswith('.zip'):
+ try:
+ content = part.get_payload(decode=True)
+ if content:
+ with zipfile.ZipFile(io.BytesIO(content)) as zf:
+ for name in zf.namelist():
+ name_lower = name.lower()
+ if name_lower.endswith('.xml'):
+ logger.info(f"Found XML in ZIP {filename}, identifying as DMARC")
+ return 'dmarc'
+ if name_lower.endswith('.json'):
+ logger.info(f"Found JSON in ZIP {filename}, identifying as TLS-RPT")
+ return 'tls-rpt'
+ except Exception as e:
+ logger.warning(f"Failed to inspect ZIP {filename}: {e}")
+
+ # Fallback to subject/header heuristics if no clear attachment type found
+ # but reject ambiguous ZIPs that we couldn't inspect or were empty of relevant files
+
+ if self.is_valid_tls_rpt_email(msg):
+ return 'tls-rpt'
+ elif self.is_valid_dmarc_email(msg):
+ return 'dmarc'
+
+ return 'unknown'
+
+ except Exception as e:
+ logger.error(f"Error detecting email type: {e}")
+ return 'unknown'
+
+ def extract_attachments(self, msg: EmailMessage, include_json: bool = False) -> List[Tuple[str, bytes]]:
"""
Extract compressed attachments from email
+ Args:
+ msg: Email message
+ include_json: If True, also extract JSON files (for TLS-RPT)
+
Returns list of (filename, content) tuples
"""
attachments = []
try:
for part in msg.walk():
+ # Try to get filename from Content-Disposition header
filename = part.get_filename()
+
+ # If no filename, try to get from Content-Type 'name' parameter
+ if not filename:
+ content_type = part.get_content_type()
+ # Check if this is a potential attachment by content type
+ if content_type in ['application/gzip', 'application/x-gzip', 'application/zip',
+ 'application/x-zip-compressed', 'text/xml', 'application/xml',
+ 'application/json', 'application/octet-stream']:
+ # Try to get name from content-type params
+ params = part.get_params()
+ if params:
+ for key, value in params:
+ if key.lower() == 'name':
+ filename = value
+ break
+
if not filename:
continue
filename_lower = filename.lower()
- if not (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip')):
+
+ # Support DMARC files: .xml.gz, .zip, .xml
+ # Support TLS-RPT files: .json, .json.gz
+ valid_extensions = ['.xml.gz', '.zip', '.xml']
+ if include_json:
+ valid_extensions.extend(['.json', '.json.gz'])
+
+ if not any(filename_lower.endswith(ext) for ext in valid_extensions):
continue
content = part.get_payload(decode=True)
if content:
attachments.append((filename, content))
- logger.debug(f"Extracted attachment: {filename}")
+ logger.debug(f"Extracted attachment: {filename} ({len(content)} bytes)")
except Exception as e:
logger.error(f"Error extracting attachments: {e}")
+ if not attachments:
+ # Log all parts for debugging
+ logger.debug(f"No attachments found. Email parts:")
+ for i, part in enumerate(msg.walk()):
+ ct = part.get_content_type()
+ fn = part.get_filename()
+ logger.debug(f" Part {i}: type={ct}, filename={fn}")
+
return attachments
def process_email(self, email_id: str, db: SessionLocal) -> Dict:
"""
- Process a single DMARC email
+ Process a single DMARC or TLS-RPT email
Returns dict with:
- success: bool
- reports_created: int
- reports_duplicate: int
- error: str or None
+ - report_type: 'dmarc' or 'tls-rpt'
"""
result = {
'success': False,
@@ -190,12 +351,13 @@ class DMARCImapService:
'reports_duplicate': 0,
'error': None,
'message_id': None,
- 'subject': None
+ 'subject': None,
+ 'report_type': None
}
try:
# Fetch email (email_id is already a string)
- status, msg_data = self.connection.fetch(email_id, '(RFC822)')
+ status, msg_data = self.connection.uid('FETCH', email_id, '(RFC822)')
if status != 'OK':
result['error'] = f"Failed to fetch email {email_id}"
@@ -206,80 +368,183 @@ class DMARCImapService:
result['message_id'] = msg.get('message-id', 'unknown')
result['subject'] = msg.get('subject', 'unknown')
- # Validate it's a DMARC email
- if not self.is_valid_dmarc_email(msg):
- result['error'] = "Not a valid DMARC report email"
+ # Detect email type
+ email_type = self.detect_email_type(msg)
+ result['report_type'] = email_type
+
+ if email_type == 'tls-rpt':
+ return self._process_tls_rpt_email(msg, db, result)
+ elif email_type == 'dmarc':
+ return self._process_dmarc_email(msg, db, result)
+ else:
+ result['error'] = "Not a valid DMARC or TLS-RPT report email"
return result
- # Extract attachments
- attachments = self.extract_attachments(msg)
-
- if not attachments:
- result['error'] = "No DMARC attachments found"
- return result
-
- # Process each attachment
- for filename, content in attachments:
- try:
- # Parse DMARC report
- parsed_data = parse_dmarc_file(content, filename)
-
- if not parsed_data:
- logger.warning(f"Failed to parse attachment: {filename}")
- continue
-
- # Extract records
- records_data = parsed_data.pop('records', [])
- report_data = parsed_data
-
- # Check for duplicate
- existing = db.query(DMARCReport).filter(
- DMARCReport.report_id == report_data['report_id']
- ).first()
-
- if existing:
- result['reports_duplicate'] += 1
- logger.info(f"Duplicate report: {report_data['report_id']}")
- continue
-
- # Create report
- report = DMARCReport(**report_data)
- db.add(report)
- db.flush()
-
- # Create records with GeoIP enrichment
- for record_data in records_data:
- record_data['dmarc_report_id'] = report.id
- enriched = enrich_dmarc_record(record_data)
- record = DMARCRecord(**enriched)
- db.add(record)
-
- db.commit()
- result['reports_created'] += 1
- logger.info(f"Created DMARC report: {report_data['report_id']}")
-
- except Exception as e:
- db.rollback()
- logger.error(f"Error processing attachment {filename}: {e}")
- if not result['error']:
- result['error'] = str(e)
-
- # Mark as success if at least one report was created
- if result['reports_created'] > 0:
- result['success'] = True
-
- return result
-
except Exception as e:
logger.error(f"Error processing email {email_id}: {e}")
result['error'] = str(e)
return result
+ def _process_dmarc_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict:
+ """Process a DMARC email"""
+ # Extract attachments (DMARC: XML files)
+ attachments = self.extract_attachments(msg, include_json=False)
+
+ if not attachments:
+ result['error'] = "No DMARC attachments found"
+ return result
+
+ # Process each attachment
+ attachment_errors = []
+ for filename, content in attachments:
+ try:
+ # Parse DMARC report
+ parsed_data = parse_dmarc_file(content, filename)
+
+ if not parsed_data:
+ attachment_errors.append(f"Failed to parse: {filename}")
+ logger.warning(f"Failed to parse attachment: {filename}")
+ continue
+
+ # Extract records
+ records_data = parsed_data.pop('records', [])
+ report_data = parsed_data
+
+ # Check for duplicate
+ existing = db.query(DMARCReport).filter(
+ DMARCReport.report_id == report_data['report_id']
+ ).first()
+
+ if existing:
+ result['reports_duplicate'] += 1
+ logger.info(f"Duplicate report: {report_data['report_id']}")
+ continue
+
+ # Create report
+ report = DMARCReport(**report_data)
+ db.add(report)
+ db.flush()
+
+ # Create records with GeoIP enrichment
+ for record_data in records_data:
+ record_data['dmarc_report_id'] = report.id
+ enriched = enrich_dmarc_record(record_data)
+ record = DMARCRecord(**enriched)
+ db.add(record)
+
+ db.commit()
+ result['reports_created'] += 1
+ logger.info(f"Created DMARC report: {report_data['report_id']}")
+
+ except Exception as e:
+ db.rollback()
+ error_msg = f"Error processing {filename}: {str(e)}"
+ attachment_errors.append(error_msg)
+ logger.error(error_msg)
+
+ # Determine success
+ return self._finalize_result(result, attachment_errors, "DMARC")
+
+ def _process_tls_rpt_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict:
+ """Process a TLS-RPT email"""
+ # Extract attachments (TLS-RPT: JSON files)
+ attachments = self.extract_attachments(msg, include_json=True)
+
+ # Filter to only JSON files (and ZIPs containing JSON)
+ json_attachments = [(f, c) for f, c in attachments if f.lower().endswith('.json') or f.lower().endswith('.json.gz') or f.lower().endswith('.zip')]
+
+ if not json_attachments:
+ result['error'] = "No TLS-RPT JSON attachments found"
+ return result
+
+ # Process each attachment
+ attachment_errors = []
+ for filename, content in json_attachments:
+ try:
+ # Parse TLS-RPT report
+ parsed_data = parse_tls_rpt_file(content, filename)
+
+ if not parsed_data:
+ attachment_errors.append(f"Failed to parse: {filename}")
+ logger.warning(f"Failed to parse TLS-RPT attachment: {filename}")
+ continue
+
+ # Extract policies
+ policies_data = parsed_data.pop('policies', [])
+
+ # Check for duplicate
+ existing = db.query(TLSReport).filter(
+ TLSReport.report_id == parsed_data['report_id']
+ ).first()
+
+ if existing:
+ result['reports_duplicate'] += 1
+ logger.info(f"Duplicate TLS-RPT report: {parsed_data['report_id']}")
+ continue
+
+ # Create TLS report
+ tls_report = TLSReport(
+ report_id=parsed_data['report_id'],
+ organization_name=parsed_data.get('organization_name', 'Unknown'),
+ contact_info=parsed_data.get('contact_info', ''),
+ policy_domain=parsed_data['policy_domain'],
+ start_datetime=parsed_data['start_datetime'],
+ end_datetime=parsed_data['end_datetime'],
+ raw_json=parsed_data.get('raw_json', '')
+ )
+ db.add(tls_report)
+ db.flush()
+
+ # Create policy records
+ for policy_data in policies_data:
+ policy = TLSReportPolicy(
+ tls_report_id=tls_report.id,
+ policy_type=policy_data.get('policy_type', 'unknown'),
+ policy_domain=policy_data.get('policy_domain', ''),
+ policy_string=policy_data.get('policy_string', []),
+ mx_host=policy_data.get('mx_host', []),
+ successful_session_count=policy_data.get('successful_session_count', 0),
+ failed_session_count=policy_data.get('failed_session_count', 0),
+ failure_details=policy_data.get('failure_details', [])
+ )
+ db.add(policy)
+
+ db.commit()
+ result['reports_created'] += 1
+ logger.info(f"Created TLS-RPT report: {parsed_data['report_id']}")
+
+ except Exception as e:
+ db.rollback()
+ error_msg = f"Error processing TLS-RPT {filename}: {str(e)}"
+ attachment_errors.append(error_msg)
+ logger.error(error_msg)
+
+ # Determine success
+ return self._finalize_result(result, attachment_errors, "TLS-RPT")
+
+ def _finalize_result(self, result: Dict, attachment_errors: List[str], report_type: str) -> Dict:
+ """Finalize the result based on processing outcome"""
+ if result['reports_created'] > 0:
+ result['success'] = True
+ elif result['reports_duplicate'] > 0 and result['reports_created'] == 0:
+ # All reports were duplicates - this is actually OK, mark as success
+ result['success'] = True
+ result['error'] = None # No error - duplicates are expected
+ else:
+ # No reports created and no duplicates - something went wrong
+ result['success'] = False
+ if attachment_errors:
+ result['error'] = "; ".join(attachment_errors)
+ else:
+ result['error'] = f"No valid {report_type} reports found in attachments"
+
+ return result
+
def mark_as_processed(self, email_id: str):
"""Mark email as processed (flag or move)"""
try:
# Add a flag to mark as processed
- self.connection.store(email_id, '+FLAGS', '\\Seen')
+ self.connection.uid('STORE', email_id, '+FLAGS', '\\Seen')
logger.debug(f"Marked email {email_id} as seen")
except Exception as e:
@@ -288,7 +553,7 @@ class DMARCImapService:
def delete_email(self, email_id: str):
"""Delete email from server"""
try:
- self.connection.store(email_id, '+FLAGS', '\\Deleted')
+ self.connection.uid('STORE', email_id, '+FLAGS', '\\Deleted')
self.connection.expunge()
logger.debug(f"Deleted email {email_id}")
@@ -297,7 +562,11 @@ class DMARCImapService:
def sync_reports(self, sync_type: str = 'auto') -> Dict:
"""
- Main sync function
+ Main sync function with batch processing
+
+ Processes emails in batches to prevent memory issues with large mailboxes.
+ After each batch, emails are deleted/marked and the search is re-run
+ to get the next batch of unprocessed emails.
Returns statistics about the sync operation
"""
@@ -308,6 +577,7 @@ class DMARCImapService:
)
db = SessionLocal()
+ batch_size = settings.dmarc_imap_batch_size
try:
db.add(sync_record)
@@ -321,45 +591,74 @@ class DMARCImapService:
if not self.select_folder():
raise Exception(f"Failed to select folder {self.folder}")
- # Search for DMARC emails
- email_ids = self.search_dmarc_emails()
- sync_record.emails_found = len(email_ids)
+ # Initial search to count total emails
+ all_email_ids = self.search_dmarc_emails()
+ total_emails = len(all_email_ids)
+ sync_record.emails_found = total_emails
db.commit()
- if not email_ids:
+ if not all_email_ids:
logger.info("No DMARC emails found")
sync_record.status = 'success'
sync_record.completed_at = datetime.now(timezone.utc)
db.commit()
return self._build_result(sync_record)
- # Process each email
- failed_emails = []
+ logger.info(f"Found {total_emails} DMARC emails, processing in batches of {batch_size}")
- for email_id in email_ids:
- email_id = email_id.decode() if isinstance(email_id, bytes) else email_id
- result = self.process_email(email_id, db)
- sync_record.emails_processed += 1
+ # Process in batches
+ failed_emails = []
+ batch_number = 0
+
+ while True:
+ batch_number += 1
- if result['success']:
- sync_record.reports_created += result['reports_created']
- sync_record.reports_duplicate += result['reports_duplicate']
+ # Re-search to get current unprocessed emails (since we delete/mark after each batch)
+ email_ids = self.search_dmarc_emails()
+
+ if not email_ids:
+ logger.info(f"Batch {batch_number}: No more emails to process")
+ break
+
+ # Take only batch_size emails
+ batch_emails = email_ids[:batch_size]
+ logger.info(f"Batch {batch_number}: Processing {len(batch_emails)} emails (remaining: {len(email_ids)})")
+
+ for email_id in batch_emails:
+ email_id = email_id.decode() if isinstance(email_id, bytes) else email_id
+ result = self.process_email(email_id, db)
+ sync_record.emails_processed += 1
- # Delete or mark as processed
- if self.delete_after:
- self.delete_email(email_id)
+ if result['success']:
+ sync_record.reports_created += result['reports_created']
+ sync_record.reports_duplicate += result['reports_duplicate']
+
+ # Delete or mark as processed
+ if self.delete_after:
+ self.delete_email(email_id)
+ else:
+ self.mark_as_processed(email_id)
else:
+ sync_record.reports_failed += 1
+ error_msg = result.get('error', 'Unknown error')
+ logger.warning(f"Failed to process email {email_id}: {error_msg}")
+
+ failed_emails.append({
+ 'email_id': email_id,
+ 'message_id': result['message_id'],
+ 'subject': result['subject'],
+ 'error': error_msg
+ })
+ # Also mark failed emails as processed to avoid re-processing
self.mark_as_processed(email_id)
- else:
- sync_record.reports_failed += 1
- failed_emails.append({
- 'email_id': email_id,
- 'message_id': result['message_id'],
- 'subject': result['subject'],
- 'error': result['error']
- })
+
+ db.commit()
- db.commit()
+ # Log batch progress
+ logger.info(f"Batch {batch_number} complete: "
+ f"{sync_record.emails_processed}/{total_emails} processed, "
+ f"{sync_record.reports_created} created, "
+ f"{sync_record.reports_failed} failed")
# Update sync record
sync_record.status = 'success'
@@ -375,6 +674,10 @@ class DMARCImapService:
f"{sync_record.reports_duplicate} duplicates, "
f"{sync_record.reports_failed} failed")
+ # Clear cache if any reports were created
+ if sync_record.reports_created > 0:
+ clear_dmarc_cache(db)
+
# Send email notification if there were failures
if failed_emails and settings.notification_smtp_configured:
logger.info(f"Sending error notification for {len(failed_emails)} failed emails")
diff --git a/backend/app/services/dmarc_parser.py b/backend/app/services/dmarc_parser.py
index 6808a7e..256d164 100644
--- a/backend/app/services/dmarc_parser.py
+++ b/backend/app/services/dmarc_parser.py
@@ -58,6 +58,57 @@ def parse_dmarc_file(file_content: bytes, filename: str) -> Optional[Dict[str, A
return None
+def find_element(parent: ET.Element, tag: str, namespaces: List[str]) -> Optional[ET.Element]:
+ """
+ Find an element trying multiple namespaces
+
+ Args:
+ parent: Parent XML element
+ tag: Tag name to find
+ namespaces: List of namespace prefixes to try (empty string for no namespace)
+
+ Returns:
+ Found element or None
+ """
+ for ns in namespaces:
+ if ns:
+ elem = parent.find(f'{{{ns}}}{tag}')
+ else:
+ elem = parent.find(tag)
+ if elem is not None:
+ return elem
+ return None
+
+
+def find_all_elements(parent: ET.Element, tag: str, namespaces: List[str]) -> List[ET.Element]:
+ """
+ Find all elements matching tag, trying multiple namespaces
+
+ Args:
+ parent: Parent XML element
+ tag: Tag name to find
+ namespaces: List of namespace prefixes to try
+
+ Returns:
+ List of found elements
+ """
+ for ns in namespaces:
+ if ns:
+ elems = parent.findall(f'{{{ns}}}{tag}')
+ else:
+ elems = parent.findall(tag)
+ if elems:
+ return elems
+ return []
+
+
+# Known DMARC XML namespaces
+DMARC_NAMESPACES = [
+ '', # No namespace (DMARC 1.0)
+ 'urn:ietf:params:xml:ns:dmarc-2.0', # DMARC 2.0
+]
+
+
def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
"""
Parse DMARC XML content
@@ -72,38 +123,38 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
try:
root = ET.fromstring(xml_string)
- # Parse report metadata
- metadata = root.find('report_metadata')
+ # Parse report metadata (try with and without namespace)
+ metadata = find_element(root, 'report_metadata', DMARC_NAMESPACES)
if metadata is None:
raise ValueError("Missing report_metadata element")
- org_name = get_element_text(metadata, 'org_name')
- email = get_element_text(metadata, 'email')
- extra_contact_info = get_element_text(metadata, 'extra_contact_info')
- report_id = get_element_text(metadata, 'report_id')
+ org_name = get_element_text(metadata, 'org_name', DMARC_NAMESPACES)
+ email = get_element_text(metadata, 'email', DMARC_NAMESPACES)
+ extra_contact_info = get_element_text(metadata, 'extra_contact_info', DMARC_NAMESPACES)
+ report_id = get_element_text(metadata, 'report_id', DMARC_NAMESPACES)
- date_range = metadata.find('date_range')
+ date_range = find_element(metadata, 'date_range', DMARC_NAMESPACES)
if date_range is None:
raise ValueError("Missing date_range element")
- begin_date = int(get_element_text(date_range, 'begin'))
- end_date = int(get_element_text(date_range, 'end'))
+ begin_date = int(get_element_text(date_range, 'begin', DMARC_NAMESPACES))
+ end_date = int(get_element_text(date_range, 'end', DMARC_NAMESPACES))
# Parse published policy
- policy = root.find('policy_published')
+ policy = find_element(root, 'policy_published', DMARC_NAMESPACES)
if policy is None:
raise ValueError("Missing policy_published element")
- domain = get_element_text(policy, 'domain')
+ domain = get_element_text(policy, 'domain', DMARC_NAMESPACES)
policy_published = {
- 'adkim': get_element_text(policy, 'adkim'),
- 'aspf': get_element_text(policy, 'aspf'),
- 'p': get_element_text(policy, 'p'),
- 'sp': get_element_text(policy, 'sp'),
- 'pct': get_element_text(policy, 'pct'),
- 'fo': get_element_text(policy, 'fo'),
- 'np': get_element_text(policy, 'np'),
+ 'adkim': get_element_text(policy, 'adkim', DMARC_NAMESPACES),
+ 'aspf': get_element_text(policy, 'aspf', DMARC_NAMESPACES),
+ 'p': get_element_text(policy, 'p', DMARC_NAMESPACES),
+ 'sp': get_element_text(policy, 'sp', DMARC_NAMESPACES),
+ 'pct': get_element_text(policy, 'pct', DMARC_NAMESPACES),
+ 'fo': get_element_text(policy, 'fo', DMARC_NAMESPACES),
+ 'np': get_element_text(policy, 'np', DMARC_NAMESPACES),
}
# Remove None values
@@ -111,7 +162,7 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
# Parse records
records = []
- for record_elem in root.findall('record'):
+ for record_elem in find_all_elements(root, 'record', DMARC_NAMESPACES):
record_data = parse_dmarc_record(record_elem)
if record_data:
records.append(record_data)
@@ -145,38 +196,38 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
Dictionary with parsed record data
"""
try:
- row = record_elem.find('row')
+ row = find_element(record_elem, 'row', DMARC_NAMESPACES)
if row is None:
return None
# Source and count
- source_ip = get_element_text(row, 'source_ip')
- count = int(get_element_text(row, 'count', '0'))
+ source_ip = get_element_text(row, 'source_ip', DMARC_NAMESPACES)
+ count = int(get_element_text(row, 'count', DMARC_NAMESPACES, '0'))
# Policy evaluation
- policy_eval = row.find('policy_evaluated')
- disposition = get_element_text(policy_eval, 'disposition') if policy_eval else None
- dkim_result = get_element_text(policy_eval, 'dkim') if policy_eval else None
- spf_result = get_element_text(policy_eval, 'spf') if policy_eval else None
+ policy_eval = find_element(row, 'policy_evaluated', DMARC_NAMESPACES)
+ disposition = get_element_text(policy_eval, 'disposition', DMARC_NAMESPACES) if policy_eval else None
+ dkim_result = get_element_text(policy_eval, 'dkim', DMARC_NAMESPACES) if policy_eval else None
+ spf_result = get_element_text(policy_eval, 'spf', DMARC_NAMESPACES) if policy_eval else None
# Identifiers
- identifiers = record_elem.find('identifiers')
- header_from = get_element_text(identifiers, 'header_from') if identifiers else None
- envelope_from = get_element_text(identifiers, 'envelope_from') if identifiers else None
- envelope_to = get_element_text(identifiers, 'envelope_to') if identifiers else None
+ identifiers = find_element(record_elem, 'identifiers', DMARC_NAMESPACES)
+ header_from = get_element_text(identifiers, 'header_from', DMARC_NAMESPACES) if identifiers else None
+ envelope_from = get_element_text(identifiers, 'envelope_from', DMARC_NAMESPACES) if identifiers else None
+ envelope_to = get_element_text(identifiers, 'envelope_to', DMARC_NAMESPACES) if identifiers else None
# Auth results
auth_results = {}
- auth_results_elem = record_elem.find('auth_results')
+ auth_results_elem = find_element(record_elem, 'auth_results', DMARC_NAMESPACES)
if auth_results_elem:
# Parse DKIM results
dkim_results = []
- for dkim_elem in auth_results_elem.findall('dkim'):
+ for dkim_elem in find_all_elements(auth_results_elem, 'dkim', DMARC_NAMESPACES):
dkim_data = {
- 'domain': get_element_text(dkim_elem, 'domain'),
- 'selector': get_element_text(dkim_elem, 'selector'),
- 'result': get_element_text(dkim_elem, 'r') or get_element_text(dkim_elem, 'result')
+ 'domain': get_element_text(dkim_elem, 'domain', DMARC_NAMESPACES),
+ 'selector': get_element_text(dkim_elem, 'selector', DMARC_NAMESPACES),
+ 'result': get_element_text(dkim_elem, 'r', DMARC_NAMESPACES) or get_element_text(dkim_elem, 'result', DMARC_NAMESPACES)
}
dkim_results.append({k: v for k, v in dkim_data.items() if v})
@@ -185,11 +236,11 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
# Parse SPF results
spf_results = []
- for spf_elem in auth_results_elem.findall('spf'):
+ for spf_elem in find_all_elements(auth_results_elem, 'spf', DMARC_NAMESPACES):
spf_data = {
- 'domain': get_element_text(spf_elem, 'domain'),
- 'scope': get_element_text(spf_elem, 'scope'),
- 'result': get_element_text(spf_elem, 'r') or get_element_text(spf_elem, 'result')
+ 'domain': get_element_text(spf_elem, 'domain', DMARC_NAMESPACES),
+ 'scope': get_element_text(spf_elem, 'scope', DMARC_NAMESPACES),
+ 'result': get_element_text(spf_elem, 'r', DMARC_NAMESPACES) or get_element_text(spf_elem, 'result', DMARC_NAMESPACES)
}
spf_results.append({k: v for k, v in spf_data.items() if v})
@@ -213,13 +264,14 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
return None
-def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[str] = None) -> Optional[str]:
+def get_element_text(parent: Optional[ET.Element], tag: str, namespaces: List[str] = None, default: Optional[str] = None) -> Optional[str]:
"""
- Safely get text from XML element
+ Safely get text from XML element with namespace support
Args:
parent: Parent XML element
tag: Tag name to find
+ namespaces: List of namespace prefixes to try
default: Default value if not found
Returns:
@@ -228,7 +280,11 @@ def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[s
if parent is None:
return default
- elem = parent.find(tag)
+ if namespaces:
+ elem = find_element(parent, tag, namespaces)
+ else:
+ elem = parent.find(tag)
+
if elem is not None and elem.text:
return elem.text.strip()
diff --git a/backend/app/services/smtp_service.py b/backend/app/services/smtp_service.py
index 2800dae..255ba9c 100644
--- a/backend/app/services/smtp_service.py
+++ b/backend/app/services/smtp_service.py
@@ -21,8 +21,10 @@ class SmtpService:
self.host = settings.smtp_host
self.port = settings.smtp_port
self.use_tls = settings.smtp_use_tls
+ self.use_ssl = settings.smtp_use_ssl
self.user = settings.smtp_user
self.password = settings.smtp_password
+ self.relay_mode = settings.smtp_relay_mode
self.from_address = settings.smtp_from or settings.smtp_user
def is_configured(self) -> bool:
@@ -70,13 +72,22 @@ class SmtpService:
part2 = MIMEText(html_content, 'html')
msg.attach(part2)
- if self.use_tls:
+ # Determine connection mode
+ # Priority 1: Implicit SSL (if configured or using port 465)
+ if self.use_ssl or self.port == 465:
+ server = smtplib.SMTP_SSL(self.host, self.port)
+ # Priority 2: STARTTLS (if configured)
+ elif self.use_tls:
server = smtplib.SMTP(self.host, self.port)
server.starttls()
+ # Priority 3: Plaintext
else:
- server = smtplib.SMTP_SSL(self.host, self.port)
+ server = smtplib.SMTP(self.host, self.port)
+
+ # Skip login in relay mode
+ if not self.relay_mode:
+ server.login(self.user, self.password)
- server.login(self.user, self.password)
server.sendmail(self.from_address, [recipient], msg.as_string())
server.quit()
diff --git a/backend/app/services/tls_rpt_parser.py b/backend/app/services/tls_rpt_parser.py
new file mode 100644
index 0000000..d66bcdf
--- /dev/null
+++ b/backend/app/services/tls_rpt_parser.py
@@ -0,0 +1,218 @@
+"""
+TLS-RPT (SMTP TLS Reporting) Parser
+Handles parsing of TLS-RPT reports in JSON format
+"""
+import json
+import gzip
+import zipfile
+import logging
+from typing import Dict, Any, Optional
+from datetime import datetime
+from io import BytesIO
+
+logger = logging.getLogger(__name__)
+
+
+def parse_tls_rpt_file(file_content: bytes, filename: str) -> Optional[Dict[str, Any]]:
+ """
+ Parse TLS-RPT report from file content (JSON, GZ compressed, or ZIP)
+
+ Args:
+ file_content: Raw bytes of the file
+ filename: Original filename (to determine compression type)
+
+ Returns:
+ Parsed TLS-RPT data or None if parsing failed
+ """
+ try:
+ filename_lower = filename.lower()
+
+ # Handle ZIP files (.json.zip)
+ if filename_lower.endswith('.zip'):
+ try:
+ with zipfile.ZipFile(BytesIO(file_content)) as zf:
+ # Find JSON file inside ZIP
+ json_files = [f for f in zf.namelist() if f.lower().endswith('.json')]
+ if not json_files:
+ logger.error(f"No JSON file found in ZIP: {filename}")
+ return None
+
+ # Read the first JSON file
+ json_content = zf.read(json_files[0]).decode('utf-8')
+ except zipfile.BadZipFile as e:
+ logger.error(f"Invalid ZIP file {filename}: {e}")
+ return None
+ # Handle GZIP files (.json.gz)
+ elif filename_lower.endswith('.gz'):
+ try:
+ json_content = gzip.decompress(file_content).decode('utf-8')
+ except Exception as e:
+ logger.error(f"Failed to decompress gzip TLS-RPT file: {e}")
+ return None
+ # Handle plain JSON files
+ elif filename_lower.endswith('.json'):
+ json_content = file_content.decode('utf-8')
+ else:
+ # Try to decode as JSON directly
+ try:
+ json_content = file_content.decode('utf-8')
+ except Exception:
+ logger.error(f"Unknown TLS-RPT file format: {filename}")
+ return None
+
+ return parse_tls_rpt_json(json_content)
+
+ except Exception as e:
+ logger.error(f"Error parsing TLS-RPT file {filename}: {e}")
+ return None
+
+
+
+def parse_tls_rpt_json(json_content: str) -> Optional[Dict[str, Any]]:
+ """
+ Parse TLS-RPT JSON content
+
+ Expected format (RFC 8460):
+ {
+ "organization-name": "Google Inc.",
+ "date-range": {
+ "start-datetime": "2026-01-12T00:00:00Z",
+ "end-datetime": "2026-01-12T23:59:59Z"
+ },
+ "contact-info": "smtp-tls-reporting@google.com",
+ "report-id": "2026-01-12T00:00:00Z_boubou.me",
+ "policies": [{
+ "policy": {
+ "policy-type": "sts",
+ "policy-string": ["version: STSv1", "mode: enforce", ...],
+ "policy-domain": "boubou.me",
+ "mx-host": ["mail.tiboxs.com"]
+ },
+ "summary": {
+ "total-successful-session-count": 1,
+ "total-failure-session-count": 0
+ },
+ "failure-details": [...] # Optional
+ }]
+ }
+
+ Returns:
+ Dictionary with parsed TLS-RPT data
+ """
+ try:
+ data = json.loads(json_content)
+
+ # Extract report metadata
+ report_id = data.get('report-id', '')
+ if not report_id:
+ logger.error("TLS-RPT report missing report-id")
+ return None
+
+ organization_name = data.get('organization-name', 'Unknown')
+ contact_info = data.get('contact-info', '')
+
+ # Parse date range
+ date_range = data.get('date-range', {})
+ start_datetime = parse_iso_datetime(date_range.get('start-datetime', ''))
+ end_datetime = parse_iso_datetime(date_range.get('end-datetime', ''))
+
+ if not start_datetime or not end_datetime:
+ logger.error("TLS-RPT report missing or invalid date-range")
+ return None
+
+ # Parse policies
+ policies = []
+ policy_domain = None
+
+ for policy_entry in data.get('policies', []):
+ policy_data = policy_entry.get('policy', {})
+ summary = policy_entry.get('summary', {})
+
+ # Get the policy domain from the first policy
+ if not policy_domain:
+ policy_domain = policy_data.get('policy-domain', '')
+
+ parsed_policy = {
+ 'policy_type': policy_data.get('policy-type', 'unknown'),
+ 'policy_domain': policy_data.get('policy-domain', ''),
+ 'policy_string': policy_data.get('policy-string', []),
+ 'mx_host': policy_data.get('mx-host', []),
+ 'successful_session_count': summary.get('total-successful-session-count', 0),
+ 'failed_session_count': summary.get('total-failure-session-count', 0),
+ 'failure_details': policy_entry.get('failure-details', [])
+ }
+ policies.append(parsed_policy)
+
+ if not policy_domain:
+ logger.error("TLS-RPT report missing policy-domain")
+ return None
+
+ return {
+ 'report_id': report_id,
+ 'organization_name': organization_name,
+ 'contact_info': contact_info,
+ 'policy_domain': policy_domain,
+ 'start_datetime': start_datetime,
+ 'end_datetime': end_datetime,
+ 'policies': policies,
+ 'raw_json': json_content
+ }
+
+ except json.JSONDecodeError as e:
+ logger.error(f"Invalid JSON in TLS-RPT report: {e}")
+ return None
+ except Exception as e:
+ logger.error(f"Error parsing TLS-RPT JSON: {e}")
+ return None
+
+
+def parse_iso_datetime(datetime_str: str) -> Optional[datetime]:
+ """
+ Parse ISO 8601 datetime string
+
+ Supports formats:
+ - 2026-01-12T00:00:00Z
+ - 2026-01-12T00:00:00+00:00
+ """
+ if not datetime_str:
+ return None
+
+ try:
+ # Remove 'Z' suffix and replace with +00:00 for parsing
+ if datetime_str.endswith('Z'):
+ datetime_str = datetime_str[:-1] + '+00:00'
+
+ # Parse with timezone
+ from datetime import timezone
+ dt = datetime.fromisoformat(datetime_str)
+
+ # Convert to UTC naive datetime for storage
+ if dt.tzinfo is not None:
+ dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+ return dt
+
+ except Exception as e:
+ logger.error(f"Error parsing datetime '{datetime_str}': {e}")
+ return None
+
+
+def is_tls_rpt_json(json_content: str) -> bool:
+ """
+ Check if JSON content is a valid TLS-RPT report
+
+ Used to detect TLS-RPT vs other JSON files
+ """
+ try:
+ data = json.loads(json_content)
+
+ # Check for required TLS-RPT fields
+ has_report_id = 'report-id' in data
+ has_date_range = 'date-range' in data
+ has_policies = 'policies' in data
+
+ # At minimum, should have policies and date-range
+ return has_policies and (has_date_range or has_report_id)
+
+ except Exception:
+ return False
diff --git a/documentation/API.md b/documentation/API.md
index 7d12d12..a3e9b06 100644
--- a/documentation/API.md
+++ b/documentation/API.md
@@ -14,18 +14,19 @@ This document describes all available API endpoints for the Mailcow Logs Viewer
2. [Health & Info](#health--info)
3. [Job Status Tracking](#job-status-tracking)
4. [Domains](#domains)
-5. [Messages (Unified View)](#messages-unified-view)
-6. [Logs](#logs)
+5. [Mailbox Statistics](#mailbox-statistics)
+6. [Messages (Unified View)](#messages-unified-view)
+7. [Logs](#logs)
- [Postfix Logs](#postfix-logs)
- [Rspamd Logs](#rspamd-logs)
- [Netfilter Logs](#netfilter-logs)
-7. [Queue & Quarantine](#queue--quarantine)
-8. [Statistics](#statistics)
-9. [Status](#status)
-10. [Settings](#settings)
+8. [Queue & Quarantine](#queue--quarantine)
+9. [Statistics](#statistics)
+10. [Status](#status)
+11. [Settings](#settings)
- [SMTP & IMAP Test](#smtp--imap-test)
-11. [Export](#export)
-12. [DMARC](#dmarc)
+12. [Export](#export)
+13. [DMARC](#dmarc)
- [DMARC IMAP Auto-Import](#dmarc-imap-auto-import)
---
@@ -736,6 +737,200 @@ POST /api/domains/example.com/check-dns
---
+## Mailbox Statistics
+
+### GET /api/mailbox-stats/summary
+
+Get summary statistics for all mailboxes.
+
+**Query Parameters:**
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) |
+
+**Response:**
+```json
+{
+ "total_mailboxes": 25,
+ "active_mailboxes": 23,
+ "inactive_mailboxes": 2,
+ "total_sent": 1234,
+ "total_received": 5678,
+ "sent_failed": 45,
+ "failure_rate": 3.6,
+ "date_range": "30days",
+ "start_date": "2026-01-16T00:00:00Z",
+ "end_date": "2026-02-16T00:00:00Z"
+}
+```
+
+---
+
+### GET /api/mailbox-stats/all
+
+Get all mailbox statistics with message counts and aliases (paginated).
+
+**Query Parameters:**
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `domain` | string | Filter by domain name |
+| `active_only` | bool | Only show active mailboxes (default: `true`) |
+| `hide_zero` | bool | Hide mailboxes with zero activity (default: `false`) |
+| `search` | string | Search mailbox username, name, or alias address |
+| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) |
+| `sort_by` | string | Sort by: `sent_total`, `received_total`, `failure_rate`, `quota_used`, `username` |
+| `sort_order` | string | Sort order: `asc`, `desc` (default: `desc`) |
+| `page` | int | Page number (default: `1`) |
+| `page_size` | int | Items per page, 10-100 (default: `50`) |
+
+**Example Request:**
+```
+GET /api/mailbox-stats/all?date_range=30days&active_only=true&hide_zero=true&sort_by=sent_total&sort_order=desc&page=1
+```
+
+**Response:**
+```json
+{
+ "total": 25,
+ "page": 1,
+ "page_size": 50,
+ "total_pages": 1,
+ "date_range": "30days",
+ "start_date": "2026-01-16T00:00:00Z",
+ "end_date": "2026-02-16T00:00:00Z",
+ "mailboxes": [
+ {
+ "id": 1,
+ "username": "user@example.com",
+ "domain": "example.com",
+ "name": "John Doe",
+ "active": true,
+ "quota": 1073741824,
+ "quota_formatted": "1.0 GB",
+ "quota_used": 536870912,
+ "quota_used_formatted": "512 MB",
+ "percent_in_use": 50.0,
+ "messages_in_mailbox": 1234,
+ "last_imap_login": "2026-01-15T10:30:00Z",
+ "last_pop3_login": null,
+ "last_smtp_login": "2026-01-16T08:45:00Z",
+ "rl_value": 100,
+ "rl_frame": "m",
+ "attributes": {
+ "imap_access": "1",
+ "pop3_access": "0",
+ "smtp_access": "1",
+ "sieve_access": "1",
+ "sogo_access": "1",
+ "tls_enforce_in": "0",
+ "tls_enforce_out": "0"
+ },
+ "mailbox_counts": {
+ "sent_total": 150,
+ "sent_delivered": 145,
+ "sent_bounced": 3,
+ "sent_deferred": 2,
+ "sent_rejected": 0,
+ "sent_failed": 5,
+ "received_total": 320,
+ "failure_rate": 3.3
+ },
+ "aliases": [
+ {
+ "alias_address": "info@example.com",
+ "active": true,
+ "is_catch_all": false,
+ "sent_total": 50,
+ "sent_delivered": 48,
+ "sent_bounced": 2,
+ "sent_deferred": 0,
+ "sent_rejected": 0,
+ "sent_failed": 2,
+ "received_total": 100,
+ "failure_rate": 4.0
+ }
+ ],
+ "alias_count": 1,
+ "combined_sent": 200,
+ "combined_received": 420,
+ "combined_total": 620,
+ "combined_failed": 7,
+ "combined_failure_rate": 3.5,
+ "created": "2025-01-01T00:00:00Z",
+ "modified": "2026-01-15T12:00:00Z"
+ }
+ ]
+}
+```
+
+**Response Fields:**
+
+| Field | Description |
+|-------|-------------|
+| `username` | Email address of the mailbox |
+| `name` | Display name |
+| `active` | Whether mailbox is active in Mailcow |
+| `quota` / `quota_used` | Quota in bytes |
+| `percent_in_use` | Quota usage percentage |
+| `messages_in_mailbox` | Number of messages stored |
+| `last_*_login` | Last login timestamps (null if never) |
+| `rl_value` / `rl_frame` | Rate limiting (e.g., 100/m = 100 per minute) |
+| `attributes` | Access permissions from Mailcow |
+| `mailbox_counts` | Message statistics for mailbox only |
+| `aliases` | Array of alias statistics |
+| `combined_*` | Combined totals (mailbox + all aliases) |
+| `created` / `modified` | Mailbox creation and last update timestamps |
+
+---
+
+### GET /api/mailbox-stats/domains
+
+Get list of domains with mailbox counts for filter dropdown.
+
+**Response:**
+```json
+{
+ "domains": [
+ {
+ "domain": "example.com",
+ "mailbox_count": 15
+ },
+ {
+ "domain": "company.org",
+ "mailbox_count": 10
+ }
+ ]
+}
+```
+
+### Caching
+
+The Mailbox Statistics API uses in-memory caching to improve performance:
+
+| Setting | Value |
+|---------|-------|
+| **Cache TTL** | 5 minutes (300 seconds) |
+| **Cache Scope** | Per unique query parameter combination |
+| **Cached Parameters** | domain, active_only, hide_zero, search, date_range, start_date, end_date, sort_by, sort_order, page, page_size |
+
+**Cache Behavior:**
+- First request with specific parameters fetches from database and caches result
+- Subsequent requests with identical parameters return cached data
+- Cache automatically expires after 5 minutes
+- Changing any parameter results in a cache miss (new database query)
+
+**Cache Management:**
+```python
+from app.routers.mailbox_stats import clear_stats_cache
+
+# Clear all stats cache (e.g., after data import)
+clear_stats_cache()
+```
+
+---
+
## Messages (Unified View)
### GET /messages
@@ -2505,6 +2700,165 @@ Get history of IMAP sync operations.
---
+## TLS-RPT (TLS Reporting)
+
+### Overview
+
+TLS-RPT (TLS Reporting) provides visibility into TLS connection failures when other mail servers attempt to deliver emails to your domain. This helps identify MTA-STS policy issues and certificate problems.
+
+---
+
+### GET /api/dmarc/domains/{domain}/tls-reports
+
+Get TLS reports for a specific domain (individual reports).
+
+**Path Parameters:**
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `domain` | string | Domain name |
+
+**Query Parameters:**
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `days` | integer | 30 | Number of days to look back |
+
+**Response:**
+```json
+{
+ "domain": "example.com",
+ "total": 15,
+ "data": [
+ {
+ "id": 1,
+ "report_id": "2026-01-14T00:00:00Z!example.com!google.com",
+ "organization_name": "Google Inc.",
+ "start_datetime": "2026-01-14T00:00:00Z",
+ "end_datetime": "2026-01-15T00:00:00Z",
+ "total_success": 1250,
+ "total_fail": 5,
+ "success_rate": 99.6
+ }
+ ]
+}
+```
+
+---
+
+### GET /api/dmarc/domains/{domain}/tls-reports/daily
+
+Get TLS reports aggregated by date (daily view).
+
+**Path Parameters:**
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `domain` | string | Domain name |
+
+**Query Parameters:**
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `days` | integer | 30 | Number of days to look back |
+| `page` | integer | 1 | Page number |
+| `page_size` | integer | 20 | Items per page |
+
+**Response:**
+```json
+{
+ "domain": "example.com",
+ "totals": {
+ "total_days": 14,
+ "total_reports": 28,
+ "total_successful_sessions": 15000,
+ "total_failed_sessions": 25,
+ "overall_success_rate": 99.83
+ },
+ "data": [
+ {
+ "date": "2026-01-17",
+ "report_count": 3,
+ "organization_count": 2,
+ "organizations": ["Google Inc.", "Microsoft Corporation"],
+ "total_success": 1500,
+ "total_fail": 2,
+ "success_rate": 99.87
+ }
+ ]
+}
+```
+
+---
+
+### GET /api/dmarc/domains/{domain}/tls-reports/{report_date}/details
+
+Get detailed TLS reports for a specific date.
+
+**Path Parameters:**
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `domain` | string | Domain name |
+| `report_date` | string | Date in YYYY-MM-DD format |
+
+**Response:**
+```json
+{
+ "domain": "example.com",
+ "date": "2026-01-17",
+ "stats": {
+ "total_reports": 3,
+ "total_providers": 2,
+ "total_success": 1500,
+ "total_fail": 2,
+ "total_sessions": 1502,
+ "success_rate": 99.87
+ },
+ "providers": [
+ {
+ "report_id": "2026-01-17T00:00:00Z!example.com!google.com",
+ "organization_name": "Google Inc.",
+ "contact_info": "smtp-tls-reporting@google.com",
+ "start_datetime": "2026-01-17T00:00:00Z",
+ "end_datetime": "2026-01-18T00:00:00Z",
+ "successful_sessions": 1200,
+ "failed_sessions": 1,
+ "total_sessions": 1201,
+ "success_rate": 99.92,
+ "policies": [
+ {
+ "policy_type": "sts",
+ "policy_domain": "example.com",
+ "mx_host": "mail.example.com",
+ "successful_sessions": 1200,
+ "failed_sessions": 1,
+ "total_sessions": 1201,
+ "success_rate": 99.92,
+ "failure_details": null
+ }
+ ]
+ }
+ ]
+}
+```
+
+---
+
+### POST /api/dmarc/upload (TLS-RPT Support)
+
+The existing DMARC upload endpoint also accepts TLS-RPT reports.
+
+**Supported TLS-RPT Formats:**
+- `.json.gz` - Gzip-compressed JSON (standard format)
+- `.json` - Plain JSON
+
+**Detection:**
+- File is identified as TLS-RPT if JSON contains `"policies"` array
+- TLS-RPT reports use RFC 8460 JSON format
+
+---
+
## Error Responses
All endpoints may return the following error responses:
diff --git a/documentation/Email_Authentication_Monitoring.md b/documentation/Email_Authentication_Monitoring.md
index 4bbce32..bddeeaa 100644
--- a/documentation/Email_Authentication_Monitoring.md
+++ b/documentation/Email_Authentication_Monitoring.md
@@ -2,7 +2,7 @@
To maintain high deliverability and robust domain security, **mailcow-logs-viewer** provides deep inspection and automated monitoring of the three core email authentication protocols: **SPF**, **DKIM**, and **DMARC**.
-### The Authentication Stack
+## The Authentication Stack
| Protocol | Technical Purpose | System Validation Logic |
| --- | --- | --- |
@@ -12,7 +12,7 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe
---
-### Advanced Monitoring & Intelligence
+## Advanced Monitoring & Intelligence
**mailcow-logs-viewer** goes beyond basic record checking by providing a comprehensive analysis of your mail flow:
@@ -23,26 +23,48 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe
---
-### 🚀 Implementation: Enabling DMARC Reporting
+## 🚀 Implementation: Enabling DMARC Reporting
To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system.
-#### 1. DNS Configuration
+### 1. DNS Configuration
Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`):
```text
-v=DMARC1; p=none; rua=mailto:dmarc-reports@yourdomain.com;
+v=DMARC1; p=none; rua=mailto:dmarc@example.net;
```
-#### 2. Parameter Details
+### 2. Parameter Details
* **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated.
* **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer.
* **`v=DMARC1`:** Required version prefix.
-#### 3. Transitioning to Enforcement
+### 3. External Domain Reporting (Verification)
+
+If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports.
+
+Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam.
+
+#### Option 1: Specific Domain Authorization (Recommended for security)
+Add a TXT record to the DNS of the **receiving domain** (`example.net`):
+
+| Host / Name | Value |
+| :--- | :--- |
+| `example.com._report._dmarc.example.net` | `v=DMARC1;` |
+
+#### Option 2: Wildcard Authorization (Recommended for multiple domains)
+If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once:
+
+| Host / Name | Value |
+| :--- | :--- |
+| `*._report._dmarc.example.net` | `v=DMARC1;` |
+
+*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.*
+
+### 4. Transitioning to Enforcement
Once the dashboard confirms that your legitimate traffic (including third-party SaaS) is passing SPF/DKIM alignment, you should update your policy to `p=quarantine` or `p=reject` to fully secure your domain against spoofing.
diff --git a/documentation/HelpDocs/DMARC.md b/documentation/HelpDocs/DMARC.md
index df0a0ec..b8fb505 100644
--- a/documentation/HelpDocs/DMARC.md
+++ b/documentation/HelpDocs/DMARC.md
@@ -10,6 +10,8 @@ The DMARC Reports page provides detailed analysis of DMARC aggregate reports rec
- Tells receiving servers what to do with emails that fail validation
- Provides reports about email authentication results
+---
+
## Report Types
### Aggregate Reports (XML)
@@ -64,6 +66,8 @@ Click an IP address to see:
- **Volume**: Number of messages from this source
- **Reverse DNS**: Hostname associated with the IP
+---
+
## Understanding Report Data
### DMARC Alignment
@@ -81,6 +85,8 @@ What the receiving server did with the email:
- **Policy**: What your DMARC record tells servers to do
- **Disposition**: What servers actually did (they may override your policy)
+---
+
## Key Features
### Geographic Visualization
@@ -103,6 +109,8 @@ What the receiving server did with the email:
- DMARC policy effectiveness
- Recommendations for policy adjustments
+---
+
## Common Scenarios
### Legitimate Sources Failing
@@ -138,6 +146,51 @@ What the receiving server did with the email:
- Verify DKIM is configured on all sending systems
- Look for email forwarding issues
+---
+
+## 🚀 Implementation: Enabling DMARC Reporting
+
+To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system.
+
+### 1. DNS Configuration
+
+Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`):
+
+```text
+v=DMARC1; p=none; rua=mailto:dmarc@example.net;
+
+```
+
+### 2. Parameter Details
+
+* **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated.
+* **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer.
+* **`v=DMARC1`:** Required version prefix.
+
+### 3. External Domain Reporting (Verification)
+
+If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports.
+
+Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam.
+
+#### Option 1: Specific Domain Authorization (Recommended for security)
+Add a TXT record to the DNS of the **receiving domain** (`example.net`):
+
+| Host / Name | Value |
+| :--- | :--- |
+| `example.com._report._dmarc.example.net` | `v=DMARC1;` |
+
+#### Option 2: Wildcard Authorization (Recommended for multiple domains)
+If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once:
+
+| Host / Name | Value |
+| :--- | :--- |
+| `*._report._dmarc.example.net` | `v=DMARC1;` |
+
+*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.*
+
+---
+
## Best Practices
### Policy Progression
@@ -162,6 +215,8 @@ When using email services (marketing, support desk, etc.):
- Test before going live
- Monitor their authentication success
+---
+
## Troubleshooting
### No Reports Appearing
@@ -185,6 +240,8 @@ When using email services (marketing, support desk, etc.):
- Older reports are automatically deleted to save space
- Export reports before they're deleted if long-term analysis is needed
+---
+
## Security Considerations
### Identifying Spoofing
diff --git a/documentation/HelpDocs/Domains.md b/documentation/HelpDocs/Domains.md
index cb72f54..21051a1 100644
--- a/documentation/HelpDocs/Domains.md
+++ b/documentation/HelpDocs/Domains.md
@@ -36,6 +36,8 @@ The system automatically validates three critical DNS records:
- `none`: Monitoring only (weakest)
- **Status**: Same indicators as SPF
+---
+
## How to Use
### Viewing Domains
@@ -59,6 +61,8 @@ When you expand a domain, the DNS Security section shows:
- Specific warnings or recommendations
- Time of last validation
+---
+
## Best Practices
1. **Regular Monitoring**: Review DNS status regularly, especially after DNS changes
@@ -67,6 +71,8 @@ When you expand a domain, the DNS Security section shows:
4. **SPF Optimization**: Keep SPF records concise (under 10 DNS lookups)
5. **DKIM Key Rotation**: Periodically rotate DKIM keys for security
+---
+
## Troubleshooting
### DNS Changes Not Reflected
diff --git a/env.example b/env.example
index 644e090..4947748 100644
--- a/env.example
+++ b/env.example
@@ -69,11 +69,18 @@ CORRELATION_CHECK_INTERVAL=120
SMTP_ENABLED=false
SMTP_HOST=
SMTP_PORT=
+# Use TLS instead of SSL (recommended)
SMTP_USE_TLS=true
+# Use SSL instead of TLS
+SMTP_USE_SSL=false
SMTP_USER=
SMTP_PASSWORD=
SMTP_FROM=noreply@yourdomain.com
+# SMTP Relay Mode - Set to true for local relay servers that don't require authentication
+# When enabled, username and password are not required
+SMTP_RELAY_MODE=false
+
# =============================================================================
# Admin Email
# =============================================================================
@@ -102,6 +109,7 @@ DMARC_IMAP_FOLDER=INBOX
DMARC_IMAP_DELETE_AFTER=true
DMARC_IMAP_INTERVAL=3600
DMARC_IMAP_RUN_ON_STARTUP=true
+DMARC_IMAP_BATCH_SIZE=10
# DMARC Error Email Override (optional - uses ADMIN_EMAIL if not set)
DMARC_ERROR_EMAIL=
diff --git a/frontend/app.js b/frontend/app.js
index 6c91b66..e4749c6 100644
--- a/frontend/app.js
+++ b/frontend/app.js
@@ -3,6 +3,174 @@
// Part 1: Core, Global State, Dashboard, Postfix, Rspamd, Netfilter
// =============================================================================
+// =============================================================================
+// GLOBAL COLOR CONFIGURATION
+// Edit these values to customize colors across the entire application
+// =============================================================================
+
+const APP_COLORS = {
+ // Email Direction Colors
+ directions: {
+ inbound: {
+ // Indigo
+ badge: 'bg-indigo-100 dark:bg-indigo-500/10 text-indigo-700 dark:text-indigo-300 border border-indigo-200 dark:border-indigo-500/20',
+ bg: 'bg-indigo-100 dark:bg-indigo-500/25',
+ text: 'text-indigo-700 dark:text-indigo-400'
+ },
+ outbound: {
+ // Blue
+ badge: 'bg-blue-100 dark:bg-blue-500/10 text-blue-700 dark:text-blue-300 border border-blue-200 dark:border-blue-500/20',
+ bg: 'bg-blue-100 dark:bg-blue-500/25',
+ text: 'text-blue-700 dark:text-blue-400'
+ },
+ internal: {
+ // Teal
+ badge: 'bg-teal-100 dark:bg-teal-500/10 text-teal-800 dark:text-teal-300 border border-teal-200 dark:border-teal-500/20',
+ bg: 'bg-teal-100 dark:bg-teal-500/25',
+ text: 'text-teal-700 dark:text-teal-400'
+ }
+ },
+ statuses: {
+ delivered: {
+ // Emerald
+ badge: 'bg-emerald-100 dark:bg-emerald-500/10 text-emerald-700 dark:text-emerald-300 border border-emerald-200 dark:border-emerald-500/20',
+ bg: 'bg-emerald-100 dark:bg-emerald-500/25',
+ text: 'text-emerald-700 dark:text-emerald-400'
+ },
+ sent: {
+ // Green
+ badge: 'bg-green-100 dark:bg-green-500/10 text-green-700 dark:text-green-300 border border-green-200 dark:border-green-500/20',
+ bg: 'bg-green-100 dark:bg-green-500/25',
+ text: 'text-green-700 dark:text-green-400'
+ },
+ deferred: {
+ // Yellow (Fixed: Changed from Amber to Yellow)
+ badge: 'bg-yellow-100 dark:bg-yellow-500/10 text-yellow-700 dark:text-yellow-300 border border-yellow-200 dark:border-yellow-500/20',
+ bg: 'bg-yellow-100 dark:bg-yellow-500/25',
+ text: 'text-yellow-700 dark:text-yellow-400'
+ },
+ bounced: {
+ // Orange
+ badge: 'bg-orange-100 dark:bg-orange-500/10 text-orange-700 dark:text-orange-300 border border-orange-200 dark:border-orange-500/20',
+ bg: 'bg-orange-100 dark:bg-orange-500/25',
+ text: 'text-orange-700 dark:text-orange-400'
+ },
+ rejected: {
+ // Red
+ badge: 'bg-red-100 dark:bg-red-500/10 text-red-700 dark:text-red-300 border border-red-200 dark:border-red-500/20',
+ bg: 'bg-red-100 dark:bg-red-500/25',
+ text: 'text-red-700 dark:text-red-400'
+ },
+ spam: {
+ // Fuchsia
+ badge: 'bg-fuchsia-100 dark:bg-fuchsia-500/10 text-fuchsia-700 dark:text-fuchsia-300 border border-fuchsia-200 dark:border-fuchsia-500/20',
+ bg: 'bg-fuchsia-100 dark:bg-fuchsia-500/25',
+ text: 'text-fuchsia-700 dark:text-fuchsia-400'
+ },
+ expired: {
+ // Zinc
+ badge: 'bg-zinc-100 dark:bg-zinc-500/10 text-zinc-700 dark:text-zinc-300 border border-zinc-200 dark:border-zinc-500/20',
+ bg: 'bg-zinc-100 dark:bg-zinc-500/25',
+ text: 'text-zinc-700 dark:text-zinc-400'
+ }
+ },
+ // Default color for unknown values
+ default: {
+ badge: 'bg-gray-100 dark:bg-gray-700 text-gray-800 dark:text-gray-300',
+ bg: 'bg-gray-100 dark:bg-gray-700',
+ text: 'text-gray-600 dark:text-gray-400'
+ }
+};
+
+// Helper functions for accessing colors
+function getDirectionBadgeClass(direction) {
+ return APP_COLORS.directions[direction]?.badge || APP_COLORS.default.badge;
+}
+
+function getDirectionBgClass(direction) {
+ return APP_COLORS.directions[direction]?.bg || APP_COLORS.default.bg;
+}
+
+function getDirectionTextClass(direction) {
+ return APP_COLORS.directions[direction]?.text || APP_COLORS.default.text;
+}
+
+function getStatusBadgeClass(status) {
+ return APP_COLORS.statuses[status]?.badge || APP_COLORS.default.badge;
+}
+
+function getStatusBgClass(status) {
+ return APP_COLORS.statuses[status]?.bg || APP_COLORS.default.bg;
+}
+
+function getStatusTextClass(status) {
+ return APP_COLORS.statuses[status]?.text || APP_COLORS.default.text;
+}
+
+// =============================================================================
+// NAVIGATION HELPERS
+// =============================================================================
+
+/**
+ * Navigate to Messages page with pre-filled filters
+ * @param {Object} options - Filter options
+ * @param {string} options.email - Email address to filter by
+ * @param {string} options.filterType - 'sender' | 'recipient' | 'search'
+ * @param {string} options.direction - 'inbound' | 'outbound' | 'internal'
+ * @param {string} options.status - 'delivered' | 'bounced' | 'deferred' | 'rejected'
+ */
+function navigateToMessagesWithFilter(options) {
+ // Clear existing filters first
+ const filterSearch = document.getElementById('messages-filter-search');
+ const filterSender = document.getElementById('messages-filter-sender');
+ const filterRecipient = document.getElementById('messages-filter-recipient');
+ const filterDirection = document.getElementById('messages-filter-direction');
+ const filterStatus = document.getElementById('messages-filter-status');
+ const filterUser = document.getElementById('messages-filter-user');
+ const filterIp = document.getElementById('messages-filter-ip');
+
+ // Reset all filters
+ if (filterSearch) filterSearch.value = '';
+ if (filterSender) filterSender.value = '';
+ if (filterRecipient) filterRecipient.value = '';
+ if (filterDirection) filterDirection.value = '';
+ if (filterStatus) filterStatus.value = '';
+ if (filterUser) filterUser.value = '';
+ if (filterIp) filterIp.value = '';
+
+ // Set email filter based on type
+ if (options.email) {
+ if (options.filterType === 'sender') {
+ if (filterSender) filterSender.value = options.email;
+ } else if (options.filterType === 'recipient') {
+ if (filterRecipient) filterRecipient.value = options.email;
+ } else {
+ // Default: use search field
+ if (filterSearch) filterSearch.value = options.email;
+ }
+ }
+
+ // Set direction filter
+ if (options.direction && filterDirection) {
+ filterDirection.value = options.direction;
+ }
+
+ // Set status filter
+ if (options.status && filterStatus) {
+ filterStatus.value = options.status;
+ }
+
+ // Navigate to Messages tab
+ navigateTo('messages');
+
+ // Apply filters after navigation
+ setTimeout(() => {
+ if (typeof applyMessagesFilters === 'function') {
+ applyMessagesFilters();
+ }
+ }, 100);
+}
+
// =============================================================================
// AUTHENTICATION SYSTEM
// =============================================================================
@@ -61,26 +229,26 @@ async function authenticatedFetch(url, options = {}) {
...options.headers,
...getAuthHeader()
};
-
+
const response = await fetch(url, {
...options,
headers
});
-
+
// Handle 401 Unauthorized
if (response.status === 401) {
clearAuthCredentials();
showLoginModal();
throw new Error('Authentication required');
}
-
+
return response;
}
// Handle login form submission (not used in main app, only in login.html)
async function handleLogin(event) {
event.preventDefault();
-
+
const username = document.getElementById('login-username').value;
const password = document.getElementById('login-password').value;
const errorDiv = document.getElementById('login-error');
@@ -88,22 +256,22 @@ async function handleLogin(event) {
const submitBtn = document.getElementById('login-submit');
const submitText = document.getElementById('login-submit-text');
const submitLoading = document.getElementById('login-submit-loading');
-
+
// Hide error
if (errorDiv) errorDiv.classList.add('hidden');
-
+
// Show loading
if (submitText) submitText.classList.add('hidden');
if (submitLoading) submitLoading.classList.remove('hidden');
if (submitBtn) submitBtn.disabled = true;
-
+
try {
// Save credentials
saveAuthCredentials(username, password);
-
+
// Test authentication with a simple API call
const response = await authenticatedFetch('/api/info');
-
+
if (response.ok) {
// Success - redirect to main app
window.location.href = '/';
@@ -118,11 +286,11 @@ async function handleLogin(event) {
errorText.textContent = error.message || 'Invalid username or password';
}
}
-
+
// Clear password field
const passwordField = document.getElementById('login-password');
if (passwordField) passwordField.value = '';
-
+
// Clear credentials
clearAuthCredentials();
} finally {
@@ -156,16 +324,16 @@ async function checkAuthentication() {
// If we can't check, assume auth is enabled for safety
console.warn('Could not check auth status, assuming enabled');
}
-
+
// Authentication is enabled, check credentials
loadAuthCredentials();
-
+
if (!authCredentials) {
// No credentials saved, redirect to login
window.location.href = '/login';
return false;
}
-
+
try {
// Test if credentials are still valid
const response = await authenticatedFetch('/api/info');
@@ -218,14 +386,14 @@ let autoRefreshTimer = null;
// Initialize on page load
document.addEventListener('DOMContentLoaded', async () => {
console.log('=== Mailcow Logs Viewer Initializing ===');
-
+
// Check authentication first
const isAuthenticated = await checkAuthentication();
if (!isAuthenticated) {
console.log('Authentication required - showing login modal');
return;
}
-
+
// Check if all required elements exist
const requiredElements = [
'app-title',
@@ -238,20 +406,28 @@ document.addEventListener('DOMContentLoaded', async () => {
'content-settings',
'content-domains'
];
-
+
const missing = requiredElements.filter(id => !document.getElementById(id));
if (missing.length > 0) {
console.error('Missing required elements:', missing);
} else {
console.log('[OK] All required DOM elements found');
}
-
+
loadAppInfo();
- loadDashboard();
-
+
+ // Initialize router and get initial route from URL
+ const routeInfo = typeof initRouter === 'function' ? initRouter() : { baseRoute: 'dashboard', params: {} };
+ const initialTab = routeInfo.baseRoute || routeInfo;
+ const initialParams = routeInfo.params || {};
+ console.log('Initial tab from URL:', initialTab, 'params:', initialParams);
+
+ // Load the initial tab (use switchTab to ensure proper initialization)
+ switchTab(initialTab, initialParams);
+
// Start auto-refresh for all tabs
startAutoRefresh();
-
+
console.log('=== Initialization Complete ===');
});
@@ -264,31 +440,31 @@ async function loadAppInfo() {
// Use regular fetch since this is called after authentication check
const response = await authenticatedFetch('/api/info');
const data = await response.json();
-
+
if (data.app_title) {
document.getElementById('app-title').textContent = data.app_title;
document.title = data.app_title;
-
+
// Update footer app name
const footerName = document.getElementById('app-name-footer');
if (footerName) {
footerName.textContent = data.app_title;
}
}
-
+
if (data.app_logo_url) {
const logoImg = document.getElementById('app-logo');
logoImg.src = data.app_logo_url;
logoImg.classList.remove('hidden');
document.getElementById('default-logo').classList.add('hidden');
}
-
+
// Update footer version
const footerVersion = document.getElementById('app-version-footer');
if (footerVersion && data.version) {
footerVersion.textContent = `v${data.version}`;
}
-
+
// Show/hide logout button based on auth status
const logoutBtn = document.getElementById('logout-btn');
if (logoutBtn) {
@@ -298,7 +474,7 @@ async function loadAppInfo() {
logoutBtn.classList.add('hidden');
}
}
-
+
// Store timezone for date formatting
if (data.timezone) {
appTimezone = data.timezone;
@@ -306,10 +482,10 @@ async function loadAppInfo() {
} else {
console.warn('No timezone in API response, using default:', appTimezone);
}
-
+
// Load app version status for update check
await loadAppVersionStatus();
-
+
// Load mailcow connection status
await loadMailcowConnectionStatus();
} catch (error) {
@@ -321,10 +497,10 @@ async function loadMailcowConnectionStatus() {
try {
const response = await authenticatedFetch('/api/status/mailcow-connection');
if (!response.ok) return;
-
+
const data = await response.json();
const indicator = document.getElementById('mailcow-connection-indicator');
-
+
if (indicator) {
indicator.classList.remove('hidden');
if (data.connected) {
@@ -363,10 +539,10 @@ async function loadAppVersionStatus() {
try {
const response = await authenticatedFetch('/api/status/app-version');
if (!response.ok) return;
-
+
const data = await response.json();
const updateBadge = document.getElementById('update-badge');
-
+
if (updateBadge && data.update_available) {
updateBadge.classList.remove('hidden');
updateBadge.title = `Update available: v${data.latest_version}`;
@@ -403,13 +579,13 @@ function startAutoRefresh() {
if (autoRefreshTimer) {
clearInterval(autoRefreshTimer);
}
-
+
// Set up auto-refresh interval
autoRefreshTimer = setInterval(() => {
smartRefreshCurrentTab();
}, AUTO_REFRESH_INTERVAL);
-
- console.log(`[OK] Auto-refresh started (every ${AUTO_REFRESH_INTERVAL/1000}s)`);
+
+ console.log(`[OK] Auto-refresh started (every ${AUTO_REFRESH_INTERVAL / 1000}s)`);
}
function stopAutoRefresh() {
@@ -427,7 +603,7 @@ async function smartRefreshCurrentTab() {
if (modal && !modal.classList.contains('hidden')) {
return;
}
-
+
try {
switch (currentTab) {
case 'dashboard':
@@ -461,7 +637,7 @@ async function smartRefreshCurrentTab() {
function hasDataChanged(newData, cacheKey) {
const oldData = lastDataCache[cacheKey];
if (!oldData) return true;
-
+
// Compare JSON strings for simple change detection
const newJson = JSON.stringify(newData);
const oldJson = JSON.stringify(oldData);
@@ -472,21 +648,21 @@ function hasDataChanged(newData, cacheKey) {
// Only refreshes if there are no active filters/search (to avoid disrupting user's view)
async function smartRefreshMessages() {
const filters = currentFilters.messages || {};
-
+
// Don't refresh if user has active search or filters
- const hasActiveFilters = filters.search || filters.sender || filters.recipient ||
- filters.direction || filters.status || filters.user || filters.ip;
-
+ const hasActiveFilters = filters.search || filters.sender || filters.recipient ||
+ filters.direction || filters.status || filters.user || filters.ip;
+
// Don't refresh if user is not on first page
if (hasActiveFilters || currentPage.messages > 1) {
return; // Skip refresh to avoid disrupting user's view
}
-
+
const params = new URLSearchParams({
page: currentPage.messages,
limit: 50
});
-
+
if (filters.search) params.append('search', filters.search);
if (filters.sender) params.append('sender', filters.sender);
if (filters.recipient) params.append('recipient', filters.recipient);
@@ -494,12 +670,12 @@ async function smartRefreshMessages() {
if (filters.status) params.append('status', filters.status);
if (filters.user) params.append('user', filters.user);
if (filters.ip) params.append('ip', filters.ip);
-
+
const response = await authenticatedFetch(`/api/messages?${params}`);
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'messages')) {
console.log('[REFRESH] Messages data changed, updating UI');
lastDataCache.messages = data;
@@ -511,12 +687,12 @@ async function smartRefreshMessages() {
function renderMessagesData(data) {
const container = document.getElementById('messages-logs');
if (!container) return;
-
+
if (!data.data || data.data.length === 0) {
container.innerHTML = 'No messages found
';
return;
}
-
+
container.innerHTML = `
${data.data.map(msg => `
@@ -534,12 +710,12 @@ function renderMessagesData(data) {
${(() => {
- const correlationStatus = getCorrelationStatusDisplay(msg);
- if (correlationStatus) {
- return `${correlationStatus.display}`;
- }
- return '';
- })()}
+ const correlationStatus = getCorrelationStatusDisplay(msg);
+ if (correlationStatus) {
+ return `${correlationStatus.display}`;
+ }
+ return '';
+ })()}
${msg.direction ? `${msg.direction}` : ''}
${msg.is_spam !== null ? `${msg.is_spam ? 'SPAM' : 'CLEAN'}` : ''}
@@ -562,20 +738,20 @@ function renderMessagesData(data) {
// Deduplicate netfilter logs based on message + time + priority
function deduplicateNetfilterLogs(logs) {
if (!logs || logs.length === 0) return [];
-
+
const seen = new Set();
const uniqueLogs = [];
-
+
for (const log of logs) {
// Create unique key from message + time + priority
const key = `${log.message || ''}|${log.time || ''}|${log.priority || ''}`;
-
+
if (!seen.has(key)) {
seen.add(key);
uniqueLogs.push(log);
}
}
-
+
return uniqueLogs;
}
@@ -583,21 +759,21 @@ function deduplicateNetfilterLogs(logs) {
function renderNetfilterData(data) {
const container = document.getElementById('netfilter-logs');
if (!container) return;
-
+
if (!data.data || data.data.length === 0) {
container.innerHTML = 'No logs found
';
return;
}
-
+
// Deduplicate logs
const uniqueLogs = deduplicateNetfilterLogs(data.data);
-
+
// Update count display with total count from API (like Messages page)
const countEl = document.getElementById('security-count');
if (countEl) {
countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : '';
}
-
+
container.innerHTML = `
${uniqueLogs.map(log => `
@@ -627,12 +803,12 @@ async function smartRefreshNetfilter() {
limit: 50,
...filters
});
-
+
const response = await authenticatedFetch(`/api/logs/netfilter?${params}`);
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'netfilter')) {
console.log('[REFRESH] Netfilter data changed, updating UI');
lastDataCache.netfilter = data;
@@ -646,9 +822,9 @@ async function smartRefreshNetfilter() {
async function smartRefreshQueue() {
const response = await authenticatedFetch('/api/queue');
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'queue')) {
console.log('[REFRESH] Queue data changed, updating UI');
lastDataCache.queue = data;
@@ -661,9 +837,9 @@ async function smartRefreshQueue() {
async function smartRefreshQuarantine() {
const response = await authenticatedFetch('/api/quarantine');
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'quarantine')) {
console.log('[REFRESH] Quarantine data changed, updating UI');
lastDataCache.quarantine = data;
@@ -675,12 +851,12 @@ async function smartRefreshQuarantine() {
function renderQuarantineData(data) {
const container = document.getElementById('quarantine-logs');
if (!container) return;
-
+
if (!data.data || data.data.length === 0) {
container.innerHTML = '
No quarantined messages
';
return;
}
-
+
container.innerHTML = `
${data.data.map(item => `
@@ -704,13 +880,13 @@ async function smartRefreshDashboard() {
try {
const response = await authenticatedFetch('/api/stats/dashboard');
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'dashboard')) {
console.log('[REFRESH] Dashboard data changed, updating UI');
lastDataCache.dashboard = data;
-
+
// Update stats without full reload
document.getElementById('stat-messages-24h').textContent = data.messages['24h'].toLocaleString();
document.getElementById('stat-messages-7d').textContent = data.messages['7d'].toLocaleString();
@@ -722,7 +898,7 @@ async function smartRefreshDashboard() {
document.getElementById('stat-auth-failures-24h').textContent = data.auth_failures['24h'].toLocaleString();
document.getElementById('stat-auth-failures-7d').textContent = data.auth_failures['7d'].toLocaleString();
}
-
+
// Also refresh recent activity and status summary
loadRecentActivity();
loadDashboardStatusSummary();
@@ -736,13 +912,13 @@ async function smartRefreshSettings() {
try {
const response = await authenticatedFetch('/api/settings/info');
if (!response.ok) return;
-
+
const data = await response.json();
-
+
if (hasDataChanged(data, 'settings')) {
console.log('[REFRESH] Settings data changed, updating UI');
lastDataCache.settings = data;
-
+
const content = document.getElementById('settings-content');
if (content && !content.classList.contains('hidden')) {
// Preserve version info from cache (don't reload it on smart refresh)
@@ -752,7 +928,7 @@ async function smartRefreshSettings() {
if (versionInfoCache.version_info) {
data.version_info = versionInfoCache.version_info;
}
-
+
renderSettings(content, data);
}
}
@@ -765,11 +941,11 @@ async function smartRefreshSettings() {
// TAB SWITCHING
// =============================================================================
-function switchTab(tab) {
- console.log('Switching to tab:', tab);
+function switchTab(tab, params = {}) {
+ console.log('Switching to tab:', tab, 'params:', params);
currentTab = tab;
-
- // Update active tab button
+
+ // Update active tab button (desktop)
document.querySelectorAll('[id^="tab-"]').forEach(btn => {
btn.classList.remove('tab-active');
btn.classList.add('text-gray-500', 'dark:text-gray-400');
@@ -779,12 +955,20 @@ function switchTab(tab) {
activeBtn.classList.add('tab-active');
activeBtn.classList.remove('text-gray-500', 'dark:text-gray-400');
}
-
+
+ // Update mobile menu state and label
+ if (typeof updateMobileMenuActiveState === 'function') {
+ updateMobileMenuActiveState(tab);
+ }
+ if (typeof updateCurrentTabLabel === 'function') {
+ updateCurrentTabLabel(tab);
+ }
+
// Hide all tab contents
document.querySelectorAll('.tab-content').forEach(content => {
content.classList.add('hidden');
});
-
+
// Show current tab content
const tabContent = document.getElementById(`content-${tab}`);
if (tabContent) {
@@ -818,7 +1002,10 @@ function switchTab(tab) {
loadDomains();
break;
case 'dmarc':
- loadDmarc();
+ handleDmarcRoute(params);
+ break;
+ case 'mailbox-stats':
+ loadMailboxStats();
break;
case 'settings':
loadSettings();
@@ -828,7 +1015,15 @@ function switchTab(tab) {
}
}
-function refreshAllData() {
+async function refreshAllData() {
+ if (currentTab === 'dmarc') {
+ try {
+ await authenticatedFetch('/api/dmarc/cache/clear', { method: 'POST' });
+ console.log('DMARC cache cleared');
+ } catch (e) {
+ console.error('Failed to clear DMARC cache:', e);
+ }
+ }
switchTab(currentTab);
}
@@ -839,15 +1034,15 @@ function refreshAllData() {
async function loadDashboard() {
try {
console.log('Loading Dashboard...');
-
+
const response = await authenticatedFetch('/api/stats/dashboard');
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
-
+
const data = await response.json();
console.log('Dashboard data:', data);
-
+
document.getElementById('stat-messages-24h').textContent = data.messages['24h'].toLocaleString();
document.getElementById('stat-messages-7d').textContent = data.messages['7d'].toLocaleString();
document.getElementById('stat-blocked-24h').textContent = data.blocked['24h'].toLocaleString();
@@ -857,7 +1052,7 @@ async function loadDashboard() {
document.getElementById('stat-deferred-7d').textContent = data.deferred['7d'].toLocaleString();
document.getElementById('stat-auth-failures-24h').textContent = data.auth_failures['24h'].toLocaleString();
document.getElementById('stat-auth-failures-7d').textContent = data.auth_failures['7d'].toLocaleString();
-
+
loadRecentActivity();
loadDashboardStatusSummary();
} catch (error) {
@@ -868,15 +1063,15 @@ async function loadDashboard() {
async function loadDashboardStatusSummary() {
try {
console.log('Loading Dashboard Status Summary...');
-
+
const response = await authenticatedFetch('/api/status/summary');
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
-
+
const data = await response.json();
console.log('Status summary data:', data);
-
+
const containersDiv = document.getElementById('dashboard-containers-summary');
const containers = data.containers || {};
containersDiv.innerHTML = `
@@ -893,13 +1088,13 @@ async function loadDashboardStatusSummary() {
${containers.total || 0}
`;
-
+
const storageDiv = document.getElementById('dashboard-storage-summary');
const storage = data.storage || {};
const usedPercent = parseInt(storage.used_percent) || 0;
- const storageColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' :
- usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' :
- 'text-green-600 dark:text-green-400';
+ const storageColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' :
+ usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' :
+ 'text-green-600 dark:text-green-400';
storageDiv.innerHTML = `
Used
@@ -915,7 +1110,7 @@ async function loadDashboardStatusSummary() {
`;
-
+
const systemDiv = document.getElementById('dashboard-system-summary');
const system = data.system || {};
systemDiv.innerHTML = `
@@ -939,23 +1134,23 @@ async function loadDashboardStatusSummary() {
async function loadRecentActivity() {
const container = document.getElementById('recent-activity');
-
+
try {
console.log('Loading Recent Activity...');
-
+
const response = await authenticatedFetch('/api/stats/recent-activity?limit=10');
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
-
+
const data = await response.json();
console.log('Recent Activity data:', data);
-
+
if (data.activity.length === 0) {
container.innerHTML = 'No recent activity
';
return;
}
-
+
container.innerHTML = data.activity.map(msg => `
@@ -986,7 +1181,7 @@ async function loadRecentActivity() {
function performDashboardSearch() {
const query = document.getElementById('dashboard-search-query').value;
const status = document.getElementById('dashboard-search-status').value;
-
+
// Set filters on Messages page
document.getElementById('messages-filter-search').value = query;
document.getElementById('messages-filter-sender').value = '';
@@ -994,14 +1189,14 @@ function performDashboardSearch() {
document.getElementById('messages-filter-direction').value = '';
document.getElementById('messages-filter-status').value = status;
document.getElementById('messages-filter-user').value = '';
-
+
// Apply filters
currentFilters.messages = {
search: query,
status: status
};
currentPage.messages = 1;
-
+
// Switch to Messages tab and load
switchTab('messages');
}
@@ -1031,38 +1226,38 @@ function clearPostfixFilters() {
async function loadPostfixLogs(page = 1) {
const container = document.getElementById('postfix-logs');
-
+
// Show loading immediately
container.innerHTML = '
Loading Postfix logs... This may take a few moments.
';
-
+
try {
const filters = currentFilters.postfix || {};
const params = new URLSearchParams({
page: page,
limit: 50
});
-
+
if (filters.search) params.append('search', filters.search);
if (filters.sender) params.append('sender', filters.sender);
if (filters.recipient) params.append('recipient', filters.recipient);
-
+
console.log('Loading Postfix logs:', `/api/logs/postfix?${params}`);
const startTime = performance.now();
-
+
const response = await authenticatedFetch(`/api/logs/postfix?${params}`);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
-
+
const data = await response.json();
const loadTime = ((performance.now() - startTime) / 1000).toFixed(2);
console.log(`Postfix data loaded in ${loadTime}s:`, data);
-
+
if (!data.data || data.data.length === 0) {
container.innerHTML = '
No logs found
';
return;
}
-
+
container.innerHTML = `