Release version 2.1.0
This commit is contained in:
Shlomi
2
.gitignore
vendored
2
.gitignore
vendored
@@ -62,6 +62,7 @@ backup/
|
||||
htmlcov/
|
||||
.tox/
|
||||
docker-compose-dev.yml
|
||||
Dev/
|
||||
|
||||
# Documentation
|
||||
docs/_build/
|
||||
@@ -77,3 +78,4 @@ Thumbs.db
|
||||
|
||||
# Maxmind
|
||||
data
|
||||
old-data
|
||||
199
CHANGELOG.md
199
CHANGELOG.md
@@ -5,6 +5,205 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [2.1.0] - 2026-01-20
|
||||
|
||||
### Added
|
||||
|
||||
#### Mailbox Statistics Page
|
||||
- **Complete Mailbox Statistics Feature**: New page showing per-mailbox message statistics
|
||||
- Summary cards: Total Sent, Received, Failed, and Failure Rate
|
||||
- Accordion-style mailbox list with expandable details
|
||||
- Message counts aggregated from MessageCorrelation table (not Mailcow API)
|
||||
- Per-alias message statistics with sent/received/failed counts
|
||||
- Combined totals (mailbox + all its aliases)
|
||||
|
||||
- **Clickable Statistics Links**: All stat cards and alias table cells are clickable
|
||||
- Click on any stat (Sent, Received, Internal, Delivered, Deferred, Bounced, Rejected) to navigate to Messages page
|
||||
- Search automatically pre-filled with the mailbox/alias email address
|
||||
|
||||
- **Mailbox Details Display (Domains-style)**:
|
||||
- Quota usage with percentage
|
||||
- Messages in mailbox count
|
||||
- Last IMAP/SMTP/POP3 login times
|
||||
- Created and Modified dates
|
||||
- Rate limiting settings (value/frame)
|
||||
- Access permissions indicators: IMAP, POP3, SMTP, Sieve, SOGo, TLS Enforce
|
||||
- Color-coded status dots (green=enabled, red=disabled)
|
||||
|
||||
- **Filtering & Search**:
|
||||
- **Date Range Picker with Presets**:
|
||||
- Quick select preset buttons: Today, 7 Days, 30 Days, 90 Days
|
||||
- Custom date range with From/To date inputs
|
||||
- Domain filter dropdown
|
||||
- Search by mailbox username, name, or alias address
|
||||
- "Active Only" checkbox (default: checked)
|
||||
- "Hide Zero Activity" checkbox (default: checked) - filters mailboxes and aliases with no messages
|
||||
- Sort by: Sent, Received, Failure Rate, Quota Used, Username
|
||||
|
||||
- **Pagination**: 50 mailboxes per page with navigation controls
|
||||
|
||||
#### Background Jobs
|
||||
- **Mailbox Statistics Job**: Fetches mailbox data from Mailcow API every 5 minutes
|
||||
- Syncs quota, messages, login times, rate limits, and attributes
|
||||
- Marks deleted mailboxes as inactive (preserves historical data)
|
||||
|
||||
- **Alias Statistics Job**: Fetches alias data from Mailcow API every 5 minutes
|
||||
- Links aliases to their target mailboxes
|
||||
- Marks deleted aliases as inactive (preserves historical data)
|
||||
|
||||
#### SMTP Relay Mode
|
||||
- **No-Authentication SMTP Support**: New option for sending emails through local relay servers without credentials
|
||||
- Enable via `SMTP_RELAY_MODE=true` in environment variables
|
||||
- When enabled, `SMTP_USER` and `SMTP_PASSWORD` are not required
|
||||
- Useful for local Postfix relay servers, internal mail gateways, or trusted SMTP relays
|
||||
- Connection test in Settings page properly handles relay mode authentication bypass
|
||||
|
||||
#### Clean URL Routing (History API)
|
||||
- **Shareable URLs for All Pages**: Implemented History API-based routing for the SPA
|
||||
- Direct navigation to any tab via clean URLs (e.g., `/dashboard`, `/messages`, `/dmarc`, `/settings`)
|
||||
- Browser Back/Forward buttons now work correctly between pages
|
||||
- URLs can be bookmarked and shared
|
||||
|
||||
- **DMARC Nested Routes**: Deep linking support for all DMARC views
|
||||
- `/dmarc` - Domains list
|
||||
- `/dmarc/{domain}` - Domain overview
|
||||
- `/dmarc/{domain}/reports` - Daily Reports tab
|
||||
- `/dmarc/{domain}/sources` - Source IPs tab
|
||||
- `/dmarc/{domain}/tls` - TLS Reports tab
|
||||
- `/dmarc/{domain}/report/{date}` - Specific daily report details
|
||||
- `/dmarc/{domain}/source/{ip}` - Specific source IP details
|
||||
|
||||
- **Removed Internal Back Button**: DMARC section no longer uses custom back button
|
||||
- Users now use browser's native Back button
|
||||
- Cleaner UI without duplicate navigation controls
|
||||
|
||||
#### TLS-RPT (TLS Reporting) Support
|
||||
- **Complete TLS-RPT Implementation**: Full support for TLS aggregate reports (RFC 8460)
|
||||
- TLS-RPT parser for JSON reports (gzip compressed)
|
||||
- Database models for TLS reports and policies
|
||||
- IMAP auto-import support for TLS-RPT emails
|
||||
- Manual upload support for TLS-RPT files
|
||||
|
||||
- **TLS Reports Tab in DMARC Page**:
|
||||
- New "TLS Reports" sub-tab alongside Daily Reports and Source IPs
|
||||
- Daily aggregated view showing reports grouped by date
|
||||
- Success rate with color-coded progress bars (green ≥95%, yellow ≥80%, red <80%)
|
||||
- Provider breakdown with session counts
|
||||
|
||||
- **TLS Report Details View**:
|
||||
- Click any daily report to see detailed breakdown
|
||||
- Stats cards: Sessions, Success Rate, Successful, Failed
|
||||
- Providers table with per-provider success rates
|
||||
|
||||
- **TLS in Domain List**:
|
||||
- TLS Success Rate column in DMARC domains table
|
||||
- TLS report count displayed per domain
|
||||
- Domains with only TLS reports (no DMARC) now included in list
|
||||
|
||||
#### DMARC Navigation Improvements
|
||||
- **Breadcrumb Navigation**: Clear path indicator for all DMARC views
|
||||
- Shows current location: `domain.com > Daily Reports > Jan 14, 2026`
|
||||
- Clickable links to navigate back to any level
|
||||
- Displayed below page description
|
||||
|
||||
#### Mobile Navigation Hamburger Menu
|
||||
- **Hamburger Menu for Mobile**:
|
||||
- Replaced horizontal scrolling tabs with a proper hamburger menu on mobile devices
|
||||
|
||||
### Fixed
|
||||
|
||||
#### DMARC Source IPs - Broken Flag Images
|
||||
- **Fixed broken flag images when MAXMIND is not configured**: When GeoIP integration is not set up, the Source IPs tab was showing broken images
|
||||
- Now displays a generic server icon instead of a broken image when country data is unavailable
|
||||
- Flag is completely hidden in source details view when no GeoIP data exists
|
||||
- Added `onerror` fallback handlers to gracefully handle missing flag files
|
||||
- Improves UX for users who haven't configured MAXMIND integration
|
||||
|
||||
#### DMARC Parser
|
||||
- **DMARC 2.0 XML Namespace Support**: Fixed parsing error for DMARC reports using XML namespaces
|
||||
- Reports from GMX and other providers using the new format now parse correctly
|
||||
- Parser now tries both namespaced and non-namespaced element lookups
|
||||
|
||||
### Improved
|
||||
|
||||
#### Backend API Performance
|
||||
- **In-Memory Caching for Statistics API**: Added 5-minute TTL cache for `/api/mailbox-stats/all` endpoint
|
||||
- Cache key generated from all query parameters
|
||||
- First request fetches from database, subsequent requests return from cache
|
||||
- Cache automatically expires after 5 minutes for fresh data
|
||||
- Significantly reduces database load and improves response times
|
||||
|
||||
- **In-Memory Caching for DMARC API**: Added 5-minute TTL cache for `/api/dmarc/domains` endpoint
|
||||
- Reduces heavy database queries for domain statistics
|
||||
- Cache cleared on new report imports
|
||||
|
||||
#### DMARC IMAP Auto-Import
|
||||
- **Batch Processing**: Emails are now processed in configurable batches to prevent memory issues
|
||||
- New `DMARC_IMAP_BATCH_SIZE` environment variable (default: 10)
|
||||
- Processes emails in chunks, re-searching after each batch
|
||||
- Progress logging shows batch number and completion status
|
||||
- Prevents application crashes when syncing mailboxes with lots of emails
|
||||
|
||||
- **UID-Based Email Handling**: Fixed "Invalid messageset" IMAP errors
|
||||
- Changed from sequence numbers to UIDs for all IMAP operations
|
||||
- UIDs remain stable even after deleting emails during sync
|
||||
- Affects SEARCH, FETCH, STORE operations
|
||||
|
||||
- **Flexible DMARC Email Detection**: Now supports more email providers
|
||||
- Yahoo and other providers that don't include "Report-ID:" now detected correctly
|
||||
- Primary validation is now attachment-based (.xml.gz or .zip files)
|
||||
- Accepts: "Report Domain:" only, "Report Domain:" + "Submitter:", or "DMARC" keyword
|
||||
|
||||
- **Improved Error Reporting in Notifications**:
|
||||
- Error notification emails now show actual error messages
|
||||
- Parse failures show: "Failed to parse: filename.xml.gz"
|
||||
- Processing errors show: "Error processing filename: exception details"
|
||||
- Duplicate reports no longer counted as failures
|
||||
|
||||
- **Infinite Loop Prevention**: Fixed sync running endlessly when emails fail validation
|
||||
- Added `UNSEEN` filter to search criteria
|
||||
- Failed or processed emails are marked as Seen and excluded from next search
|
||||
- Prevents re-processing the same emails repeatedly
|
||||
|
||||
- **Microsoft Outlook Support**: Fixed DMARC reports from Microsoft not being recognized
|
||||
- Now detects DMARC reports by filename pattern (contains `!` separator)
|
||||
- Supports filenames like: `enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz`
|
||||
|
||||
- **Enhanced Attachment Extraction**: More robust attachment detection
|
||||
- Now supports plain `.xml` files in addition to `.xml.gz` and `.zip`
|
||||
- Falls back to Content-Type `name` parameter when filename header is missing
|
||||
- Recognizes attachments by Content-Type: `application/gzip`, `application/zip`, `text/xml`, etc.
|
||||
- Added debug logging to help troubleshoot attachment detection issues
|
||||
|
||||
#### Domains Page - DKIM View Record
|
||||
- **DKIM Record Viewer**: Added "View Record" functionality for DKIM, similar to SPF
|
||||
- Displays the full DNS record name including selector (e.g., `dkim._domainkey.example.com`)
|
||||
- Shows the DKIM public key record value
|
||||
- Helps users identify exactly which DNS record to configure
|
||||
|
||||
### Technical
|
||||
|
||||
#### New API Endpoints
|
||||
```
|
||||
GET /api/mailbox-stats/summary
|
||||
GET /api/mailbox-stats/all
|
||||
GET /api/mailbox-stats/domains
|
||||
```
|
||||
|
||||
#### API Parameters
|
||||
- `date_range`: today, 7days, 30days, 90days, custom
|
||||
- `start_date`: Custom start date (YYYY-MM-DD) - required when date_range is 'custom'
|
||||
- `end_date`: Custom end date (YYYY-MM-DD) - required when date_range is 'custom'
|
||||
- `domain`: Filter by specific domain
|
||||
- `active_only`: true/false
|
||||
- `hide_zero`: true/false (filter zero-activity mailboxes)
|
||||
- `search`: Search mailbox/alias addresses
|
||||
- `sort_by`: sent_total, received_total, failure_rate, quota_used, username
|
||||
- `sort_order`: asc, desc
|
||||
- `page`, `page_size`: Pagination
|
||||
|
||||
---
|
||||
|
||||
## [2.0.4] - 2026-01-15
|
||||
|
||||
### Fixed
|
||||
|
||||
25
README.md
25
README.md
@@ -59,6 +59,31 @@ A modern, self-hosted dashboard for viewing and analyzing Mailcow mail server lo
|
||||
- Quarantined emails
|
||||
- Search and filter
|
||||
|
||||
### 🌐 Domains Overview
|
||||
- Complete domains overview with DNS validation
|
||||
- SPF validation with DNS lookup counter and server IP authorization check
|
||||
- DKIM validation with Mailcow configuration comparison
|
||||
- DMARC policy checking and recommendations
|
||||
- Automated DNS checks every 6 hours
|
||||
- Domain info: mailboxes, aliases, storage, relay settings
|
||||
- Color-coded status indicators (✓ green, ⚠ amber, ✗ red)
|
||||
|
||||
### 📊 Mailbox Statistics
|
||||
- Per-mailbox message statistics (sent, received, failed, failure rate)
|
||||
- Accordion-style mailbox list with expandable details
|
||||
- Per-alias statistics with message counts
|
||||
- Quota usage, login times, rate limits
|
||||
- Filtering by date range, domain, search, active only, hide zero activity
|
||||
- Sorting by sent, received, failure rate, quota, username
|
||||
|
||||
### 📧 DMARC Reports
|
||||
- DMARC report viewing and analysis
|
||||
- GeoIP enrichment with MaxMind (City + ASN)
|
||||
- Daily aggregated reports with compliance rates
|
||||
- Manual upload (XML, GZ, ZIP formats)
|
||||
- IMAP auto-import with configurable sync interval
|
||||
- Sync history tracking with error notifications
|
||||
|
||||
### 📈 Status
|
||||
- All container states (running/stopped count)
|
||||
- Storage usage with percentage bar
|
||||
|
||||
@@ -50,10 +50,11 @@ class BasicAuthMiddleware(BaseHTTPMiddleware):
|
||||
if not settings.auth_enabled:
|
||||
return await call_next(request)
|
||||
|
||||
# Allow access to login page, static files, and health check without authentication
|
||||
# Allow access to login page, static files, health check, and info endpoint without authentication
|
||||
# Health check endpoint must be accessible for Docker health monitoring
|
||||
# Info endpoint is used to check if authentication is enabled
|
||||
path = request.url.path
|
||||
if path == "/login" or path.startswith("/static/") or path == "/api/health":
|
||||
if path == "/login" or path.startswith("/static/") or path == "/api/health" or path == "/api/info":
|
||||
return await call_next(request)
|
||||
|
||||
# Check if password is configured
|
||||
@@ -67,9 +68,10 @@ class BasicAuthMiddleware(BaseHTTPMiddleware):
|
||||
# Extract credentials from Authorization header
|
||||
authorization = request.headers.get("Authorization", "")
|
||||
|
||||
# For root page, allow access without Authorization header
|
||||
# For frontend routes (not API), allow access without Authorization header
|
||||
# The frontend JavaScript will handle authentication and redirect if needed
|
||||
if path == "/":
|
||||
# This enables clean URLs like /dashboard, /messages, /dmarc etc.
|
||||
if not path.startswith("/api/"):
|
||||
return await call_next(request)
|
||||
|
||||
# For all other paths (API endpoints), require authentication
|
||||
|
||||
@@ -162,6 +162,12 @@ class Settings(BaseSettings):
|
||||
description='Run IMAP sync once on application startup'
|
||||
)
|
||||
|
||||
dmarc_imap_batch_size: int = Field(
|
||||
default=10,
|
||||
env='DMARC_IMAP_BATCH_SIZE',
|
||||
description='Number of emails to process per batch (prevents memory issues with large mailboxes)'
|
||||
)
|
||||
|
||||
dmarc_error_email: Optional[str] = Field(
|
||||
default=None,
|
||||
env='DMARC_ERROR_EMAIL',
|
||||
@@ -188,11 +194,17 @@ class Settings(BaseSettings):
|
||||
)
|
||||
|
||||
smtp_use_tls: bool = Field(
|
||||
default=True,
|
||||
default=False,
|
||||
env='SMTP_USE_TLS',
|
||||
description='Use STARTTLS for SMTP connection'
|
||||
)
|
||||
|
||||
smtp_use_ssl: bool = Field(
|
||||
default=False,
|
||||
env='SMTP_USE_SSL',
|
||||
description='Use Implicit SSL/TLS for SMTP connection (usually port 465)'
|
||||
)
|
||||
|
||||
smtp_user: Optional[str] = Field(
|
||||
default=None,
|
||||
env='SMTP_USER',
|
||||
@@ -211,6 +223,12 @@ class Settings(BaseSettings):
|
||||
description='From address for emails (defaults to SMTP user if not set)'
|
||||
)
|
||||
|
||||
smtp_relay_mode: bool = Field(
|
||||
default=False,
|
||||
env='SMTP_RELAY_MODE',
|
||||
description='Relay mode - send emails without authentication (for local relay servers)'
|
||||
)
|
||||
|
||||
# Global Admin Email
|
||||
admin_email: Optional[str] = Field(
|
||||
default=None,
|
||||
@@ -260,6 +278,15 @@ class Settings(BaseSettings):
|
||||
@property
|
||||
def notification_smtp_configured(self) -> bool:
|
||||
"""Check if SMTP is properly configured for notifications"""
|
||||
if self.smtp_relay_mode:
|
||||
# Relay mode - only need host and from address
|
||||
return (
|
||||
self.smtp_enabled and
|
||||
self.smtp_host is not None and
|
||||
self.smtp_from is not None
|
||||
)
|
||||
else:
|
||||
# Standard mode - need authentication
|
||||
return (
|
||||
self.smtp_enabled and
|
||||
self.smtp_host is not None and
|
||||
|
||||
@@ -20,6 +20,7 @@ from .routers import logs, stats
|
||||
from .routers import export as export_router
|
||||
from .routers import domains as domains_router
|
||||
from .routers import dmarc as dmarc_router
|
||||
from .routers import mailbox_stats as mailbox_stats_router
|
||||
from .routers import documentation
|
||||
from .migrations import run_migrations
|
||||
from .auth import BasicAuthMiddleware
|
||||
@@ -172,6 +173,7 @@ app.include_router(messages_router.router, prefix="/api", tags=["Messages"])
|
||||
app.include_router(settings_router.router, prefix="/api", tags=["Settings"])
|
||||
app.include_router(domains_router.router, prefix="/api", tags=["Domains"])
|
||||
app.include_router(dmarc_router.router, prefix="/api", tags=["DMARC"])
|
||||
app.include_router(mailbox_stats_router.router, prefix="/api", tags=["Mailbox Stats"])
|
||||
app.include_router(documentation.router, prefix="/api", tags=["Documentation"])
|
||||
|
||||
# Mount static files (frontend)
|
||||
@@ -256,6 +258,23 @@ async def global_exception_handler(request: Request, exc: Exception):
|
||||
)
|
||||
|
||||
|
||||
# SPA catch-all route - must be AFTER all other routes and exception handlers
|
||||
# Returns index.html for all frontend routes (e.g., /dashboard, /messages, /dmarc)
|
||||
@app.get("/{full_path:path}", response_class=HTMLResponse)
|
||||
async def spa_catch_all(full_path: str):
|
||||
"""Serve the SPA for all frontend routes - enables clean URLs"""
|
||||
# API and static routes are handled by their respective routers/mounts
|
||||
# This catch-all only receives unmatched routes
|
||||
try:
|
||||
with open("/app/frontend/index.html", "r") as f:
|
||||
return HTMLResponse(content=f.read())
|
||||
except FileNotFoundError:
|
||||
return HTMLResponse(
|
||||
content="<h1>Mailcow Logs Viewer</h1><p>Frontend not found. Please check installation.</p>",
|
||||
status_code=500
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(
|
||||
|
||||
@@ -795,6 +795,59 @@ def create_dmarc_sync_table(db: Session):
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
def ensure_mailbox_statistics_table(db: Session):
|
||||
"""Ensure mailbox_statistics table exists with proper structure"""
|
||||
try:
|
||||
# Check if table exists
|
||||
result = db.execute(text("""
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_name = 'mailbox_statistics'
|
||||
)
|
||||
""")).scalar()
|
||||
|
||||
if not result:
|
||||
logger.info("Creating mailbox_statistics table...")
|
||||
db.execute(text("""
|
||||
CREATE TABLE mailbox_statistics (
|
||||
id SERIAL PRIMARY KEY,
|
||||
username VARCHAR(255) UNIQUE NOT NULL,
|
||||
domain VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255),
|
||||
quota BIGINT DEFAULT 0,
|
||||
quota_used BIGINT DEFAULT 0,
|
||||
percent_in_use FLOAT DEFAULT 0.0,
|
||||
messages INTEGER DEFAULT 0,
|
||||
active BOOLEAN DEFAULT TRUE,
|
||||
last_imap_login BIGINT,
|
||||
last_pop3_login BIGINT,
|
||||
last_smtp_login BIGINT,
|
||||
spam_aliases INTEGER DEFAULT 0,
|
||||
rl_value INTEGER,
|
||||
rl_frame VARCHAR(20),
|
||||
attributes JSONB,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
)
|
||||
"""))
|
||||
|
||||
# Create indexes
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_domain ON mailbox_statistics(domain)"))
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_active ON mailbox_statistics(active)"))
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_quota_used ON mailbox_statistics(quota_used)"))
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_username ON mailbox_statistics(username)"))
|
||||
|
||||
db.commit()
|
||||
logger.info("mailbox_statistics table created successfully")
|
||||
else:
|
||||
logger.debug("mailbox_statistics table already exists")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error ensuring mailbox_statistics table: {e}")
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def run_migrations():
|
||||
"""
|
||||
Run all database migrations and maintenance tasks
|
||||
@@ -825,6 +878,17 @@ def run_migrations():
|
||||
create_dmarc_sync_table(db)
|
||||
|
||||
# GeoIP fields
|
||||
add_geoip_fields_to_dmarc(db)
|
||||
|
||||
# Mailbox statistics table
|
||||
ensure_mailbox_statistics_table(db)
|
||||
|
||||
# Alias statistics table
|
||||
ensure_alias_statistics_table(db)
|
||||
|
||||
# System settings (for cache signaling)
|
||||
ensure_system_settings_table(db)
|
||||
|
||||
add_geoip_fields_to_dmarc(db)
|
||||
add_geoip_fields_to_rspamd(db)
|
||||
|
||||
@@ -838,3 +902,81 @@ def run_migrations():
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def ensure_alias_statistics_table(db: Session):
|
||||
"""Ensure alias_statistics table exists with all required fields"""
|
||||
try:
|
||||
# Check if table exists
|
||||
result = db.execute(text("""
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_name = 'alias_statistics'
|
||||
)
|
||||
"""))
|
||||
table_exists = result.scalar()
|
||||
|
||||
if not table_exists:
|
||||
logger.info("Creating alias_statistics table...")
|
||||
db.execute(text("""
|
||||
CREATE TABLE alias_statistics (
|
||||
id SERIAL PRIMARY KEY,
|
||||
alias_address VARCHAR(255) NOT NULL UNIQUE,
|
||||
goto TEXT,
|
||||
domain VARCHAR(255) NOT NULL,
|
||||
active BOOLEAN DEFAULT TRUE,
|
||||
is_catch_all BOOLEAN DEFAULT FALSE,
|
||||
primary_mailbox VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
)
|
||||
"""))
|
||||
|
||||
# Create indexes
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_domain ON alias_statistics(domain)"))
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_active ON alias_statistics(active)"))
|
||||
db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_primary_mailbox ON alias_statistics(primary_mailbox)"))
|
||||
db.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_alias_address ON alias_statistics(alias_address)"))
|
||||
|
||||
db.commit()
|
||||
logger.info("alias_statistics table created successfully")
|
||||
else:
|
||||
logger.debug("alias_statistics table already exists")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error ensuring alias_statistics table: {e}")
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def ensure_system_settings_table(db: Session):
|
||||
"""Ensure system_settings table exists"""
|
||||
try:
|
||||
# Check if table exists
|
||||
result = db.execute(text("""
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_name = 'system_settings'
|
||||
)
|
||||
"""))
|
||||
table_exists = result.scalar()
|
||||
|
||||
if not table_exists:
|
||||
logger.info("Creating system_settings table...")
|
||||
db.execute(text("""
|
||||
CREATE TABLE system_settings (
|
||||
key VARCHAR(255) PRIMARY KEY,
|
||||
value TEXT,
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
)
|
||||
"""))
|
||||
|
||||
db.commit()
|
||||
logger.info("system_settings table created successfully")
|
||||
else:
|
||||
logger.debug("system_settings table already exists")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error ensuring system_settings table: {e}")
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
@@ -6,7 +6,7 @@ SIMPLIFIED VERSION:
|
||||
- Removed old generate_correlation_key function
|
||||
- Correlation key is now SHA256 of Message-ID
|
||||
"""
|
||||
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint
|
||||
from sqlalchemy import Column, Integer, BigInteger, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from datetime import datetime
|
||||
|
||||
@@ -293,3 +293,173 @@ class DMARCSync(Base):
|
||||
|
||||
def __repr__(self):
|
||||
return f"<DMARCSync(type={self.sync_type}, status={self.status}, reports={self.reports_created})>"
|
||||
|
||||
|
||||
class MailboxStatistics(Base):
|
||||
"""
|
||||
Mailbox statistics fetched from Mailcow API
|
||||
Tracks quota usage, message counts, and last access times for each mailbox
|
||||
"""
|
||||
__tablename__ = "mailbox_statistics"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
# Mailbox identification
|
||||
username = Column(String(255), unique=True, index=True, nullable=False) # email address
|
||||
domain = Column(String(255), index=True, nullable=False)
|
||||
name = Column(String(255)) # Display name
|
||||
|
||||
# Quota information (in bytes)
|
||||
quota = Column(BigInteger, default=0) # Allocated quota
|
||||
quota_used = Column(BigInteger, default=0) # Used quota
|
||||
percent_in_use = Column(Float, default=0.0) # Percentage used
|
||||
|
||||
# Message counts
|
||||
messages = Column(Integer, default=0) # Total messages in mailbox
|
||||
|
||||
# Status
|
||||
active = Column(Boolean, default=True, index=True)
|
||||
|
||||
# Access times (Unix timestamps from API, stored as integers)
|
||||
last_imap_login = Column(BigInteger, nullable=True)
|
||||
last_pop3_login = Column(BigInteger, nullable=True)
|
||||
last_smtp_login = Column(BigInteger, nullable=True)
|
||||
|
||||
# Spam filter settings
|
||||
spam_aliases = Column(Integer, default=0)
|
||||
|
||||
# Rate limits
|
||||
rl_value = Column(Integer, nullable=True) # Rate limit value
|
||||
rl_frame = Column(String(20), nullable=True) # Rate limit time frame (e.g., "s", "m", "h")
|
||||
|
||||
# Attributes from API
|
||||
attributes = Column(JSONB) # Store full attributes for reference
|
||||
|
||||
# Timestamps
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_mailbox_domain', 'domain'),
|
||||
Index('idx_mailbox_active', 'active'),
|
||||
Index('idx_mailbox_quota_used', 'quota_used'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<MailboxStatistics(username={self.username}, quota_used={self.quota_used}/{self.quota})>"
|
||||
|
||||
|
||||
class AliasStatistics(Base):
|
||||
"""
|
||||
Alias statistics for tracking message counts per alias
|
||||
Links aliases to their target mailboxes
|
||||
"""
|
||||
__tablename__ = "alias_statistics"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
# Alias identification
|
||||
alias_address = Column(String(255), unique=True, index=True, nullable=False) # The alias email
|
||||
goto = Column(Text) # Target mailbox(es), comma-separated
|
||||
domain = Column(String(255), index=True, nullable=False)
|
||||
|
||||
# Status
|
||||
active = Column(Boolean, default=True, index=True)
|
||||
is_catch_all = Column(Boolean, default=False) # Is this a catch-all alias
|
||||
|
||||
# Link to primary mailbox (if applicable)
|
||||
primary_mailbox = Column(String(255), index=True, nullable=True) # Main target mailbox
|
||||
|
||||
# Timestamps
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_alias_domain', 'domain'),
|
||||
Index('idx_alias_active', 'active'),
|
||||
Index('idx_alias_primary_mailbox', 'primary_mailbox'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<AliasStatistics(alias={self.alias_address}, goto={self.goto})>"
|
||||
|
||||
|
||||
class TLSReport(Base):
|
||||
"""TLS-RPT (SMTP TLS Reporting) reports received from email providers"""
|
||||
__tablename__ = "tls_reports"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
# Report identification
|
||||
report_id = Column(String(255), unique=True, index=True, nullable=False)
|
||||
|
||||
# Organization that sent the report
|
||||
organization_name = Column(String(255), index=True)
|
||||
contact_info = Column(String(255))
|
||||
|
||||
# Domain being reported on
|
||||
policy_domain = Column(String(255), index=True, nullable=False)
|
||||
|
||||
# Date range of the report
|
||||
start_datetime = Column(DateTime, nullable=False)
|
||||
end_datetime = Column(DateTime, nullable=False)
|
||||
|
||||
# Raw JSON for reference
|
||||
raw_json = Column(Text)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_tls_report_domain_date', 'policy_domain', 'start_datetime'),
|
||||
Index('idx_tls_report_org', 'organization_name'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<TLSReport(report_id={self.report_id}, domain={self.policy_domain}, org={self.organization_name})>"
|
||||
|
||||
|
||||
class TLSReportPolicy(Base):
|
||||
"""Individual policy records within a TLS-RPT report"""
|
||||
__tablename__ = "tls_report_policies"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
tls_report_id = Column(Integer, index=True, nullable=False)
|
||||
|
||||
# Policy information
|
||||
policy_type = Column(String(50)) # "sts", "no-policy-found", etc.
|
||||
policy_domain = Column(String(255))
|
||||
policy_string = Column(JSONB) # The policy string array
|
||||
mx_host = Column(JSONB) # List of MX hosts
|
||||
|
||||
# Session counts
|
||||
successful_session_count = Column(Integer, default=0)
|
||||
failed_session_count = Column(Integer, default=0)
|
||||
|
||||
# Failure details if any
|
||||
failure_details = Column(JSONB)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_tls_policy_report', 'tls_report_id'),
|
||||
Index('idx_tls_policy_type', 'policy_type'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<TLSReportPolicy(type={self.policy_type}, success={self.successful_session_count}, fail={self.failed_session_count})>"
|
||||
|
||||
|
||||
class SystemSetting(Base):
|
||||
"""
|
||||
Global system settings and state
|
||||
Used for inter-process signaling (e.g., cache invalidation)
|
||||
"""
|
||||
__tablename__ = "system_settings"
|
||||
|
||||
key = Column(String(255), primary_key=True, index=True)
|
||||
value = Column(Text) # JSON string or simple text
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<SystemSetting(key={self.key}, updated={self.updated_at})>"
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
DMARC Router - Domain-centric view (Cloudflare style)
|
||||
"""
|
||||
import logging
|
||||
import hashlib
|
||||
import json
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, BackgroundTasks
|
||||
@@ -9,10 +11,16 @@ from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, and_, or_, case
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import DMARCReport, DMARCRecord, DMARCSync
|
||||
from ..models import DMARCReport, DMARCRecord, DMARCSync, TLSReport, TLSReportPolicy
|
||||
from ..services.dmarc_parser import parse_dmarc_file
|
||||
from ..services.geoip_service import enrich_dmarc_record
|
||||
from ..services.dmarc_imap_service import sync_dmarc_reports_from_imap
|
||||
from ..services.dmarc_cache import (
|
||||
get_dmarc_cache_key,
|
||||
get_dmarc_cached,
|
||||
set_dmarc_cache,
|
||||
clear_dmarc_cache
|
||||
)
|
||||
from ..config import settings
|
||||
from ..scheduler import update_job_status
|
||||
|
||||
@@ -20,21 +28,52 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# =============================================================================
|
||||
# CACHING SYSTEM
|
||||
# =============================================================================
|
||||
|
||||
# =============================================================================
|
||||
# CACHING SYSTEM (Delegated to services.dmarc_cache)
|
||||
# =============================================================================
|
||||
|
||||
# Cache functions imported from ..services.dmarc_cache
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DOMAINS LIST
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/dmarc/cache/clear")
|
||||
async def clear_cache(
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Clear all DMARC related cache
|
||||
"""
|
||||
try:
|
||||
clear_dmarc_cache(db)
|
||||
return {"status": "success", "message": "Cache cleared"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing DMARC cache: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/dmarc/domains")
|
||||
async def get_domains_list(
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get list of all domains with DMARC reports and their statistics
|
||||
Similar to Cloudflare's domain list
|
||||
Get list of all domains with DMARC and/or TLS-RPT reports and their statistics
|
||||
"""
|
||||
try:
|
||||
domains_query = db.query(
|
||||
# Check cache first
|
||||
cache_key = get_dmarc_cache_key("domains_list")
|
||||
cached_result = get_dmarc_cached(cache_key, db)
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
|
||||
# Get domains from DMARC reports
|
||||
dmarc_domains = db.query(
|
||||
DMARCReport.domain,
|
||||
func.count(DMARCReport.id).label('report_count'),
|
||||
func.min(DMARCReport.begin_date).label('first_report'),
|
||||
@@ -43,11 +82,57 @@ async def get_domains_list(
|
||||
DMARCReport.domain
|
||||
).all()
|
||||
|
||||
# Get domains from TLS reports
|
||||
tls_domains = db.query(
|
||||
TLSReport.policy_domain
|
||||
).distinct().all()
|
||||
tls_domain_set = {d[0] for d in tls_domains}
|
||||
|
||||
# Combine domains
|
||||
all_domains = set()
|
||||
dmarc_domain_data = {}
|
||||
|
||||
for domain, report_count, first_report, last_report in dmarc_domains:
|
||||
all_domains.add(domain)
|
||||
dmarc_domain_data[domain] = {
|
||||
'report_count': report_count,
|
||||
'first_report': first_report,
|
||||
'last_report': last_report
|
||||
}
|
||||
|
||||
# Add TLS-only domains
|
||||
all_domains.update(tls_domain_set)
|
||||
|
||||
domains_list = []
|
||||
|
||||
for domain, report_count, first_report, last_report in domains_query:
|
||||
for domain in all_domains:
|
||||
thirty_days_ago = int((datetime.now() - timedelta(days=30)).timestamp())
|
||||
thirty_days_ago_datetime = datetime.now() - timedelta(days=30)
|
||||
|
||||
# Get TLS stats for this domain
|
||||
tls_stats = db.query(
|
||||
func.count(TLSReport.id).label('tls_report_count'),
|
||||
func.sum(TLSReportPolicy.successful_session_count).label('tls_success'),
|
||||
func.sum(TLSReportPolicy.failed_session_count).label('tls_fail')
|
||||
).join(
|
||||
TLSReportPolicy,
|
||||
TLSReportPolicy.tls_report_id == TLSReport.id
|
||||
).filter(
|
||||
and_(
|
||||
TLSReport.policy_domain == domain,
|
||||
TLSReport.start_datetime >= thirty_days_ago_datetime
|
||||
)
|
||||
).first()
|
||||
|
||||
tls_report_count = tls_stats.tls_report_count or 0
|
||||
tls_success = tls_stats.tls_success or 0
|
||||
tls_fail = tls_stats.tls_fail or 0
|
||||
tls_total = tls_success + tls_fail
|
||||
tls_success_pct = round((tls_success / tls_total * 100) if tls_total > 0 else 100, 2)
|
||||
|
||||
# Get DMARC stats
|
||||
dmarc_data = dmarc_domain_data.get(domain)
|
||||
if dmarc_data:
|
||||
stats = db.query(
|
||||
func.sum(DMARCRecord.count).label('total_messages'),
|
||||
func.count(func.distinct(DMARCRecord.source_ip)).label('unique_ips'),
|
||||
@@ -72,26 +157,64 @@ async def get_domains_list(
|
||||
|
||||
domains_list.append({
|
||||
'domain': domain,
|
||||
'report_count': report_count,
|
||||
'first_report': first_report,
|
||||
'last_report': last_report,
|
||||
'report_count': dmarc_data['report_count'],
|
||||
'tls_report_count': tls_report_count,
|
||||
'first_report': dmarc_data['first_report'],
|
||||
'last_report': dmarc_data['last_report'],
|
||||
'has_dmarc': True,
|
||||
'has_tls': domain in tls_domain_set,
|
||||
'stats_30d': {
|
||||
'total_messages': total_msgs,
|
||||
'unique_ips': stats.unique_ips or 0,
|
||||
'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2)
|
||||
'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2),
|
||||
'tls_success_pct': tls_success_pct
|
||||
}
|
||||
})
|
||||
else:
|
||||
# TLS-only domain
|
||||
tls_report = db.query(TLSReport).filter(
|
||||
TLSReport.policy_domain == domain
|
||||
).order_by(TLSReport.end_datetime.desc()).first()
|
||||
|
||||
first_tls = db.query(func.min(TLSReport.start_datetime)).filter(
|
||||
TLSReport.policy_domain == domain
|
||||
).scalar()
|
||||
|
||||
domains_list.append({
|
||||
'domain': domain,
|
||||
'report_count': 0,
|
||||
'tls_report_count': tls_report_count,
|
||||
'first_report': int(first_tls.timestamp()) if first_tls else None,
|
||||
'last_report': int(tls_report.end_datetime.timestamp()) if tls_report and tls_report.end_datetime else None,
|
||||
'has_dmarc': False,
|
||||
'has_tls': True,
|
||||
'stats_30d': {
|
||||
'total_messages': 0,
|
||||
'unique_ips': 0,
|
||||
'dmarc_pass_pct': 0,
|
||||
'tls_success_pct': tls_success_pct
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
'domains': sorted(domains_list, key=lambda x: x['last_report'], reverse=True),
|
||||
# Sort by last_report, handling None values
|
||||
domains_list.sort(key=lambda x: x['last_report'] or 0, reverse=True)
|
||||
|
||||
response = {
|
||||
'domains': domains_list,
|
||||
'total': len(domains_list)
|
||||
}
|
||||
|
||||
# Cache the result
|
||||
set_dmarc_cache(cache_key, response)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching domains list: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DOMAIN OVERVIEW
|
||||
# =============================================================================
|
||||
@@ -624,6 +747,297 @@ async def get_source_details(
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching source details: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
# =============================================================================
|
||||
# TLS-RPT REPORTS
|
||||
# =============================================================================
|
||||
|
||||
@router.get("/dmarc/domains/{domain}/tls-reports")
|
||||
async def get_domain_tls_reports(
|
||||
domain: str,
|
||||
days: int = 30,
|
||||
page: int = 1,
|
||||
limit: int = 50,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get TLS-RPT reports for a domain
|
||||
Returns list of TLS reports with organization, date range, success/fail counts
|
||||
"""
|
||||
try:
|
||||
cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
|
||||
# Query TLS reports for domain
|
||||
reports_query = db.query(TLSReport).filter(
|
||||
and_(
|
||||
TLSReport.policy_domain == domain,
|
||||
TLSReport.start_datetime >= cutoff_date
|
||||
)
|
||||
).order_by(TLSReport.start_datetime.desc())
|
||||
|
||||
total = reports_query.count()
|
||||
reports = reports_query.offset((page - 1) * limit).limit(limit).all()
|
||||
|
||||
# Get aggregated totals
|
||||
total_success = 0
|
||||
total_fail = 0
|
||||
organizations = set()
|
||||
|
||||
reports_list = []
|
||||
for report in reports:
|
||||
# Get policies for this report
|
||||
policies = db.query(TLSReportPolicy).filter(
|
||||
TLSReportPolicy.tls_report_id == report.id
|
||||
).all()
|
||||
|
||||
report_success = sum(p.successful_session_count for p in policies)
|
||||
report_fail = sum(p.failed_session_count for p in policies)
|
||||
|
||||
total_success += report_success
|
||||
total_fail += report_fail
|
||||
organizations.add(report.organization_name)
|
||||
|
||||
policies_list = [{
|
||||
'policy_type': p.policy_type,
|
||||
'policy_domain': p.policy_domain,
|
||||
'mx_host': p.mx_host,
|
||||
'successful_sessions': p.successful_session_count,
|
||||
'failed_sessions': p.failed_session_count,
|
||||
'failure_details': p.failure_details
|
||||
} for p in policies]
|
||||
|
||||
reports_list.append({
|
||||
'id': report.id,
|
||||
'report_id': report.report_id,
|
||||
'organization_name': report.organization_name,
|
||||
'contact_info': report.contact_info,
|
||||
'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None,
|
||||
'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None,
|
||||
'successful_sessions': report_success,
|
||||
'failed_sessions': report_fail,
|
||||
'total_sessions': report_success + report_fail,
|
||||
'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2),
|
||||
'policies': policies_list
|
||||
})
|
||||
|
||||
return {
|
||||
'domain': domain,
|
||||
'total': total,
|
||||
'page': page,
|
||||
'limit': limit,
|
||||
'pages': (total + limit - 1) // limit if total > 0 else 0,
|
||||
'totals': {
|
||||
'total_reports': total,
|
||||
'total_successful_sessions': total_success,
|
||||
'total_failed_sessions': total_fail,
|
||||
'overall_success_rate': round((total_success / (total_success + total_fail) * 100) if (total_success + total_fail) > 0 else 100, 2),
|
||||
'unique_organizations': len(organizations)
|
||||
},
|
||||
'data': reports_list
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching TLS reports for domain {domain}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/dmarc/domains/{domain}/tls-reports/daily")
|
||||
async def get_domain_tls_daily_reports(
|
||||
domain: str,
|
||||
days: int = 30,
|
||||
page: int = 1,
|
||||
limit: int = 30,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get TLS-RPT reports aggregated by date
|
||||
Groups multiple reports from same day together, like DMARC daily reports
|
||||
"""
|
||||
try:
|
||||
cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
|
||||
# Query all TLS reports for domain
|
||||
reports = db.query(TLSReport).filter(
|
||||
and_(
|
||||
TLSReport.policy_domain == domain,
|
||||
TLSReport.start_datetime >= cutoff_date
|
||||
)
|
||||
).order_by(TLSReport.start_datetime.desc()).all()
|
||||
|
||||
# Group by date
|
||||
daily_data = {}
|
||||
for report in reports:
|
||||
if report.start_datetime:
|
||||
date_key = report.start_datetime.strftime('%Y-%m-%d')
|
||||
if date_key not in daily_data:
|
||||
daily_data[date_key] = {
|
||||
'date': date_key,
|
||||
'report_count': 0,
|
||||
'organizations': set(),
|
||||
'total_success': 0,
|
||||
'total_fail': 0,
|
||||
'reports': []
|
||||
}
|
||||
|
||||
# Get policies for this report
|
||||
policies = db.query(TLSReportPolicy).filter(
|
||||
TLSReportPolicy.tls_report_id == report.id
|
||||
).all()
|
||||
|
||||
report_success = sum(p.successful_session_count for p in policies)
|
||||
report_fail = sum(p.failed_session_count for p in policies)
|
||||
|
||||
daily_data[date_key]['report_count'] += 1
|
||||
daily_data[date_key]['organizations'].add(report.organization_name)
|
||||
daily_data[date_key]['total_success'] += report_success
|
||||
daily_data[date_key]['total_fail'] += report_fail
|
||||
daily_data[date_key]['reports'].append({
|
||||
'id': report.id,
|
||||
'organization_name': report.organization_name,
|
||||
'successful_sessions': report_success,
|
||||
'failed_sessions': report_fail
|
||||
})
|
||||
|
||||
# Convert to list and add success rate
|
||||
daily_list = []
|
||||
for date_key in sorted(daily_data.keys(), reverse=True):
|
||||
data = daily_data[date_key]
|
||||
total = data['total_success'] + data['total_fail']
|
||||
success_rate = round((data['total_success'] / total * 100) if total > 0 else 100, 2)
|
||||
|
||||
daily_list.append({
|
||||
'date': date_key,
|
||||
'report_count': data['report_count'],
|
||||
'organizations': list(data['organizations']),
|
||||
'organization_count': len(data['organizations']),
|
||||
'total_success': data['total_success'],
|
||||
'total_fail': data['total_fail'],
|
||||
'total_sessions': total,
|
||||
'success_rate': success_rate,
|
||||
'reports': data['reports']
|
||||
})
|
||||
|
||||
# Pagination
|
||||
total = len(daily_list)
|
||||
start_idx = (page - 1) * limit
|
||||
end_idx = start_idx + limit
|
||||
paginated = daily_list[start_idx:end_idx]
|
||||
|
||||
# Calculate overall totals
|
||||
overall_success = sum(d['total_success'] for d in daily_list)
|
||||
overall_fail = sum(d['total_fail'] for d in daily_list)
|
||||
overall_total = overall_success + overall_fail
|
||||
|
||||
return {
|
||||
'domain': domain,
|
||||
'total': total,
|
||||
'page': page,
|
||||
'limit': limit,
|
||||
'pages': (total + limit - 1) // limit if total > 0 else 0,
|
||||
'totals': {
|
||||
'total_days': total,
|
||||
'total_reports': sum(d['report_count'] for d in daily_list),
|
||||
'total_successful_sessions': overall_success,
|
||||
'total_failed_sessions': overall_fail,
|
||||
'overall_success_rate': round((overall_success / overall_total * 100) if overall_total > 0 else 100, 2)
|
||||
},
|
||||
'data': paginated
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching TLS daily reports for domain {domain}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/dmarc/domains/{domain}/tls-reports/{report_date}/details")
|
||||
async def get_tls_report_details(
|
||||
domain: str,
|
||||
report_date: str,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get detailed TLS-RPT reports for a specific date
|
||||
Shows all reports and policies from that day with breakdown by provider
|
||||
"""
|
||||
try:
|
||||
date_obj = datetime.strptime(report_date, '%Y-%m-%d').date()
|
||||
start_dt = datetime.combine(date_obj, datetime.min.time(), tzinfo=timezone.utc)
|
||||
end_dt = datetime.combine(date_obj, datetime.max.time(), tzinfo=timezone.utc)
|
||||
|
||||
reports = db.query(TLSReport).filter(
|
||||
and_(
|
||||
TLSReport.policy_domain == domain,
|
||||
TLSReport.start_datetime >= start_dt,
|
||||
TLSReport.start_datetime <= end_dt
|
||||
)
|
||||
).all()
|
||||
|
||||
if not reports:
|
||||
raise HTTPException(status_code=404, detail="No TLS reports found for this date")
|
||||
|
||||
# Aggregate stats
|
||||
total_success = 0
|
||||
total_fail = 0
|
||||
providers = []
|
||||
|
||||
for report in reports:
|
||||
policies = db.query(TLSReportPolicy).filter(
|
||||
TLSReportPolicy.tls_report_id == report.id
|
||||
).all()
|
||||
|
||||
report_success = sum(p.successful_session_count for p in policies)
|
||||
report_fail = sum(p.failed_session_count for p in policies)
|
||||
|
||||
total_success += report_success
|
||||
total_fail += report_fail
|
||||
|
||||
# Add provider details
|
||||
policies_list = []
|
||||
for p in policies:
|
||||
policies_list.append({
|
||||
'policy_type': p.policy_type,
|
||||
'policy_domain': p.policy_domain,
|
||||
'mx_host': p.mx_host,
|
||||
'successful_sessions': p.successful_session_count,
|
||||
'failed_sessions': p.failed_session_count,
|
||||
'total_sessions': p.successful_session_count + p.failed_session_count,
|
||||
'success_rate': round((p.successful_session_count / (p.successful_session_count + p.failed_session_count) * 100) if (p.successful_session_count + p.failed_session_count) > 0 else 100, 2),
|
||||
'failure_details': p.failure_details
|
||||
})
|
||||
|
||||
providers.append({
|
||||
'report_id': report.report_id,
|
||||
'organization_name': report.organization_name,
|
||||
'contact_info': report.contact_info,
|
||||
'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None,
|
||||
'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None,
|
||||
'successful_sessions': report_success,
|
||||
'failed_sessions': report_fail,
|
||||
'total_sessions': report_success + report_fail,
|
||||
'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2),
|
||||
'policies': policies_list
|
||||
})
|
||||
|
||||
total = total_success + total_fail
|
||||
|
||||
return {
|
||||
'domain': domain,
|
||||
'date': report_date,
|
||||
'stats': {
|
||||
'total_reports': len(reports),
|
||||
'total_providers': len(set(r.organization_name for r in reports)),
|
||||
'total_success': total_success,
|
||||
'total_fail': total_fail,
|
||||
'total_sessions': total,
|
||||
'success_rate': round((total_success / total * 100) if total > 0 else 100, 2)
|
||||
},
|
||||
'providers': providers
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching TLS report details for {domain}/{report_date}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -796,27 +1210,54 @@ async def get_sync_history(
|
||||
logger.error(f"Error fetching sync history: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# UPLOAD
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/dmarc/upload")
|
||||
async def upload_dmarc_report(
|
||||
async def upload_report(
|
||||
file: UploadFile = File(...),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Upload and parse DMARC or TLS-RPT report file
|
||||
|
||||
Supported formats:
|
||||
- DMARC: .xml, .xml.gz, .zip (XML content)
|
||||
- TLS-RPT: .json, .json.gz
|
||||
"""
|
||||
if not settings.dmarc_manual_upload_enabled:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Manual DMARC report upload is disabled"
|
||||
detail="Manual report upload is disabled"
|
||||
)
|
||||
|
||||
"""Upload and parse DMARC report file (GZ or ZIP)"""
|
||||
try:
|
||||
file_content = await file.read()
|
||||
filename = file.filename.lower()
|
||||
|
||||
parsed_data = parse_dmarc_file(file_content, file.filename)
|
||||
# Detect file type based on extension
|
||||
# TLS-RPT files: .json, .json.gz, .json.zip
|
||||
is_tls_rpt = filename.endswith('.json') or filename.endswith('.json.gz') or filename.endswith('.json.zip')
|
||||
|
||||
if is_tls_rpt:
|
||||
# Process TLS-RPT report
|
||||
return await _upload_tls_rpt_report(file_content, file.filename, db)
|
||||
else:
|
||||
# Process DMARC report (default)
|
||||
return await _upload_dmarc_report(file_content, file.filename, db)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Error uploading report: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
async def _upload_dmarc_report(file_content: bytes, filename: str, db: Session):
|
||||
"""Handle DMARC report upload"""
|
||||
parsed_data = parse_dmarc_file(file_content, filename)
|
||||
|
||||
if not parsed_data:
|
||||
raise HTTPException(status_code=400, detail="Failed to parse DMARC report")
|
||||
@@ -831,6 +1272,7 @@ async def upload_dmarc_report(
|
||||
if existing:
|
||||
return {
|
||||
'status': 'duplicate',
|
||||
'report_type': 'dmarc',
|
||||
'message': f'Report {report_data["report_id"]} already exists'
|
||||
}
|
||||
|
||||
@@ -846,16 +1288,78 @@ async def upload_dmarc_report(
|
||||
|
||||
db.commit()
|
||||
|
||||
# Clear cache
|
||||
clear_dmarc_cache(db)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'message': f'Uploaded report for {report.domain} from {report.org_name}',
|
||||
'report_type': 'dmarc',
|
||||
'message': f'Uploaded DMARC report for {report.domain} from {report.org_name}',
|
||||
'report_id': report.id,
|
||||
'records_count': len(records_data)
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Error uploading DMARC report: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
async def _upload_tls_rpt_report(file_content: bytes, filename: str, db: Session):
|
||||
"""Handle TLS-RPT report upload"""
|
||||
from ..services.tls_rpt_parser import parse_tls_rpt_file
|
||||
|
||||
parsed_data = parse_tls_rpt_file(file_content, filename)
|
||||
|
||||
if not parsed_data:
|
||||
raise HTTPException(status_code=400, detail="Failed to parse TLS-RPT report")
|
||||
|
||||
# Extract policies
|
||||
policies_data = parsed_data.pop('policies', [])
|
||||
|
||||
# Check for duplicate
|
||||
existing = db.query(TLSReport).filter(
|
||||
TLSReport.report_id == parsed_data['report_id']
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
return {
|
||||
'status': 'duplicate',
|
||||
'report_type': 'tls-rpt',
|
||||
'message': f'TLS-RPT report {parsed_data["report_id"]} already exists'
|
||||
}
|
||||
|
||||
# Create TLS report
|
||||
tls_report = TLSReport(
|
||||
report_id=parsed_data['report_id'],
|
||||
organization_name=parsed_data.get('organization_name', 'Unknown'),
|
||||
contact_info=parsed_data.get('contact_info', ''),
|
||||
policy_domain=parsed_data['policy_domain'],
|
||||
start_datetime=parsed_data['start_datetime'],
|
||||
end_datetime=parsed_data['end_datetime'],
|
||||
raw_json=parsed_data.get('raw_json', '')
|
||||
)
|
||||
db.add(tls_report)
|
||||
db.flush()
|
||||
|
||||
# Create policy records
|
||||
for policy_data in policies_data:
|
||||
policy = TLSReportPolicy(
|
||||
tls_report_id=tls_report.id,
|
||||
policy_type=policy_data.get('policy_type', 'unknown'),
|
||||
policy_domain=policy_data.get('policy_domain', ''),
|
||||
policy_string=policy_data.get('policy_string', []),
|
||||
mx_host=policy_data.get('mx_host', []),
|
||||
successful_session_count=policy_data.get('successful_session_count', 0),
|
||||
failed_session_count=policy_data.get('failed_session_count', 0),
|
||||
failure_details=policy_data.get('failure_details', [])
|
||||
)
|
||||
db.add(policy)
|
||||
|
||||
db.commit()
|
||||
|
||||
# Clear cache
|
||||
clear_dmarc_cache(db)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'report_type': 'tls-rpt',
|
||||
'message': f'Uploaded TLS-RPT report for {tls_report.policy_domain} from {tls_report.organization_name}',
|
||||
'report_id': tls_report.id,
|
||||
'policies_count': len(policies_data)
|
||||
}
|
||||
@@ -404,88 +404,6 @@ async def count_spf_dns_lookups(domain: str, spf_record: str, resolver, visited_
|
||||
return lookup_count
|
||||
|
||||
|
||||
async def check_ip_in_spf(domain: str, ip_to_check: str, spf_record: str, resolver, visited_domains: set = None, depth: int = 0) -> tuple:
|
||||
"""
|
||||
Check if IP is authorized in SPF record recursively
|
||||
Returns: (authorized: bool, method: str or None)
|
||||
"""
|
||||
if depth > 10:
|
||||
return False, None
|
||||
|
||||
if visited_domains is None:
|
||||
visited_domains = set()
|
||||
|
||||
if domain in visited_domains:
|
||||
return False, None
|
||||
|
||||
visited_domains.add(domain)
|
||||
|
||||
parts = spf_record.split()
|
||||
|
||||
for part in parts:
|
||||
clean_part = part.lstrip('+-~?')
|
||||
|
||||
if clean_part.startswith('ip4:'):
|
||||
ip_spec = clean_part.replace('ip4:', '')
|
||||
try:
|
||||
if '/' in ip_spec:
|
||||
network = ipaddress.ip_network(ip_spec, strict=False)
|
||||
if ipaddress.ip_address(ip_to_check) in network:
|
||||
return True, f'ip4:{ip_spec}'
|
||||
else:
|
||||
if ip_to_check == ip_spec:
|
||||
return True, f'ip4:{ip_spec}'
|
||||
except:
|
||||
pass
|
||||
|
||||
elif clean_part in ['a'] or clean_part.startswith('a:'):
|
||||
check_domain = domain if clean_part == 'a' else clean_part.split(':', 1)[1]
|
||||
try:
|
||||
a_records = await resolver.resolve(check_domain, 'A')
|
||||
for rdata in a_records:
|
||||
if str(rdata) == ip_to_check:
|
||||
return True, f'a:{check_domain}' if clean_part.startswith('a:') else 'a'
|
||||
except:
|
||||
pass
|
||||
|
||||
elif clean_part in ['mx'] or clean_part.startswith('mx:'):
|
||||
check_domain = domain if clean_part == 'mx' else clean_part.split(':', 1)[1]
|
||||
try:
|
||||
mx_records = await resolver.resolve(check_domain, 'MX')
|
||||
for mx in mx_records:
|
||||
try:
|
||||
mx_a_records = await resolver.resolve(str(mx.exchange), 'A')
|
||||
for rdata in mx_a_records:
|
||||
if str(rdata) == ip_to_check:
|
||||
return True, f'mx:{check_domain}' if clean_part.startswith('mx:') else 'mx'
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
|
||||
elif clean_part.startswith('include:'):
|
||||
include_domain = clean_part.replace('include:', '')
|
||||
try:
|
||||
include_answers = await resolver.resolve(include_domain, 'TXT')
|
||||
for rdata in include_answers:
|
||||
include_spf = b''.join(rdata.strings).decode('utf-8')
|
||||
if include_spf.startswith('v=spf1'):
|
||||
authorized, method = await check_ip_in_spf(
|
||||
include_domain,
|
||||
ip_to_check,
|
||||
include_spf,
|
||||
resolver,
|
||||
visited_domains.copy(),
|
||||
depth + 1
|
||||
)
|
||||
if authorized:
|
||||
return True, f'include:{include_domain} ({method})'
|
||||
except:
|
||||
pass
|
||||
|
||||
return False, None
|
||||
|
||||
|
||||
def parse_dkim_parameters(dkim_record: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse and validate DKIM record parameters
|
||||
|
||||
553
backend/app/routers/mailbox_stats.py
Normal file
553
backend/app/routers/mailbox_stats.py
Normal file
@@ -0,0 +1,553 @@
|
||||
"""
|
||||
API endpoints for mailbox statistics with message counts
|
||||
Shows per-mailbox/per-alias message statistics from MessageCorrelation table
|
||||
"""
|
||||
import logging
|
||||
import hashlib
|
||||
import json
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, case, and_, or_
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import MailboxStatistics, AliasStatistics, MessageCorrelation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# =============================================================================
|
||||
# CACHING SYSTEM
|
||||
# =============================================================================
|
||||
|
||||
# In-memory cache for mailbox stats
|
||||
_stats_cache = {}
|
||||
_cache_ttl_seconds = 300 # 5 minutes cache TTL
|
||||
|
||||
|
||||
def _get_cache_key(prefix: str, **params) -> str:
|
||||
"""Generate a cache key from parameters"""
|
||||
param_str = json.dumps(params, sort_keys=True, default=str)
|
||||
hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16]
|
||||
return f"{prefix}:{hash_val}"
|
||||
|
||||
|
||||
def _get_cached(key: str):
|
||||
"""Get cached value if not expired"""
|
||||
if key in _stats_cache:
|
||||
cached_data, cached_time = _stats_cache[key]
|
||||
if datetime.now(timezone.utc) - cached_time < timedelta(seconds=_cache_ttl_seconds):
|
||||
logger.debug(f"Cache hit for key: {key}")
|
||||
return cached_data
|
||||
else:
|
||||
# Cache expired, remove it
|
||||
del _stats_cache[key]
|
||||
return None
|
||||
|
||||
|
||||
def _set_cache(key: str, data):
|
||||
"""Set cached value with current timestamp"""
|
||||
_stats_cache[key] = (data, datetime.now(timezone.utc))
|
||||
logger.debug(f"Cache set for key: {key}")
|
||||
|
||||
|
||||
def clear_stats_cache():
|
||||
"""Clear all stats cache - call after data changes"""
|
||||
global _stats_cache
|
||||
_stats_cache = {}
|
||||
logger.info("Stats cache cleared")
|
||||
|
||||
|
||||
def format_datetime_utc(dt: Optional[datetime]) -> Optional[str]:
|
||||
"""Format datetime for API response with proper UTC timezone"""
|
||||
if dt is None:
|
||||
return None
|
||||
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
|
||||
dt_utc = dt.astimezone(timezone.utc)
|
||||
return dt_utc.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def format_bytes(bytes_value) -> str:
|
||||
"""Format bytes into human-readable format"""
|
||||
if bytes_value is None:
|
||||
return "0 B"
|
||||
|
||||
bytes_value = float(bytes_value)
|
||||
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||
if abs(bytes_value) < 1024.0:
|
||||
return f"{bytes_value:.1f} {unit}"
|
||||
bytes_value /= 1024.0
|
||||
return f"{bytes_value:.1f} PB"
|
||||
|
||||
|
||||
def format_unix_timestamp(timestamp: Optional[int]) -> Optional[str]:
|
||||
"""Convert Unix timestamp to ISO format string"""
|
||||
if timestamp is None or timestamp == 0:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
|
||||
return dt.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_date_range(date_range: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> tuple[datetime, datetime]:
|
||||
"""Parse date range string into start and end datetimes
|
||||
|
||||
For custom date ranges, start_date and end_date should be ISO format strings (YYYY-MM-DD)
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if date_range == "custom" and start_date and end_date:
|
||||
# Parse custom date range
|
||||
try:
|
||||
start = datetime.fromisoformat(start_date.replace('Z', '+00:00'))
|
||||
if start.tzinfo is None:
|
||||
start = start.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc)
|
||||
|
||||
end = datetime.fromisoformat(end_date.replace('Z', '+00:00'))
|
||||
if end.tzinfo is None:
|
||||
# Set end time to end of day
|
||||
end = end.replace(hour=23, minute=59, second=59, microsecond=999999, tzinfo=timezone.utc)
|
||||
|
||||
return start, end
|
||||
except ValueError:
|
||||
# If parsing fails, fall back to 30 days
|
||||
logger.warning(f"Failed to parse custom date range: {start_date} - {end_date}, falling back to 30 days")
|
||||
start = now - timedelta(days=30)
|
||||
end = now
|
||||
elif date_range == "today":
|
||||
start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
end = now
|
||||
elif date_range == "7days":
|
||||
start = now - timedelta(days=7)
|
||||
end = now
|
||||
elif date_range == "30days":
|
||||
start = now - timedelta(days=30)
|
||||
end = now
|
||||
elif date_range == "90days":
|
||||
start = now - timedelta(days=90)
|
||||
end = now
|
||||
else:
|
||||
# Default to 30 days
|
||||
start = now - timedelta(days=30)
|
||||
end = now
|
||||
|
||||
return start, end
|
||||
|
||||
|
||||
def get_message_counts_for_email(db: Session, email: str, start_date: datetime, end_date: datetime) -> dict:
|
||||
"""
|
||||
Get message counts for a specific email address (mailbox or alias)
|
||||
Counts sent (as sender) and received (as recipient) with status breakdown
|
||||
Also counts by direction (inbound, outbound, internal)
|
||||
"""
|
||||
# Count sent messages (this email as sender) by status
|
||||
sent_query = db.query(
|
||||
MessageCorrelation.final_status,
|
||||
func.count(MessageCorrelation.id).label('count')
|
||||
).filter(
|
||||
MessageCorrelation.sender == email,
|
||||
MessageCorrelation.first_seen >= start_date,
|
||||
MessageCorrelation.first_seen <= end_date
|
||||
).group_by(MessageCorrelation.final_status).all()
|
||||
|
||||
# Count received messages (this email as recipient) by status
|
||||
received_query = db.query(
|
||||
MessageCorrelation.final_status,
|
||||
func.count(MessageCorrelation.id).label('count')
|
||||
).filter(
|
||||
MessageCorrelation.recipient == email,
|
||||
MessageCorrelation.first_seen >= start_date,
|
||||
MessageCorrelation.first_seen <= end_date
|
||||
).group_by(MessageCorrelation.final_status).all()
|
||||
|
||||
# Count by direction (inbound, outbound, internal)
|
||||
direction_query = db.query(
|
||||
MessageCorrelation.direction,
|
||||
func.count(MessageCorrelation.id).label('count')
|
||||
).filter(
|
||||
or_(
|
||||
MessageCorrelation.sender == email,
|
||||
MessageCorrelation.recipient == email
|
||||
),
|
||||
MessageCorrelation.first_seen >= start_date,
|
||||
MessageCorrelation.first_seen <= end_date
|
||||
).group_by(MessageCorrelation.direction).all()
|
||||
|
||||
# Process results
|
||||
sent_by_status = {row.final_status or 'unknown': row.count for row in sent_query}
|
||||
received_by_status = {row.final_status or 'unknown': row.count for row in received_query}
|
||||
direction_counts = {row.direction or 'unknown': row.count for row in direction_query}
|
||||
|
||||
sent_total = sum(sent_by_status.values())
|
||||
received_total = sum(received_by_status.values())
|
||||
|
||||
# Calculate failures (only for sent messages - bounces and rejections are outbound failures)
|
||||
sent_failed = sent_by_status.get('bounced', 0) + sent_by_status.get('rejected', 0)
|
||||
|
||||
total = sent_total + received_total
|
||||
failure_rate = round((sent_failed / sent_total * 100) if sent_total > 0 else 0, 1)
|
||||
|
||||
return {
|
||||
"sent_total": sent_total,
|
||||
"sent_delivered": sent_by_status.get('delivered', 0) + sent_by_status.get('sent', 0),
|
||||
"sent_bounced": sent_by_status.get('bounced', 0),
|
||||
"sent_rejected": sent_by_status.get('rejected', 0),
|
||||
"sent_deferred": sent_by_status.get('deferred', 0),
|
||||
"sent_expired": sent_by_status.get('expired', 0),
|
||||
"sent_failed": sent_failed,
|
||||
"received_total": received_total,
|
||||
"total_messages": total,
|
||||
"failure_rate": failure_rate,
|
||||
# Direction counts
|
||||
"direction_inbound": direction_counts.get('inbound', 0),
|
||||
"direction_outbound": direction_counts.get('outbound', 0),
|
||||
"direction_internal": direction_counts.get('internal', 0)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/mailbox-stats/summary")
|
||||
async def get_mailbox_stats_summary(
|
||||
date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"),
|
||||
start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"),
|
||||
end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get summary statistics for all mailboxes
|
||||
"""
|
||||
try:
|
||||
parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date)
|
||||
|
||||
# Total mailboxes
|
||||
total_mailboxes = db.query(func.count(MailboxStatistics.id)).scalar() or 0
|
||||
active_mailboxes = db.query(func.count(MailboxStatistics.id)).filter(
|
||||
MailboxStatistics.active == True
|
||||
).scalar() or 0
|
||||
|
||||
# Total aliases
|
||||
total_aliases = db.query(func.count(AliasStatistics.id)).scalar() or 0
|
||||
active_aliases = db.query(func.count(AliasStatistics.id)).filter(
|
||||
AliasStatistics.active == True
|
||||
).scalar() or 0
|
||||
|
||||
# Unique domains
|
||||
unique_domains = db.query(func.count(func.distinct(MailboxStatistics.domain))).scalar() or 0
|
||||
|
||||
# Get total storage used
|
||||
total_quota_used = db.query(func.sum(MailboxStatistics.quota_used)).scalar() or 0
|
||||
|
||||
# Get last update time
|
||||
last_update = db.query(func.max(MailboxStatistics.updated_at)).scalar()
|
||||
|
||||
# Get all local mailbox emails and alias emails
|
||||
mailbox_emails = [m.username for m in db.query(MailboxStatistics.username).all()]
|
||||
alias_emails = [a.alias_address for a in db.query(AliasStatistics.alias_address).all()]
|
||||
all_local_emails = set(mailbox_emails + alias_emails)
|
||||
|
||||
# Count total messages for all local emails
|
||||
total_sent = 0
|
||||
total_received = 0
|
||||
total_failed = 0
|
||||
|
||||
if all_local_emails:
|
||||
# Sent messages
|
||||
sent_result = db.query(func.count(MessageCorrelation.id)).filter(
|
||||
MessageCorrelation.sender.in_(all_local_emails),
|
||||
MessageCorrelation.first_seen >= parsed_start,
|
||||
MessageCorrelation.first_seen <= parsed_end
|
||||
).scalar() or 0
|
||||
total_sent = sent_result
|
||||
|
||||
# Received messages
|
||||
received_result = db.query(func.count(MessageCorrelation.id)).filter(
|
||||
MessageCorrelation.recipient.in_(all_local_emails),
|
||||
MessageCorrelation.first_seen >= parsed_start,
|
||||
MessageCorrelation.first_seen <= parsed_end
|
||||
).scalar() or 0
|
||||
total_received = received_result
|
||||
|
||||
# Failed messages (only sent that bounced/rejected - failures are outbound)
|
||||
failed_result = db.query(func.count(MessageCorrelation.id)).filter(
|
||||
MessageCorrelation.sender.in_(all_local_emails),
|
||||
MessageCorrelation.first_seen >= parsed_start,
|
||||
MessageCorrelation.first_seen <= parsed_end,
|
||||
MessageCorrelation.final_status.in_(['bounced', 'rejected'])
|
||||
).scalar() or 0
|
||||
total_sent_failed = failed_result
|
||||
|
||||
total_messages = total_sent + total_received
|
||||
failure_rate = round((total_sent_failed / total_sent * 100) if total_sent > 0 else 0, 1)
|
||||
|
||||
return {
|
||||
"total_sent": total_sent,
|
||||
"total_received": total_received,
|
||||
"total_messages": total_messages,
|
||||
"sent_failed": total_sent_failed,
|
||||
"failure_rate": failure_rate,
|
||||
"date_range": date_range,
|
||||
"start_date": format_datetime_utc(parsed_start),
|
||||
"end_date": format_datetime_utc(parsed_end),
|
||||
"last_update": format_datetime_utc(last_update)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching mailbox stats summary: {e}")
|
||||
return {"error": str(e), "total_mailboxes": 0}
|
||||
|
||||
|
||||
@router.get("/mailbox-stats/all")
|
||||
async def get_all_mailbox_stats(
|
||||
domain: Optional[str] = None,
|
||||
active_only: bool = True, # Changed default to True
|
||||
hide_zero: bool = False, # Filter out mailboxes with zero activity
|
||||
search: Optional[str] = None,
|
||||
date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"),
|
||||
start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"),
|
||||
end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"),
|
||||
sort_by: str = "sent_total",
|
||||
sort_order: str = "desc",
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
page_size: int = Query(50, ge=10, le=100, description="Items per page"),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get all mailbox statistics with message counts and aliases (paginated)
|
||||
"""
|
||||
try:
|
||||
# Check cache first
|
||||
cache_key = _get_cache_key(
|
||||
"mailbox_stats_all",
|
||||
domain=domain,
|
||||
active_only=active_only,
|
||||
hide_zero=hide_zero,
|
||||
search=search,
|
||||
date_range=date_range,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
page=page,
|
||||
page_size=page_size
|
||||
)
|
||||
|
||||
cached_result = _get_cached(cache_key)
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
|
||||
parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date)
|
||||
|
||||
query = db.query(MailboxStatistics)
|
||||
|
||||
# Apply domain filter
|
||||
if domain:
|
||||
query = query.filter(MailboxStatistics.domain == domain)
|
||||
|
||||
# Apply active filter
|
||||
if active_only:
|
||||
query = query.filter(MailboxStatistics.active == True)
|
||||
|
||||
# Apply search filter on mailbox username/name OR mailboxes that have matching aliases
|
||||
if search:
|
||||
mailbox_search_term = f"%{search}%"
|
||||
|
||||
# Find mailboxes that have matching aliases
|
||||
alias_matched_usernames = db.query(AliasStatistics.primary_mailbox).filter(
|
||||
AliasStatistics.alias_address.ilike(mailbox_search_term)
|
||||
).distinct().subquery()
|
||||
|
||||
query = query.filter(
|
||||
or_(
|
||||
MailboxStatistics.username.ilike(mailbox_search_term),
|
||||
MailboxStatistics.name.ilike(mailbox_search_term),
|
||||
MailboxStatistics.username.in_(alias_matched_usernames)
|
||||
)
|
||||
)
|
||||
|
||||
# Get total count before pagination
|
||||
total_count = query.count()
|
||||
|
||||
# Get all for sorting (we need to calculate counts before pagination)
|
||||
mailboxes = query.all()
|
||||
|
||||
# Build result with message counts for each mailbox
|
||||
result = []
|
||||
for mb in mailboxes:
|
||||
# Get message counts for this mailbox
|
||||
counts = get_message_counts_for_email(db, mb.username, parsed_start, parsed_end)
|
||||
|
||||
# Get aliases for this mailbox
|
||||
aliases = db.query(AliasStatistics).filter(
|
||||
AliasStatistics.primary_mailbox == mb.username
|
||||
).all()
|
||||
|
||||
# Get message counts for each alias
|
||||
alias_list = []
|
||||
alias_sent_total = 0
|
||||
alias_received_total = 0
|
||||
alias_failed_total = 0
|
||||
alias_internal_total = 0
|
||||
alias_delivered_total = 0
|
||||
|
||||
for alias in aliases:
|
||||
alias_counts = get_message_counts_for_email(db, alias.alias_address, parsed_start, parsed_end)
|
||||
alias_sent_total += alias_counts['sent_total']
|
||||
alias_received_total += alias_counts['received_total']
|
||||
alias_failed_total += alias_counts['sent_failed']
|
||||
alias_internal_total += alias_counts['direction_internal']
|
||||
alias_delivered_total += alias_counts['sent_delivered']
|
||||
|
||||
alias_list.append({
|
||||
"alias_address": alias.alias_address,
|
||||
"active": alias.active,
|
||||
"is_catch_all": alias.is_catch_all,
|
||||
**alias_counts
|
||||
})
|
||||
|
||||
# Calculate combined totals (mailbox + all aliases)
|
||||
combined_sent = counts['sent_total'] + alias_sent_total
|
||||
combined_received = counts['received_total'] + alias_received_total
|
||||
combined_total = combined_sent + combined_received
|
||||
combined_failed = counts['sent_failed'] + alias_failed_total
|
||||
combined_failure_rate = round((combined_failed / combined_sent * 100) if combined_sent > 0 else 0, 1)
|
||||
combined_internal = counts['direction_internal'] + alias_internal_total
|
||||
combined_delivered = counts['sent_delivered'] + alias_delivered_total
|
||||
combined_inbound = counts['direction_inbound']
|
||||
combined_outbound = counts['direction_outbound']
|
||||
|
||||
result.append({
|
||||
"id": mb.id,
|
||||
"username": mb.username,
|
||||
"domain": mb.domain,
|
||||
"name": mb.name,
|
||||
"active": mb.active,
|
||||
# Quota info
|
||||
"quota": float(mb.quota or 0),
|
||||
"quota_formatted": format_bytes(mb.quota),
|
||||
"quota_used": float(mb.quota_used or 0),
|
||||
"quota_used_formatted": format_bytes(mb.quota_used),
|
||||
"percent_in_use": round(float(mb.percent_in_use or 0), 1),
|
||||
"messages_in_mailbox": mb.messages or 0,
|
||||
# Last login times
|
||||
"last_imap_login": format_unix_timestamp(mb.last_imap_login),
|
||||
"last_pop3_login": format_unix_timestamp(mb.last_pop3_login),
|
||||
"last_smtp_login": format_unix_timestamp(mb.last_smtp_login),
|
||||
# Rate limiting
|
||||
"rl_value": mb.rl_value,
|
||||
"rl_frame": mb.rl_frame,
|
||||
# Attributes (access permissions)
|
||||
"attributes": mb.attributes or {},
|
||||
# Message counts for mailbox only
|
||||
"mailbox_counts": counts,
|
||||
# Aliases
|
||||
"aliases": alias_list,
|
||||
"alias_count": len(alias_list),
|
||||
# Combined totals (mailbox + aliases)
|
||||
"combined_sent": combined_sent,
|
||||
"combined_received": combined_received,
|
||||
"combined_total": combined_total,
|
||||
"combined_failed": combined_failed,
|
||||
"combined_failure_rate": combined_failure_rate,
|
||||
# Direction and status combined counts
|
||||
"combined_internal": combined_internal,
|
||||
"combined_delivered": combined_delivered,
|
||||
"combined_inbound": combined_inbound,
|
||||
"combined_outbound": combined_outbound,
|
||||
# Metadata
|
||||
"created": format_datetime_utc(mb.created_at),
|
||||
"modified": format_datetime_utc(mb.updated_at)
|
||||
})
|
||||
|
||||
# Sort results
|
||||
reverse = sort_order.lower() == "desc"
|
||||
if sort_by == "sent_total":
|
||||
result.sort(key=lambda x: x['combined_sent'], reverse=reverse)
|
||||
elif sort_by == "received_total":
|
||||
result.sort(key=lambda x: x['combined_received'], reverse=reverse)
|
||||
elif sort_by == "failure_rate":
|
||||
result.sort(key=lambda x: x['combined_failure_rate'], reverse=reverse)
|
||||
elif sort_by == "username":
|
||||
result.sort(key=lambda x: x['username'].lower(), reverse=reverse)
|
||||
elif sort_by == "quota_used":
|
||||
result.sort(key=lambda x: x['quota_used'], reverse=reverse)
|
||||
else:
|
||||
result.sort(key=lambda x: x['combined_total'], reverse=reverse)
|
||||
|
||||
# Apply hide_zero filter - remove mailboxes with no activity
|
||||
if hide_zero:
|
||||
result = [r for r in result if r['combined_total'] > 0]
|
||||
|
||||
# Apply pagination
|
||||
total_pages = (len(result) + page_size - 1) // page_size
|
||||
start_index = (page - 1) * page_size
|
||||
end_index = start_index + page_size
|
||||
paginated_result = result[start_index:end_index]
|
||||
|
||||
response = {
|
||||
"total": len(result),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"total_pages": total_pages,
|
||||
"date_range": date_range,
|
||||
"start_date": format_datetime_utc(parsed_start),
|
||||
"end_date": format_datetime_utc(parsed_end),
|
||||
"mailboxes": paginated_result
|
||||
}
|
||||
|
||||
# Cache the result
|
||||
_set_cache(cache_key, response)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching all mailbox stats: {e}")
|
||||
return {"error": str(e), "total": 0, "mailboxes": []}
|
||||
|
||||
|
||||
@router.get("/mailbox-stats/domains")
|
||||
async def get_mailbox_domains(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get list of all domains for filtering
|
||||
"""
|
||||
try:
|
||||
domains = db.query(
|
||||
MailboxStatistics.domain,
|
||||
func.count(MailboxStatistics.id).label('count')
|
||||
).group_by(
|
||||
MailboxStatistics.domain
|
||||
).order_by(
|
||||
MailboxStatistics.domain
|
||||
).all()
|
||||
|
||||
return {
|
||||
"domains": [
|
||||
{"domain": d.domain, "mailbox_count": d.count}
|
||||
for d in domains
|
||||
]
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching mailbox domains: {e}")
|
||||
return {"error": str(e), "domains": []}
|
||||
|
||||
|
||||
@router.get("/mailbox-stats/refresh")
|
||||
async def refresh_mailbox_stats(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get last update time for mailbox statistics
|
||||
"""
|
||||
try:
|
||||
last_mailbox_update = db.query(func.max(MailboxStatistics.updated_at)).scalar()
|
||||
last_alias_update = db.query(func.max(AliasStatistics.updated_at)).scalar()
|
||||
|
||||
return {
|
||||
"last_mailbox_update": format_datetime_utc(last_mailbox_update),
|
||||
"last_alias_update": format_datetime_utc(last_alias_update)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting refresh info: {e}")
|
||||
return {"error": str(e)}
|
||||
@@ -224,6 +224,20 @@ async def get_settings_info(db: Session = Depends(get_db)):
|
||||
"status": jobs_status.get('update_geoip', {}).get('status', 'idle') if is_license_configured() else 'disabled',
|
||||
"last_run": format_datetime_utc(jobs_status.get('update_geoip', {}).get('last_run')) if is_license_configured() else None,
|
||||
"error": jobs_status.get('update_geoip', {}).get('error') if is_license_configured() else None
|
||||
},
|
||||
"mailbox_stats": {
|
||||
"interval": "5 minutes",
|
||||
"description": "Fetches mailbox statistics from Mailcow API",
|
||||
"status": jobs_status.get('mailbox_stats', {}).get('status', 'unknown'),
|
||||
"last_run": format_datetime_utc(jobs_status.get('mailbox_stats', {}).get('last_run')),
|
||||
"error": jobs_status.get('mailbox_stats', {}).get('error')
|
||||
},
|
||||
"alias_stats": {
|
||||
"interval": "5 minutes",
|
||||
"description": "Syncs alias data from Mailcow API",
|
||||
"status": jobs_status.get('alias_stats', {}).get('status', 'unknown'),
|
||||
"last_run": format_datetime_utc(jobs_status.get('alias_stats', {}).get('last_run')),
|
||||
"error": jobs_status.get('alias_stats', {}).get('error')
|
||||
}
|
||||
},
|
||||
"smtp_configuration": {
|
||||
|
||||
@@ -18,7 +18,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from .config import settings, set_cached_active_domains
|
||||
from .database import get_db_context, SessionLocal
|
||||
from .mailcow_api import mailcow_api
|
||||
from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck
|
||||
from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck, MailboxStatistics, AliasStatistics
|
||||
from .correlation import detect_direction, parse_postfix_message
|
||||
from .routers.domains import check_domain_dns, save_dns_check_to_db
|
||||
from .services.dmarc_imap_service import sync_dmarc_reports_from_imap
|
||||
@@ -44,7 +44,9 @@ job_status = {
|
||||
'check_app_version': {'last_run': None, 'status': 'idle', 'error': None},
|
||||
'dns_check': {'last_run': None, 'status': 'idle', 'error': None},
|
||||
'update_geoip': {'last_run': None, 'status': 'idle', 'error': None},
|
||||
'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None}
|
||||
'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None},
|
||||
'mailbox_stats': {'last_run': None, 'status': 'idle', 'error': None},
|
||||
'alias_stats': {'last_run': None, 'status': 'idle', 'error': None}
|
||||
}
|
||||
|
||||
def update_job_status(job_name: str, status: str, error: str = None):
|
||||
@@ -1404,6 +1406,266 @@ async def sync_local_domains():
|
||||
update_job_status('sync_local_domains', 'failed', str(e))
|
||||
return False
|
||||
|
||||
# =============================================================================
|
||||
# MAILBOX STATISTICS
|
||||
# =============================================================================
|
||||
|
||||
def safe_int(value, default=0):
|
||||
"""Safely convert a value to int, handling '- ', None, and other invalid values"""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if value in ('', '-', '- '):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
def safe_float(value, default=0.0):
|
||||
"""Safely convert a value to float, handling '- ', None, and other invalid values"""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if value in ('', '-', '- '):
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
async def update_mailbox_statistics():
|
||||
"""
|
||||
Fetch mailbox statistics from Mailcow API and update the database.
|
||||
Runs every 5 minutes.
|
||||
Also removes mailboxes that no longer exist in Mailcow.
|
||||
"""
|
||||
update_job_status('mailbox_stats', 'running')
|
||||
logger.info("Starting mailbox statistics update...")
|
||||
|
||||
try:
|
||||
# Fetch mailboxes from Mailcow API
|
||||
mailboxes = await mailcow_api.get_mailboxes()
|
||||
|
||||
if not mailboxes:
|
||||
logger.warning("No mailboxes retrieved from Mailcow API")
|
||||
update_job_status('mailbox_stats', 'success')
|
||||
return
|
||||
|
||||
# Get set of current mailbox usernames from API
|
||||
api_mailbox_usernames = {mb.get('username') for mb in mailboxes if mb.get('username')}
|
||||
|
||||
with get_db_context() as db:
|
||||
updated = 0
|
||||
created = 0
|
||||
deleted = 0
|
||||
|
||||
# First, mark mailboxes that no longer exist in Mailcow as inactive
|
||||
db_mailboxes = db.query(MailboxStatistics).all()
|
||||
for db_mb in db_mailboxes:
|
||||
if db_mb.username not in api_mailbox_usernames:
|
||||
if db_mb.active: # Only log and count if it was previously active
|
||||
logger.info(f"Marking deleted mailbox as inactive: {db_mb.username}")
|
||||
db_mb.active = False
|
||||
db_mb.updated_at = datetime.now(timezone.utc)
|
||||
deleted += 1
|
||||
|
||||
for mb in mailboxes:
|
||||
try:
|
||||
username = mb.get('username')
|
||||
if not username:
|
||||
continue
|
||||
|
||||
# Extract domain from username
|
||||
domain = username.split('@')[-1] if '@' in username else ''
|
||||
|
||||
# Check if mailbox exists
|
||||
existing = db.query(MailboxStatistics).filter(
|
||||
MailboxStatistics.username == username
|
||||
).first()
|
||||
|
||||
# Prepare data - safely convert values
|
||||
attributes = mb.get('attributes', {})
|
||||
rl_value_raw = mb.get('rl_value')
|
||||
rl_value = safe_int(rl_value_raw) if rl_value_raw not in (None, '', '-', '- ') else None
|
||||
rl_frame = mb.get('rl_frame')
|
||||
if rl_frame in ('', '-', '- '):
|
||||
rl_frame = None
|
||||
|
||||
if existing:
|
||||
# Update existing record
|
||||
existing.domain = domain
|
||||
existing.name = mb.get('name', '') or ''
|
||||
existing.quota = safe_int(mb.get('quota'), 0)
|
||||
existing.quota_used = safe_int(mb.get('quota_used'), 0)
|
||||
existing.percent_in_use = safe_float(mb.get('percent_in_use'), 0.0)
|
||||
existing.messages = safe_int(mb.get('messages'), 0)
|
||||
existing.active = mb.get('active', 1) == 1
|
||||
existing.last_imap_login = safe_int(mb.get('last_imap_login'), 0) or None
|
||||
existing.last_pop3_login = safe_int(mb.get('last_pop3_login'), 0) or None
|
||||
existing.last_smtp_login = safe_int(mb.get('last_smtp_login'), 0) or None
|
||||
existing.spam_aliases = safe_int(mb.get('spam_aliases'), 0)
|
||||
existing.rl_value = rl_value
|
||||
existing.rl_frame = rl_frame
|
||||
existing.attributes = attributes
|
||||
existing.updated_at = datetime.now(timezone.utc)
|
||||
updated += 1
|
||||
else:
|
||||
# Create new record
|
||||
new_mailbox = MailboxStatistics(
|
||||
username=username,
|
||||
domain=domain,
|
||||
name=mb.get('name', '') or '',
|
||||
quota=safe_int(mb.get('quota'), 0),
|
||||
quota_used=safe_int(mb.get('quota_used'), 0),
|
||||
percent_in_use=safe_float(mb.get('percent_in_use'), 0.0),
|
||||
messages=safe_int(mb.get('messages'), 0),
|
||||
active=mb.get('active', 1) == 1,
|
||||
last_imap_login=safe_int(mb.get('last_imap_login'), 0) or None,
|
||||
last_pop3_login=safe_int(mb.get('last_pop3_login'), 0) or None,
|
||||
last_smtp_login=safe_int(mb.get('last_smtp_login'), 0) or None,
|
||||
spam_aliases=safe_int(mb.get('spam_aliases'), 0),
|
||||
rl_value=rl_value,
|
||||
rl_frame=rl_frame,
|
||||
attributes=attributes
|
||||
)
|
||||
db.add(new_mailbox)
|
||||
created += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing mailbox {mb.get('username', 'unknown')}: {e}")
|
||||
continue
|
||||
|
||||
db.commit()
|
||||
logger.info(f"✓ Mailbox statistics updated: {updated} updated, {created} created, {deleted} deactivated")
|
||||
update_job_status('mailbox_stats', 'success')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Failed to update mailbox statistics: {e}")
|
||||
update_job_status('mailbox_stats', 'failed', str(e))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ALIAS STATISTICS
|
||||
# =============================================================================
|
||||
|
||||
async def update_alias_statistics():
|
||||
"""
|
||||
Fetch aliases from Mailcow API and update the database.
|
||||
Links aliases to their target mailboxes.
|
||||
Runs every 5 minutes.
|
||||
Also removes aliases that no longer exist in Mailcow.
|
||||
"""
|
||||
update_job_status('alias_stats', 'running')
|
||||
logger.info("Starting alias statistics update...")
|
||||
|
||||
try:
|
||||
# Fetch aliases from Mailcow API
|
||||
aliases = await mailcow_api.get_aliases()
|
||||
|
||||
if not aliases:
|
||||
logger.warning("No aliases retrieved from Mailcow API")
|
||||
update_job_status('alias_stats', 'success')
|
||||
return
|
||||
|
||||
# Get set of current alias addresses from API
|
||||
api_alias_addresses = {alias.get('address') for alias in aliases if alias.get('address')}
|
||||
|
||||
with get_db_context() as db:
|
||||
updated = 0
|
||||
created = 0
|
||||
deleted = 0
|
||||
|
||||
# First, mark aliases that no longer exist in Mailcow as inactive
|
||||
db_aliases = db.query(AliasStatistics).all()
|
||||
for db_alias in db_aliases:
|
||||
if db_alias.alias_address not in api_alias_addresses:
|
||||
if db_alias.active: # Only log and count if it was previously active
|
||||
logger.info(f"Marking deleted alias as inactive: {db_alias.alias_address}")
|
||||
db_alias.active = False
|
||||
db_alias.updated_at = datetime.now(timezone.utc)
|
||||
deleted += 1
|
||||
|
||||
for alias in aliases:
|
||||
try:
|
||||
alias_address = alias.get('address')
|
||||
if not alias_address:
|
||||
continue
|
||||
|
||||
# Skip if this is a mailbox address (not an alias)
|
||||
if alias.get('is_catch_all') is None and not alias.get('goto'):
|
||||
continue
|
||||
|
||||
# Extract domain from alias address
|
||||
domain = alias_address.split('@')[-1] if '@' in alias_address else ''
|
||||
|
||||
# Get the target mailbox(es)
|
||||
goto = alias.get('goto', '')
|
||||
|
||||
# Determine primary mailbox (first in goto list)
|
||||
primary_mailbox = None
|
||||
if goto:
|
||||
goto_list = [g.strip() for g in goto.split(',') if g.strip()]
|
||||
if goto_list:
|
||||
primary_mailbox = goto_list[0]
|
||||
|
||||
# Check if alias exists
|
||||
existing = db.query(AliasStatistics).filter(
|
||||
AliasStatistics.alias_address == alias_address
|
||||
).first()
|
||||
|
||||
is_catch_all = alias.get('is_catch_all', 0) == 1
|
||||
is_active = alias.get('active', 1) == 1
|
||||
|
||||
if existing:
|
||||
# Update existing record
|
||||
existing.goto = goto
|
||||
existing.domain = domain
|
||||
existing.active = is_active
|
||||
existing.is_catch_all = is_catch_all
|
||||
existing.primary_mailbox = primary_mailbox
|
||||
existing.updated_at = datetime.now(timezone.utc)
|
||||
updated += 1
|
||||
else:
|
||||
# Create new record
|
||||
new_alias = AliasStatistics(
|
||||
alias_address=alias_address,
|
||||
goto=goto,
|
||||
domain=domain,
|
||||
active=is_active,
|
||||
is_catch_all=is_catch_all,
|
||||
primary_mailbox=primary_mailbox
|
||||
)
|
||||
db.add(new_alias)
|
||||
created += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing alias {alias.get('address', 'unknown')}: {e}")
|
||||
continue
|
||||
|
||||
db.commit()
|
||||
logger.info(f"✓ Alias statistics updated: {updated} updated, {created} created, {deleted} deactivated")
|
||||
update_job_status('alias_stats', 'success')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Failed to update alias statistics: {e}")
|
||||
update_job_status('alias_stats', 'failed', str(e))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SCHEDULER SETUP
|
||||
# =============================================================================
|
||||
@@ -1562,6 +1824,46 @@ def start_scheduler():
|
||||
)
|
||||
logger.info("Scheduled initial DMARC IMAP sync on startup")
|
||||
|
||||
# Job 13: Mailbox Statistics (every 5 minutes)
|
||||
scheduler.add_job(
|
||||
update_mailbox_statistics,
|
||||
IntervalTrigger(minutes=5),
|
||||
id='mailbox_stats',
|
||||
name='Update Mailbox Statistics',
|
||||
replace_existing=True,
|
||||
max_instances=1
|
||||
)
|
||||
|
||||
# Run once on startup (after 45 seconds)
|
||||
scheduler.add_job(
|
||||
update_mailbox_statistics,
|
||||
'date',
|
||||
run_date=datetime.now(timezone.utc) + timedelta(seconds=45),
|
||||
id='mailbox_stats_startup',
|
||||
name='Mailbox Statistics (Startup)'
|
||||
)
|
||||
logger.info("Scheduled mailbox statistics job (interval: 5 minutes)")
|
||||
|
||||
# Job 14: Alias Statistics (every 5 minutes)
|
||||
scheduler.add_job(
|
||||
update_alias_statistics,
|
||||
IntervalTrigger(minutes=5),
|
||||
id='alias_stats',
|
||||
name='Update Alias Statistics',
|
||||
replace_existing=True,
|
||||
max_instances=1
|
||||
)
|
||||
|
||||
# Run once on startup (after 50 seconds)
|
||||
scheduler.add_job(
|
||||
update_alias_statistics,
|
||||
'date',
|
||||
run_date=datetime.now(timezone.utc) + timedelta(seconds=50),
|
||||
id='alias_stats_startup',
|
||||
name='Alias Statistics (Startup)'
|
||||
)
|
||||
logger.info("Scheduled alias statistics job (interval: 5 minutes)")
|
||||
|
||||
scheduler.start()
|
||||
|
||||
logger.info("[OK] Scheduler started")
|
||||
|
||||
@@ -20,8 +20,15 @@ def test_smtp_connection() -> Dict:
|
||||
logs.append(f"Host: {settings.smtp_host}")
|
||||
logs.append(f"Port: {settings.smtp_port}")
|
||||
logs.append(f"Use TLS: {settings.smtp_use_tls}")
|
||||
logs.append(f"Relay Mode: {settings.smtp_relay_mode}")
|
||||
logs.append(f"User: {settings.smtp_user}")
|
||||
|
||||
# Different validation for relay mode
|
||||
if settings.smtp_relay_mode:
|
||||
if not settings.smtp_host or not settings.smtp_from:
|
||||
logs.append("ERROR: SMTP relay mode requires host and from address")
|
||||
return {"success": False, "logs": logs}
|
||||
else:
|
||||
if not settings.smtp_host or not settings.smtp_user or not settings.smtp_password:
|
||||
logs.append("ERROR: SMTP not fully configured")
|
||||
return {"success": False, "logs": logs}
|
||||
@@ -40,9 +47,13 @@ def test_smtp_connection() -> Dict:
|
||||
server.starttls()
|
||||
logs.append("TLS established")
|
||||
|
||||
# Skip login in relay mode
|
||||
if not settings.smtp_relay_mode:
|
||||
logs.append("Logging in...")
|
||||
server.login(settings.smtp_user, settings.smtp_password)
|
||||
logs.append("Login successful")
|
||||
else:
|
||||
logs.append("Relay mode - skipping authentication")
|
||||
|
||||
logs.append("Sending test email...")
|
||||
msg = MIMEMultipart()
|
||||
|
||||
95
backend/app/services/dmarc_cache.py
Normal file
95
backend/app/services/dmarc_cache.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
Shared DMARC Caching Service
|
||||
Used to share cache state between API router and background services
|
||||
"""
|
||||
import logging
|
||||
import json
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional, Any
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import SystemSetting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# In-memory cache for DMARC stats
|
||||
_dmarc_cache = {}
|
||||
_dmarc_cache_ttl_seconds = 300 # 5 minutes cache TTL
|
||||
|
||||
# Global signal tracking
|
||||
_cache_valid_since = datetime.now(timezone.utc)
|
||||
_last_db_check = datetime.min.replace(tzinfo=timezone.utc)
|
||||
_db_check_interval_seconds = 5
|
||||
|
||||
|
||||
def get_dmarc_cache_key(prefix: str, **params) -> str:
|
||||
"""Generate a cache key from parameters"""
|
||||
param_str = json.dumps(params, sort_keys=True, default=str)
|
||||
hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16]
|
||||
return f"dmarc:{prefix}:{hash_val}"
|
||||
|
||||
|
||||
def get_dmarc_cached(key: str, db: Session) -> Optional[Any]:
|
||||
"""Get cached value if not expired and not invalidated globally"""
|
||||
global _dmarc_cache, _last_db_check, _cache_valid_since
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Periodically check DB for invalidation signal
|
||||
if (now - _last_db_check).total_seconds() > _db_check_interval_seconds:
|
||||
_last_db_check = now
|
||||
try:
|
||||
setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first()
|
||||
if setting and setting.updated_at:
|
||||
# Ensure timezone awareness
|
||||
db_updated_at = setting.updated_at
|
||||
if db_updated_at.tzinfo is None:
|
||||
db_updated_at = db_updated_at.replace(tzinfo=timezone.utc)
|
||||
|
||||
# If DB signal is newer than our local validity, clear cache
|
||||
if db_updated_at > _cache_valid_since:
|
||||
logger.info("DMARC cache invalidated by another process")
|
||||
_dmarc_cache = {}
|
||||
_cache_valid_since = now
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking cache signal: {e}")
|
||||
|
||||
if key in _dmarc_cache:
|
||||
cached_data, cached_time = _dmarc_cache[key]
|
||||
if now - cached_time < timedelta(seconds=_dmarc_cache_ttl_seconds):
|
||||
logger.debug(f"DMARC cache hit for key: {key}")
|
||||
return cached_data
|
||||
else:
|
||||
# Cache expired, remove it
|
||||
del _dmarc_cache[key]
|
||||
return None
|
||||
|
||||
|
||||
def set_dmarc_cache(key: str, data: Any) -> None:
|
||||
"""Set cached value with current timestamp"""
|
||||
_dmarc_cache[key] = (data, datetime.now(timezone.utc))
|
||||
logger.debug(f"DMARC cache set for key: {key}")
|
||||
|
||||
|
||||
def clear_dmarc_cache(db: Session) -> None:
|
||||
"""Clear all DMARC cache locally and signal other processes via DB"""
|
||||
global _dmarc_cache, _cache_valid_since
|
||||
|
||||
# local clear
|
||||
_dmarc_cache = {}
|
||||
_cache_valid_since = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
# DB signal
|
||||
setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first()
|
||||
if not setting:
|
||||
setting = SystemSetting(key="dmarc_last_update", value="signal")
|
||||
db.add(setting)
|
||||
|
||||
setting.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
logger.info("DMARC cache cleared and signaled to DB")
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing cache signal: {e}")
|
||||
db.rollback()
|
||||
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
DMARC IMAP Service
|
||||
Automatically fetches and processes DMARC reports from email inbox
|
||||
DMARC & TLS-RPT IMAP Service
|
||||
Automatically fetches and processes DMARC and TLS-RPT reports from email inbox
|
||||
"""
|
||||
import logging
|
||||
import imaplib
|
||||
@@ -8,16 +8,19 @@ import email
|
||||
import gzip
|
||||
import zipfile
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from email.message import EmailMessage
|
||||
|
||||
from ..config import settings
|
||||
from ..database import SessionLocal
|
||||
from ..models import DMARCSync, DMARCReport, DMARCRecord
|
||||
from ..models import DMARCSync, DMARCReport, DMARCRecord, TLSReport, TLSReportPolicy
|
||||
from ..services.dmarc_parser import parse_dmarc_file
|
||||
from ..services.tls_rpt_parser import parse_tls_rpt_file, is_tls_rpt_json
|
||||
from ..services.geoip_service import enrich_dmarc_record
|
||||
from ..services.dmarc_notifications import send_dmarc_error_notification
|
||||
from ..services.dmarc_cache import clear_dmarc_cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -75,30 +78,33 @@ class DMARCImapService:
|
||||
logger.error(f"Error selecting folder: {e}")
|
||||
return False
|
||||
|
||||
def search_dmarc_emails(self) -> List[bytes]:
|
||||
def search_report_emails(self) -> List[bytes]:
|
||||
"""
|
||||
Search for DMARC report emails
|
||||
Search for DMARC and TLS-RPT report emails
|
||||
|
||||
Looking for emails with subject containing:
|
||||
- "Report Domain:"
|
||||
- "Submitter:"
|
||||
- "Report-ID:"
|
||||
- "Report Domain:" (DMARC)
|
||||
- "DMARC" (DMARC)
|
||||
- "Report-ID:" (DMARC)
|
||||
- "TLS-RPT" (TLS-RPT)
|
||||
- "TLS Report" (TLS-RPT)
|
||||
|
||||
Returns list of email IDs
|
||||
"""
|
||||
try:
|
||||
# Search for emails with DMARC-related subject
|
||||
# Search for emails with DMARC or TLS-RPT related subjects
|
||||
# Using OR to be more flexible
|
||||
search_criteria = '(OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (SUBJECT "Report-ID:")))'
|
||||
# UNSEEN ensures we don't re-process emails that were already handled (marked as Seen)
|
||||
search_criteria = '(UNSEEN (OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (OR (SUBJECT "Report-ID:") (OR (SUBJECT "TLS-RPT") (SUBJECT "TLS Report"))))))'
|
||||
|
||||
status, messages = self.connection.search(None, search_criteria)
|
||||
status, messages = self.connection.uid('SEARCH', None, search_criteria)
|
||||
|
||||
if status != 'OK':
|
||||
logger.error("Failed to search for DMARC emails")
|
||||
logger.error("Failed to search for report emails")
|
||||
return []
|
||||
|
||||
email_ids = messages[0].split()
|
||||
logger.info(f"Found {len(email_ids)} potential DMARC emails")
|
||||
logger.info(f"Found {len(email_ids)} potential DMARC/TLS-RPT emails")
|
||||
|
||||
return email_ids
|
||||
|
||||
@@ -106,83 +112,238 @@ class DMARCImapService:
|
||||
logger.error(f"Error searching for emails: {e}")
|
||||
return []
|
||||
|
||||
def search_dmarc_emails(self) -> List[bytes]:
|
||||
"""Alias for backward compatibility"""
|
||||
return self.search_report_emails()
|
||||
|
||||
def is_valid_dmarc_email(self, msg: EmailMessage) -> bool:
|
||||
"""
|
||||
Validate that this is a genuine DMARC report email
|
||||
|
||||
Checks:
|
||||
1. Subject contains "Report Domain:" AND ("Submitter:" OR "Report-ID:")
|
||||
2. Has at least one compressed attachment (.xml.gz or .zip)
|
||||
Accepts multiple DMARC email formats:
|
||||
- Standard: "Report Domain: X Submitter: Y Report-ID: Z"
|
||||
- Yahoo format: "Report Domain: X Submitter: Y" (no Report-ID)
|
||||
- Alternative: Contains "DMARC" in subject
|
||||
- Microsoft Outlook: DMARC-like attachment filename pattern
|
||||
|
||||
Primary validation is the attachment (.xml.gz or .zip with DMARC content)
|
||||
"""
|
||||
try:
|
||||
subject = msg.get('subject', '').lower()
|
||||
|
||||
# Check subject format
|
||||
has_report_domain = 'report domain:' in subject
|
||||
has_submitter = 'submitter:' in subject
|
||||
has_report_id = 'report-id:' in subject
|
||||
# Check for compressed DMARC attachments FIRST (most reliable indicator)
|
||||
has_dmarc_attachment = False
|
||||
has_dmarc_filename = False
|
||||
|
||||
if not (has_report_domain and (has_submitter or has_report_id)):
|
||||
logger.debug(f"Email does not match DMARC subject pattern: {subject}")
|
||||
return False
|
||||
|
||||
# Check for compressed attachments
|
||||
has_attachment = False
|
||||
for part in msg.walk():
|
||||
filename = part.get_filename()
|
||||
if filename:
|
||||
filename_lower = filename.lower()
|
||||
if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip'):
|
||||
has_attachment = True
|
||||
# DMARC reports come as .xml.gz, .xml, or .zip files
|
||||
if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip') or filename_lower.endswith('.xml'):
|
||||
has_dmarc_attachment = True
|
||||
# Check if filename looks like a DMARC report
|
||||
# Microsoft format: enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz
|
||||
# Standard format: domain!report-domain!timestamp!timestamp.xml.gz
|
||||
if '!' in filename and (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml') or filename_lower.endswith('.zip')):
|
||||
has_dmarc_filename = True
|
||||
break
|
||||
|
||||
if not has_attachment:
|
||||
if not has_dmarc_attachment:
|
||||
logger.debug(f"Email has no compressed DMARC attachment: {subject}")
|
||||
return False
|
||||
|
||||
# Check subject format - be flexible to support different providers
|
||||
has_report_domain = 'report domain:' in subject
|
||||
has_submitter = 'submitter:' in subject
|
||||
has_report_id = 'report-id:' in subject
|
||||
has_dmarc_keyword = 'dmarc' in subject
|
||||
|
||||
# Accept if:
|
||||
# 1. Has "Report Domain:" and ("Submitter:" or "Report-ID:") - standard format
|
||||
# 2. Has "Report Domain:" only (Yahoo and others) - we have verified attachment
|
||||
# 3. Has "DMARC" keyword in subject with valid attachment
|
||||
# 4. Has DMARC-like filename pattern (Microsoft Outlook and others) - attachment name contains '!'
|
||||
is_valid_subject = (
|
||||
(has_report_domain and (has_submitter or has_report_id)) or # Standard format
|
||||
(has_report_domain) or # Yahoo/minimal format (attachment already verified)
|
||||
(has_dmarc_keyword) or # DMARC keyword with attachment
|
||||
(has_dmarc_filename) # Microsoft Outlook format - DMARC filename pattern
|
||||
)
|
||||
|
||||
if not is_valid_subject:
|
||||
logger.debug(f"Email does not match DMARC subject/filename pattern: {subject}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error validating DMARC email: {e}")
|
||||
return False
|
||||
|
||||
def extract_attachments(self, msg: EmailMessage) -> List[Tuple[str, bytes]]:
|
||||
def is_valid_tls_rpt_email(self, msg: EmailMessage) -> bool:
|
||||
"""
|
||||
Validate that this is a TLS-RPT report email
|
||||
|
||||
TLS-RPT emails typically have:
|
||||
- Subject containing "TLS-RPT" or "TLS Report"
|
||||
- JSON or JSON.GZ attachment
|
||||
- Some providers send with generic subjects like "Report Domain: ..."
|
||||
"""
|
||||
try:
|
||||
subject = msg.get('subject', '').lower()
|
||||
|
||||
# Check for JSON/ZIP attachments
|
||||
has_json_attachment = False
|
||||
|
||||
for part in msg.walk():
|
||||
filename = part.get_filename()
|
||||
if filename:
|
||||
filename_lower = filename.lower()
|
||||
if filename_lower.endswith('.json') or filename_lower.endswith('.json.gz') or filename_lower.endswith('.zip'):
|
||||
has_json_attachment = True
|
||||
break
|
||||
|
||||
if not has_json_attachment:
|
||||
return False
|
||||
|
||||
# Trust the attachment if it looks like a TLS report
|
||||
# If it has a json/gz/zip attachment, we should try to process it as potential TLS-RPT
|
||||
# The parser will validate the content anyway
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error validating TLS-RPT email: {e}")
|
||||
return False
|
||||
|
||||
def detect_email_type(self, msg: EmailMessage) -> str:
|
||||
"""
|
||||
Detect if email is DMARC or TLS-RPT by inspecting attachments
|
||||
|
||||
Returns: 'dmarc', 'tls-rpt', or 'unknown'
|
||||
"""
|
||||
try:
|
||||
# Check attachments FIRST - content is king
|
||||
for part in msg.walk():
|
||||
filename = part.get_filename()
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
filename_lower = filename.lower()
|
||||
content = None
|
||||
|
||||
# Check explicit extensions
|
||||
if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml'):
|
||||
return 'dmarc'
|
||||
|
||||
if filename_lower.endswith('.json.gz') or filename_lower.endswith('.json'):
|
||||
return 'tls-rpt'
|
||||
|
||||
# Check ZIP content
|
||||
if filename_lower.endswith('.zip'):
|
||||
try:
|
||||
content = part.get_payload(decode=True)
|
||||
if content:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||||
for name in zf.namelist():
|
||||
name_lower = name.lower()
|
||||
if name_lower.endswith('.xml'):
|
||||
logger.info(f"Found XML in ZIP {filename}, identifying as DMARC")
|
||||
return 'dmarc'
|
||||
if name_lower.endswith('.json'):
|
||||
logger.info(f"Found JSON in ZIP {filename}, identifying as TLS-RPT")
|
||||
return 'tls-rpt'
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to inspect ZIP {filename}: {e}")
|
||||
|
||||
# Fallback to subject/header heuristics if no clear attachment type found
|
||||
# but reject ambiguous ZIPs that we couldn't inspect or were empty of relevant files
|
||||
|
||||
if self.is_valid_tls_rpt_email(msg):
|
||||
return 'tls-rpt'
|
||||
elif self.is_valid_dmarc_email(msg):
|
||||
return 'dmarc'
|
||||
|
||||
return 'unknown'
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error detecting email type: {e}")
|
||||
return 'unknown'
|
||||
|
||||
def extract_attachments(self, msg: EmailMessage, include_json: bool = False) -> List[Tuple[str, bytes]]:
|
||||
"""
|
||||
Extract compressed attachments from email
|
||||
|
||||
Args:
|
||||
msg: Email message
|
||||
include_json: If True, also extract JSON files (for TLS-RPT)
|
||||
|
||||
Returns list of (filename, content) tuples
|
||||
"""
|
||||
attachments = []
|
||||
|
||||
try:
|
||||
for part in msg.walk():
|
||||
# Try to get filename from Content-Disposition header
|
||||
filename = part.get_filename()
|
||||
|
||||
# If no filename, try to get from Content-Type 'name' parameter
|
||||
if not filename:
|
||||
content_type = part.get_content_type()
|
||||
# Check if this is a potential attachment by content type
|
||||
if content_type in ['application/gzip', 'application/x-gzip', 'application/zip',
|
||||
'application/x-zip-compressed', 'text/xml', 'application/xml',
|
||||
'application/json', 'application/octet-stream']:
|
||||
# Try to get name from content-type params
|
||||
params = part.get_params()
|
||||
if params:
|
||||
for key, value in params:
|
||||
if key.lower() == 'name':
|
||||
filename = value
|
||||
break
|
||||
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
filename_lower = filename.lower()
|
||||
if not (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip')):
|
||||
|
||||
# Support DMARC files: .xml.gz, .zip, .xml
|
||||
# Support TLS-RPT files: .json, .json.gz
|
||||
valid_extensions = ['.xml.gz', '.zip', '.xml']
|
||||
if include_json:
|
||||
valid_extensions.extend(['.json', '.json.gz'])
|
||||
|
||||
if not any(filename_lower.endswith(ext) for ext in valid_extensions):
|
||||
continue
|
||||
|
||||
content = part.get_payload(decode=True)
|
||||
if content:
|
||||
attachments.append((filename, content))
|
||||
logger.debug(f"Extracted attachment: {filename}")
|
||||
logger.debug(f"Extracted attachment: {filename} ({len(content)} bytes)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting attachments: {e}")
|
||||
|
||||
if not attachments:
|
||||
# Log all parts for debugging
|
||||
logger.debug(f"No attachments found. Email parts:")
|
||||
for i, part in enumerate(msg.walk()):
|
||||
ct = part.get_content_type()
|
||||
fn = part.get_filename()
|
||||
logger.debug(f" Part {i}: type={ct}, filename={fn}")
|
||||
|
||||
return attachments
|
||||
|
||||
def process_email(self, email_id: str, db: SessionLocal) -> Dict:
|
||||
"""
|
||||
Process a single DMARC email
|
||||
Process a single DMARC or TLS-RPT email
|
||||
|
||||
Returns dict with:
|
||||
- success: bool
|
||||
- reports_created: int
|
||||
- reports_duplicate: int
|
||||
- error: str or None
|
||||
- report_type: 'dmarc' or 'tls-rpt'
|
||||
"""
|
||||
result = {
|
||||
'success': False,
|
||||
@@ -190,12 +351,13 @@ class DMARCImapService:
|
||||
'reports_duplicate': 0,
|
||||
'error': None,
|
||||
'message_id': None,
|
||||
'subject': None
|
||||
'subject': None,
|
||||
'report_type': None
|
||||
}
|
||||
|
||||
try:
|
||||
# Fetch email (email_id is already a string)
|
||||
status, msg_data = self.connection.fetch(email_id, '(RFC822)')
|
||||
status, msg_data = self.connection.uid('FETCH', email_id, '(RFC822)')
|
||||
|
||||
if status != 'OK':
|
||||
result['error'] = f"Failed to fetch email {email_id}"
|
||||
@@ -206,25 +368,41 @@ class DMARCImapService:
|
||||
result['message_id'] = msg.get('message-id', 'unknown')
|
||||
result['subject'] = msg.get('subject', 'unknown')
|
||||
|
||||
# Validate it's a DMARC email
|
||||
if not self.is_valid_dmarc_email(msg):
|
||||
result['error'] = "Not a valid DMARC report email"
|
||||
# Detect email type
|
||||
email_type = self.detect_email_type(msg)
|
||||
result['report_type'] = email_type
|
||||
|
||||
if email_type == 'tls-rpt':
|
||||
return self._process_tls_rpt_email(msg, db, result)
|
||||
elif email_type == 'dmarc':
|
||||
return self._process_dmarc_email(msg, db, result)
|
||||
else:
|
||||
result['error'] = "Not a valid DMARC or TLS-RPT report email"
|
||||
return result
|
||||
|
||||
# Extract attachments
|
||||
attachments = self.extract_attachments(msg)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing email {email_id}: {e}")
|
||||
result['error'] = str(e)
|
||||
return result
|
||||
|
||||
def _process_dmarc_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict:
|
||||
"""Process a DMARC email"""
|
||||
# Extract attachments (DMARC: XML files)
|
||||
attachments = self.extract_attachments(msg, include_json=False)
|
||||
|
||||
if not attachments:
|
||||
result['error'] = "No DMARC attachments found"
|
||||
return result
|
||||
|
||||
# Process each attachment
|
||||
attachment_errors = []
|
||||
for filename, content in attachments:
|
||||
try:
|
||||
# Parse DMARC report
|
||||
parsed_data = parse_dmarc_file(content, filename)
|
||||
|
||||
if not parsed_data:
|
||||
attachment_errors.append(f"Failed to parse: {filename}")
|
||||
logger.warning(f"Failed to parse attachment: {filename}")
|
||||
continue
|
||||
|
||||
@@ -260,26 +438,113 @@ class DMARCImapService:
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Error processing attachment {filename}: {e}")
|
||||
if not result['error']:
|
||||
result['error'] = str(e)
|
||||
error_msg = f"Error processing {filename}: {str(e)}"
|
||||
attachment_errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
# Mark as success if at least one report was created
|
||||
if result['reports_created'] > 0:
|
||||
result['success'] = True
|
||||
# Determine success
|
||||
return self._finalize_result(result, attachment_errors, "DMARC")
|
||||
|
||||
def _process_tls_rpt_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict:
|
||||
"""Process a TLS-RPT email"""
|
||||
# Extract attachments (TLS-RPT: JSON files)
|
||||
attachments = self.extract_attachments(msg, include_json=True)
|
||||
|
||||
# Filter to only JSON files (and ZIPs containing JSON)
|
||||
json_attachments = [(f, c) for f, c in attachments if f.lower().endswith('.json') or f.lower().endswith('.json.gz') or f.lower().endswith('.zip')]
|
||||
|
||||
if not json_attachments:
|
||||
result['error'] = "No TLS-RPT JSON attachments found"
|
||||
return result
|
||||
|
||||
# Process each attachment
|
||||
attachment_errors = []
|
||||
for filename, content in json_attachments:
|
||||
try:
|
||||
# Parse TLS-RPT report
|
||||
parsed_data = parse_tls_rpt_file(content, filename)
|
||||
|
||||
if not parsed_data:
|
||||
attachment_errors.append(f"Failed to parse: {filename}")
|
||||
logger.warning(f"Failed to parse TLS-RPT attachment: {filename}")
|
||||
continue
|
||||
|
||||
# Extract policies
|
||||
policies_data = parsed_data.pop('policies', [])
|
||||
|
||||
# Check for duplicate
|
||||
existing = db.query(TLSReport).filter(
|
||||
TLSReport.report_id == parsed_data['report_id']
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
result['reports_duplicate'] += 1
|
||||
logger.info(f"Duplicate TLS-RPT report: {parsed_data['report_id']}")
|
||||
continue
|
||||
|
||||
# Create TLS report
|
||||
tls_report = TLSReport(
|
||||
report_id=parsed_data['report_id'],
|
||||
organization_name=parsed_data.get('organization_name', 'Unknown'),
|
||||
contact_info=parsed_data.get('contact_info', ''),
|
||||
policy_domain=parsed_data['policy_domain'],
|
||||
start_datetime=parsed_data['start_datetime'],
|
||||
end_datetime=parsed_data['end_datetime'],
|
||||
raw_json=parsed_data.get('raw_json', '')
|
||||
)
|
||||
db.add(tls_report)
|
||||
db.flush()
|
||||
|
||||
# Create policy records
|
||||
for policy_data in policies_data:
|
||||
policy = TLSReportPolicy(
|
||||
tls_report_id=tls_report.id,
|
||||
policy_type=policy_data.get('policy_type', 'unknown'),
|
||||
policy_domain=policy_data.get('policy_domain', ''),
|
||||
policy_string=policy_data.get('policy_string', []),
|
||||
mx_host=policy_data.get('mx_host', []),
|
||||
successful_session_count=policy_data.get('successful_session_count', 0),
|
||||
failed_session_count=policy_data.get('failed_session_count', 0),
|
||||
failure_details=policy_data.get('failure_details', [])
|
||||
)
|
||||
db.add(policy)
|
||||
|
||||
db.commit()
|
||||
result['reports_created'] += 1
|
||||
logger.info(f"Created TLS-RPT report: {parsed_data['report_id']}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing email {email_id}: {e}")
|
||||
result['error'] = str(e)
|
||||
db.rollback()
|
||||
error_msg = f"Error processing TLS-RPT {filename}: {str(e)}"
|
||||
attachment_errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
# Determine success
|
||||
return self._finalize_result(result, attachment_errors, "TLS-RPT")
|
||||
|
||||
def _finalize_result(self, result: Dict, attachment_errors: List[str], report_type: str) -> Dict:
|
||||
"""Finalize the result based on processing outcome"""
|
||||
if result['reports_created'] > 0:
|
||||
result['success'] = True
|
||||
elif result['reports_duplicate'] > 0 and result['reports_created'] == 0:
|
||||
# All reports were duplicates - this is actually OK, mark as success
|
||||
result['success'] = True
|
||||
result['error'] = None # No error - duplicates are expected
|
||||
else:
|
||||
# No reports created and no duplicates - something went wrong
|
||||
result['success'] = False
|
||||
if attachment_errors:
|
||||
result['error'] = "; ".join(attachment_errors)
|
||||
else:
|
||||
result['error'] = f"No valid {report_type} reports found in attachments"
|
||||
|
||||
return result
|
||||
|
||||
def mark_as_processed(self, email_id: str):
|
||||
"""Mark email as processed (flag or move)"""
|
||||
try:
|
||||
# Add a flag to mark as processed
|
||||
self.connection.store(email_id, '+FLAGS', '\\Seen')
|
||||
self.connection.uid('STORE', email_id, '+FLAGS', '\\Seen')
|
||||
logger.debug(f"Marked email {email_id} as seen")
|
||||
|
||||
except Exception as e:
|
||||
@@ -288,7 +553,7 @@ class DMARCImapService:
|
||||
def delete_email(self, email_id: str):
|
||||
"""Delete email from server"""
|
||||
try:
|
||||
self.connection.store(email_id, '+FLAGS', '\\Deleted')
|
||||
self.connection.uid('STORE', email_id, '+FLAGS', '\\Deleted')
|
||||
self.connection.expunge()
|
||||
logger.debug(f"Deleted email {email_id}")
|
||||
|
||||
@@ -297,7 +562,11 @@ class DMARCImapService:
|
||||
|
||||
def sync_reports(self, sync_type: str = 'auto') -> Dict:
|
||||
"""
|
||||
Main sync function
|
||||
Main sync function with batch processing
|
||||
|
||||
Processes emails in batches to prevent memory issues with large mailboxes.
|
||||
After each batch, emails are deleted/marked and the search is re-run
|
||||
to get the next batch of unprocessed emails.
|
||||
|
||||
Returns statistics about the sync operation
|
||||
"""
|
||||
@@ -308,6 +577,7 @@ class DMARCImapService:
|
||||
)
|
||||
|
||||
db = SessionLocal()
|
||||
batch_size = settings.dmarc_imap_batch_size
|
||||
|
||||
try:
|
||||
db.add(sync_record)
|
||||
@@ -321,22 +591,40 @@ class DMARCImapService:
|
||||
if not self.select_folder():
|
||||
raise Exception(f"Failed to select folder {self.folder}")
|
||||
|
||||
# Search for DMARC emails
|
||||
email_ids = self.search_dmarc_emails()
|
||||
sync_record.emails_found = len(email_ids)
|
||||
# Initial search to count total emails
|
||||
all_email_ids = self.search_dmarc_emails()
|
||||
total_emails = len(all_email_ids)
|
||||
sync_record.emails_found = total_emails
|
||||
db.commit()
|
||||
|
||||
if not email_ids:
|
||||
if not all_email_ids:
|
||||
logger.info("No DMARC emails found")
|
||||
sync_record.status = 'success'
|
||||
sync_record.completed_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return self._build_result(sync_record)
|
||||
|
||||
# Process each email
|
||||
failed_emails = []
|
||||
logger.info(f"Found {total_emails} DMARC emails, processing in batches of {batch_size}")
|
||||
|
||||
for email_id in email_ids:
|
||||
# Process in batches
|
||||
failed_emails = []
|
||||
batch_number = 0
|
||||
|
||||
while True:
|
||||
batch_number += 1
|
||||
|
||||
# Re-search to get current unprocessed emails (since we delete/mark after each batch)
|
||||
email_ids = self.search_dmarc_emails()
|
||||
|
||||
if not email_ids:
|
||||
logger.info(f"Batch {batch_number}: No more emails to process")
|
||||
break
|
||||
|
||||
# Take only batch_size emails
|
||||
batch_emails = email_ids[:batch_size]
|
||||
logger.info(f"Batch {batch_number}: Processing {len(batch_emails)} emails (remaining: {len(email_ids)})")
|
||||
|
||||
for email_id in batch_emails:
|
||||
email_id = email_id.decode() if isinstance(email_id, bytes) else email_id
|
||||
result = self.process_email(email_id, db)
|
||||
sync_record.emails_processed += 1
|
||||
@@ -352,15 +640,26 @@ class DMARCImapService:
|
||||
self.mark_as_processed(email_id)
|
||||
else:
|
||||
sync_record.reports_failed += 1
|
||||
error_msg = result.get('error', 'Unknown error')
|
||||
logger.warning(f"Failed to process email {email_id}: {error_msg}")
|
||||
|
||||
failed_emails.append({
|
||||
'email_id': email_id,
|
||||
'message_id': result['message_id'],
|
||||
'subject': result['subject'],
|
||||
'error': result['error']
|
||||
'error': error_msg
|
||||
})
|
||||
# Also mark failed emails as processed to avoid re-processing
|
||||
self.mark_as_processed(email_id)
|
||||
|
||||
db.commit()
|
||||
|
||||
# Log batch progress
|
||||
logger.info(f"Batch {batch_number} complete: "
|
||||
f"{sync_record.emails_processed}/{total_emails} processed, "
|
||||
f"{sync_record.reports_created} created, "
|
||||
f"{sync_record.reports_failed} failed")
|
||||
|
||||
# Update sync record
|
||||
sync_record.status = 'success'
|
||||
sync_record.completed_at = datetime.now(timezone.utc)
|
||||
@@ -375,6 +674,10 @@ class DMARCImapService:
|
||||
f"{sync_record.reports_duplicate} duplicates, "
|
||||
f"{sync_record.reports_failed} failed")
|
||||
|
||||
# Clear cache if any reports were created
|
||||
if sync_record.reports_created > 0:
|
||||
clear_dmarc_cache(db)
|
||||
|
||||
# Send email notification if there were failures
|
||||
if failed_emails and settings.notification_smtp_configured:
|
||||
logger.info(f"Sending error notification for {len(failed_emails)} failed emails")
|
||||
|
||||
@@ -58,6 +58,57 @@ def parse_dmarc_file(file_content: bytes, filename: str) -> Optional[Dict[str, A
|
||||
return None
|
||||
|
||||
|
||||
def find_element(parent: ET.Element, tag: str, namespaces: List[str]) -> Optional[ET.Element]:
|
||||
"""
|
||||
Find an element trying multiple namespaces
|
||||
|
||||
Args:
|
||||
parent: Parent XML element
|
||||
tag: Tag name to find
|
||||
namespaces: List of namespace prefixes to try (empty string for no namespace)
|
||||
|
||||
Returns:
|
||||
Found element or None
|
||||
"""
|
||||
for ns in namespaces:
|
||||
if ns:
|
||||
elem = parent.find(f'{{{ns}}}{tag}')
|
||||
else:
|
||||
elem = parent.find(tag)
|
||||
if elem is not None:
|
||||
return elem
|
||||
return None
|
||||
|
||||
|
||||
def find_all_elements(parent: ET.Element, tag: str, namespaces: List[str]) -> List[ET.Element]:
|
||||
"""
|
||||
Find all elements matching tag, trying multiple namespaces
|
||||
|
||||
Args:
|
||||
parent: Parent XML element
|
||||
tag: Tag name to find
|
||||
namespaces: List of namespace prefixes to try
|
||||
|
||||
Returns:
|
||||
List of found elements
|
||||
"""
|
||||
for ns in namespaces:
|
||||
if ns:
|
||||
elems = parent.findall(f'{{{ns}}}{tag}')
|
||||
else:
|
||||
elems = parent.findall(tag)
|
||||
if elems:
|
||||
return elems
|
||||
return []
|
||||
|
||||
|
||||
# Known DMARC XML namespaces
|
||||
DMARC_NAMESPACES = [
|
||||
'', # No namespace (DMARC 1.0)
|
||||
'urn:ietf:params:xml:ns:dmarc-2.0', # DMARC 2.0
|
||||
]
|
||||
|
||||
|
||||
def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse DMARC XML content
|
||||
@@ -72,38 +123,38 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
|
||||
try:
|
||||
root = ET.fromstring(xml_string)
|
||||
|
||||
# Parse report metadata
|
||||
metadata = root.find('report_metadata')
|
||||
# Parse report metadata (try with and without namespace)
|
||||
metadata = find_element(root, 'report_metadata', DMARC_NAMESPACES)
|
||||
if metadata is None:
|
||||
raise ValueError("Missing report_metadata element")
|
||||
|
||||
org_name = get_element_text(metadata, 'org_name')
|
||||
email = get_element_text(metadata, 'email')
|
||||
extra_contact_info = get_element_text(metadata, 'extra_contact_info')
|
||||
report_id = get_element_text(metadata, 'report_id')
|
||||
org_name = get_element_text(metadata, 'org_name', DMARC_NAMESPACES)
|
||||
email = get_element_text(metadata, 'email', DMARC_NAMESPACES)
|
||||
extra_contact_info = get_element_text(metadata, 'extra_contact_info', DMARC_NAMESPACES)
|
||||
report_id = get_element_text(metadata, 'report_id', DMARC_NAMESPACES)
|
||||
|
||||
date_range = metadata.find('date_range')
|
||||
date_range = find_element(metadata, 'date_range', DMARC_NAMESPACES)
|
||||
if date_range is None:
|
||||
raise ValueError("Missing date_range element")
|
||||
|
||||
begin_date = int(get_element_text(date_range, 'begin'))
|
||||
end_date = int(get_element_text(date_range, 'end'))
|
||||
begin_date = int(get_element_text(date_range, 'begin', DMARC_NAMESPACES))
|
||||
end_date = int(get_element_text(date_range, 'end', DMARC_NAMESPACES))
|
||||
|
||||
# Parse published policy
|
||||
policy = root.find('policy_published')
|
||||
policy = find_element(root, 'policy_published', DMARC_NAMESPACES)
|
||||
if policy is None:
|
||||
raise ValueError("Missing policy_published element")
|
||||
|
||||
domain = get_element_text(policy, 'domain')
|
||||
domain = get_element_text(policy, 'domain', DMARC_NAMESPACES)
|
||||
|
||||
policy_published = {
|
||||
'adkim': get_element_text(policy, 'adkim'),
|
||||
'aspf': get_element_text(policy, 'aspf'),
|
||||
'p': get_element_text(policy, 'p'),
|
||||
'sp': get_element_text(policy, 'sp'),
|
||||
'pct': get_element_text(policy, 'pct'),
|
||||
'fo': get_element_text(policy, 'fo'),
|
||||
'np': get_element_text(policy, 'np'),
|
||||
'adkim': get_element_text(policy, 'adkim', DMARC_NAMESPACES),
|
||||
'aspf': get_element_text(policy, 'aspf', DMARC_NAMESPACES),
|
||||
'p': get_element_text(policy, 'p', DMARC_NAMESPACES),
|
||||
'sp': get_element_text(policy, 'sp', DMARC_NAMESPACES),
|
||||
'pct': get_element_text(policy, 'pct', DMARC_NAMESPACES),
|
||||
'fo': get_element_text(policy, 'fo', DMARC_NAMESPACES),
|
||||
'np': get_element_text(policy, 'np', DMARC_NAMESPACES),
|
||||
}
|
||||
|
||||
# Remove None values
|
||||
@@ -111,7 +162,7 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]:
|
||||
|
||||
# Parse records
|
||||
records = []
|
||||
for record_elem in root.findall('record'):
|
||||
for record_elem in find_all_elements(root, 'record', DMARC_NAMESPACES):
|
||||
record_data = parse_dmarc_record(record_elem)
|
||||
if record_data:
|
||||
records.append(record_data)
|
||||
@@ -145,38 +196,38 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
|
||||
Dictionary with parsed record data
|
||||
"""
|
||||
try:
|
||||
row = record_elem.find('row')
|
||||
row = find_element(record_elem, 'row', DMARC_NAMESPACES)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
# Source and count
|
||||
source_ip = get_element_text(row, 'source_ip')
|
||||
count = int(get_element_text(row, 'count', '0'))
|
||||
source_ip = get_element_text(row, 'source_ip', DMARC_NAMESPACES)
|
||||
count = int(get_element_text(row, 'count', DMARC_NAMESPACES, '0'))
|
||||
|
||||
# Policy evaluation
|
||||
policy_eval = row.find('policy_evaluated')
|
||||
disposition = get_element_text(policy_eval, 'disposition') if policy_eval else None
|
||||
dkim_result = get_element_text(policy_eval, 'dkim') if policy_eval else None
|
||||
spf_result = get_element_text(policy_eval, 'spf') if policy_eval else None
|
||||
policy_eval = find_element(row, 'policy_evaluated', DMARC_NAMESPACES)
|
||||
disposition = get_element_text(policy_eval, 'disposition', DMARC_NAMESPACES) if policy_eval else None
|
||||
dkim_result = get_element_text(policy_eval, 'dkim', DMARC_NAMESPACES) if policy_eval else None
|
||||
spf_result = get_element_text(policy_eval, 'spf', DMARC_NAMESPACES) if policy_eval else None
|
||||
|
||||
# Identifiers
|
||||
identifiers = record_elem.find('identifiers')
|
||||
header_from = get_element_text(identifiers, 'header_from') if identifiers else None
|
||||
envelope_from = get_element_text(identifiers, 'envelope_from') if identifiers else None
|
||||
envelope_to = get_element_text(identifiers, 'envelope_to') if identifiers else None
|
||||
identifiers = find_element(record_elem, 'identifiers', DMARC_NAMESPACES)
|
||||
header_from = get_element_text(identifiers, 'header_from', DMARC_NAMESPACES) if identifiers else None
|
||||
envelope_from = get_element_text(identifiers, 'envelope_from', DMARC_NAMESPACES) if identifiers else None
|
||||
envelope_to = get_element_text(identifiers, 'envelope_to', DMARC_NAMESPACES) if identifiers else None
|
||||
|
||||
# Auth results
|
||||
auth_results = {}
|
||||
auth_results_elem = record_elem.find('auth_results')
|
||||
auth_results_elem = find_element(record_elem, 'auth_results', DMARC_NAMESPACES)
|
||||
|
||||
if auth_results_elem:
|
||||
# Parse DKIM results
|
||||
dkim_results = []
|
||||
for dkim_elem in auth_results_elem.findall('dkim'):
|
||||
for dkim_elem in find_all_elements(auth_results_elem, 'dkim', DMARC_NAMESPACES):
|
||||
dkim_data = {
|
||||
'domain': get_element_text(dkim_elem, 'domain'),
|
||||
'selector': get_element_text(dkim_elem, 'selector'),
|
||||
'result': get_element_text(dkim_elem, 'r') or get_element_text(dkim_elem, 'result')
|
||||
'domain': get_element_text(dkim_elem, 'domain', DMARC_NAMESPACES),
|
||||
'selector': get_element_text(dkim_elem, 'selector', DMARC_NAMESPACES),
|
||||
'result': get_element_text(dkim_elem, 'r', DMARC_NAMESPACES) or get_element_text(dkim_elem, 'result', DMARC_NAMESPACES)
|
||||
}
|
||||
dkim_results.append({k: v for k, v in dkim_data.items() if v})
|
||||
|
||||
@@ -185,11 +236,11 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
|
||||
|
||||
# Parse SPF results
|
||||
spf_results = []
|
||||
for spf_elem in auth_results_elem.findall('spf'):
|
||||
for spf_elem in find_all_elements(auth_results_elem, 'spf', DMARC_NAMESPACES):
|
||||
spf_data = {
|
||||
'domain': get_element_text(spf_elem, 'domain'),
|
||||
'scope': get_element_text(spf_elem, 'scope'),
|
||||
'result': get_element_text(spf_elem, 'r') or get_element_text(spf_elem, 'result')
|
||||
'domain': get_element_text(spf_elem, 'domain', DMARC_NAMESPACES),
|
||||
'scope': get_element_text(spf_elem, 'scope', DMARC_NAMESPACES),
|
||||
'result': get_element_text(spf_elem, 'r', DMARC_NAMESPACES) or get_element_text(spf_elem, 'result', DMARC_NAMESPACES)
|
||||
}
|
||||
spf_results.append({k: v for k, v in spf_data.items() if v})
|
||||
|
||||
@@ -213,13 +264,14 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]:
|
||||
return None
|
||||
|
||||
|
||||
def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[str] = None) -> Optional[str]:
|
||||
def get_element_text(parent: Optional[ET.Element], tag: str, namespaces: List[str] = None, default: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Safely get text from XML element
|
||||
Safely get text from XML element with namespace support
|
||||
|
||||
Args:
|
||||
parent: Parent XML element
|
||||
tag: Tag name to find
|
||||
namespaces: List of namespace prefixes to try
|
||||
default: Default value if not found
|
||||
|
||||
Returns:
|
||||
@@ -228,7 +280,11 @@ def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[s
|
||||
if parent is None:
|
||||
return default
|
||||
|
||||
if namespaces:
|
||||
elem = find_element(parent, tag, namespaces)
|
||||
else:
|
||||
elem = parent.find(tag)
|
||||
|
||||
if elem is not None and elem.text:
|
||||
return elem.text.strip()
|
||||
|
||||
|
||||
@@ -21,8 +21,10 @@ class SmtpService:
|
||||
self.host = settings.smtp_host
|
||||
self.port = settings.smtp_port
|
||||
self.use_tls = settings.smtp_use_tls
|
||||
self.use_ssl = settings.smtp_use_ssl
|
||||
self.user = settings.smtp_user
|
||||
self.password = settings.smtp_password
|
||||
self.relay_mode = settings.smtp_relay_mode
|
||||
self.from_address = settings.smtp_from or settings.smtp_user
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
@@ -70,13 +72,22 @@ class SmtpService:
|
||||
part2 = MIMEText(html_content, 'html')
|
||||
msg.attach(part2)
|
||||
|
||||
if self.use_tls:
|
||||
# Determine connection mode
|
||||
# Priority 1: Implicit SSL (if configured or using port 465)
|
||||
if self.use_ssl or self.port == 465:
|
||||
server = smtplib.SMTP_SSL(self.host, self.port)
|
||||
# Priority 2: STARTTLS (if configured)
|
||||
elif self.use_tls:
|
||||
server = smtplib.SMTP(self.host, self.port)
|
||||
server.starttls()
|
||||
# Priority 3: Plaintext
|
||||
else:
|
||||
server = smtplib.SMTP_SSL(self.host, self.port)
|
||||
server = smtplib.SMTP(self.host, self.port)
|
||||
|
||||
# Skip login in relay mode
|
||||
if not self.relay_mode:
|
||||
server.login(self.user, self.password)
|
||||
|
||||
server.sendmail(self.from_address, [recipient], msg.as_string())
|
||||
server.quit()
|
||||
|
||||
|
||||
218
backend/app/services/tls_rpt_parser.py
Normal file
218
backend/app/services/tls_rpt_parser.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
TLS-RPT (SMTP TLS Reporting) Parser
|
||||
Handles parsing of TLS-RPT reports in JSON format
|
||||
"""
|
||||
import json
|
||||
import gzip
|
||||
import zipfile
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_tls_rpt_file(file_content: bytes, filename: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse TLS-RPT report from file content (JSON, GZ compressed, or ZIP)
|
||||
|
||||
Args:
|
||||
file_content: Raw bytes of the file
|
||||
filename: Original filename (to determine compression type)
|
||||
|
||||
Returns:
|
||||
Parsed TLS-RPT data or None if parsing failed
|
||||
"""
|
||||
try:
|
||||
filename_lower = filename.lower()
|
||||
|
||||
# Handle ZIP files (.json.zip)
|
||||
if filename_lower.endswith('.zip'):
|
||||
try:
|
||||
with zipfile.ZipFile(BytesIO(file_content)) as zf:
|
||||
# Find JSON file inside ZIP
|
||||
json_files = [f for f in zf.namelist() if f.lower().endswith('.json')]
|
||||
if not json_files:
|
||||
logger.error(f"No JSON file found in ZIP: {filename}")
|
||||
return None
|
||||
|
||||
# Read the first JSON file
|
||||
json_content = zf.read(json_files[0]).decode('utf-8')
|
||||
except zipfile.BadZipFile as e:
|
||||
logger.error(f"Invalid ZIP file {filename}: {e}")
|
||||
return None
|
||||
# Handle GZIP files (.json.gz)
|
||||
elif filename_lower.endswith('.gz'):
|
||||
try:
|
||||
json_content = gzip.decompress(file_content).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to decompress gzip TLS-RPT file: {e}")
|
||||
return None
|
||||
# Handle plain JSON files
|
||||
elif filename_lower.endswith('.json'):
|
||||
json_content = file_content.decode('utf-8')
|
||||
else:
|
||||
# Try to decode as JSON directly
|
||||
try:
|
||||
json_content = file_content.decode('utf-8')
|
||||
except Exception:
|
||||
logger.error(f"Unknown TLS-RPT file format: {filename}")
|
||||
return None
|
||||
|
||||
return parse_tls_rpt_json(json_content)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing TLS-RPT file {filename}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
def parse_tls_rpt_json(json_content: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse TLS-RPT JSON content
|
||||
|
||||
Expected format (RFC 8460):
|
||||
{
|
||||
"organization-name": "Google Inc.",
|
||||
"date-range": {
|
||||
"start-datetime": "2026-01-12T00:00:00Z",
|
||||
"end-datetime": "2026-01-12T23:59:59Z"
|
||||
},
|
||||
"contact-info": "smtp-tls-reporting@google.com",
|
||||
"report-id": "2026-01-12T00:00:00Z_boubou.me",
|
||||
"policies": [{
|
||||
"policy": {
|
||||
"policy-type": "sts",
|
||||
"policy-string": ["version: STSv1", "mode: enforce", ...],
|
||||
"policy-domain": "boubou.me",
|
||||
"mx-host": ["mail.tiboxs.com"]
|
||||
},
|
||||
"summary": {
|
||||
"total-successful-session-count": 1,
|
||||
"total-failure-session-count": 0
|
||||
},
|
||||
"failure-details": [...] # Optional
|
||||
}]
|
||||
}
|
||||
|
||||
Returns:
|
||||
Dictionary with parsed TLS-RPT data
|
||||
"""
|
||||
try:
|
||||
data = json.loads(json_content)
|
||||
|
||||
# Extract report metadata
|
||||
report_id = data.get('report-id', '')
|
||||
if not report_id:
|
||||
logger.error("TLS-RPT report missing report-id")
|
||||
return None
|
||||
|
||||
organization_name = data.get('organization-name', 'Unknown')
|
||||
contact_info = data.get('contact-info', '')
|
||||
|
||||
# Parse date range
|
||||
date_range = data.get('date-range', {})
|
||||
start_datetime = parse_iso_datetime(date_range.get('start-datetime', ''))
|
||||
end_datetime = parse_iso_datetime(date_range.get('end-datetime', ''))
|
||||
|
||||
if not start_datetime or not end_datetime:
|
||||
logger.error("TLS-RPT report missing or invalid date-range")
|
||||
return None
|
||||
|
||||
# Parse policies
|
||||
policies = []
|
||||
policy_domain = None
|
||||
|
||||
for policy_entry in data.get('policies', []):
|
||||
policy_data = policy_entry.get('policy', {})
|
||||
summary = policy_entry.get('summary', {})
|
||||
|
||||
# Get the policy domain from the first policy
|
||||
if not policy_domain:
|
||||
policy_domain = policy_data.get('policy-domain', '')
|
||||
|
||||
parsed_policy = {
|
||||
'policy_type': policy_data.get('policy-type', 'unknown'),
|
||||
'policy_domain': policy_data.get('policy-domain', ''),
|
||||
'policy_string': policy_data.get('policy-string', []),
|
||||
'mx_host': policy_data.get('mx-host', []),
|
||||
'successful_session_count': summary.get('total-successful-session-count', 0),
|
||||
'failed_session_count': summary.get('total-failure-session-count', 0),
|
||||
'failure_details': policy_entry.get('failure-details', [])
|
||||
}
|
||||
policies.append(parsed_policy)
|
||||
|
||||
if not policy_domain:
|
||||
logger.error("TLS-RPT report missing policy-domain")
|
||||
return None
|
||||
|
||||
return {
|
||||
'report_id': report_id,
|
||||
'organization_name': organization_name,
|
||||
'contact_info': contact_info,
|
||||
'policy_domain': policy_domain,
|
||||
'start_datetime': start_datetime,
|
||||
'end_datetime': end_datetime,
|
||||
'policies': policies,
|
||||
'raw_json': json_content
|
||||
}
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Invalid JSON in TLS-RPT report: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing TLS-RPT JSON: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_iso_datetime(datetime_str: str) -> Optional[datetime]:
|
||||
"""
|
||||
Parse ISO 8601 datetime string
|
||||
|
||||
Supports formats:
|
||||
- 2026-01-12T00:00:00Z
|
||||
- 2026-01-12T00:00:00+00:00
|
||||
"""
|
||||
if not datetime_str:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove 'Z' suffix and replace with +00:00 for parsing
|
||||
if datetime_str.endswith('Z'):
|
||||
datetime_str = datetime_str[:-1] + '+00:00'
|
||||
|
||||
# Parse with timezone
|
||||
from datetime import timezone
|
||||
dt = datetime.fromisoformat(datetime_str)
|
||||
|
||||
# Convert to UTC naive datetime for storage
|
||||
if dt.tzinfo is not None:
|
||||
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
return dt
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing datetime '{datetime_str}': {e}")
|
||||
return None
|
||||
|
||||
|
||||
def is_tls_rpt_json(json_content: str) -> bool:
|
||||
"""
|
||||
Check if JSON content is a valid TLS-RPT report
|
||||
|
||||
Used to detect TLS-RPT vs other JSON files
|
||||
"""
|
||||
try:
|
||||
data = json.loads(json_content)
|
||||
|
||||
# Check for required TLS-RPT fields
|
||||
has_report_id = 'report-id' in data
|
||||
has_date_range = 'date-range' in data
|
||||
has_policies = 'policies' in data
|
||||
|
||||
# At minimum, should have policies and date-range
|
||||
return has_policies and (has_date_range or has_report_id)
|
||||
|
||||
except Exception:
|
||||
return False
|
||||
@@ -14,18 +14,19 @@ This document describes all available API endpoints for the Mailcow Logs Viewer
|
||||
2. [Health & Info](#health--info)
|
||||
3. [Job Status Tracking](#job-status-tracking)
|
||||
4. [Domains](#domains)
|
||||
5. [Messages (Unified View)](#messages-unified-view)
|
||||
6. [Logs](#logs)
|
||||
5. [Mailbox Statistics](#mailbox-statistics)
|
||||
6. [Messages (Unified View)](#messages-unified-view)
|
||||
7. [Logs](#logs)
|
||||
- [Postfix Logs](#postfix-logs)
|
||||
- [Rspamd Logs](#rspamd-logs)
|
||||
- [Netfilter Logs](#netfilter-logs)
|
||||
7. [Queue & Quarantine](#queue--quarantine)
|
||||
8. [Statistics](#statistics)
|
||||
9. [Status](#status)
|
||||
10. [Settings](#settings)
|
||||
8. [Queue & Quarantine](#queue--quarantine)
|
||||
9. [Statistics](#statistics)
|
||||
10. [Status](#status)
|
||||
11. [Settings](#settings)
|
||||
- [SMTP & IMAP Test](#smtp--imap-test)
|
||||
11. [Export](#export)
|
||||
12. [DMARC](#dmarc)
|
||||
12. [Export](#export)
|
||||
13. [DMARC](#dmarc)
|
||||
- [DMARC IMAP Auto-Import](#dmarc-imap-auto-import)
|
||||
|
||||
---
|
||||
@@ -736,6 +737,200 @@ POST /api/domains/example.com/check-dns
|
||||
|
||||
---
|
||||
|
||||
## Mailbox Statistics
|
||||
|
||||
### GET /api/mailbox-stats/summary
|
||||
|
||||
Get summary statistics for all mailboxes.
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"total_mailboxes": 25,
|
||||
"active_mailboxes": 23,
|
||||
"inactive_mailboxes": 2,
|
||||
"total_sent": 1234,
|
||||
"total_received": 5678,
|
||||
"sent_failed": 45,
|
||||
"failure_rate": 3.6,
|
||||
"date_range": "30days",
|
||||
"start_date": "2026-01-16T00:00:00Z",
|
||||
"end_date": "2026-02-16T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### GET /api/mailbox-stats/all
|
||||
|
||||
Get all mailbox statistics with message counts and aliases (paginated).
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `domain` | string | Filter by domain name |
|
||||
| `active_only` | bool | Only show active mailboxes (default: `true`) |
|
||||
| `hide_zero` | bool | Hide mailboxes with zero activity (default: `false`) |
|
||||
| `search` | string | Search mailbox username, name, or alias address |
|
||||
| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) |
|
||||
| `sort_by` | string | Sort by: `sent_total`, `received_total`, `failure_rate`, `quota_used`, `username` |
|
||||
| `sort_order` | string | Sort order: `asc`, `desc` (default: `desc`) |
|
||||
| `page` | int | Page number (default: `1`) |
|
||||
| `page_size` | int | Items per page, 10-100 (default: `50`) |
|
||||
|
||||
**Example Request:**
|
||||
```
|
||||
GET /api/mailbox-stats/all?date_range=30days&active_only=true&hide_zero=true&sort_by=sent_total&sort_order=desc&page=1
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"total": 25,
|
||||
"page": 1,
|
||||
"page_size": 50,
|
||||
"total_pages": 1,
|
||||
"date_range": "30days",
|
||||
"start_date": "2026-01-16T00:00:00Z",
|
||||
"end_date": "2026-02-16T00:00:00Z",
|
||||
"mailboxes": [
|
||||
{
|
||||
"id": 1,
|
||||
"username": "user@example.com",
|
||||
"domain": "example.com",
|
||||
"name": "John Doe",
|
||||
"active": true,
|
||||
"quota": 1073741824,
|
||||
"quota_formatted": "1.0 GB",
|
||||
"quota_used": 536870912,
|
||||
"quota_used_formatted": "512 MB",
|
||||
"percent_in_use": 50.0,
|
||||
"messages_in_mailbox": 1234,
|
||||
"last_imap_login": "2026-01-15T10:30:00Z",
|
||||
"last_pop3_login": null,
|
||||
"last_smtp_login": "2026-01-16T08:45:00Z",
|
||||
"rl_value": 100,
|
||||
"rl_frame": "m",
|
||||
"attributes": {
|
||||
"imap_access": "1",
|
||||
"pop3_access": "0",
|
||||
"smtp_access": "1",
|
||||
"sieve_access": "1",
|
||||
"sogo_access": "1",
|
||||
"tls_enforce_in": "0",
|
||||
"tls_enforce_out": "0"
|
||||
},
|
||||
"mailbox_counts": {
|
||||
"sent_total": 150,
|
||||
"sent_delivered": 145,
|
||||
"sent_bounced": 3,
|
||||
"sent_deferred": 2,
|
||||
"sent_rejected": 0,
|
||||
"sent_failed": 5,
|
||||
"received_total": 320,
|
||||
"failure_rate": 3.3
|
||||
},
|
||||
"aliases": [
|
||||
{
|
||||
"alias_address": "info@example.com",
|
||||
"active": true,
|
||||
"is_catch_all": false,
|
||||
"sent_total": 50,
|
||||
"sent_delivered": 48,
|
||||
"sent_bounced": 2,
|
||||
"sent_deferred": 0,
|
||||
"sent_rejected": 0,
|
||||
"sent_failed": 2,
|
||||
"received_total": 100,
|
||||
"failure_rate": 4.0
|
||||
}
|
||||
],
|
||||
"alias_count": 1,
|
||||
"combined_sent": 200,
|
||||
"combined_received": 420,
|
||||
"combined_total": 620,
|
||||
"combined_failed": 7,
|
||||
"combined_failure_rate": 3.5,
|
||||
"created": "2025-01-01T00:00:00Z",
|
||||
"modified": "2026-01-15T12:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Response Fields:**
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `username` | Email address of the mailbox |
|
||||
| `name` | Display name |
|
||||
| `active` | Whether mailbox is active in Mailcow |
|
||||
| `quota` / `quota_used` | Quota in bytes |
|
||||
| `percent_in_use` | Quota usage percentage |
|
||||
| `messages_in_mailbox` | Number of messages stored |
|
||||
| `last_*_login` | Last login timestamps (null if never) |
|
||||
| `rl_value` / `rl_frame` | Rate limiting (e.g., 100/m = 100 per minute) |
|
||||
| `attributes` | Access permissions from Mailcow |
|
||||
| `mailbox_counts` | Message statistics for mailbox only |
|
||||
| `aliases` | Array of alias statistics |
|
||||
| `combined_*` | Combined totals (mailbox + all aliases) |
|
||||
| `created` / `modified` | Mailbox creation and last update timestamps |
|
||||
|
||||
---
|
||||
|
||||
### GET /api/mailbox-stats/domains
|
||||
|
||||
Get list of domains with mailbox counts for filter dropdown.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"domains": [
|
||||
{
|
||||
"domain": "example.com",
|
||||
"mailbox_count": 15
|
||||
},
|
||||
{
|
||||
"domain": "company.org",
|
||||
"mailbox_count": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Caching
|
||||
|
||||
The Mailbox Statistics API uses in-memory caching to improve performance:
|
||||
|
||||
| Setting | Value |
|
||||
|---------|-------|
|
||||
| **Cache TTL** | 5 minutes (300 seconds) |
|
||||
| **Cache Scope** | Per unique query parameter combination |
|
||||
| **Cached Parameters** | domain, active_only, hide_zero, search, date_range, start_date, end_date, sort_by, sort_order, page, page_size |
|
||||
|
||||
**Cache Behavior:**
|
||||
- First request with specific parameters fetches from database and caches result
|
||||
- Subsequent requests with identical parameters return cached data
|
||||
- Cache automatically expires after 5 minutes
|
||||
- Changing any parameter results in a cache miss (new database query)
|
||||
|
||||
**Cache Management:**
|
||||
```python
|
||||
from app.routers.mailbox_stats import clear_stats_cache
|
||||
|
||||
# Clear all stats cache (e.g., after data import)
|
||||
clear_stats_cache()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Messages (Unified View)
|
||||
|
||||
### GET /messages
|
||||
@@ -2505,6 +2700,165 @@ Get history of IMAP sync operations.
|
||||
|
||||
---
|
||||
|
||||
## TLS-RPT (TLS Reporting)
|
||||
|
||||
### Overview
|
||||
|
||||
TLS-RPT (TLS Reporting) provides visibility into TLS connection failures when other mail servers attempt to deliver emails to your domain. This helps identify MTA-STS policy issues and certificate problems.
|
||||
|
||||
---
|
||||
|
||||
### GET /api/dmarc/domains/{domain}/tls-reports
|
||||
|
||||
Get TLS reports for a specific domain (individual reports).
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `domain` | string | Domain name |
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `days` | integer | 30 | Number of days to look back |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"domain": "example.com",
|
||||
"total": 15,
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"report_id": "2026-01-14T00:00:00Z!example.com!google.com",
|
||||
"organization_name": "Google Inc.",
|
||||
"start_datetime": "2026-01-14T00:00:00Z",
|
||||
"end_datetime": "2026-01-15T00:00:00Z",
|
||||
"total_success": 1250,
|
||||
"total_fail": 5,
|
||||
"success_rate": 99.6
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### GET /api/dmarc/domains/{domain}/tls-reports/daily
|
||||
|
||||
Get TLS reports aggregated by date (daily view).
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `domain` | string | Domain name |
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `days` | integer | 30 | Number of days to look back |
|
||||
| `page` | integer | 1 | Page number |
|
||||
| `page_size` | integer | 20 | Items per page |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"domain": "example.com",
|
||||
"totals": {
|
||||
"total_days": 14,
|
||||
"total_reports": 28,
|
||||
"total_successful_sessions": 15000,
|
||||
"total_failed_sessions": 25,
|
||||
"overall_success_rate": 99.83
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"date": "2026-01-17",
|
||||
"report_count": 3,
|
||||
"organization_count": 2,
|
||||
"organizations": ["Google Inc.", "Microsoft Corporation"],
|
||||
"total_success": 1500,
|
||||
"total_fail": 2,
|
||||
"success_rate": 99.87
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### GET /api/dmarc/domains/{domain}/tls-reports/{report_date}/details
|
||||
|
||||
Get detailed TLS reports for a specific date.
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `domain` | string | Domain name |
|
||||
| `report_date` | string | Date in YYYY-MM-DD format |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"domain": "example.com",
|
||||
"date": "2026-01-17",
|
||||
"stats": {
|
||||
"total_reports": 3,
|
||||
"total_providers": 2,
|
||||
"total_success": 1500,
|
||||
"total_fail": 2,
|
||||
"total_sessions": 1502,
|
||||
"success_rate": 99.87
|
||||
},
|
||||
"providers": [
|
||||
{
|
||||
"report_id": "2026-01-17T00:00:00Z!example.com!google.com",
|
||||
"organization_name": "Google Inc.",
|
||||
"contact_info": "smtp-tls-reporting@google.com",
|
||||
"start_datetime": "2026-01-17T00:00:00Z",
|
||||
"end_datetime": "2026-01-18T00:00:00Z",
|
||||
"successful_sessions": 1200,
|
||||
"failed_sessions": 1,
|
||||
"total_sessions": 1201,
|
||||
"success_rate": 99.92,
|
||||
"policies": [
|
||||
{
|
||||
"policy_type": "sts",
|
||||
"policy_domain": "example.com",
|
||||
"mx_host": "mail.example.com",
|
||||
"successful_sessions": 1200,
|
||||
"failed_sessions": 1,
|
||||
"total_sessions": 1201,
|
||||
"success_rate": 99.92,
|
||||
"failure_details": null
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### POST /api/dmarc/upload (TLS-RPT Support)
|
||||
|
||||
The existing DMARC upload endpoint also accepts TLS-RPT reports.
|
||||
|
||||
**Supported TLS-RPT Formats:**
|
||||
- `.json.gz` - Gzip-compressed JSON (standard format)
|
||||
- `.json` - Plain JSON
|
||||
|
||||
**Detection:**
|
||||
- File is identified as TLS-RPT if JSON contains `"policies"` array
|
||||
- TLS-RPT reports use RFC 8460 JSON format
|
||||
|
||||
---
|
||||
|
||||
## Error Responses
|
||||
|
||||
All endpoints may return the following error responses:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
To maintain high deliverability and robust domain security, **mailcow-logs-viewer** provides deep inspection and automated monitoring of the three core email authentication protocols: **SPF**, **DKIM**, and **DMARC**.
|
||||
|
||||
### The Authentication Stack
|
||||
## The Authentication Stack
|
||||
|
||||
| Protocol | Technical Purpose | System Validation Logic |
|
||||
| --- | --- | --- |
|
||||
@@ -12,7 +12,7 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe
|
||||
|
||||
---
|
||||
|
||||
### Advanced Monitoring & Intelligence
|
||||
## Advanced Monitoring & Intelligence
|
||||
|
||||
**mailcow-logs-viewer** goes beyond basic record checking by providing a comprehensive analysis of your mail flow:
|
||||
|
||||
@@ -23,26 +23,48 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe
|
||||
|
||||
---
|
||||
|
||||
### 🚀 Implementation: Enabling DMARC Reporting
|
||||
## 🚀 Implementation: Enabling DMARC Reporting
|
||||
|
||||
To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system.
|
||||
|
||||
#### 1. DNS Configuration
|
||||
### 1. DNS Configuration
|
||||
|
||||
Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`):
|
||||
|
||||
```text
|
||||
v=DMARC1; p=none; rua=mailto:dmarc-reports@yourdomain.com;
|
||||
v=DMARC1; p=none; rua=mailto:dmarc@example.net;
|
||||
|
||||
```
|
||||
|
||||
#### 2. Parameter Details
|
||||
### 2. Parameter Details
|
||||
|
||||
* **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated.
|
||||
* **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer.
|
||||
* **`v=DMARC1`:** Required version prefix.
|
||||
|
||||
#### 3. Transitioning to Enforcement
|
||||
### 3. External Domain Reporting (Verification)
|
||||
|
||||
If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports.
|
||||
|
||||
Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam.
|
||||
|
||||
#### Option 1: Specific Domain Authorization (Recommended for security)
|
||||
Add a TXT record to the DNS of the **receiving domain** (`example.net`):
|
||||
|
||||
| Host / Name | Value |
|
||||
| :--- | :--- |
|
||||
| `example.com._report._dmarc.example.net` | `v=DMARC1;` |
|
||||
|
||||
#### Option 2: Wildcard Authorization (Recommended for multiple domains)
|
||||
If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once:
|
||||
|
||||
| Host / Name | Value |
|
||||
| :--- | :--- |
|
||||
| `*._report._dmarc.example.net` | `v=DMARC1;` |
|
||||
|
||||
*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.*
|
||||
|
||||
### 4. Transitioning to Enforcement
|
||||
|
||||
Once the dashboard confirms that your legitimate traffic (including third-party SaaS) is passing SPF/DKIM alignment, you should update your policy to `p=quarantine` or `p=reject` to fully secure your domain against spoofing.
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ The DMARC Reports page provides detailed analysis of DMARC aggregate reports rec
|
||||
- Tells receiving servers what to do with emails that fail validation
|
||||
- Provides reports about email authentication results
|
||||
|
||||
---
|
||||
|
||||
## Report Types
|
||||
|
||||
### Aggregate Reports (XML)
|
||||
@@ -64,6 +66,8 @@ Click an IP address to see:
|
||||
- **Volume**: Number of messages from this source
|
||||
- **Reverse DNS**: Hostname associated with the IP
|
||||
|
||||
---
|
||||
|
||||
## Understanding Report Data
|
||||
|
||||
### DMARC Alignment
|
||||
@@ -81,6 +85,8 @@ What the receiving server did with the email:
|
||||
- **Policy**: What your DMARC record tells servers to do
|
||||
- **Disposition**: What servers actually did (they may override your policy)
|
||||
|
||||
---
|
||||
|
||||
## Key Features
|
||||
|
||||
### Geographic Visualization
|
||||
@@ -103,6 +109,8 @@ What the receiving server did with the email:
|
||||
- DMARC policy effectiveness
|
||||
- Recommendations for policy adjustments
|
||||
|
||||
---
|
||||
|
||||
## Common Scenarios
|
||||
|
||||
### Legitimate Sources Failing
|
||||
@@ -138,6 +146,51 @@ What the receiving server did with the email:
|
||||
- Verify DKIM is configured on all sending systems
|
||||
- Look for email forwarding issues
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Implementation: Enabling DMARC Reporting
|
||||
|
||||
To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system.
|
||||
|
||||
### 1. DNS Configuration
|
||||
|
||||
Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`):
|
||||
|
||||
```text
|
||||
v=DMARC1; p=none; rua=mailto:dmarc@example.net;
|
||||
|
||||
```
|
||||
|
||||
### 2. Parameter Details
|
||||
|
||||
* **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated.
|
||||
* **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer.
|
||||
* **`v=DMARC1`:** Required version prefix.
|
||||
|
||||
### 3. External Domain Reporting (Verification)
|
||||
|
||||
If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports.
|
||||
|
||||
Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam.
|
||||
|
||||
#### Option 1: Specific Domain Authorization (Recommended for security)
|
||||
Add a TXT record to the DNS of the **receiving domain** (`example.net`):
|
||||
|
||||
| Host / Name | Value |
|
||||
| :--- | :--- |
|
||||
| `example.com._report._dmarc.example.net` | `v=DMARC1;` |
|
||||
|
||||
#### Option 2: Wildcard Authorization (Recommended for multiple domains)
|
||||
If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once:
|
||||
|
||||
| Host / Name | Value |
|
||||
| :--- | :--- |
|
||||
| `*._report._dmarc.example.net` | `v=DMARC1;` |
|
||||
|
||||
*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.*
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Policy Progression
|
||||
@@ -162,6 +215,8 @@ When using email services (marketing, support desk, etc.):
|
||||
- Test before going live
|
||||
- Monitor their authentication success
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### No Reports Appearing
|
||||
@@ -185,6 +240,8 @@ When using email services (marketing, support desk, etc.):
|
||||
- Older reports are automatically deleted to save space
|
||||
- Export reports before they're deleted if long-term analysis is needed
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Identifying Spoofing
|
||||
|
||||
@@ -36,6 +36,8 @@ The system automatically validates three critical DNS records:
|
||||
- `none`: Monitoring only (weakest)
|
||||
- **Status**: Same indicators as SPF
|
||||
|
||||
---
|
||||
|
||||
## How to Use
|
||||
|
||||
### Viewing Domains
|
||||
@@ -59,6 +61,8 @@ When you expand a domain, the DNS Security section shows:
|
||||
- Specific warnings or recommendations
|
||||
- Time of last validation
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Regular Monitoring**: Review DNS status regularly, especially after DNS changes
|
||||
@@ -67,6 +71,8 @@ When you expand a domain, the DNS Security section shows:
|
||||
4. **SPF Optimization**: Keep SPF records concise (under 10 DNS lookups)
|
||||
5. **DKIM Key Rotation**: Periodically rotate DKIM keys for security
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### DNS Changes Not Reflected
|
||||
|
||||
@@ -69,11 +69,18 @@ CORRELATION_CHECK_INTERVAL=120
|
||||
SMTP_ENABLED=false
|
||||
SMTP_HOST=
|
||||
SMTP_PORT=
|
||||
# Use TLS instead of SSL (recommended)
|
||||
SMTP_USE_TLS=true
|
||||
# Use SSL instead of TLS
|
||||
SMTP_USE_SSL=false
|
||||
SMTP_USER=
|
||||
SMTP_PASSWORD=
|
||||
SMTP_FROM=noreply@yourdomain.com
|
||||
|
||||
# SMTP Relay Mode - Set to true for local relay servers that don't require authentication
|
||||
# When enabled, username and password are not required
|
||||
SMTP_RELAY_MODE=false
|
||||
|
||||
# =============================================================================
|
||||
# Admin Email
|
||||
# =============================================================================
|
||||
@@ -102,6 +109,7 @@ DMARC_IMAP_FOLDER=INBOX
|
||||
DMARC_IMAP_DELETE_AFTER=true
|
||||
DMARC_IMAP_INTERVAL=3600
|
||||
DMARC_IMAP_RUN_ON_STARTUP=true
|
||||
DMARC_IMAP_BATCH_SIZE=10
|
||||
|
||||
# DMARC Error Email Override (optional - uses ADMIN_EMAIL if not set)
|
||||
DMARC_ERROR_EMAIL=
|
||||
|
||||
1683
frontend/app.js
1683
frontend/app.js
File diff suppressed because it is too large
Load Diff
1074
frontend/index.html
1074
frontend/index.html
File diff suppressed because it is too large
Load Diff
325
frontend/router.js
Normal file
325
frontend/router.js
Normal file
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Router module for SPA clean URL navigation
|
||||
* Handles History API based routing for the Mailcow Logs Viewer
|
||||
*/
|
||||
|
||||
// Valid base routes for the SPA
|
||||
const VALID_ROUTES = [
|
||||
'dashboard',
|
||||
'messages',
|
||||
'netfilter',
|
||||
'queue',
|
||||
'quarantine',
|
||||
'status',
|
||||
'domains',
|
||||
'dmarc',
|
||||
'mailbox-stats',
|
||||
'settings'
|
||||
];
|
||||
|
||||
/**
|
||||
* Parse the current URL path into route components
|
||||
* @returns {Object} Route info with baseRoute and optional params
|
||||
*/
|
||||
function parseRoute() {
|
||||
const path = window.location.pathname;
|
||||
|
||||
// Root path = dashboard
|
||||
if (path === '/' || path === '') {
|
||||
return { baseRoute: 'dashboard', params: {} };
|
||||
}
|
||||
|
||||
// Split path into segments
|
||||
const segments = path.split('/').filter(s => s.length > 0);
|
||||
|
||||
if (segments.length === 0) {
|
||||
return { baseRoute: 'dashboard', params: {} };
|
||||
}
|
||||
|
||||
const baseRoute = segments[0];
|
||||
|
||||
// Special handling for DMARC nested routes
|
||||
if (baseRoute === 'dmarc' && segments.length > 1) {
|
||||
return parseDmarcRoute(segments);
|
||||
}
|
||||
|
||||
// Validate base route
|
||||
if (!VALID_ROUTES.includes(baseRoute)) {
|
||||
console.warn(`Unknown route: ${baseRoute}, defaulting to dashboard`);
|
||||
return { baseRoute: 'dashboard', params: {} };
|
||||
}
|
||||
|
||||
return { baseRoute, params: {} };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse DMARC-specific nested routes
|
||||
* @param {string[]} segments - URL path segments
|
||||
* @returns {Object} Route info for DMARC
|
||||
*/
|
||||
function parseDmarcRoute(segments) {
|
||||
// segments[0] = 'dmarc'
|
||||
// segments[1] = domain (e.g., 'example.com')
|
||||
// segments[2] = type ('report', 'source', 'tls', 'reports', 'sources')
|
||||
// segments[3] = id (date or IP)
|
||||
|
||||
const params = { domain: null, type: null, id: null };
|
||||
|
||||
if (segments.length >= 2) {
|
||||
params.domain = decodeURIComponent(segments[1]);
|
||||
}
|
||||
|
||||
if (segments.length >= 3) {
|
||||
params.type = segments[2];
|
||||
}
|
||||
|
||||
if (segments.length >= 4) {
|
||||
params.id = decodeURIComponent(segments[3]);
|
||||
}
|
||||
|
||||
return { baseRoute: 'dmarc', params };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a URL path from route components
|
||||
* @param {string} baseRoute - The base route
|
||||
* @param {Object} params - Optional parameters
|
||||
* @returns {string} The URL path
|
||||
*/
|
||||
function buildPath(baseRoute, params = {}) {
|
||||
if (baseRoute === 'dashboard') {
|
||||
return '/';
|
||||
}
|
||||
|
||||
let path = `/${baseRoute}`;
|
||||
|
||||
// Handle DMARC nested routes
|
||||
if (baseRoute === 'dmarc' && params.domain) {
|
||||
path += `/${encodeURIComponent(params.domain)}`;
|
||||
|
||||
if (params.type) {
|
||||
path += `/${params.type}`;
|
||||
|
||||
if (params.id) {
|
||||
path += `/${encodeURIComponent(params.id)}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a route - updates URL and switches tab
|
||||
* @param {string} route - The base route to navigate to
|
||||
* @param {Object} params - Optional route parameters (for nested routes)
|
||||
* @param {boolean} updateHistory - Whether to push to browser history (default: true)
|
||||
*/
|
||||
function navigateTo(route, params = {}, updateHistory = true) {
|
||||
// Handle legacy calls with just route string
|
||||
if (typeof params === 'boolean') {
|
||||
updateHistory = params;
|
||||
params = {};
|
||||
}
|
||||
|
||||
// Validate base route
|
||||
if (!VALID_ROUTES.includes(route)) {
|
||||
console.warn(`Invalid route: ${route}, defaulting to dashboard`);
|
||||
route = 'dashboard';
|
||||
params = {};
|
||||
}
|
||||
|
||||
// Build the new path
|
||||
const newPath = buildPath(route, params);
|
||||
|
||||
// Update history if path actually changed
|
||||
if (updateHistory && window.location.pathname !== newPath) {
|
||||
history.pushState({ route, params }, '', newPath);
|
||||
}
|
||||
|
||||
// Always switch to the tab (even if URL is same, to handle returning to main view)
|
||||
if (typeof switchTab === 'function') {
|
||||
switchTab(route, params);
|
||||
} else {
|
||||
console.error('switchTab function not found');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate specifically within DMARC section
|
||||
* @param {string} domain - Domain name (null for domains list)
|
||||
* @param {string} type - Type: 'reports', 'sources', 'tls', 'report', 'source'
|
||||
* @param {string} id - ID: date for report/tls, IP for source
|
||||
*/
|
||||
function navigateToDmarc(domain = null, type = null, id = null) {
|
||||
const params = {};
|
||||
if (domain) params.domain = domain;
|
||||
if (type) params.type = type;
|
||||
if (id) params.id = id;
|
||||
|
||||
navigateTo('dmarc', params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current route from URL path
|
||||
* @returns {string} The current base route name
|
||||
*/
|
||||
function getCurrentRoute() {
|
||||
return parseRoute().baseRoute;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current route with full parameters
|
||||
* @returns {Object} Route info with baseRoute and params
|
||||
*/
|
||||
function getFullRoute() {
|
||||
return parseRoute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the router
|
||||
* Sets up popstate listener and returns initial route info
|
||||
* @returns {Object} The initial route info { baseRoute, params }
|
||||
*/
|
||||
function initRouter() {
|
||||
console.log('Initializing SPA router...');
|
||||
|
||||
// Handle browser back/forward buttons
|
||||
window.addEventListener('popstate', (event) => {
|
||||
const routeInfo = event.state || parseRoute();
|
||||
const route = routeInfo.route || routeInfo.baseRoute || getCurrentRoute();
|
||||
const params = routeInfo.params || {};
|
||||
|
||||
console.log('Popstate event, navigating to:', route, params);
|
||||
|
||||
// Use switchTab directly to avoid pushing duplicate history entries
|
||||
if (typeof switchTab === 'function') {
|
||||
switchTab(route, params);
|
||||
}
|
||||
});
|
||||
|
||||
// Get initial route from URL
|
||||
const routeInfo = parseRoute();
|
||||
|
||||
// Replace current history state with route info
|
||||
history.replaceState({ route: routeInfo.baseRoute, params: routeInfo.params }, '', window.location.pathname);
|
||||
|
||||
console.log('Router initialized, initial route:', routeInfo);
|
||||
return routeInfo;
|
||||
}
|
||||
|
||||
// Tab labels for mobile menu display
|
||||
const TAB_LABELS = {
|
||||
'dashboard': 'Dashboard',
|
||||
'messages': 'Messages',
|
||||
'netfilter': 'Security',
|
||||
'queue': 'Queue',
|
||||
'quarantine': 'Quarantine',
|
||||
'status': 'Status',
|
||||
'domains': 'Domains',
|
||||
'dmarc': 'DMARC',
|
||||
'mailbox-stats': 'Mailbox Stats',
|
||||
'settings': 'Settings'
|
||||
};
|
||||
|
||||
/**
|
||||
* Toggle mobile menu open/close
|
||||
*/
|
||||
function toggleMobileMenu() {
|
||||
const mobileMenu = document.getElementById('mobile-menu');
|
||||
const hamburgerBtn = document.getElementById('hamburger-btn');
|
||||
|
||||
if (mobileMenu && hamburgerBtn) {
|
||||
mobileMenu.classList.toggle('active');
|
||||
hamburgerBtn.classList.toggle('active');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close mobile menu
|
||||
*/
|
||||
function closeMobileMenu() {
|
||||
const mobileMenu = document.getElementById('mobile-menu');
|
||||
const hamburgerBtn = document.getElementById('hamburger-btn');
|
||||
|
||||
if (mobileMenu && hamburgerBtn) {
|
||||
mobileMenu.classList.remove('active');
|
||||
hamburgerBtn.classList.remove('active');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate from mobile menu - closes menu and navigates
|
||||
* @param {string} route - The route to navigate to
|
||||
*/
|
||||
function navigateToMobile(route) {
|
||||
// Close the mobile menu first
|
||||
closeMobileMenu();
|
||||
|
||||
// Update mobile menu active state
|
||||
updateMobileMenuActiveState(route);
|
||||
|
||||
// Update the current tab label
|
||||
updateCurrentTabLabel(route);
|
||||
|
||||
// Navigate to the route
|
||||
navigateTo(route);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update mobile menu item active states
|
||||
* @param {string} activeRoute - The currently active route
|
||||
*/
|
||||
function updateMobileMenuActiveState(activeRoute) {
|
||||
// Remove active class from all mobile menu items
|
||||
document.querySelectorAll('.mobile-menu-item').forEach(item => {
|
||||
item.classList.remove('active');
|
||||
});
|
||||
|
||||
// Add active class to the new active item
|
||||
const activeItem = document.getElementById(`mobile-tab-${activeRoute}`);
|
||||
if (activeItem) {
|
||||
activeItem.classList.add('active');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the current tab label shown on mobile
|
||||
* @param {string} route - The current route
|
||||
*/
|
||||
function updateCurrentTabLabel(route) {
|
||||
const label = document.getElementById('current-tab-label');
|
||||
if (label) {
|
||||
label.textContent = TAB_LABELS[route] || route;
|
||||
}
|
||||
}
|
||||
|
||||
// Close mobile menu when clicking outside
|
||||
document.addEventListener('click', (event) => {
|
||||
const mobileMenu = document.getElementById('mobile-menu');
|
||||
const hamburgerBtn = document.getElementById('hamburger-btn');
|
||||
|
||||
if (mobileMenu && mobileMenu.classList.contains('active')) {
|
||||
// Check if click is outside the menu content and hamburger button
|
||||
const menuContent = mobileMenu.querySelector('.mobile-menu-content');
|
||||
if (!menuContent.contains(event.target) && !hamburgerBtn.contains(event.target)) {
|
||||
closeMobileMenu();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Expose functions globally
|
||||
window.navigateTo = navigateTo;
|
||||
window.navigateToDmarc = navigateToDmarc;
|
||||
window.getCurrentRoute = getCurrentRoute;
|
||||
window.getFullRoute = getFullRoute;
|
||||
window.parseRoute = parseRoute;
|
||||
window.buildPath = buildPath;
|
||||
window.initRouter = initRouter;
|
||||
window.VALID_ROUTES = VALID_ROUTES;
|
||||
window.toggleMobileMenu = toggleMobileMenu;
|
||||
window.closeMobileMenu = closeMobileMenu;
|
||||
window.navigateToMobile = navigateToMobile;
|
||||
window.updateMobileMenuActiveState = updateMobileMenuActiveState;
|
||||
window.updateCurrentTabLabel = updateCurrentTabLabel;
|
||||
window.TAB_LABELS = TAB_LABELS;
|
||||
Reference in New Issue
Block a user