diff --git a/.gitignore b/.gitignore index 8148291..7ad41a5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ backup/ htmlcov/ .tox/ docker-compose-dev.yml +Dev/ # Documentation docs/_build/ @@ -76,4 +77,5 @@ Thumbs.db .DS_Store # Maxmind -data \ No newline at end of file +data +old-data \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f96f32e..980a6ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,205 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.0] - 2026-01-20 + +### Added + +#### Mailbox Statistics Page +- **Complete Mailbox Statistics Feature**: New page showing per-mailbox message statistics + - Summary cards: Total Sent, Received, Failed, and Failure Rate + - Accordion-style mailbox list with expandable details + - Message counts aggregated from MessageCorrelation table (not Mailcow API) + - Per-alias message statistics with sent/received/failed counts + - Combined totals (mailbox + all its aliases) + +- **Clickable Statistics Links**: All stat cards and alias table cells are clickable + - Click on any stat (Sent, Received, Internal, Delivered, Deferred, Bounced, Rejected) to navigate to Messages page + - Search automatically pre-filled with the mailbox/alias email address + +- **Mailbox Details Display (Domains-style)**: + - Quota usage with percentage + - Messages in mailbox count + - Last IMAP/SMTP/POP3 login times + - Created and Modified dates + - Rate limiting settings (value/frame) + - Access permissions indicators: IMAP, POP3, SMTP, Sieve, SOGo, TLS Enforce + - Color-coded status dots (green=enabled, red=disabled) + +- **Filtering & Search**: + - **Date Range Picker with Presets**: + - Quick select preset buttons: Today, 7 Days, 30 Days, 90 Days + - Custom date range with From/To date inputs + - Domain filter dropdown + - Search by mailbox username, name, or alias address + - "Active Only" checkbox (default: checked) + - "Hide Zero Activity" checkbox (default: checked) - filters mailboxes and aliases with no messages + - Sort by: Sent, Received, Failure Rate, Quota Used, Username + +- **Pagination**: 50 mailboxes per page with navigation controls + +#### Background Jobs +- **Mailbox Statistics Job**: Fetches mailbox data from Mailcow API every 5 minutes + - Syncs quota, messages, login times, rate limits, and attributes + - Marks deleted mailboxes as inactive (preserves historical data) + +- **Alias Statistics Job**: Fetches alias data from Mailcow API every 5 minutes + - Links aliases to their target mailboxes + - Marks deleted aliases as inactive (preserves historical data) + +#### SMTP Relay Mode +- **No-Authentication SMTP Support**: New option for sending emails through local relay servers without credentials + - Enable via `SMTP_RELAY_MODE=true` in environment variables + - When enabled, `SMTP_USER` and `SMTP_PASSWORD` are not required + - Useful for local Postfix relay servers, internal mail gateways, or trusted SMTP relays + - Connection test in Settings page properly handles relay mode authentication bypass + +#### Clean URL Routing (History API) +- **Shareable URLs for All Pages**: Implemented History API-based routing for the SPA + - Direct navigation to any tab via clean URLs (e.g., `/dashboard`, `/messages`, `/dmarc`, `/settings`) + - Browser Back/Forward buttons now work correctly between pages + - URLs can be bookmarked and shared + +- **DMARC Nested Routes**: Deep linking support for all DMARC views + - `/dmarc` - Domains list + - `/dmarc/{domain}` - Domain overview + - `/dmarc/{domain}/reports` - Daily Reports tab + - `/dmarc/{domain}/sources` - Source IPs tab + - `/dmarc/{domain}/tls` - TLS Reports tab + - `/dmarc/{domain}/report/{date}` - Specific daily report details + - `/dmarc/{domain}/source/{ip}` - Specific source IP details + +- **Removed Internal Back Button**: DMARC section no longer uses custom back button + - Users now use browser's native Back button + - Cleaner UI without duplicate navigation controls + +#### TLS-RPT (TLS Reporting) Support +- **Complete TLS-RPT Implementation**: Full support for TLS aggregate reports (RFC 8460) + - TLS-RPT parser for JSON reports (gzip compressed) + - Database models for TLS reports and policies + - IMAP auto-import support for TLS-RPT emails + - Manual upload support for TLS-RPT files + +- **TLS Reports Tab in DMARC Page**: + - New "TLS Reports" sub-tab alongside Daily Reports and Source IPs + - Daily aggregated view showing reports grouped by date + - Success rate with color-coded progress bars (green ≥95%, yellow ≥80%, red <80%) + - Provider breakdown with session counts + +- **TLS Report Details View**: + - Click any daily report to see detailed breakdown + - Stats cards: Sessions, Success Rate, Successful, Failed + - Providers table with per-provider success rates + +- **TLS in Domain List**: + - TLS Success Rate column in DMARC domains table + - TLS report count displayed per domain + - Domains with only TLS reports (no DMARC) now included in list + +#### DMARC Navigation Improvements +- **Breadcrumb Navigation**: Clear path indicator for all DMARC views + - Shows current location: `domain.com > Daily Reports > Jan 14, 2026` + - Clickable links to navigate back to any level + - Displayed below page description + +#### Mobile Navigation Hamburger Menu +- **Hamburger Menu for Mobile**: + - Replaced horizontal scrolling tabs with a proper hamburger menu on mobile devices + +### Fixed + +#### DMARC Source IPs - Broken Flag Images +- **Fixed broken flag images when MAXMIND is not configured**: When GeoIP integration is not set up, the Source IPs tab was showing broken images + - Now displays a generic server icon instead of a broken image when country data is unavailable + - Flag is completely hidden in source details view when no GeoIP data exists + - Added `onerror` fallback handlers to gracefully handle missing flag files + - Improves UX for users who haven't configured MAXMIND integration + +#### DMARC Parser +- **DMARC 2.0 XML Namespace Support**: Fixed parsing error for DMARC reports using XML namespaces + - Reports from GMX and other providers using the new format now parse correctly + - Parser now tries both namespaced and non-namespaced element lookups + +### Improved + +#### Backend API Performance +- **In-Memory Caching for Statistics API**: Added 5-minute TTL cache for `/api/mailbox-stats/all` endpoint + - Cache key generated from all query parameters + - First request fetches from database, subsequent requests return from cache + - Cache automatically expires after 5 minutes for fresh data + - Significantly reduces database load and improves response times + +- **In-Memory Caching for DMARC API**: Added 5-minute TTL cache for `/api/dmarc/domains` endpoint + - Reduces heavy database queries for domain statistics + - Cache cleared on new report imports + +#### DMARC IMAP Auto-Import +- **Batch Processing**: Emails are now processed in configurable batches to prevent memory issues + - New `DMARC_IMAP_BATCH_SIZE` environment variable (default: 10) + - Processes emails in chunks, re-searching after each batch + - Progress logging shows batch number and completion status + - Prevents application crashes when syncing mailboxes with lots of emails + +- **UID-Based Email Handling**: Fixed "Invalid messageset" IMAP errors + - Changed from sequence numbers to UIDs for all IMAP operations + - UIDs remain stable even after deleting emails during sync + - Affects SEARCH, FETCH, STORE operations + +- **Flexible DMARC Email Detection**: Now supports more email providers + - Yahoo and other providers that don't include "Report-ID:" now detected correctly + - Primary validation is now attachment-based (.xml.gz or .zip files) + - Accepts: "Report Domain:" only, "Report Domain:" + "Submitter:", or "DMARC" keyword + +- **Improved Error Reporting in Notifications**: + - Error notification emails now show actual error messages + - Parse failures show: "Failed to parse: filename.xml.gz" + - Processing errors show: "Error processing filename: exception details" + - Duplicate reports no longer counted as failures + +- **Infinite Loop Prevention**: Fixed sync running endlessly when emails fail validation + - Added `UNSEEN` filter to search criteria + - Failed or processed emails are marked as Seen and excluded from next search + - Prevents re-processing the same emails repeatedly + +- **Microsoft Outlook Support**: Fixed DMARC reports from Microsoft not being recognized + - Now detects DMARC reports by filename pattern (contains `!` separator) + - Supports filenames like: `enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz` + +- **Enhanced Attachment Extraction**: More robust attachment detection + - Now supports plain `.xml` files in addition to `.xml.gz` and `.zip` + - Falls back to Content-Type `name` parameter when filename header is missing + - Recognizes attachments by Content-Type: `application/gzip`, `application/zip`, `text/xml`, etc. + - Added debug logging to help troubleshoot attachment detection issues + +#### Domains Page - DKIM View Record +- **DKIM Record Viewer**: Added "View Record" functionality for DKIM, similar to SPF + - Displays the full DNS record name including selector (e.g., `dkim._domainkey.example.com`) + - Shows the DKIM public key record value + - Helps users identify exactly which DNS record to configure + +### Technical + +#### New API Endpoints +``` +GET /api/mailbox-stats/summary +GET /api/mailbox-stats/all +GET /api/mailbox-stats/domains +``` + +#### API Parameters +- `date_range`: today, 7days, 30days, 90days, custom +- `start_date`: Custom start date (YYYY-MM-DD) - required when date_range is 'custom' +- `end_date`: Custom end date (YYYY-MM-DD) - required when date_range is 'custom' +- `domain`: Filter by specific domain +- `active_only`: true/false +- `hide_zero`: true/false (filter zero-activity mailboxes) +- `search`: Search mailbox/alias addresses +- `sort_by`: sent_total, received_total, failure_rate, quota_used, username +- `sort_order`: asc, desc +- `page`, `page_size`: Pagination + +--- + ## [2.0.4] - 2026-01-15 ### Fixed diff --git a/README.md b/README.md index b3ed1bf..77f70ff 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,31 @@ A modern, self-hosted dashboard for viewing and analyzing Mailcow mail server lo - Quarantined emails - Search and filter +### 🌐 Domains Overview +- Complete domains overview with DNS validation +- SPF validation with DNS lookup counter and server IP authorization check +- DKIM validation with Mailcow configuration comparison +- DMARC policy checking and recommendations +- Automated DNS checks every 6 hours +- Domain info: mailboxes, aliases, storage, relay settings +- Color-coded status indicators (✓ green, ⚠ amber, ✗ red) + +### 📊 Mailbox Statistics +- Per-mailbox message statistics (sent, received, failed, failure rate) +- Accordion-style mailbox list with expandable details +- Per-alias statistics with message counts +- Quota usage, login times, rate limits +- Filtering by date range, domain, search, active only, hide zero activity +- Sorting by sent, received, failure rate, quota, username + +### 📧 DMARC Reports +- DMARC report viewing and analysis +- GeoIP enrichment with MaxMind (City + ASN) +- Daily aggregated reports with compliance rates +- Manual upload (XML, GZ, ZIP formats) +- IMAP auto-import with configurable sync interval +- Sync history tracking with error notifications + ### 📈 Status - All container states (running/stopped count) - Storage usage with percentage bar diff --git a/VERSION b/VERSION index 26e3379..50aea0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.4 \ No newline at end of file +2.1.0 \ No newline at end of file diff --git a/backend/app/auth.py b/backend/app/auth.py index 5fb46a9..c8030ce 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -50,10 +50,11 @@ class BasicAuthMiddleware(BaseHTTPMiddleware): if not settings.auth_enabled: return await call_next(request) - # Allow access to login page, static files, and health check without authentication + # Allow access to login page, static files, health check, and info endpoint without authentication # Health check endpoint must be accessible for Docker health monitoring + # Info endpoint is used to check if authentication is enabled path = request.url.path - if path == "/login" or path.startswith("/static/") or path == "/api/health": + if path == "/login" or path.startswith("/static/") or path == "/api/health" or path == "/api/info": return await call_next(request) # Check if password is configured @@ -67,9 +68,10 @@ class BasicAuthMiddleware(BaseHTTPMiddleware): # Extract credentials from Authorization header authorization = request.headers.get("Authorization", "") - # For root page, allow access without Authorization header + # For frontend routes (not API), allow access without Authorization header # The frontend JavaScript will handle authentication and redirect if needed - if path == "/": + # This enables clean URLs like /dashboard, /messages, /dmarc etc. + if not path.startswith("/api/"): return await call_next(request) # For all other paths (API endpoints), require authentication diff --git a/backend/app/config.py b/backend/app/config.py index 026ab2f..32ca7ca 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -162,6 +162,12 @@ class Settings(BaseSettings): description='Run IMAP sync once on application startup' ) + dmarc_imap_batch_size: int = Field( + default=10, + env='DMARC_IMAP_BATCH_SIZE', + description='Number of emails to process per batch (prevents memory issues with large mailboxes)' + ) + dmarc_error_email: Optional[str] = Field( default=None, env='DMARC_ERROR_EMAIL', @@ -188,11 +194,17 @@ class Settings(BaseSettings): ) smtp_use_tls: bool = Field( - default=True, + default=False, env='SMTP_USE_TLS', description='Use STARTTLS for SMTP connection' ) + smtp_use_ssl: bool = Field( + default=False, + env='SMTP_USE_SSL', + description='Use Implicit SSL/TLS for SMTP connection (usually port 465)' + ) + smtp_user: Optional[str] = Field( default=None, env='SMTP_USER', @@ -211,6 +223,12 @@ class Settings(BaseSettings): description='From address for emails (defaults to SMTP user if not set)' ) + smtp_relay_mode: bool = Field( + default=False, + env='SMTP_RELAY_MODE', + description='Relay mode - send emails without authentication (for local relay servers)' + ) + # Global Admin Email admin_email: Optional[str] = Field( default=None, @@ -260,12 +278,21 @@ class Settings(BaseSettings): @property def notification_smtp_configured(self) -> bool: """Check if SMTP is properly configured for notifications""" - return ( - self.smtp_enabled and - self.smtp_host is not None and - self.smtp_user is not None and - self.smtp_password is not None - ) + if self.smtp_relay_mode: + # Relay mode - only need host and from address + return ( + self.smtp_enabled and + self.smtp_host is not None and + self.smtp_from is not None + ) + else: + # Standard mode - need authentication + return ( + self.smtp_enabled and + self.smtp_host is not None and + self.smtp_user is not None and + self.smtp_password is not None + ) @property def database_url(self) -> str: diff --git a/backend/app/main.py b/backend/app/main.py index 08fd274..2970abf 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -20,6 +20,7 @@ from .routers import logs, stats from .routers import export as export_router from .routers import domains as domains_router from .routers import dmarc as dmarc_router +from .routers import mailbox_stats as mailbox_stats_router from .routers import documentation from .migrations import run_migrations from .auth import BasicAuthMiddleware @@ -172,6 +173,7 @@ app.include_router(messages_router.router, prefix="/api", tags=["Messages"]) app.include_router(settings_router.router, prefix="/api", tags=["Settings"]) app.include_router(domains_router.router, prefix="/api", tags=["Domains"]) app.include_router(dmarc_router.router, prefix="/api", tags=["DMARC"]) +app.include_router(mailbox_stats_router.router, prefix="/api", tags=["Mailbox Stats"]) app.include_router(documentation.router, prefix="/api", tags=["Documentation"]) # Mount static files (frontend) @@ -256,6 +258,23 @@ async def global_exception_handler(request: Request, exc: Exception): ) +# SPA catch-all route - must be AFTER all other routes and exception handlers +# Returns index.html for all frontend routes (e.g., /dashboard, /messages, /dmarc) +@app.get("/{full_path:path}", response_class=HTMLResponse) +async def spa_catch_all(full_path: str): + """Serve the SPA for all frontend routes - enables clean URLs""" + # API and static routes are handled by their respective routers/mounts + # This catch-all only receives unmatched routes + try: + with open("/app/frontend/index.html", "r") as f: + return HTMLResponse(content=f.read()) + except FileNotFoundError: + return HTMLResponse( + content="

Mailcow Logs Viewer

Frontend not found. Please check installation.

", + status_code=500 + ) + + if __name__ == "__main__": import uvicorn uvicorn.run( diff --git a/backend/app/migrations.py b/backend/app/migrations.py index bd7af7a..50b8850 100644 --- a/backend/app/migrations.py +++ b/backend/app/migrations.py @@ -795,6 +795,59 @@ def create_dmarc_sync_table(db: Session): db.rollback() raise +def ensure_mailbox_statistics_table(db: Session): + """Ensure mailbox_statistics table exists with proper structure""" + try: + # Check if table exists + result = db.execute(text(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'mailbox_statistics' + ) + """)).scalar() + + if not result: + logger.info("Creating mailbox_statistics table...") + db.execute(text(""" + CREATE TABLE mailbox_statistics ( + id SERIAL PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + domain VARCHAR(255) NOT NULL, + name VARCHAR(255), + quota BIGINT DEFAULT 0, + quota_used BIGINT DEFAULT 0, + percent_in_use FLOAT DEFAULT 0.0, + messages INTEGER DEFAULT 0, + active BOOLEAN DEFAULT TRUE, + last_imap_login BIGINT, + last_pop3_login BIGINT, + last_smtp_login BIGINT, + spam_aliases INTEGER DEFAULT 0, + rl_value INTEGER, + rl_frame VARCHAR(20), + attributes JSONB, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + ) + """)) + + # Create indexes + db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_domain ON mailbox_statistics(domain)")) + db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_active ON mailbox_statistics(active)")) + db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_quota_used ON mailbox_statistics(quota_used)")) + db.execute(text("CREATE INDEX IF NOT EXISTS idx_mailbox_username ON mailbox_statistics(username)")) + + db.commit() + logger.info("mailbox_statistics table created successfully") + else: + logger.debug("mailbox_statistics table already exists") + + except Exception as e: + logger.error(f"Error ensuring mailbox_statistics table: {e}") + db.rollback() + raise + + def run_migrations(): """ Run all database migrations and maintenance tasks @@ -825,6 +878,17 @@ def run_migrations(): create_dmarc_sync_table(db) # GeoIP fields + add_geoip_fields_to_dmarc(db) + + # Mailbox statistics table + ensure_mailbox_statistics_table(db) + + # Alias statistics table + ensure_alias_statistics_table(db) + + # System settings (for cache signaling) + ensure_system_settings_table(db) + add_geoip_fields_to_dmarc(db) add_geoip_fields_to_rspamd(db) @@ -837,4 +901,82 @@ def run_migrations(): logger.error(f"Migration failed: {e}") raise finally: - db.close() \ No newline at end of file + db.close() + + +def ensure_alias_statistics_table(db: Session): + """Ensure alias_statistics table exists with all required fields""" + try: + # Check if table exists + result = db.execute(text(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'alias_statistics' + ) + """)) + table_exists = result.scalar() + + if not table_exists: + logger.info("Creating alias_statistics table...") + db.execute(text(""" + CREATE TABLE alias_statistics ( + id SERIAL PRIMARY KEY, + alias_address VARCHAR(255) NOT NULL UNIQUE, + goto TEXT, + domain VARCHAR(255) NOT NULL, + active BOOLEAN DEFAULT TRUE, + is_catch_all BOOLEAN DEFAULT FALSE, + primary_mailbox VARCHAR(255), + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + ) + """)) + + # Create indexes + db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_domain ON alias_statistics(domain)")) + db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_active ON alias_statistics(active)")) + db.execute(text("CREATE INDEX IF NOT EXISTS idx_alias_primary_mailbox ON alias_statistics(primary_mailbox)")) + db.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_alias_address ON alias_statistics(alias_address)")) + + db.commit() + logger.info("alias_statistics table created successfully") + else: + logger.debug("alias_statistics table already exists") + + except Exception as e: + logger.error(f"Error ensuring alias_statistics table: {e}") + db.rollback() + raise + + +def ensure_system_settings_table(db: Session): + """Ensure system_settings table exists""" + try: + # Check if table exists + result = db.execute(text(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'system_settings' + ) + """)) + table_exists = result.scalar() + + if not table_exists: + logger.info("Creating system_settings table...") + db.execute(text(""" + CREATE TABLE system_settings ( + key VARCHAR(255) PRIMARY KEY, + value TEXT, + updated_at TIMESTAMP DEFAULT NOW() + ) + """)) + + db.commit() + logger.info("system_settings table created successfully") + else: + logger.debug("system_settings table already exists") + + except Exception as e: + logger.error(f"Error ensuring system_settings table: {e}") + db.rollback() + raise diff --git a/backend/app/models.py b/backend/app/models.py index c7c3376..41a5be6 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -6,7 +6,7 @@ SIMPLIFIED VERSION: - Removed old generate_correlation_key function - Correlation key is now SHA256 of Message-ID """ -from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint +from sqlalchemy import Column, Integer, BigInteger, String, Float, DateTime, Boolean, Text, Index, JSON, UniqueConstraint from sqlalchemy.dialects.postgresql import JSONB from datetime import datetime @@ -292,4 +292,174 @@ class DMARCSync(Base): ) def __repr__(self): - return f"" \ No newline at end of file + return f"" + + +class MailboxStatistics(Base): + """ + Mailbox statistics fetched from Mailcow API + Tracks quota usage, message counts, and last access times for each mailbox + """ + __tablename__ = "mailbox_statistics" + + id = Column(Integer, primary_key=True, index=True) + + # Mailbox identification + username = Column(String(255), unique=True, index=True, nullable=False) # email address + domain = Column(String(255), index=True, nullable=False) + name = Column(String(255)) # Display name + + # Quota information (in bytes) + quota = Column(BigInteger, default=0) # Allocated quota + quota_used = Column(BigInteger, default=0) # Used quota + percent_in_use = Column(Float, default=0.0) # Percentage used + + # Message counts + messages = Column(Integer, default=0) # Total messages in mailbox + + # Status + active = Column(Boolean, default=True, index=True) + + # Access times (Unix timestamps from API, stored as integers) + last_imap_login = Column(BigInteger, nullable=True) + last_pop3_login = Column(BigInteger, nullable=True) + last_smtp_login = Column(BigInteger, nullable=True) + + # Spam filter settings + spam_aliases = Column(Integer, default=0) + + # Rate limits + rl_value = Column(Integer, nullable=True) # Rate limit value + rl_frame = Column(String(20), nullable=True) # Rate limit time frame (e.g., "s", "m", "h") + + # Attributes from API + attributes = Column(JSONB) # Store full attributes for reference + + # Timestamps + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = ( + Index('idx_mailbox_domain', 'domain'), + Index('idx_mailbox_active', 'active'), + Index('idx_mailbox_quota_used', 'quota_used'), + ) + + def __repr__(self): + return f"" + + +class AliasStatistics(Base): + """ + Alias statistics for tracking message counts per alias + Links aliases to their target mailboxes + """ + __tablename__ = "alias_statistics" + + id = Column(Integer, primary_key=True, index=True) + + # Alias identification + alias_address = Column(String(255), unique=True, index=True, nullable=False) # The alias email + goto = Column(Text) # Target mailbox(es), comma-separated + domain = Column(String(255), index=True, nullable=False) + + # Status + active = Column(Boolean, default=True, index=True) + is_catch_all = Column(Boolean, default=False) # Is this a catch-all alias + + # Link to primary mailbox (if applicable) + primary_mailbox = Column(String(255), index=True, nullable=True) # Main target mailbox + + # Timestamps + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = ( + Index('idx_alias_domain', 'domain'), + Index('idx_alias_active', 'active'), + Index('idx_alias_primary_mailbox', 'primary_mailbox'), + ) + + def __repr__(self): + return f"" + + +class TLSReport(Base): + """TLS-RPT (SMTP TLS Reporting) reports received from email providers""" + __tablename__ = "tls_reports" + + id = Column(Integer, primary_key=True, index=True) + + # Report identification + report_id = Column(String(255), unique=True, index=True, nullable=False) + + # Organization that sent the report + organization_name = Column(String(255), index=True) + contact_info = Column(String(255)) + + # Domain being reported on + policy_domain = Column(String(255), index=True, nullable=False) + + # Date range of the report + start_datetime = Column(DateTime, nullable=False) + end_datetime = Column(DateTime, nullable=False) + + # Raw JSON for reference + raw_json = Column(Text) + + created_at = Column(DateTime, default=datetime.utcnow, index=True) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = ( + Index('idx_tls_report_domain_date', 'policy_domain', 'start_datetime'), + Index('idx_tls_report_org', 'organization_name'), + ) + + def __repr__(self): + return f"" + + +class TLSReportPolicy(Base): + """Individual policy records within a TLS-RPT report""" + __tablename__ = "tls_report_policies" + + id = Column(Integer, primary_key=True, index=True) + tls_report_id = Column(Integer, index=True, nullable=False) + + # Policy information + policy_type = Column(String(50)) # "sts", "no-policy-found", etc. + policy_domain = Column(String(255)) + policy_string = Column(JSONB) # The policy string array + mx_host = Column(JSONB) # List of MX hosts + + # Session counts + successful_session_count = Column(Integer, default=0) + failed_session_count = Column(Integer, default=0) + + # Failure details if any + failure_details = Column(JSONB) + + created_at = Column(DateTime, default=datetime.utcnow) + + __table_args__ = ( + Index('idx_tls_policy_report', 'tls_report_id'), + Index('idx_tls_policy_type', 'policy_type'), + ) + + def __repr__(self): + return f"" + + +class SystemSetting(Base): + """ + Global system settings and state + Used for inter-process signaling (e.g., cache invalidation) + """ + __tablename__ = "system_settings" + + key = Column(String(255), primary_key=True, index=True) + value = Column(Text) # JSON string or simple text + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + def __repr__(self): + return f"" diff --git a/backend/app/routers/dmarc.py b/backend/app/routers/dmarc.py index a15cbbf..16ac34a 100644 --- a/backend/app/routers/dmarc.py +++ b/backend/app/routers/dmarc.py @@ -2,6 +2,8 @@ DMARC Router - Domain-centric view (Cloudflare style) """ import logging +import hashlib +import json from typing import List, Optional from datetime import datetime, timedelta, timezone from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, BackgroundTasks @@ -9,10 +11,16 @@ from sqlalchemy.orm import Session from sqlalchemy import func, and_, or_, case from ..database import get_db -from ..models import DMARCReport, DMARCRecord, DMARCSync +from ..models import DMARCReport, DMARCRecord, DMARCSync, TLSReport, TLSReportPolicy from ..services.dmarc_parser import parse_dmarc_file from ..services.geoip_service import enrich_dmarc_record from ..services.dmarc_imap_service import sync_dmarc_reports_from_imap +from ..services.dmarc_cache import ( + get_dmarc_cache_key, + get_dmarc_cached, + set_dmarc_cache, + clear_dmarc_cache +) from ..config import settings from ..scheduler import update_job_status @@ -20,21 +28,52 @@ logger = logging.getLogger(__name__) router = APIRouter() +# ============================================================================= +# CACHING SYSTEM +# ============================================================================= + +# ============================================================================= +# CACHING SYSTEM (Delegated to services.dmarc_cache) +# ============================================================================= + +# Cache functions imported from ..services.dmarc_cache + # ============================================================================= # DOMAINS LIST # ============================================================================= +@router.post("/dmarc/cache/clear") +async def clear_cache( + db: Session = Depends(get_db) +): + """ + Clear all DMARC related cache + """ + try: + clear_dmarc_cache(db) + return {"status": "success", "message": "Cache cleared"} + except Exception as e: + logger.error(f"Error clearing DMARC cache: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/dmarc/domains") async def get_domains_list( db: Session = Depends(get_db) ): """ - Get list of all domains with DMARC reports and their statistics - Similar to Cloudflare's domain list + Get list of all domains with DMARC and/or TLS-RPT reports and their statistics """ try: - domains_query = db.query( + # Check cache first + cache_key = get_dmarc_cache_key("domains_list") + cached_result = get_dmarc_cached(cache_key, db) + if cached_result is not None: + return cached_result + + # Get domains from DMARC reports + dmarc_domains = db.query( DMARCReport.domain, func.count(DMARCReport.id).label('report_count'), func.min(DMARCReport.begin_date).label('first_report'), @@ -43,55 +82,139 @@ async def get_domains_list( DMARCReport.domain ).all() + # Get domains from TLS reports + tls_domains = db.query( + TLSReport.policy_domain + ).distinct().all() + tls_domain_set = {d[0] for d in tls_domains} + + # Combine domains + all_domains = set() + dmarc_domain_data = {} + + for domain, report_count, first_report, last_report in dmarc_domains: + all_domains.add(domain) + dmarc_domain_data[domain] = { + 'report_count': report_count, + 'first_report': first_report, + 'last_report': last_report + } + + # Add TLS-only domains + all_domains.update(tls_domain_set) + domains_list = [] - for domain, report_count, first_report, last_report in domains_query: + for domain in all_domains: thirty_days_ago = int((datetime.now() - timedelta(days=30)).timestamp()) + thirty_days_ago_datetime = datetime.now() - timedelta(days=30) - stats = db.query( - func.sum(DMARCRecord.count).label('total_messages'), - func.count(func.distinct(DMARCRecord.source_ip)).label('unique_ips'), - func.sum( - case( - (and_(DMARCRecord.spf_result == 'pass', DMARCRecord.dkim_result == 'pass'), DMARCRecord.count), - else_=0 - ) - ).label('dmarc_pass_count') + # Get TLS stats for this domain + tls_stats = db.query( + func.count(TLSReport.id).label('tls_report_count'), + func.sum(TLSReportPolicy.successful_session_count).label('tls_success'), + func.sum(TLSReportPolicy.failed_session_count).label('tls_fail') ).join( - DMARCReport, - DMARCRecord.dmarc_report_id == DMARCReport.id + TLSReportPolicy, + TLSReportPolicy.tls_report_id == TLSReport.id ).filter( and_( - DMARCReport.domain == domain, - DMARCReport.begin_date >= thirty_days_ago + TLSReport.policy_domain == domain, + TLSReport.start_datetime >= thirty_days_ago_datetime ) ).first() - total_msgs = stats.total_messages or 0 - dmarc_pass = stats.dmarc_pass_count or 0 + tls_report_count = tls_stats.tls_report_count or 0 + tls_success = tls_stats.tls_success or 0 + tls_fail = tls_stats.tls_fail or 0 + tls_total = tls_success + tls_fail + tls_success_pct = round((tls_success / tls_total * 100) if tls_total > 0 else 100, 2) - domains_list.append({ - 'domain': domain, - 'report_count': report_count, - 'first_report': first_report, - 'last_report': last_report, - 'stats_30d': { - 'total_messages': total_msgs, - 'unique_ips': stats.unique_ips or 0, - 'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2) - } - }) + # Get DMARC stats + dmarc_data = dmarc_domain_data.get(domain) + if dmarc_data: + stats = db.query( + func.sum(DMARCRecord.count).label('total_messages'), + func.count(func.distinct(DMARCRecord.source_ip)).label('unique_ips'), + func.sum( + case( + (and_(DMARCRecord.spf_result == 'pass', DMARCRecord.dkim_result == 'pass'), DMARCRecord.count), + else_=0 + ) + ).label('dmarc_pass_count') + ).join( + DMARCReport, + DMARCRecord.dmarc_report_id == DMARCReport.id + ).filter( + and_( + DMARCReport.domain == domain, + DMARCReport.begin_date >= thirty_days_ago + ) + ).first() + + total_msgs = stats.total_messages or 0 + dmarc_pass = stats.dmarc_pass_count or 0 + + domains_list.append({ + 'domain': domain, + 'report_count': dmarc_data['report_count'], + 'tls_report_count': tls_report_count, + 'first_report': dmarc_data['first_report'], + 'last_report': dmarc_data['last_report'], + 'has_dmarc': True, + 'has_tls': domain in tls_domain_set, + 'stats_30d': { + 'total_messages': total_msgs, + 'unique_ips': stats.unique_ips or 0, + 'dmarc_pass_pct': round((dmarc_pass / total_msgs * 100) if total_msgs > 0 else 0, 2), + 'tls_success_pct': tls_success_pct + } + }) + else: + # TLS-only domain + tls_report = db.query(TLSReport).filter( + TLSReport.policy_domain == domain + ).order_by(TLSReport.end_datetime.desc()).first() + + first_tls = db.query(func.min(TLSReport.start_datetime)).filter( + TLSReport.policy_domain == domain + ).scalar() + + domains_list.append({ + 'domain': domain, + 'report_count': 0, + 'tls_report_count': tls_report_count, + 'first_report': int(first_tls.timestamp()) if first_tls else None, + 'last_report': int(tls_report.end_datetime.timestamp()) if tls_report and tls_report.end_datetime else None, + 'has_dmarc': False, + 'has_tls': True, + 'stats_30d': { + 'total_messages': 0, + 'unique_ips': 0, + 'dmarc_pass_pct': 0, + 'tls_success_pct': tls_success_pct + } + }) - return { - 'domains': sorted(domains_list, key=lambda x: x['last_report'], reverse=True), + # Sort by last_report, handling None values + domains_list.sort(key=lambda x: x['last_report'] or 0, reverse=True) + + response = { + 'domains': domains_list, 'total': len(domains_list) } + # Cache the result + set_dmarc_cache(cache_key, response) + + return response + except Exception as e: logger.error(f"Error fetching domains list: {e}") raise HTTPException(status_code=500, detail=str(e)) + # ============================================================================= # DOMAIN OVERVIEW # ============================================================================= @@ -624,6 +747,297 @@ async def get_source_details( except Exception as e: logger.error(f"Error fetching source details: {e}") raise HTTPException(status_code=500, detail=str(e)) +# ============================================================================= +# TLS-RPT REPORTS +# ============================================================================= + +@router.get("/dmarc/domains/{domain}/tls-reports") +async def get_domain_tls_reports( + domain: str, + days: int = 30, + page: int = 1, + limit: int = 50, + db: Session = Depends(get_db) +): + """ + Get TLS-RPT reports for a domain + Returns list of TLS reports with organization, date range, success/fail counts + """ + try: + cutoff_date = datetime.now(timezone.utc) - timedelta(days=days) + + # Query TLS reports for domain + reports_query = db.query(TLSReport).filter( + and_( + TLSReport.policy_domain == domain, + TLSReport.start_datetime >= cutoff_date + ) + ).order_by(TLSReport.start_datetime.desc()) + + total = reports_query.count() + reports = reports_query.offset((page - 1) * limit).limit(limit).all() + + # Get aggregated totals + total_success = 0 + total_fail = 0 + organizations = set() + + reports_list = [] + for report in reports: + # Get policies for this report + policies = db.query(TLSReportPolicy).filter( + TLSReportPolicy.tls_report_id == report.id + ).all() + + report_success = sum(p.successful_session_count for p in policies) + report_fail = sum(p.failed_session_count for p in policies) + + total_success += report_success + total_fail += report_fail + organizations.add(report.organization_name) + + policies_list = [{ + 'policy_type': p.policy_type, + 'policy_domain': p.policy_domain, + 'mx_host': p.mx_host, + 'successful_sessions': p.successful_session_count, + 'failed_sessions': p.failed_session_count, + 'failure_details': p.failure_details + } for p in policies] + + reports_list.append({ + 'id': report.id, + 'report_id': report.report_id, + 'organization_name': report.organization_name, + 'contact_info': report.contact_info, + 'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None, + 'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None, + 'successful_sessions': report_success, + 'failed_sessions': report_fail, + 'total_sessions': report_success + report_fail, + 'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2), + 'policies': policies_list + }) + + return { + 'domain': domain, + 'total': total, + 'page': page, + 'limit': limit, + 'pages': (total + limit - 1) // limit if total > 0 else 0, + 'totals': { + 'total_reports': total, + 'total_successful_sessions': total_success, + 'total_failed_sessions': total_fail, + 'overall_success_rate': round((total_success / (total_success + total_fail) * 100) if (total_success + total_fail) > 0 else 100, 2), + 'unique_organizations': len(organizations) + }, + 'data': reports_list + } + + except Exception as e: + logger.error(f"Error fetching TLS reports for domain {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/dmarc/domains/{domain}/tls-reports/daily") +async def get_domain_tls_daily_reports( + domain: str, + days: int = 30, + page: int = 1, + limit: int = 30, + db: Session = Depends(get_db) +): + """ + Get TLS-RPT reports aggregated by date + Groups multiple reports from same day together, like DMARC daily reports + """ + try: + cutoff_date = datetime.now(timezone.utc) - timedelta(days=days) + + # Query all TLS reports for domain + reports = db.query(TLSReport).filter( + and_( + TLSReport.policy_domain == domain, + TLSReport.start_datetime >= cutoff_date + ) + ).order_by(TLSReport.start_datetime.desc()).all() + + # Group by date + daily_data = {} + for report in reports: + if report.start_datetime: + date_key = report.start_datetime.strftime('%Y-%m-%d') + if date_key not in daily_data: + daily_data[date_key] = { + 'date': date_key, + 'report_count': 0, + 'organizations': set(), + 'total_success': 0, + 'total_fail': 0, + 'reports': [] + } + + # Get policies for this report + policies = db.query(TLSReportPolicy).filter( + TLSReportPolicy.tls_report_id == report.id + ).all() + + report_success = sum(p.successful_session_count for p in policies) + report_fail = sum(p.failed_session_count for p in policies) + + daily_data[date_key]['report_count'] += 1 + daily_data[date_key]['organizations'].add(report.organization_name) + daily_data[date_key]['total_success'] += report_success + daily_data[date_key]['total_fail'] += report_fail + daily_data[date_key]['reports'].append({ + 'id': report.id, + 'organization_name': report.organization_name, + 'successful_sessions': report_success, + 'failed_sessions': report_fail + }) + + # Convert to list and add success rate + daily_list = [] + for date_key in sorted(daily_data.keys(), reverse=True): + data = daily_data[date_key] + total = data['total_success'] + data['total_fail'] + success_rate = round((data['total_success'] / total * 100) if total > 0 else 100, 2) + + daily_list.append({ + 'date': date_key, + 'report_count': data['report_count'], + 'organizations': list(data['organizations']), + 'organization_count': len(data['organizations']), + 'total_success': data['total_success'], + 'total_fail': data['total_fail'], + 'total_sessions': total, + 'success_rate': success_rate, + 'reports': data['reports'] + }) + + # Pagination + total = len(daily_list) + start_idx = (page - 1) * limit + end_idx = start_idx + limit + paginated = daily_list[start_idx:end_idx] + + # Calculate overall totals + overall_success = sum(d['total_success'] for d in daily_list) + overall_fail = sum(d['total_fail'] for d in daily_list) + overall_total = overall_success + overall_fail + + return { + 'domain': domain, + 'total': total, + 'page': page, + 'limit': limit, + 'pages': (total + limit - 1) // limit if total > 0 else 0, + 'totals': { + 'total_days': total, + 'total_reports': sum(d['report_count'] for d in daily_list), + 'total_successful_sessions': overall_success, + 'total_failed_sessions': overall_fail, + 'overall_success_rate': round((overall_success / overall_total * 100) if overall_total > 0 else 100, 2) + }, + 'data': paginated + } + + except Exception as e: + logger.error(f"Error fetching TLS daily reports for domain {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/dmarc/domains/{domain}/tls-reports/{report_date}/details") +async def get_tls_report_details( + domain: str, + report_date: str, + db: Session = Depends(get_db) +): + """ + Get detailed TLS-RPT reports for a specific date + Shows all reports and policies from that day with breakdown by provider + """ + try: + date_obj = datetime.strptime(report_date, '%Y-%m-%d').date() + start_dt = datetime.combine(date_obj, datetime.min.time(), tzinfo=timezone.utc) + end_dt = datetime.combine(date_obj, datetime.max.time(), tzinfo=timezone.utc) + + reports = db.query(TLSReport).filter( + and_( + TLSReport.policy_domain == domain, + TLSReport.start_datetime >= start_dt, + TLSReport.start_datetime <= end_dt + ) + ).all() + + if not reports: + raise HTTPException(status_code=404, detail="No TLS reports found for this date") + + # Aggregate stats + total_success = 0 + total_fail = 0 + providers = [] + + for report in reports: + policies = db.query(TLSReportPolicy).filter( + TLSReportPolicy.tls_report_id == report.id + ).all() + + report_success = sum(p.successful_session_count for p in policies) + report_fail = sum(p.failed_session_count for p in policies) + + total_success += report_success + total_fail += report_fail + + # Add provider details + policies_list = [] + for p in policies: + policies_list.append({ + 'policy_type': p.policy_type, + 'policy_domain': p.policy_domain, + 'mx_host': p.mx_host, + 'successful_sessions': p.successful_session_count, + 'failed_sessions': p.failed_session_count, + 'total_sessions': p.successful_session_count + p.failed_session_count, + 'success_rate': round((p.successful_session_count / (p.successful_session_count + p.failed_session_count) * 100) if (p.successful_session_count + p.failed_session_count) > 0 else 100, 2), + 'failure_details': p.failure_details + }) + + providers.append({ + 'report_id': report.report_id, + 'organization_name': report.organization_name, + 'contact_info': report.contact_info, + 'start_datetime': report.start_datetime.isoformat() if report.start_datetime else None, + 'end_datetime': report.end_datetime.isoformat() if report.end_datetime else None, + 'successful_sessions': report_success, + 'failed_sessions': report_fail, + 'total_sessions': report_success + report_fail, + 'success_rate': round((report_success / (report_success + report_fail) * 100) if (report_success + report_fail) > 0 else 100, 2), + 'policies': policies_list + }) + + total = total_success + total_fail + + return { + 'domain': domain, + 'date': report_date, + 'stats': { + 'total_reports': len(reports), + 'total_providers': len(set(r.organization_name for r in reports)), + 'total_success': total_success, + 'total_fail': total_fail, + 'total_sessions': total, + 'success_rate': round((total_success / total * 100) if total > 0 else 100, 2) + }, + 'providers': providers + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching TLS report details for {domain}/{report_date}: {e}") + raise HTTPException(status_code=500, detail=str(e)) # ============================================================================= @@ -796,66 +1210,156 @@ async def get_sync_history( logger.error(f"Error fetching sync history: {e}") raise HTTPException(status_code=500, detail=str(e)) - # ============================================================================= # UPLOAD # ============================================================================= @router.post("/dmarc/upload") -async def upload_dmarc_report( +async def upload_report( file: UploadFile = File(...), db: Session = Depends(get_db) ): + """ + Upload and parse DMARC or TLS-RPT report file + + Supported formats: + - DMARC: .xml, .xml.gz, .zip (XML content) + - TLS-RPT: .json, .json.gz + """ if not settings.dmarc_manual_upload_enabled: raise HTTPException( status_code=403, - detail="Manual DMARC report upload is disabled" + detail="Manual report upload is disabled" ) - """Upload and parse DMARC report file (GZ or ZIP)""" try: file_content = await file.read() + filename = file.filename.lower() - parsed_data = parse_dmarc_file(file_content, file.filename) + # Detect file type based on extension + # TLS-RPT files: .json, .json.gz, .json.zip + is_tls_rpt = filename.endswith('.json') or filename.endswith('.json.gz') or filename.endswith('.json.zip') - if not parsed_data: - raise HTTPException(status_code=400, detail="Failed to parse DMARC report") - - records_data = parsed_data.pop('records', []) - report_data = parsed_data - - existing = db.query(DMARCReport).filter( - DMARCReport.report_id == report_data['report_id'] - ).first() - - if existing: - return { - 'status': 'duplicate', - 'message': f'Report {report_data["report_id"]} already exists' - } - - report = DMARCReport(**report_data) - db.add(report) - db.flush() - - for record_data in records_data: - record_data['dmarc_report_id'] = report.id - enriched = enrich_dmarc_record(record_data) - record = DMARCRecord(**enriched) - db.add(record) - - db.commit() - - return { - 'status': 'success', - 'message': f'Uploaded report for {report.domain} from {report.org_name}', - 'report_id': report.id, - 'records_count': len(records_data) - } + if is_tls_rpt: + # Process TLS-RPT report + return await _upload_tls_rpt_report(file_content, file.filename, db) + else: + # Process DMARC report (default) + return await _upload_dmarc_report(file_content, file.filename, db) except HTTPException: raise except Exception as e: db.rollback() - logger.error(f"Error uploading DMARC report: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file + logger.error(f"Error uploading report: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +async def _upload_dmarc_report(file_content: bytes, filename: str, db: Session): + """Handle DMARC report upload""" + parsed_data = parse_dmarc_file(file_content, filename) + + if not parsed_data: + raise HTTPException(status_code=400, detail="Failed to parse DMARC report") + + records_data = parsed_data.pop('records', []) + report_data = parsed_data + + existing = db.query(DMARCReport).filter( + DMARCReport.report_id == report_data['report_id'] + ).first() + + if existing: + return { + 'status': 'duplicate', + 'report_type': 'dmarc', + 'message': f'Report {report_data["report_id"]} already exists' + } + + report = DMARCReport(**report_data) + db.add(report) + db.flush() + + for record_data in records_data: + record_data['dmarc_report_id'] = report.id + enriched = enrich_dmarc_record(record_data) + record = DMARCRecord(**enriched) + db.add(record) + + db.commit() + + # Clear cache + clear_dmarc_cache(db) + + return { + 'status': 'success', + 'report_type': 'dmarc', + 'message': f'Uploaded DMARC report for {report.domain} from {report.org_name}', + 'report_id': report.id, + 'records_count': len(records_data) + } + + +async def _upload_tls_rpt_report(file_content: bytes, filename: str, db: Session): + """Handle TLS-RPT report upload""" + from ..services.tls_rpt_parser import parse_tls_rpt_file + + parsed_data = parse_tls_rpt_file(file_content, filename) + + if not parsed_data: + raise HTTPException(status_code=400, detail="Failed to parse TLS-RPT report") + + # Extract policies + policies_data = parsed_data.pop('policies', []) + + # Check for duplicate + existing = db.query(TLSReport).filter( + TLSReport.report_id == parsed_data['report_id'] + ).first() + + if existing: + return { + 'status': 'duplicate', + 'report_type': 'tls-rpt', + 'message': f'TLS-RPT report {parsed_data["report_id"]} already exists' + } + + # Create TLS report + tls_report = TLSReport( + report_id=parsed_data['report_id'], + organization_name=parsed_data.get('organization_name', 'Unknown'), + contact_info=parsed_data.get('contact_info', ''), + policy_domain=parsed_data['policy_domain'], + start_datetime=parsed_data['start_datetime'], + end_datetime=parsed_data['end_datetime'], + raw_json=parsed_data.get('raw_json', '') + ) + db.add(tls_report) + db.flush() + + # Create policy records + for policy_data in policies_data: + policy = TLSReportPolicy( + tls_report_id=tls_report.id, + policy_type=policy_data.get('policy_type', 'unknown'), + policy_domain=policy_data.get('policy_domain', ''), + policy_string=policy_data.get('policy_string', []), + mx_host=policy_data.get('mx_host', []), + successful_session_count=policy_data.get('successful_session_count', 0), + failed_session_count=policy_data.get('failed_session_count', 0), + failure_details=policy_data.get('failure_details', []) + ) + db.add(policy) + + db.commit() + + # Clear cache + clear_dmarc_cache(db) + + return { + 'status': 'success', + 'report_type': 'tls-rpt', + 'message': f'Uploaded TLS-RPT report for {tls_report.policy_domain} from {tls_report.organization_name}', + 'report_id': tls_report.id, + 'policies_count': len(policies_data) + } \ No newline at end of file diff --git a/backend/app/routers/domains.py b/backend/app/routers/domains.py index da01008..bb1f04f 100644 --- a/backend/app/routers/domains.py +++ b/backend/app/routers/domains.py @@ -404,88 +404,6 @@ async def count_spf_dns_lookups(domain: str, spf_record: str, resolver, visited_ return lookup_count -async def check_ip_in_spf(domain: str, ip_to_check: str, spf_record: str, resolver, visited_domains: set = None, depth: int = 0) -> tuple: - """ - Check if IP is authorized in SPF record recursively - Returns: (authorized: bool, method: str or None) - """ - if depth > 10: - return False, None - - if visited_domains is None: - visited_domains = set() - - if domain in visited_domains: - return False, None - - visited_domains.add(domain) - - parts = spf_record.split() - - for part in parts: - clean_part = part.lstrip('+-~?') - - if clean_part.startswith('ip4:'): - ip_spec = clean_part.replace('ip4:', '') - try: - if '/' in ip_spec: - network = ipaddress.ip_network(ip_spec, strict=False) - if ipaddress.ip_address(ip_to_check) in network: - return True, f'ip4:{ip_spec}' - else: - if ip_to_check == ip_spec: - return True, f'ip4:{ip_spec}' - except: - pass - - elif clean_part in ['a'] or clean_part.startswith('a:'): - check_domain = domain if clean_part == 'a' else clean_part.split(':', 1)[1] - try: - a_records = await resolver.resolve(check_domain, 'A') - for rdata in a_records: - if str(rdata) == ip_to_check: - return True, f'a:{check_domain}' if clean_part.startswith('a:') else 'a' - except: - pass - - elif clean_part in ['mx'] or clean_part.startswith('mx:'): - check_domain = domain if clean_part == 'mx' else clean_part.split(':', 1)[1] - try: - mx_records = await resolver.resolve(check_domain, 'MX') - for mx in mx_records: - try: - mx_a_records = await resolver.resolve(str(mx.exchange), 'A') - for rdata in mx_a_records: - if str(rdata) == ip_to_check: - return True, f'mx:{check_domain}' if clean_part.startswith('mx:') else 'mx' - except: - pass - except: - pass - - elif clean_part.startswith('include:'): - include_domain = clean_part.replace('include:', '') - try: - include_answers = await resolver.resolve(include_domain, 'TXT') - for rdata in include_answers: - include_spf = b''.join(rdata.strings).decode('utf-8') - if include_spf.startswith('v=spf1'): - authorized, method = await check_ip_in_spf( - include_domain, - ip_to_check, - include_spf, - resolver, - visited_domains.copy(), - depth + 1 - ) - if authorized: - return True, f'include:{include_domain} ({method})' - except: - pass - - return False, None - - def parse_dkim_parameters(dkim_record: str) -> Dict[str, Any]: """ Parse and validate DKIM record parameters diff --git a/backend/app/routers/mailbox_stats.py b/backend/app/routers/mailbox_stats.py new file mode 100644 index 0000000..0bf424a --- /dev/null +++ b/backend/app/routers/mailbox_stats.py @@ -0,0 +1,553 @@ +""" +API endpoints for mailbox statistics with message counts +Shows per-mailbox/per-alias message statistics from MessageCorrelation table +""" +import logging +import hashlib +import json +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session +from sqlalchemy import func, case, and_, or_ +from datetime import datetime, timezone, timedelta +from typing import Optional, List + +from ..database import get_db +from ..models import MailboxStatistics, AliasStatistics, MessageCorrelation + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# ============================================================================= +# CACHING SYSTEM +# ============================================================================= + +# In-memory cache for mailbox stats +_stats_cache = {} +_cache_ttl_seconds = 300 # 5 minutes cache TTL + + +def _get_cache_key(prefix: str, **params) -> str: + """Generate a cache key from parameters""" + param_str = json.dumps(params, sort_keys=True, default=str) + hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16] + return f"{prefix}:{hash_val}" + + +def _get_cached(key: str): + """Get cached value if not expired""" + if key in _stats_cache: + cached_data, cached_time = _stats_cache[key] + if datetime.now(timezone.utc) - cached_time < timedelta(seconds=_cache_ttl_seconds): + logger.debug(f"Cache hit for key: {key}") + return cached_data + else: + # Cache expired, remove it + del _stats_cache[key] + return None + + +def _set_cache(key: str, data): + """Set cached value with current timestamp""" + _stats_cache[key] = (data, datetime.now(timezone.utc)) + logger.debug(f"Cache set for key: {key}") + + +def clear_stats_cache(): + """Clear all stats cache - call after data changes""" + global _stats_cache + _stats_cache = {} + logger.info("Stats cache cleared") + + +def format_datetime_utc(dt: Optional[datetime]) -> Optional[str]: + """Format datetime for API response with proper UTC timezone""" + if dt is None: + return None + + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + + dt_utc = dt.astimezone(timezone.utc) + return dt_utc.replace(microsecond=0).isoformat().replace('+00:00', 'Z') + + +def format_bytes(bytes_value) -> str: + """Format bytes into human-readable format""" + if bytes_value is None: + return "0 B" + + bytes_value = float(bytes_value) + + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if abs(bytes_value) < 1024.0: + return f"{bytes_value:.1f} {unit}" + bytes_value /= 1024.0 + return f"{bytes_value:.1f} PB" + + +def format_unix_timestamp(timestamp: Optional[int]) -> Optional[str]: + """Convert Unix timestamp to ISO format string""" + if timestamp is None or timestamp == 0: + return None + try: + dt = datetime.fromtimestamp(timestamp, tz=timezone.utc) + return dt.replace(microsecond=0).isoformat().replace('+00:00', 'Z') + except (ValueError, OSError): + return None + + +def parse_date_range(date_range: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> tuple[datetime, datetime]: + """Parse date range string into start and end datetimes + + For custom date ranges, start_date and end_date should be ISO format strings (YYYY-MM-DD) + """ + now = datetime.now(timezone.utc) + + if date_range == "custom" and start_date and end_date: + # Parse custom date range + try: + start = datetime.fromisoformat(start_date.replace('Z', '+00:00')) + if start.tzinfo is None: + start = start.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) + + end = datetime.fromisoformat(end_date.replace('Z', '+00:00')) + if end.tzinfo is None: + # Set end time to end of day + end = end.replace(hour=23, minute=59, second=59, microsecond=999999, tzinfo=timezone.utc) + + return start, end + except ValueError: + # If parsing fails, fall back to 30 days + logger.warning(f"Failed to parse custom date range: {start_date} - {end_date}, falling back to 30 days") + start = now - timedelta(days=30) + end = now + elif date_range == "today": + start = now.replace(hour=0, minute=0, second=0, microsecond=0) + end = now + elif date_range == "7days": + start = now - timedelta(days=7) + end = now + elif date_range == "30days": + start = now - timedelta(days=30) + end = now + elif date_range == "90days": + start = now - timedelta(days=90) + end = now + else: + # Default to 30 days + start = now - timedelta(days=30) + end = now + + return start, end + + +def get_message_counts_for_email(db: Session, email: str, start_date: datetime, end_date: datetime) -> dict: + """ + Get message counts for a specific email address (mailbox or alias) + Counts sent (as sender) and received (as recipient) with status breakdown + Also counts by direction (inbound, outbound, internal) + """ + # Count sent messages (this email as sender) by status + sent_query = db.query( + MessageCorrelation.final_status, + func.count(MessageCorrelation.id).label('count') + ).filter( + MessageCorrelation.sender == email, + MessageCorrelation.first_seen >= start_date, + MessageCorrelation.first_seen <= end_date + ).group_by(MessageCorrelation.final_status).all() + + # Count received messages (this email as recipient) by status + received_query = db.query( + MessageCorrelation.final_status, + func.count(MessageCorrelation.id).label('count') + ).filter( + MessageCorrelation.recipient == email, + MessageCorrelation.first_seen >= start_date, + MessageCorrelation.first_seen <= end_date + ).group_by(MessageCorrelation.final_status).all() + + # Count by direction (inbound, outbound, internal) + direction_query = db.query( + MessageCorrelation.direction, + func.count(MessageCorrelation.id).label('count') + ).filter( + or_( + MessageCorrelation.sender == email, + MessageCorrelation.recipient == email + ), + MessageCorrelation.first_seen >= start_date, + MessageCorrelation.first_seen <= end_date + ).group_by(MessageCorrelation.direction).all() + + # Process results + sent_by_status = {row.final_status or 'unknown': row.count for row in sent_query} + received_by_status = {row.final_status or 'unknown': row.count for row in received_query} + direction_counts = {row.direction or 'unknown': row.count for row in direction_query} + + sent_total = sum(sent_by_status.values()) + received_total = sum(received_by_status.values()) + + # Calculate failures (only for sent messages - bounces and rejections are outbound failures) + sent_failed = sent_by_status.get('bounced', 0) + sent_by_status.get('rejected', 0) + + total = sent_total + received_total + failure_rate = round((sent_failed / sent_total * 100) if sent_total > 0 else 0, 1) + + return { + "sent_total": sent_total, + "sent_delivered": sent_by_status.get('delivered', 0) + sent_by_status.get('sent', 0), + "sent_bounced": sent_by_status.get('bounced', 0), + "sent_rejected": sent_by_status.get('rejected', 0), + "sent_deferred": sent_by_status.get('deferred', 0), + "sent_expired": sent_by_status.get('expired', 0), + "sent_failed": sent_failed, + "received_total": received_total, + "total_messages": total, + "failure_rate": failure_rate, + # Direction counts + "direction_inbound": direction_counts.get('inbound', 0), + "direction_outbound": direction_counts.get('outbound', 0), + "direction_internal": direction_counts.get('internal', 0) + } + + +@router.get("/mailbox-stats/summary") +async def get_mailbox_stats_summary( + date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"), + start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"), + end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"), + db: Session = Depends(get_db) +): + """ + Get summary statistics for all mailboxes + """ + try: + parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date) + + # Total mailboxes + total_mailboxes = db.query(func.count(MailboxStatistics.id)).scalar() or 0 + active_mailboxes = db.query(func.count(MailboxStatistics.id)).filter( + MailboxStatistics.active == True + ).scalar() or 0 + + # Total aliases + total_aliases = db.query(func.count(AliasStatistics.id)).scalar() or 0 + active_aliases = db.query(func.count(AliasStatistics.id)).filter( + AliasStatistics.active == True + ).scalar() or 0 + + # Unique domains + unique_domains = db.query(func.count(func.distinct(MailboxStatistics.domain))).scalar() or 0 + + # Get total storage used + total_quota_used = db.query(func.sum(MailboxStatistics.quota_used)).scalar() or 0 + + # Get last update time + last_update = db.query(func.max(MailboxStatistics.updated_at)).scalar() + + # Get all local mailbox emails and alias emails + mailbox_emails = [m.username for m in db.query(MailboxStatistics.username).all()] + alias_emails = [a.alias_address for a in db.query(AliasStatistics.alias_address).all()] + all_local_emails = set(mailbox_emails + alias_emails) + + # Count total messages for all local emails + total_sent = 0 + total_received = 0 + total_failed = 0 + + if all_local_emails: + # Sent messages + sent_result = db.query(func.count(MessageCorrelation.id)).filter( + MessageCorrelation.sender.in_(all_local_emails), + MessageCorrelation.first_seen >= parsed_start, + MessageCorrelation.first_seen <= parsed_end + ).scalar() or 0 + total_sent = sent_result + + # Received messages + received_result = db.query(func.count(MessageCorrelation.id)).filter( + MessageCorrelation.recipient.in_(all_local_emails), + MessageCorrelation.first_seen >= parsed_start, + MessageCorrelation.first_seen <= parsed_end + ).scalar() or 0 + total_received = received_result + + # Failed messages (only sent that bounced/rejected - failures are outbound) + failed_result = db.query(func.count(MessageCorrelation.id)).filter( + MessageCorrelation.sender.in_(all_local_emails), + MessageCorrelation.first_seen >= parsed_start, + MessageCorrelation.first_seen <= parsed_end, + MessageCorrelation.final_status.in_(['bounced', 'rejected']) + ).scalar() or 0 + total_sent_failed = failed_result + + total_messages = total_sent + total_received + failure_rate = round((total_sent_failed / total_sent * 100) if total_sent > 0 else 0, 1) + + return { + "total_sent": total_sent, + "total_received": total_received, + "total_messages": total_messages, + "sent_failed": total_sent_failed, + "failure_rate": failure_rate, + "date_range": date_range, + "start_date": format_datetime_utc(parsed_start), + "end_date": format_datetime_utc(parsed_end), + "last_update": format_datetime_utc(last_update) + } + except Exception as e: + logger.error(f"Error fetching mailbox stats summary: {e}") + return {"error": str(e), "total_mailboxes": 0} + + +@router.get("/mailbox-stats/all") +async def get_all_mailbox_stats( + domain: Optional[str] = None, + active_only: bool = True, # Changed default to True + hide_zero: bool = False, # Filter out mailboxes with zero activity + search: Optional[str] = None, + date_range: str = Query("30days", description="Date range: today, 7days, 30days, 90days, custom"), + start_date: Optional[str] = Query(None, description="Custom start date (YYYY-MM-DD) - required when date_range is 'custom'"), + end_date: Optional[str] = Query(None, description="Custom end date (YYYY-MM-DD) - required when date_range is 'custom'"), + sort_by: str = "sent_total", + sort_order: str = "desc", + page: int = Query(1, ge=1, description="Page number"), + page_size: int = Query(50, ge=10, le=100, description="Items per page"), + db: Session = Depends(get_db) +): + """ + Get all mailbox statistics with message counts and aliases (paginated) + """ + try: + # Check cache first + cache_key = _get_cache_key( + "mailbox_stats_all", + domain=domain, + active_only=active_only, + hide_zero=hide_zero, + search=search, + date_range=date_range, + start_date=start_date, + end_date=end_date, + sort_by=sort_by, + sort_order=sort_order, + page=page, + page_size=page_size + ) + + cached_result = _get_cached(cache_key) + if cached_result is not None: + return cached_result + + parsed_start, parsed_end = parse_date_range(date_range, start_date, end_date) + + query = db.query(MailboxStatistics) + + # Apply domain filter + if domain: + query = query.filter(MailboxStatistics.domain == domain) + + # Apply active filter + if active_only: + query = query.filter(MailboxStatistics.active == True) + + # Apply search filter on mailbox username/name OR mailboxes that have matching aliases + if search: + mailbox_search_term = f"%{search}%" + + # Find mailboxes that have matching aliases + alias_matched_usernames = db.query(AliasStatistics.primary_mailbox).filter( + AliasStatistics.alias_address.ilike(mailbox_search_term) + ).distinct().subquery() + + query = query.filter( + or_( + MailboxStatistics.username.ilike(mailbox_search_term), + MailboxStatistics.name.ilike(mailbox_search_term), + MailboxStatistics.username.in_(alias_matched_usernames) + ) + ) + + # Get total count before pagination + total_count = query.count() + + # Get all for sorting (we need to calculate counts before pagination) + mailboxes = query.all() + + # Build result with message counts for each mailbox + result = [] + for mb in mailboxes: + # Get message counts for this mailbox + counts = get_message_counts_for_email(db, mb.username, parsed_start, parsed_end) + + # Get aliases for this mailbox + aliases = db.query(AliasStatistics).filter( + AliasStatistics.primary_mailbox == mb.username + ).all() + + # Get message counts for each alias + alias_list = [] + alias_sent_total = 0 + alias_received_total = 0 + alias_failed_total = 0 + alias_internal_total = 0 + alias_delivered_total = 0 + + for alias in aliases: + alias_counts = get_message_counts_for_email(db, alias.alias_address, parsed_start, parsed_end) + alias_sent_total += alias_counts['sent_total'] + alias_received_total += alias_counts['received_total'] + alias_failed_total += alias_counts['sent_failed'] + alias_internal_total += alias_counts['direction_internal'] + alias_delivered_total += alias_counts['sent_delivered'] + + alias_list.append({ + "alias_address": alias.alias_address, + "active": alias.active, + "is_catch_all": alias.is_catch_all, + **alias_counts + }) + + # Calculate combined totals (mailbox + all aliases) + combined_sent = counts['sent_total'] + alias_sent_total + combined_received = counts['received_total'] + alias_received_total + combined_total = combined_sent + combined_received + combined_failed = counts['sent_failed'] + alias_failed_total + combined_failure_rate = round((combined_failed / combined_sent * 100) if combined_sent > 0 else 0, 1) + combined_internal = counts['direction_internal'] + alias_internal_total + combined_delivered = counts['sent_delivered'] + alias_delivered_total + combined_inbound = counts['direction_inbound'] + combined_outbound = counts['direction_outbound'] + + result.append({ + "id": mb.id, + "username": mb.username, + "domain": mb.domain, + "name": mb.name, + "active": mb.active, + # Quota info + "quota": float(mb.quota or 0), + "quota_formatted": format_bytes(mb.quota), + "quota_used": float(mb.quota_used or 0), + "quota_used_formatted": format_bytes(mb.quota_used), + "percent_in_use": round(float(mb.percent_in_use or 0), 1), + "messages_in_mailbox": mb.messages or 0, + # Last login times + "last_imap_login": format_unix_timestamp(mb.last_imap_login), + "last_pop3_login": format_unix_timestamp(mb.last_pop3_login), + "last_smtp_login": format_unix_timestamp(mb.last_smtp_login), + # Rate limiting + "rl_value": mb.rl_value, + "rl_frame": mb.rl_frame, + # Attributes (access permissions) + "attributes": mb.attributes or {}, + # Message counts for mailbox only + "mailbox_counts": counts, + # Aliases + "aliases": alias_list, + "alias_count": len(alias_list), + # Combined totals (mailbox + aliases) + "combined_sent": combined_sent, + "combined_received": combined_received, + "combined_total": combined_total, + "combined_failed": combined_failed, + "combined_failure_rate": combined_failure_rate, + # Direction and status combined counts + "combined_internal": combined_internal, + "combined_delivered": combined_delivered, + "combined_inbound": combined_inbound, + "combined_outbound": combined_outbound, + # Metadata + "created": format_datetime_utc(mb.created_at), + "modified": format_datetime_utc(mb.updated_at) + }) + + # Sort results + reverse = sort_order.lower() == "desc" + if sort_by == "sent_total": + result.sort(key=lambda x: x['combined_sent'], reverse=reverse) + elif sort_by == "received_total": + result.sort(key=lambda x: x['combined_received'], reverse=reverse) + elif sort_by == "failure_rate": + result.sort(key=lambda x: x['combined_failure_rate'], reverse=reverse) + elif sort_by == "username": + result.sort(key=lambda x: x['username'].lower(), reverse=reverse) + elif sort_by == "quota_used": + result.sort(key=lambda x: x['quota_used'], reverse=reverse) + else: + result.sort(key=lambda x: x['combined_total'], reverse=reverse) + + # Apply hide_zero filter - remove mailboxes with no activity + if hide_zero: + result = [r for r in result if r['combined_total'] > 0] + + # Apply pagination + total_pages = (len(result) + page_size - 1) // page_size + start_index = (page - 1) * page_size + end_index = start_index + page_size + paginated_result = result[start_index:end_index] + + response = { + "total": len(result), + "page": page, + "page_size": page_size, + "total_pages": total_pages, + "date_range": date_range, + "start_date": format_datetime_utc(parsed_start), + "end_date": format_datetime_utc(parsed_end), + "mailboxes": paginated_result + } + + # Cache the result + _set_cache(cache_key, response) + + return response + except Exception as e: + logger.error(f"Error fetching all mailbox stats: {e}") + return {"error": str(e), "total": 0, "mailboxes": []} + + +@router.get("/mailbox-stats/domains") +async def get_mailbox_domains(db: Session = Depends(get_db)): + """ + Get list of all domains for filtering + """ + try: + domains = db.query( + MailboxStatistics.domain, + func.count(MailboxStatistics.id).label('count') + ).group_by( + MailboxStatistics.domain + ).order_by( + MailboxStatistics.domain + ).all() + + return { + "domains": [ + {"domain": d.domain, "mailbox_count": d.count} + for d in domains + ] + } + except Exception as e: + logger.error(f"Error fetching mailbox domains: {e}") + return {"error": str(e), "domains": []} + + +@router.get("/mailbox-stats/refresh") +async def refresh_mailbox_stats(db: Session = Depends(get_db)): + """ + Get last update time for mailbox statistics + """ + try: + last_mailbox_update = db.query(func.max(MailboxStatistics.updated_at)).scalar() + last_alias_update = db.query(func.max(AliasStatistics.updated_at)).scalar() + + return { + "last_mailbox_update": format_datetime_utc(last_mailbox_update), + "last_alias_update": format_datetime_utc(last_alias_update) + } + except Exception as e: + logger.error(f"Error getting refresh info: {e}") + return {"error": str(e)} diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py index d540bf8..e6f463e 100644 --- a/backend/app/routers/settings.py +++ b/backend/app/routers/settings.py @@ -224,6 +224,20 @@ async def get_settings_info(db: Session = Depends(get_db)): "status": jobs_status.get('update_geoip', {}).get('status', 'idle') if is_license_configured() else 'disabled', "last_run": format_datetime_utc(jobs_status.get('update_geoip', {}).get('last_run')) if is_license_configured() else None, "error": jobs_status.get('update_geoip', {}).get('error') if is_license_configured() else None + }, + "mailbox_stats": { + "interval": "5 minutes", + "description": "Fetches mailbox statistics from Mailcow API", + "status": jobs_status.get('mailbox_stats', {}).get('status', 'unknown'), + "last_run": format_datetime_utc(jobs_status.get('mailbox_stats', {}).get('last_run')), + "error": jobs_status.get('mailbox_stats', {}).get('error') + }, + "alias_stats": { + "interval": "5 minutes", + "description": "Syncs alias data from Mailcow API", + "status": jobs_status.get('alias_stats', {}).get('status', 'unknown'), + "last_run": format_datetime_utc(jobs_status.get('alias_stats', {}).get('last_run')), + "error": jobs_status.get('alias_stats', {}).get('error') } }, "smtp_configuration": { diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index b00cd9d..bd2abde 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -18,7 +18,7 @@ from sqlalchemy.exc import IntegrityError from .config import settings, set_cached_active_domains from .database import get_db_context, SessionLocal from .mailcow_api import mailcow_api -from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck +from .models import PostfixLog, RspamdLog, NetfilterLog, MessageCorrelation, DMARCSync, DomainDNSCheck, MailboxStatistics, AliasStatistics from .correlation import detect_direction, parse_postfix_message from .routers.domains import check_domain_dns, save_dns_check_to_db from .services.dmarc_imap_service import sync_dmarc_reports_from_imap @@ -44,7 +44,9 @@ job_status = { 'check_app_version': {'last_run': None, 'status': 'idle', 'error': None}, 'dns_check': {'last_run': None, 'status': 'idle', 'error': None}, 'update_geoip': {'last_run': None, 'status': 'idle', 'error': None}, - 'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None} + 'dmarc_imap_sync': {'last_run': None, 'status': 'idle', 'error': None}, + 'mailbox_stats': {'last_run': None, 'status': 'idle', 'error': None}, + 'alias_stats': {'last_run': None, 'status': 'idle', 'error': None} } def update_job_status(job_name: str, status: str, error: str = None): @@ -1404,6 +1406,266 @@ async def sync_local_domains(): update_job_status('sync_local_domains', 'failed', str(e)) return False +# ============================================================================= +# MAILBOX STATISTICS +# ============================================================================= + +def safe_int(value, default=0): + """Safely convert a value to int, handling '- ', None, and other invalid values""" + if value is None: + return default + if isinstance(value, int): + return value + if isinstance(value, str): + value = value.strip() + if value in ('', '-', '- '): + return default + try: + return int(value) + except (ValueError, TypeError): + return default + try: + return int(value) + except (ValueError, TypeError): + return default + +def safe_float(value, default=0.0): + """Safely convert a value to float, handling '- ', None, and other invalid values""" + if value is None: + return default + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + value = value.strip() + if value in ('', '-', '- '): + return default + try: + return float(value) + except (ValueError, TypeError): + return default + try: + return float(value) + except (ValueError, TypeError): + return default + +async def update_mailbox_statistics(): + """ + Fetch mailbox statistics from Mailcow API and update the database. + Runs every 5 minutes. + Also removes mailboxes that no longer exist in Mailcow. + """ + update_job_status('mailbox_stats', 'running') + logger.info("Starting mailbox statistics update...") + + try: + # Fetch mailboxes from Mailcow API + mailboxes = await mailcow_api.get_mailboxes() + + if not mailboxes: + logger.warning("No mailboxes retrieved from Mailcow API") + update_job_status('mailbox_stats', 'success') + return + + # Get set of current mailbox usernames from API + api_mailbox_usernames = {mb.get('username') for mb in mailboxes if mb.get('username')} + + with get_db_context() as db: + updated = 0 + created = 0 + deleted = 0 + + # First, mark mailboxes that no longer exist in Mailcow as inactive + db_mailboxes = db.query(MailboxStatistics).all() + for db_mb in db_mailboxes: + if db_mb.username not in api_mailbox_usernames: + if db_mb.active: # Only log and count if it was previously active + logger.info(f"Marking deleted mailbox as inactive: {db_mb.username}") + db_mb.active = False + db_mb.updated_at = datetime.now(timezone.utc) + deleted += 1 + + for mb in mailboxes: + try: + username = mb.get('username') + if not username: + continue + + # Extract domain from username + domain = username.split('@')[-1] if '@' in username else '' + + # Check if mailbox exists + existing = db.query(MailboxStatistics).filter( + MailboxStatistics.username == username + ).first() + + # Prepare data - safely convert values + attributes = mb.get('attributes', {}) + rl_value_raw = mb.get('rl_value') + rl_value = safe_int(rl_value_raw) if rl_value_raw not in (None, '', '-', '- ') else None + rl_frame = mb.get('rl_frame') + if rl_frame in ('', '-', '- '): + rl_frame = None + + if existing: + # Update existing record + existing.domain = domain + existing.name = mb.get('name', '') or '' + existing.quota = safe_int(mb.get('quota'), 0) + existing.quota_used = safe_int(mb.get('quota_used'), 0) + existing.percent_in_use = safe_float(mb.get('percent_in_use'), 0.0) + existing.messages = safe_int(mb.get('messages'), 0) + existing.active = mb.get('active', 1) == 1 + existing.last_imap_login = safe_int(mb.get('last_imap_login'), 0) or None + existing.last_pop3_login = safe_int(mb.get('last_pop3_login'), 0) or None + existing.last_smtp_login = safe_int(mb.get('last_smtp_login'), 0) or None + existing.spam_aliases = safe_int(mb.get('spam_aliases'), 0) + existing.rl_value = rl_value + existing.rl_frame = rl_frame + existing.attributes = attributes + existing.updated_at = datetime.now(timezone.utc) + updated += 1 + else: + # Create new record + new_mailbox = MailboxStatistics( + username=username, + domain=domain, + name=mb.get('name', '') or '', + quota=safe_int(mb.get('quota'), 0), + quota_used=safe_int(mb.get('quota_used'), 0), + percent_in_use=safe_float(mb.get('percent_in_use'), 0.0), + messages=safe_int(mb.get('messages'), 0), + active=mb.get('active', 1) == 1, + last_imap_login=safe_int(mb.get('last_imap_login'), 0) or None, + last_pop3_login=safe_int(mb.get('last_pop3_login'), 0) or None, + last_smtp_login=safe_int(mb.get('last_smtp_login'), 0) or None, + spam_aliases=safe_int(mb.get('spam_aliases'), 0), + rl_value=rl_value, + rl_frame=rl_frame, + attributes=attributes + ) + db.add(new_mailbox) + created += 1 + + except Exception as e: + logger.error(f"Error processing mailbox {mb.get('username', 'unknown')}: {e}") + continue + + db.commit() + logger.info(f"✓ Mailbox statistics updated: {updated} updated, {created} created, {deleted} deactivated") + update_job_status('mailbox_stats', 'success') + + except Exception as e: + logger.error(f"✗ Failed to update mailbox statistics: {e}") + update_job_status('mailbox_stats', 'failed', str(e)) + + +# ============================================================================= +# ALIAS STATISTICS +# ============================================================================= + +async def update_alias_statistics(): + """ + Fetch aliases from Mailcow API and update the database. + Links aliases to their target mailboxes. + Runs every 5 minutes. + Also removes aliases that no longer exist in Mailcow. + """ + update_job_status('alias_stats', 'running') + logger.info("Starting alias statistics update...") + + try: + # Fetch aliases from Mailcow API + aliases = await mailcow_api.get_aliases() + + if not aliases: + logger.warning("No aliases retrieved from Mailcow API") + update_job_status('alias_stats', 'success') + return + + # Get set of current alias addresses from API + api_alias_addresses = {alias.get('address') for alias in aliases if alias.get('address')} + + with get_db_context() as db: + updated = 0 + created = 0 + deleted = 0 + + # First, mark aliases that no longer exist in Mailcow as inactive + db_aliases = db.query(AliasStatistics).all() + for db_alias in db_aliases: + if db_alias.alias_address not in api_alias_addresses: + if db_alias.active: # Only log and count if it was previously active + logger.info(f"Marking deleted alias as inactive: {db_alias.alias_address}") + db_alias.active = False + db_alias.updated_at = datetime.now(timezone.utc) + deleted += 1 + + for alias in aliases: + try: + alias_address = alias.get('address') + if not alias_address: + continue + + # Skip if this is a mailbox address (not an alias) + if alias.get('is_catch_all') is None and not alias.get('goto'): + continue + + # Extract domain from alias address + domain = alias_address.split('@')[-1] if '@' in alias_address else '' + + # Get the target mailbox(es) + goto = alias.get('goto', '') + + # Determine primary mailbox (first in goto list) + primary_mailbox = None + if goto: + goto_list = [g.strip() for g in goto.split(',') if g.strip()] + if goto_list: + primary_mailbox = goto_list[0] + + # Check if alias exists + existing = db.query(AliasStatistics).filter( + AliasStatistics.alias_address == alias_address + ).first() + + is_catch_all = alias.get('is_catch_all', 0) == 1 + is_active = alias.get('active', 1) == 1 + + if existing: + # Update existing record + existing.goto = goto + existing.domain = domain + existing.active = is_active + existing.is_catch_all = is_catch_all + existing.primary_mailbox = primary_mailbox + existing.updated_at = datetime.now(timezone.utc) + updated += 1 + else: + # Create new record + new_alias = AliasStatistics( + alias_address=alias_address, + goto=goto, + domain=domain, + active=is_active, + is_catch_all=is_catch_all, + primary_mailbox=primary_mailbox + ) + db.add(new_alias) + created += 1 + + except Exception as e: + logger.error(f"Error processing alias {alias.get('address', 'unknown')}: {e}") + continue + + db.commit() + logger.info(f"✓ Alias statistics updated: {updated} updated, {created} created, {deleted} deactivated") + update_job_status('alias_stats', 'success') + + except Exception as e: + logger.error(f"✗ Failed to update alias statistics: {e}") + update_job_status('alias_stats', 'failed', str(e)) + + # ============================================================================= # SCHEDULER SETUP # ============================================================================= @@ -1562,6 +1824,46 @@ def start_scheduler(): ) logger.info("Scheduled initial DMARC IMAP sync on startup") + # Job 13: Mailbox Statistics (every 5 minutes) + scheduler.add_job( + update_mailbox_statistics, + IntervalTrigger(minutes=5), + id='mailbox_stats', + name='Update Mailbox Statistics', + replace_existing=True, + max_instances=1 + ) + + # Run once on startup (after 45 seconds) + scheduler.add_job( + update_mailbox_statistics, + 'date', + run_date=datetime.now(timezone.utc) + timedelta(seconds=45), + id='mailbox_stats_startup', + name='Mailbox Statistics (Startup)' + ) + logger.info("Scheduled mailbox statistics job (interval: 5 minutes)") + + # Job 14: Alias Statistics (every 5 minutes) + scheduler.add_job( + update_alias_statistics, + IntervalTrigger(minutes=5), + id='alias_stats', + name='Update Alias Statistics', + replace_existing=True, + max_instances=1 + ) + + # Run once on startup (after 50 seconds) + scheduler.add_job( + update_alias_statistics, + 'date', + run_date=datetime.now(timezone.utc) + timedelta(seconds=50), + id='alias_stats_startup', + name='Alias Statistics (Startup)' + ) + logger.info("Scheduled alias statistics job (interval: 5 minutes)") + scheduler.start() logger.info("[OK] Scheduler started") diff --git a/backend/app/services/connection_test.py b/backend/app/services/connection_test.py index 222e10a..6e900e1 100644 --- a/backend/app/services/connection_test.py +++ b/backend/app/services/connection_test.py @@ -20,11 +20,18 @@ def test_smtp_connection() -> Dict: logs.append(f"Host: {settings.smtp_host}") logs.append(f"Port: {settings.smtp_port}") logs.append(f"Use TLS: {settings.smtp_use_tls}") + logs.append(f"Relay Mode: {settings.smtp_relay_mode}") logs.append(f"User: {settings.smtp_user}") - if not settings.smtp_host or not settings.smtp_user or not settings.smtp_password: - logs.append("ERROR: SMTP not fully configured") - return {"success": False, "logs": logs} + # Different validation for relay mode + if settings.smtp_relay_mode: + if not settings.smtp_host or not settings.smtp_from: + logs.append("ERROR: SMTP relay mode requires host and from address") + return {"success": False, "logs": logs} + else: + if not settings.smtp_host or not settings.smtp_user or not settings.smtp_password: + logs.append("ERROR: SMTP not fully configured") + return {"success": False, "logs": logs} logs.append("Connecting to SMTP server...") @@ -40,9 +47,13 @@ def test_smtp_connection() -> Dict: server.starttls() logs.append("TLS established") - logs.append("Logging in...") - server.login(settings.smtp_user, settings.smtp_password) - logs.append("Login successful") + # Skip login in relay mode + if not settings.smtp_relay_mode: + logs.append("Logging in...") + server.login(settings.smtp_user, settings.smtp_password) + logs.append("Login successful") + else: + logs.append("Relay mode - skipping authentication") logs.append("Sending test email...") msg = MIMEMultipart() diff --git a/backend/app/services/dmarc_cache.py b/backend/app/services/dmarc_cache.py new file mode 100644 index 0000000..e62d2c4 --- /dev/null +++ b/backend/app/services/dmarc_cache.py @@ -0,0 +1,95 @@ +""" +Shared DMARC Caching Service +Used to share cache state between API router and background services +""" +import logging +import json +import hashlib +from datetime import datetime, timedelta, timezone +from typing import Optional, Any +from sqlalchemy.orm import Session +from ..models import SystemSetting + +logger = logging.getLogger(__name__) + +# In-memory cache for DMARC stats +_dmarc_cache = {} +_dmarc_cache_ttl_seconds = 300 # 5 minutes cache TTL + +# Global signal tracking +_cache_valid_since = datetime.now(timezone.utc) +_last_db_check = datetime.min.replace(tzinfo=timezone.utc) +_db_check_interval_seconds = 5 + + +def get_dmarc_cache_key(prefix: str, **params) -> str: + """Generate a cache key from parameters""" + param_str = json.dumps(params, sort_keys=True, default=str) + hash_val = hashlib.md5(param_str.encode()).hexdigest()[:16] + return f"dmarc:{prefix}:{hash_val}" + + +def get_dmarc_cached(key: str, db: Session) -> Optional[Any]: + """Get cached value if not expired and not invalidated globally""" + global _dmarc_cache, _last_db_check, _cache_valid_since + + now = datetime.now(timezone.utc) + + # Periodically check DB for invalidation signal + if (now - _last_db_check).total_seconds() > _db_check_interval_seconds: + _last_db_check = now + try: + setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first() + if setting and setting.updated_at: + # Ensure timezone awareness + db_updated_at = setting.updated_at + if db_updated_at.tzinfo is None: + db_updated_at = db_updated_at.replace(tzinfo=timezone.utc) + + # If DB signal is newer than our local validity, clear cache + if db_updated_at > _cache_valid_since: + logger.info("DMARC cache invalidated by another process") + _dmarc_cache = {} + _cache_valid_since = now + return None + except Exception as e: + logger.error(f"Error checking cache signal: {e}") + + if key in _dmarc_cache: + cached_data, cached_time = _dmarc_cache[key] + if now - cached_time < timedelta(seconds=_dmarc_cache_ttl_seconds): + logger.debug(f"DMARC cache hit for key: {key}") + return cached_data + else: + # Cache expired, remove it + del _dmarc_cache[key] + return None + + +def set_dmarc_cache(key: str, data: Any) -> None: + """Set cached value with current timestamp""" + _dmarc_cache[key] = (data, datetime.now(timezone.utc)) + logger.debug(f"DMARC cache set for key: {key}") + + +def clear_dmarc_cache(db: Session) -> None: + """Clear all DMARC cache locally and signal other processes via DB""" + global _dmarc_cache, _cache_valid_since + + # local clear + _dmarc_cache = {} + _cache_valid_since = datetime.now(timezone.utc) + + try: + # DB signal + setting = db.query(SystemSetting).filter(SystemSetting.key == "dmarc_last_update").first() + if not setting: + setting = SystemSetting(key="dmarc_last_update", value="signal") + db.add(setting) + + setting.updated_at = datetime.utcnow() + db.commit() + logger.info("DMARC cache cleared and signaled to DB") + except Exception as e: + logger.error(f"Error clearing cache signal: {e}") + db.rollback() diff --git a/backend/app/services/dmarc_imap_service.py b/backend/app/services/dmarc_imap_service.py index ea7a596..5d3363c 100644 --- a/backend/app/services/dmarc_imap_service.py +++ b/backend/app/services/dmarc_imap_service.py @@ -1,6 +1,6 @@ """ -DMARC IMAP Service -Automatically fetches and processes DMARC reports from email inbox +DMARC & TLS-RPT IMAP Service +Automatically fetches and processes DMARC and TLS-RPT reports from email inbox """ import logging import imaplib @@ -8,16 +8,19 @@ import email import gzip import zipfile import io +import json from datetime import datetime, timezone from typing import List, Dict, Optional, Tuple from email.message import EmailMessage from ..config import settings from ..database import SessionLocal -from ..models import DMARCSync, DMARCReport, DMARCRecord +from ..models import DMARCSync, DMARCReport, DMARCRecord, TLSReport, TLSReportPolicy from ..services.dmarc_parser import parse_dmarc_file +from ..services.tls_rpt_parser import parse_tls_rpt_file, is_tls_rpt_json from ..services.geoip_service import enrich_dmarc_record from ..services.dmarc_notifications import send_dmarc_error_notification +from ..services.dmarc_cache import clear_dmarc_cache logger = logging.getLogger(__name__) @@ -75,30 +78,33 @@ class DMARCImapService: logger.error(f"Error selecting folder: {e}") return False - def search_dmarc_emails(self) -> List[bytes]: + def search_report_emails(self) -> List[bytes]: """ - Search for DMARC report emails + Search for DMARC and TLS-RPT report emails Looking for emails with subject containing: - - "Report Domain:" - - "Submitter:" - - "Report-ID:" + - "Report Domain:" (DMARC) + - "DMARC" (DMARC) + - "Report-ID:" (DMARC) + - "TLS-RPT" (TLS-RPT) + - "TLS Report" (TLS-RPT) Returns list of email IDs """ try: - # Search for emails with DMARC-related subject + # Search for emails with DMARC or TLS-RPT related subjects # Using OR to be more flexible - search_criteria = '(OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (SUBJECT "Report-ID:")))' + # UNSEEN ensures we don't re-process emails that were already handled (marked as Seen) + search_criteria = '(UNSEEN (OR (SUBJECT "Report Domain:") (OR (SUBJECT "DMARC") (OR (SUBJECT "Report-ID:") (OR (SUBJECT "TLS-RPT") (SUBJECT "TLS Report"))))))' - status, messages = self.connection.search(None, search_criteria) + status, messages = self.connection.uid('SEARCH', None, search_criteria) if status != 'OK': - logger.error("Failed to search for DMARC emails") + logger.error("Failed to search for report emails") return [] email_ids = messages[0].split() - logger.info(f"Found {len(email_ids)} potential DMARC emails") + logger.info(f"Found {len(email_ids)} potential DMARC/TLS-RPT emails") return email_ids @@ -106,83 +112,238 @@ class DMARCImapService: logger.error(f"Error searching for emails: {e}") return [] + def search_dmarc_emails(self) -> List[bytes]: + """Alias for backward compatibility""" + return self.search_report_emails() + def is_valid_dmarc_email(self, msg: EmailMessage) -> bool: """ Validate that this is a genuine DMARC report email - Checks: - 1. Subject contains "Report Domain:" AND ("Submitter:" OR "Report-ID:") - 2. Has at least one compressed attachment (.xml.gz or .zip) + Accepts multiple DMARC email formats: + - Standard: "Report Domain: X Submitter: Y Report-ID: Z" + - Yahoo format: "Report Domain: X Submitter: Y" (no Report-ID) + - Alternative: Contains "DMARC" in subject + - Microsoft Outlook: DMARC-like attachment filename pattern + + Primary validation is the attachment (.xml.gz or .zip with DMARC content) """ try: subject = msg.get('subject', '').lower() - # Check subject format - has_report_domain = 'report domain:' in subject - has_submitter = 'submitter:' in subject - has_report_id = 'report-id:' in subject + # Check for compressed DMARC attachments FIRST (most reliable indicator) + has_dmarc_attachment = False + has_dmarc_filename = False - if not (has_report_domain and (has_submitter or has_report_id)): - logger.debug(f"Email does not match DMARC subject pattern: {subject}") - return False - - # Check for compressed attachments - has_attachment = False for part in msg.walk(): filename = part.get_filename() if filename: filename_lower = filename.lower() - if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip'): - has_attachment = True + # DMARC reports come as .xml.gz, .xml, or .zip files + if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip') or filename_lower.endswith('.xml'): + has_dmarc_attachment = True + # Check if filename looks like a DMARC report + # Microsoft format: enterprise.protection.outlook.com!domain!timestamp!timestamp.xml.gz + # Standard format: domain!report-domain!timestamp!timestamp.xml.gz + if '!' in filename and (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml') or filename_lower.endswith('.zip')): + has_dmarc_filename = True break - if not has_attachment: + if not has_dmarc_attachment: logger.debug(f"Email has no compressed DMARC attachment: {subject}") return False + # Check subject format - be flexible to support different providers + has_report_domain = 'report domain:' in subject + has_submitter = 'submitter:' in subject + has_report_id = 'report-id:' in subject + has_dmarc_keyword = 'dmarc' in subject + + # Accept if: + # 1. Has "Report Domain:" and ("Submitter:" or "Report-ID:") - standard format + # 2. Has "Report Domain:" only (Yahoo and others) - we have verified attachment + # 3. Has "DMARC" keyword in subject with valid attachment + # 4. Has DMARC-like filename pattern (Microsoft Outlook and others) - attachment name contains '!' + is_valid_subject = ( + (has_report_domain and (has_submitter or has_report_id)) or # Standard format + (has_report_domain) or # Yahoo/minimal format (attachment already verified) + (has_dmarc_keyword) or # DMARC keyword with attachment + (has_dmarc_filename) # Microsoft Outlook format - DMARC filename pattern + ) + + if not is_valid_subject: + logger.debug(f"Email does not match DMARC subject/filename pattern: {subject}") + return False + return True except Exception as e: logger.error(f"Error validating DMARC email: {e}") return False - def extract_attachments(self, msg: EmailMessage) -> List[Tuple[str, bytes]]: + def is_valid_tls_rpt_email(self, msg: EmailMessage) -> bool: + """ + Validate that this is a TLS-RPT report email + + TLS-RPT emails typically have: + - Subject containing "TLS-RPT" or "TLS Report" + - JSON or JSON.GZ attachment + - Some providers send with generic subjects like "Report Domain: ..." + """ + try: + subject = msg.get('subject', '').lower() + + # Check for JSON/ZIP attachments + has_json_attachment = False + + for part in msg.walk(): + filename = part.get_filename() + if filename: + filename_lower = filename.lower() + if filename_lower.endswith('.json') or filename_lower.endswith('.json.gz') or filename_lower.endswith('.zip'): + has_json_attachment = True + break + + if not has_json_attachment: + return False + + # Trust the attachment if it looks like a TLS report + # If it has a json/gz/zip attachment, we should try to process it as potential TLS-RPT + # The parser will validate the content anyway + return True + + except Exception as e: + logger.error(f"Error validating TLS-RPT email: {e}") + return False + + def detect_email_type(self, msg: EmailMessage) -> str: + """ + Detect if email is DMARC or TLS-RPT by inspecting attachments + + Returns: 'dmarc', 'tls-rpt', or 'unknown' + """ + try: + # Check attachments FIRST - content is king + for part in msg.walk(): + filename = part.get_filename() + if not filename: + continue + + filename_lower = filename.lower() + content = None + + # Check explicit extensions + if filename_lower.endswith('.xml.gz') or filename_lower.endswith('.xml'): + return 'dmarc' + + if filename_lower.endswith('.json.gz') or filename_lower.endswith('.json'): + return 'tls-rpt' + + # Check ZIP content + if filename_lower.endswith('.zip'): + try: + content = part.get_payload(decode=True) + if content: + with zipfile.ZipFile(io.BytesIO(content)) as zf: + for name in zf.namelist(): + name_lower = name.lower() + if name_lower.endswith('.xml'): + logger.info(f"Found XML in ZIP {filename}, identifying as DMARC") + return 'dmarc' + if name_lower.endswith('.json'): + logger.info(f"Found JSON in ZIP {filename}, identifying as TLS-RPT") + return 'tls-rpt' + except Exception as e: + logger.warning(f"Failed to inspect ZIP {filename}: {e}") + + # Fallback to subject/header heuristics if no clear attachment type found + # but reject ambiguous ZIPs that we couldn't inspect or were empty of relevant files + + if self.is_valid_tls_rpt_email(msg): + return 'tls-rpt' + elif self.is_valid_dmarc_email(msg): + return 'dmarc' + + return 'unknown' + + except Exception as e: + logger.error(f"Error detecting email type: {e}") + return 'unknown' + + def extract_attachments(self, msg: EmailMessage, include_json: bool = False) -> List[Tuple[str, bytes]]: """ Extract compressed attachments from email + Args: + msg: Email message + include_json: If True, also extract JSON files (for TLS-RPT) + Returns list of (filename, content) tuples """ attachments = [] try: for part in msg.walk(): + # Try to get filename from Content-Disposition header filename = part.get_filename() + + # If no filename, try to get from Content-Type 'name' parameter + if not filename: + content_type = part.get_content_type() + # Check if this is a potential attachment by content type + if content_type in ['application/gzip', 'application/x-gzip', 'application/zip', + 'application/x-zip-compressed', 'text/xml', 'application/xml', + 'application/json', 'application/octet-stream']: + # Try to get name from content-type params + params = part.get_params() + if params: + for key, value in params: + if key.lower() == 'name': + filename = value + break + if not filename: continue filename_lower = filename.lower() - if not (filename_lower.endswith('.xml.gz') or filename_lower.endswith('.zip')): + + # Support DMARC files: .xml.gz, .zip, .xml + # Support TLS-RPT files: .json, .json.gz + valid_extensions = ['.xml.gz', '.zip', '.xml'] + if include_json: + valid_extensions.extend(['.json', '.json.gz']) + + if not any(filename_lower.endswith(ext) for ext in valid_extensions): continue content = part.get_payload(decode=True) if content: attachments.append((filename, content)) - logger.debug(f"Extracted attachment: {filename}") + logger.debug(f"Extracted attachment: {filename} ({len(content)} bytes)") except Exception as e: logger.error(f"Error extracting attachments: {e}") + if not attachments: + # Log all parts for debugging + logger.debug(f"No attachments found. Email parts:") + for i, part in enumerate(msg.walk()): + ct = part.get_content_type() + fn = part.get_filename() + logger.debug(f" Part {i}: type={ct}, filename={fn}") + return attachments def process_email(self, email_id: str, db: SessionLocal) -> Dict: """ - Process a single DMARC email + Process a single DMARC or TLS-RPT email Returns dict with: - success: bool - reports_created: int - reports_duplicate: int - error: str or None + - report_type: 'dmarc' or 'tls-rpt' """ result = { 'success': False, @@ -190,12 +351,13 @@ class DMARCImapService: 'reports_duplicate': 0, 'error': None, 'message_id': None, - 'subject': None + 'subject': None, + 'report_type': None } try: # Fetch email (email_id is already a string) - status, msg_data = self.connection.fetch(email_id, '(RFC822)') + status, msg_data = self.connection.uid('FETCH', email_id, '(RFC822)') if status != 'OK': result['error'] = f"Failed to fetch email {email_id}" @@ -206,80 +368,183 @@ class DMARCImapService: result['message_id'] = msg.get('message-id', 'unknown') result['subject'] = msg.get('subject', 'unknown') - # Validate it's a DMARC email - if not self.is_valid_dmarc_email(msg): - result['error'] = "Not a valid DMARC report email" + # Detect email type + email_type = self.detect_email_type(msg) + result['report_type'] = email_type + + if email_type == 'tls-rpt': + return self._process_tls_rpt_email(msg, db, result) + elif email_type == 'dmarc': + return self._process_dmarc_email(msg, db, result) + else: + result['error'] = "Not a valid DMARC or TLS-RPT report email" return result - # Extract attachments - attachments = self.extract_attachments(msg) - - if not attachments: - result['error'] = "No DMARC attachments found" - return result - - # Process each attachment - for filename, content in attachments: - try: - # Parse DMARC report - parsed_data = parse_dmarc_file(content, filename) - - if not parsed_data: - logger.warning(f"Failed to parse attachment: {filename}") - continue - - # Extract records - records_data = parsed_data.pop('records', []) - report_data = parsed_data - - # Check for duplicate - existing = db.query(DMARCReport).filter( - DMARCReport.report_id == report_data['report_id'] - ).first() - - if existing: - result['reports_duplicate'] += 1 - logger.info(f"Duplicate report: {report_data['report_id']}") - continue - - # Create report - report = DMARCReport(**report_data) - db.add(report) - db.flush() - - # Create records with GeoIP enrichment - for record_data in records_data: - record_data['dmarc_report_id'] = report.id - enriched = enrich_dmarc_record(record_data) - record = DMARCRecord(**enriched) - db.add(record) - - db.commit() - result['reports_created'] += 1 - logger.info(f"Created DMARC report: {report_data['report_id']}") - - except Exception as e: - db.rollback() - logger.error(f"Error processing attachment {filename}: {e}") - if not result['error']: - result['error'] = str(e) - - # Mark as success if at least one report was created - if result['reports_created'] > 0: - result['success'] = True - - return result - except Exception as e: logger.error(f"Error processing email {email_id}: {e}") result['error'] = str(e) return result + def _process_dmarc_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict: + """Process a DMARC email""" + # Extract attachments (DMARC: XML files) + attachments = self.extract_attachments(msg, include_json=False) + + if not attachments: + result['error'] = "No DMARC attachments found" + return result + + # Process each attachment + attachment_errors = [] + for filename, content in attachments: + try: + # Parse DMARC report + parsed_data = parse_dmarc_file(content, filename) + + if not parsed_data: + attachment_errors.append(f"Failed to parse: {filename}") + logger.warning(f"Failed to parse attachment: {filename}") + continue + + # Extract records + records_data = parsed_data.pop('records', []) + report_data = parsed_data + + # Check for duplicate + existing = db.query(DMARCReport).filter( + DMARCReport.report_id == report_data['report_id'] + ).first() + + if existing: + result['reports_duplicate'] += 1 + logger.info(f"Duplicate report: {report_data['report_id']}") + continue + + # Create report + report = DMARCReport(**report_data) + db.add(report) + db.flush() + + # Create records with GeoIP enrichment + for record_data in records_data: + record_data['dmarc_report_id'] = report.id + enriched = enrich_dmarc_record(record_data) + record = DMARCRecord(**enriched) + db.add(record) + + db.commit() + result['reports_created'] += 1 + logger.info(f"Created DMARC report: {report_data['report_id']}") + + except Exception as e: + db.rollback() + error_msg = f"Error processing {filename}: {str(e)}" + attachment_errors.append(error_msg) + logger.error(error_msg) + + # Determine success + return self._finalize_result(result, attachment_errors, "DMARC") + + def _process_tls_rpt_email(self, msg: EmailMessage, db: SessionLocal, result: Dict) -> Dict: + """Process a TLS-RPT email""" + # Extract attachments (TLS-RPT: JSON files) + attachments = self.extract_attachments(msg, include_json=True) + + # Filter to only JSON files (and ZIPs containing JSON) + json_attachments = [(f, c) for f, c in attachments if f.lower().endswith('.json') or f.lower().endswith('.json.gz') or f.lower().endswith('.zip')] + + if not json_attachments: + result['error'] = "No TLS-RPT JSON attachments found" + return result + + # Process each attachment + attachment_errors = [] + for filename, content in json_attachments: + try: + # Parse TLS-RPT report + parsed_data = parse_tls_rpt_file(content, filename) + + if not parsed_data: + attachment_errors.append(f"Failed to parse: {filename}") + logger.warning(f"Failed to parse TLS-RPT attachment: {filename}") + continue + + # Extract policies + policies_data = parsed_data.pop('policies', []) + + # Check for duplicate + existing = db.query(TLSReport).filter( + TLSReport.report_id == parsed_data['report_id'] + ).first() + + if existing: + result['reports_duplicate'] += 1 + logger.info(f"Duplicate TLS-RPT report: {parsed_data['report_id']}") + continue + + # Create TLS report + tls_report = TLSReport( + report_id=parsed_data['report_id'], + organization_name=parsed_data.get('organization_name', 'Unknown'), + contact_info=parsed_data.get('contact_info', ''), + policy_domain=parsed_data['policy_domain'], + start_datetime=parsed_data['start_datetime'], + end_datetime=parsed_data['end_datetime'], + raw_json=parsed_data.get('raw_json', '') + ) + db.add(tls_report) + db.flush() + + # Create policy records + for policy_data in policies_data: + policy = TLSReportPolicy( + tls_report_id=tls_report.id, + policy_type=policy_data.get('policy_type', 'unknown'), + policy_domain=policy_data.get('policy_domain', ''), + policy_string=policy_data.get('policy_string', []), + mx_host=policy_data.get('mx_host', []), + successful_session_count=policy_data.get('successful_session_count', 0), + failed_session_count=policy_data.get('failed_session_count', 0), + failure_details=policy_data.get('failure_details', []) + ) + db.add(policy) + + db.commit() + result['reports_created'] += 1 + logger.info(f"Created TLS-RPT report: {parsed_data['report_id']}") + + except Exception as e: + db.rollback() + error_msg = f"Error processing TLS-RPT {filename}: {str(e)}" + attachment_errors.append(error_msg) + logger.error(error_msg) + + # Determine success + return self._finalize_result(result, attachment_errors, "TLS-RPT") + + def _finalize_result(self, result: Dict, attachment_errors: List[str], report_type: str) -> Dict: + """Finalize the result based on processing outcome""" + if result['reports_created'] > 0: + result['success'] = True + elif result['reports_duplicate'] > 0 and result['reports_created'] == 0: + # All reports were duplicates - this is actually OK, mark as success + result['success'] = True + result['error'] = None # No error - duplicates are expected + else: + # No reports created and no duplicates - something went wrong + result['success'] = False + if attachment_errors: + result['error'] = "; ".join(attachment_errors) + else: + result['error'] = f"No valid {report_type} reports found in attachments" + + return result + def mark_as_processed(self, email_id: str): """Mark email as processed (flag or move)""" try: # Add a flag to mark as processed - self.connection.store(email_id, '+FLAGS', '\\Seen') + self.connection.uid('STORE', email_id, '+FLAGS', '\\Seen') logger.debug(f"Marked email {email_id} as seen") except Exception as e: @@ -288,7 +553,7 @@ class DMARCImapService: def delete_email(self, email_id: str): """Delete email from server""" try: - self.connection.store(email_id, '+FLAGS', '\\Deleted') + self.connection.uid('STORE', email_id, '+FLAGS', '\\Deleted') self.connection.expunge() logger.debug(f"Deleted email {email_id}") @@ -297,7 +562,11 @@ class DMARCImapService: def sync_reports(self, sync_type: str = 'auto') -> Dict: """ - Main sync function + Main sync function with batch processing + + Processes emails in batches to prevent memory issues with large mailboxes. + After each batch, emails are deleted/marked and the search is re-run + to get the next batch of unprocessed emails. Returns statistics about the sync operation """ @@ -308,6 +577,7 @@ class DMARCImapService: ) db = SessionLocal() + batch_size = settings.dmarc_imap_batch_size try: db.add(sync_record) @@ -321,45 +591,74 @@ class DMARCImapService: if not self.select_folder(): raise Exception(f"Failed to select folder {self.folder}") - # Search for DMARC emails - email_ids = self.search_dmarc_emails() - sync_record.emails_found = len(email_ids) + # Initial search to count total emails + all_email_ids = self.search_dmarc_emails() + total_emails = len(all_email_ids) + sync_record.emails_found = total_emails db.commit() - if not email_ids: + if not all_email_ids: logger.info("No DMARC emails found") sync_record.status = 'success' sync_record.completed_at = datetime.now(timezone.utc) db.commit() return self._build_result(sync_record) - # Process each email - failed_emails = [] + logger.info(f"Found {total_emails} DMARC emails, processing in batches of {batch_size}") - for email_id in email_ids: - email_id = email_id.decode() if isinstance(email_id, bytes) else email_id - result = self.process_email(email_id, db) - sync_record.emails_processed += 1 + # Process in batches + failed_emails = [] + batch_number = 0 + + while True: + batch_number += 1 - if result['success']: - sync_record.reports_created += result['reports_created'] - sync_record.reports_duplicate += result['reports_duplicate'] + # Re-search to get current unprocessed emails (since we delete/mark after each batch) + email_ids = self.search_dmarc_emails() + + if not email_ids: + logger.info(f"Batch {batch_number}: No more emails to process") + break + + # Take only batch_size emails + batch_emails = email_ids[:batch_size] + logger.info(f"Batch {batch_number}: Processing {len(batch_emails)} emails (remaining: {len(email_ids)})") + + for email_id in batch_emails: + email_id = email_id.decode() if isinstance(email_id, bytes) else email_id + result = self.process_email(email_id, db) + sync_record.emails_processed += 1 - # Delete or mark as processed - if self.delete_after: - self.delete_email(email_id) + if result['success']: + sync_record.reports_created += result['reports_created'] + sync_record.reports_duplicate += result['reports_duplicate'] + + # Delete or mark as processed + if self.delete_after: + self.delete_email(email_id) + else: + self.mark_as_processed(email_id) else: + sync_record.reports_failed += 1 + error_msg = result.get('error', 'Unknown error') + logger.warning(f"Failed to process email {email_id}: {error_msg}") + + failed_emails.append({ + 'email_id': email_id, + 'message_id': result['message_id'], + 'subject': result['subject'], + 'error': error_msg + }) + # Also mark failed emails as processed to avoid re-processing self.mark_as_processed(email_id) - else: - sync_record.reports_failed += 1 - failed_emails.append({ - 'email_id': email_id, - 'message_id': result['message_id'], - 'subject': result['subject'], - 'error': result['error'] - }) + + db.commit() - db.commit() + # Log batch progress + logger.info(f"Batch {batch_number} complete: " + f"{sync_record.emails_processed}/{total_emails} processed, " + f"{sync_record.reports_created} created, " + f"{sync_record.reports_failed} failed") # Update sync record sync_record.status = 'success' @@ -375,6 +674,10 @@ class DMARCImapService: f"{sync_record.reports_duplicate} duplicates, " f"{sync_record.reports_failed} failed") + # Clear cache if any reports were created + if sync_record.reports_created > 0: + clear_dmarc_cache(db) + # Send email notification if there were failures if failed_emails and settings.notification_smtp_configured: logger.info(f"Sending error notification for {len(failed_emails)} failed emails") diff --git a/backend/app/services/dmarc_parser.py b/backend/app/services/dmarc_parser.py index 6808a7e..256d164 100644 --- a/backend/app/services/dmarc_parser.py +++ b/backend/app/services/dmarc_parser.py @@ -58,6 +58,57 @@ def parse_dmarc_file(file_content: bytes, filename: str) -> Optional[Dict[str, A return None +def find_element(parent: ET.Element, tag: str, namespaces: List[str]) -> Optional[ET.Element]: + """ + Find an element trying multiple namespaces + + Args: + parent: Parent XML element + tag: Tag name to find + namespaces: List of namespace prefixes to try (empty string for no namespace) + + Returns: + Found element or None + """ + for ns in namespaces: + if ns: + elem = parent.find(f'{{{ns}}}{tag}') + else: + elem = parent.find(tag) + if elem is not None: + return elem + return None + + +def find_all_elements(parent: ET.Element, tag: str, namespaces: List[str]) -> List[ET.Element]: + """ + Find all elements matching tag, trying multiple namespaces + + Args: + parent: Parent XML element + tag: Tag name to find + namespaces: List of namespace prefixes to try + + Returns: + List of found elements + """ + for ns in namespaces: + if ns: + elems = parent.findall(f'{{{ns}}}{tag}') + else: + elems = parent.findall(tag) + if elems: + return elems + return [] + + +# Known DMARC XML namespaces +DMARC_NAMESPACES = [ + '', # No namespace (DMARC 1.0) + 'urn:ietf:params:xml:ns:dmarc-2.0', # DMARC 2.0 +] + + def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]: """ Parse DMARC XML content @@ -72,38 +123,38 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]: try: root = ET.fromstring(xml_string) - # Parse report metadata - metadata = root.find('report_metadata') + # Parse report metadata (try with and without namespace) + metadata = find_element(root, 'report_metadata', DMARC_NAMESPACES) if metadata is None: raise ValueError("Missing report_metadata element") - org_name = get_element_text(metadata, 'org_name') - email = get_element_text(metadata, 'email') - extra_contact_info = get_element_text(metadata, 'extra_contact_info') - report_id = get_element_text(metadata, 'report_id') + org_name = get_element_text(metadata, 'org_name', DMARC_NAMESPACES) + email = get_element_text(metadata, 'email', DMARC_NAMESPACES) + extra_contact_info = get_element_text(metadata, 'extra_contact_info', DMARC_NAMESPACES) + report_id = get_element_text(metadata, 'report_id', DMARC_NAMESPACES) - date_range = metadata.find('date_range') + date_range = find_element(metadata, 'date_range', DMARC_NAMESPACES) if date_range is None: raise ValueError("Missing date_range element") - begin_date = int(get_element_text(date_range, 'begin')) - end_date = int(get_element_text(date_range, 'end')) + begin_date = int(get_element_text(date_range, 'begin', DMARC_NAMESPACES)) + end_date = int(get_element_text(date_range, 'end', DMARC_NAMESPACES)) # Parse published policy - policy = root.find('policy_published') + policy = find_element(root, 'policy_published', DMARC_NAMESPACES) if policy is None: raise ValueError("Missing policy_published element") - domain = get_element_text(policy, 'domain') + domain = get_element_text(policy, 'domain', DMARC_NAMESPACES) policy_published = { - 'adkim': get_element_text(policy, 'adkim'), - 'aspf': get_element_text(policy, 'aspf'), - 'p': get_element_text(policy, 'p'), - 'sp': get_element_text(policy, 'sp'), - 'pct': get_element_text(policy, 'pct'), - 'fo': get_element_text(policy, 'fo'), - 'np': get_element_text(policy, 'np'), + 'adkim': get_element_text(policy, 'adkim', DMARC_NAMESPACES), + 'aspf': get_element_text(policy, 'aspf', DMARC_NAMESPACES), + 'p': get_element_text(policy, 'p', DMARC_NAMESPACES), + 'sp': get_element_text(policy, 'sp', DMARC_NAMESPACES), + 'pct': get_element_text(policy, 'pct', DMARC_NAMESPACES), + 'fo': get_element_text(policy, 'fo', DMARC_NAMESPACES), + 'np': get_element_text(policy, 'np', DMARC_NAMESPACES), } # Remove None values @@ -111,7 +162,7 @@ def parse_dmarc_xml(xml_string: str, raw_xml: str) -> Dict[str, Any]: # Parse records records = [] - for record_elem in root.findall('record'): + for record_elem in find_all_elements(root, 'record', DMARC_NAMESPACES): record_data = parse_dmarc_record(record_elem) if record_data: records.append(record_data) @@ -145,38 +196,38 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]: Dictionary with parsed record data """ try: - row = record_elem.find('row') + row = find_element(record_elem, 'row', DMARC_NAMESPACES) if row is None: return None # Source and count - source_ip = get_element_text(row, 'source_ip') - count = int(get_element_text(row, 'count', '0')) + source_ip = get_element_text(row, 'source_ip', DMARC_NAMESPACES) + count = int(get_element_text(row, 'count', DMARC_NAMESPACES, '0')) # Policy evaluation - policy_eval = row.find('policy_evaluated') - disposition = get_element_text(policy_eval, 'disposition') if policy_eval else None - dkim_result = get_element_text(policy_eval, 'dkim') if policy_eval else None - spf_result = get_element_text(policy_eval, 'spf') if policy_eval else None + policy_eval = find_element(row, 'policy_evaluated', DMARC_NAMESPACES) + disposition = get_element_text(policy_eval, 'disposition', DMARC_NAMESPACES) if policy_eval else None + dkim_result = get_element_text(policy_eval, 'dkim', DMARC_NAMESPACES) if policy_eval else None + spf_result = get_element_text(policy_eval, 'spf', DMARC_NAMESPACES) if policy_eval else None # Identifiers - identifiers = record_elem.find('identifiers') - header_from = get_element_text(identifiers, 'header_from') if identifiers else None - envelope_from = get_element_text(identifiers, 'envelope_from') if identifiers else None - envelope_to = get_element_text(identifiers, 'envelope_to') if identifiers else None + identifiers = find_element(record_elem, 'identifiers', DMARC_NAMESPACES) + header_from = get_element_text(identifiers, 'header_from', DMARC_NAMESPACES) if identifiers else None + envelope_from = get_element_text(identifiers, 'envelope_from', DMARC_NAMESPACES) if identifiers else None + envelope_to = get_element_text(identifiers, 'envelope_to', DMARC_NAMESPACES) if identifiers else None # Auth results auth_results = {} - auth_results_elem = record_elem.find('auth_results') + auth_results_elem = find_element(record_elem, 'auth_results', DMARC_NAMESPACES) if auth_results_elem: # Parse DKIM results dkim_results = [] - for dkim_elem in auth_results_elem.findall('dkim'): + for dkim_elem in find_all_elements(auth_results_elem, 'dkim', DMARC_NAMESPACES): dkim_data = { - 'domain': get_element_text(dkim_elem, 'domain'), - 'selector': get_element_text(dkim_elem, 'selector'), - 'result': get_element_text(dkim_elem, 'r') or get_element_text(dkim_elem, 'result') + 'domain': get_element_text(dkim_elem, 'domain', DMARC_NAMESPACES), + 'selector': get_element_text(dkim_elem, 'selector', DMARC_NAMESPACES), + 'result': get_element_text(dkim_elem, 'r', DMARC_NAMESPACES) or get_element_text(dkim_elem, 'result', DMARC_NAMESPACES) } dkim_results.append({k: v for k, v in dkim_data.items() if v}) @@ -185,11 +236,11 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]: # Parse SPF results spf_results = [] - for spf_elem in auth_results_elem.findall('spf'): + for spf_elem in find_all_elements(auth_results_elem, 'spf', DMARC_NAMESPACES): spf_data = { - 'domain': get_element_text(spf_elem, 'domain'), - 'scope': get_element_text(spf_elem, 'scope'), - 'result': get_element_text(spf_elem, 'r') or get_element_text(spf_elem, 'result') + 'domain': get_element_text(spf_elem, 'domain', DMARC_NAMESPACES), + 'scope': get_element_text(spf_elem, 'scope', DMARC_NAMESPACES), + 'result': get_element_text(spf_elem, 'r', DMARC_NAMESPACES) or get_element_text(spf_elem, 'result', DMARC_NAMESPACES) } spf_results.append({k: v for k, v in spf_data.items() if v}) @@ -213,13 +264,14 @@ def parse_dmarc_record(record_elem: ET.Element) -> Optional[Dict[str, Any]]: return None -def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[str] = None) -> Optional[str]: +def get_element_text(parent: Optional[ET.Element], tag: str, namespaces: List[str] = None, default: Optional[str] = None) -> Optional[str]: """ - Safely get text from XML element + Safely get text from XML element with namespace support Args: parent: Parent XML element tag: Tag name to find + namespaces: List of namespace prefixes to try default: Default value if not found Returns: @@ -228,7 +280,11 @@ def get_element_text(parent: Optional[ET.Element], tag: str, default: Optional[s if parent is None: return default - elem = parent.find(tag) + if namespaces: + elem = find_element(parent, tag, namespaces) + else: + elem = parent.find(tag) + if elem is not None and elem.text: return elem.text.strip() diff --git a/backend/app/services/smtp_service.py b/backend/app/services/smtp_service.py index 2800dae..255ba9c 100644 --- a/backend/app/services/smtp_service.py +++ b/backend/app/services/smtp_service.py @@ -21,8 +21,10 @@ class SmtpService: self.host = settings.smtp_host self.port = settings.smtp_port self.use_tls = settings.smtp_use_tls + self.use_ssl = settings.smtp_use_ssl self.user = settings.smtp_user self.password = settings.smtp_password + self.relay_mode = settings.smtp_relay_mode self.from_address = settings.smtp_from or settings.smtp_user def is_configured(self) -> bool: @@ -70,13 +72,22 @@ class SmtpService: part2 = MIMEText(html_content, 'html') msg.attach(part2) - if self.use_tls: + # Determine connection mode + # Priority 1: Implicit SSL (if configured or using port 465) + if self.use_ssl or self.port == 465: + server = smtplib.SMTP_SSL(self.host, self.port) + # Priority 2: STARTTLS (if configured) + elif self.use_tls: server = smtplib.SMTP(self.host, self.port) server.starttls() + # Priority 3: Plaintext else: - server = smtplib.SMTP_SSL(self.host, self.port) + server = smtplib.SMTP(self.host, self.port) + + # Skip login in relay mode + if not self.relay_mode: + server.login(self.user, self.password) - server.login(self.user, self.password) server.sendmail(self.from_address, [recipient], msg.as_string()) server.quit() diff --git a/backend/app/services/tls_rpt_parser.py b/backend/app/services/tls_rpt_parser.py new file mode 100644 index 0000000..d66bcdf --- /dev/null +++ b/backend/app/services/tls_rpt_parser.py @@ -0,0 +1,218 @@ +""" +TLS-RPT (SMTP TLS Reporting) Parser +Handles parsing of TLS-RPT reports in JSON format +""" +import json +import gzip +import zipfile +import logging +from typing import Dict, Any, Optional +from datetime import datetime +from io import BytesIO + +logger = logging.getLogger(__name__) + + +def parse_tls_rpt_file(file_content: bytes, filename: str) -> Optional[Dict[str, Any]]: + """ + Parse TLS-RPT report from file content (JSON, GZ compressed, or ZIP) + + Args: + file_content: Raw bytes of the file + filename: Original filename (to determine compression type) + + Returns: + Parsed TLS-RPT data or None if parsing failed + """ + try: + filename_lower = filename.lower() + + # Handle ZIP files (.json.zip) + if filename_lower.endswith('.zip'): + try: + with zipfile.ZipFile(BytesIO(file_content)) as zf: + # Find JSON file inside ZIP + json_files = [f for f in zf.namelist() if f.lower().endswith('.json')] + if not json_files: + logger.error(f"No JSON file found in ZIP: {filename}") + return None + + # Read the first JSON file + json_content = zf.read(json_files[0]).decode('utf-8') + except zipfile.BadZipFile as e: + logger.error(f"Invalid ZIP file {filename}: {e}") + return None + # Handle GZIP files (.json.gz) + elif filename_lower.endswith('.gz'): + try: + json_content = gzip.decompress(file_content).decode('utf-8') + except Exception as e: + logger.error(f"Failed to decompress gzip TLS-RPT file: {e}") + return None + # Handle plain JSON files + elif filename_lower.endswith('.json'): + json_content = file_content.decode('utf-8') + else: + # Try to decode as JSON directly + try: + json_content = file_content.decode('utf-8') + except Exception: + logger.error(f"Unknown TLS-RPT file format: {filename}") + return None + + return parse_tls_rpt_json(json_content) + + except Exception as e: + logger.error(f"Error parsing TLS-RPT file {filename}: {e}") + return None + + + +def parse_tls_rpt_json(json_content: str) -> Optional[Dict[str, Any]]: + """ + Parse TLS-RPT JSON content + + Expected format (RFC 8460): + { + "organization-name": "Google Inc.", + "date-range": { + "start-datetime": "2026-01-12T00:00:00Z", + "end-datetime": "2026-01-12T23:59:59Z" + }, + "contact-info": "smtp-tls-reporting@google.com", + "report-id": "2026-01-12T00:00:00Z_boubou.me", + "policies": [{ + "policy": { + "policy-type": "sts", + "policy-string": ["version: STSv1", "mode: enforce", ...], + "policy-domain": "boubou.me", + "mx-host": ["mail.tiboxs.com"] + }, + "summary": { + "total-successful-session-count": 1, + "total-failure-session-count": 0 + }, + "failure-details": [...] # Optional + }] + } + + Returns: + Dictionary with parsed TLS-RPT data + """ + try: + data = json.loads(json_content) + + # Extract report metadata + report_id = data.get('report-id', '') + if not report_id: + logger.error("TLS-RPT report missing report-id") + return None + + organization_name = data.get('organization-name', 'Unknown') + contact_info = data.get('contact-info', '') + + # Parse date range + date_range = data.get('date-range', {}) + start_datetime = parse_iso_datetime(date_range.get('start-datetime', '')) + end_datetime = parse_iso_datetime(date_range.get('end-datetime', '')) + + if not start_datetime or not end_datetime: + logger.error("TLS-RPT report missing or invalid date-range") + return None + + # Parse policies + policies = [] + policy_domain = None + + for policy_entry in data.get('policies', []): + policy_data = policy_entry.get('policy', {}) + summary = policy_entry.get('summary', {}) + + # Get the policy domain from the first policy + if not policy_domain: + policy_domain = policy_data.get('policy-domain', '') + + parsed_policy = { + 'policy_type': policy_data.get('policy-type', 'unknown'), + 'policy_domain': policy_data.get('policy-domain', ''), + 'policy_string': policy_data.get('policy-string', []), + 'mx_host': policy_data.get('mx-host', []), + 'successful_session_count': summary.get('total-successful-session-count', 0), + 'failed_session_count': summary.get('total-failure-session-count', 0), + 'failure_details': policy_entry.get('failure-details', []) + } + policies.append(parsed_policy) + + if not policy_domain: + logger.error("TLS-RPT report missing policy-domain") + return None + + return { + 'report_id': report_id, + 'organization_name': organization_name, + 'contact_info': contact_info, + 'policy_domain': policy_domain, + 'start_datetime': start_datetime, + 'end_datetime': end_datetime, + 'policies': policies, + 'raw_json': json_content + } + + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON in TLS-RPT report: {e}") + return None + except Exception as e: + logger.error(f"Error parsing TLS-RPT JSON: {e}") + return None + + +def parse_iso_datetime(datetime_str: str) -> Optional[datetime]: + """ + Parse ISO 8601 datetime string + + Supports formats: + - 2026-01-12T00:00:00Z + - 2026-01-12T00:00:00+00:00 + """ + if not datetime_str: + return None + + try: + # Remove 'Z' suffix and replace with +00:00 for parsing + if datetime_str.endswith('Z'): + datetime_str = datetime_str[:-1] + '+00:00' + + # Parse with timezone + from datetime import timezone + dt = datetime.fromisoformat(datetime_str) + + # Convert to UTC naive datetime for storage + if dt.tzinfo is not None: + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + + return dt + + except Exception as e: + logger.error(f"Error parsing datetime '{datetime_str}': {e}") + return None + + +def is_tls_rpt_json(json_content: str) -> bool: + """ + Check if JSON content is a valid TLS-RPT report + + Used to detect TLS-RPT vs other JSON files + """ + try: + data = json.loads(json_content) + + # Check for required TLS-RPT fields + has_report_id = 'report-id' in data + has_date_range = 'date-range' in data + has_policies = 'policies' in data + + # At minimum, should have policies and date-range + return has_policies and (has_date_range or has_report_id) + + except Exception: + return False diff --git a/documentation/API.md b/documentation/API.md index 7d12d12..a3e9b06 100644 --- a/documentation/API.md +++ b/documentation/API.md @@ -14,18 +14,19 @@ This document describes all available API endpoints for the Mailcow Logs Viewer 2. [Health & Info](#health--info) 3. [Job Status Tracking](#job-status-tracking) 4. [Domains](#domains) -5. [Messages (Unified View)](#messages-unified-view) -6. [Logs](#logs) +5. [Mailbox Statistics](#mailbox-statistics) +6. [Messages (Unified View)](#messages-unified-view) +7. [Logs](#logs) - [Postfix Logs](#postfix-logs) - [Rspamd Logs](#rspamd-logs) - [Netfilter Logs](#netfilter-logs) -7. [Queue & Quarantine](#queue--quarantine) -8. [Statistics](#statistics) -9. [Status](#status) -10. [Settings](#settings) +8. [Queue & Quarantine](#queue--quarantine) +9. [Statistics](#statistics) +10. [Status](#status) +11. [Settings](#settings) - [SMTP & IMAP Test](#smtp--imap-test) -11. [Export](#export) -12. [DMARC](#dmarc) +12. [Export](#export) +13. [DMARC](#dmarc) - [DMARC IMAP Auto-Import](#dmarc-imap-auto-import) --- @@ -736,6 +737,200 @@ POST /api/domains/example.com/check-dns --- +## Mailbox Statistics + +### GET /api/mailbox-stats/summary + +Get summary statistics for all mailboxes. + +**Query Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) | + +**Response:** +```json +{ + "total_mailboxes": 25, + "active_mailboxes": 23, + "inactive_mailboxes": 2, + "total_sent": 1234, + "total_received": 5678, + "sent_failed": 45, + "failure_rate": 3.6, + "date_range": "30days", + "start_date": "2026-01-16T00:00:00Z", + "end_date": "2026-02-16T00:00:00Z" +} +``` + +--- + +### GET /api/mailbox-stats/all + +Get all mailbox statistics with message counts and aliases (paginated). + +**Query Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `domain` | string | Filter by domain name | +| `active_only` | bool | Only show active mailboxes (default: `true`) | +| `hide_zero` | bool | Hide mailboxes with zero activity (default: `false`) | +| `search` | string | Search mailbox username, name, or alias address | +| `date_range` | string | Date range: `today`, `7days`, `30days`, `90days` (default: `30days`) | +| `sort_by` | string | Sort by: `sent_total`, `received_total`, `failure_rate`, `quota_used`, `username` | +| `sort_order` | string | Sort order: `asc`, `desc` (default: `desc`) | +| `page` | int | Page number (default: `1`) | +| `page_size` | int | Items per page, 10-100 (default: `50`) | + +**Example Request:** +``` +GET /api/mailbox-stats/all?date_range=30days&active_only=true&hide_zero=true&sort_by=sent_total&sort_order=desc&page=1 +``` + +**Response:** +```json +{ + "total": 25, + "page": 1, + "page_size": 50, + "total_pages": 1, + "date_range": "30days", + "start_date": "2026-01-16T00:00:00Z", + "end_date": "2026-02-16T00:00:00Z", + "mailboxes": [ + { + "id": 1, + "username": "user@example.com", + "domain": "example.com", + "name": "John Doe", + "active": true, + "quota": 1073741824, + "quota_formatted": "1.0 GB", + "quota_used": 536870912, + "quota_used_formatted": "512 MB", + "percent_in_use": 50.0, + "messages_in_mailbox": 1234, + "last_imap_login": "2026-01-15T10:30:00Z", + "last_pop3_login": null, + "last_smtp_login": "2026-01-16T08:45:00Z", + "rl_value": 100, + "rl_frame": "m", + "attributes": { + "imap_access": "1", + "pop3_access": "0", + "smtp_access": "1", + "sieve_access": "1", + "sogo_access": "1", + "tls_enforce_in": "0", + "tls_enforce_out": "0" + }, + "mailbox_counts": { + "sent_total": 150, + "sent_delivered": 145, + "sent_bounced": 3, + "sent_deferred": 2, + "sent_rejected": 0, + "sent_failed": 5, + "received_total": 320, + "failure_rate": 3.3 + }, + "aliases": [ + { + "alias_address": "info@example.com", + "active": true, + "is_catch_all": false, + "sent_total": 50, + "sent_delivered": 48, + "sent_bounced": 2, + "sent_deferred": 0, + "sent_rejected": 0, + "sent_failed": 2, + "received_total": 100, + "failure_rate": 4.0 + } + ], + "alias_count": 1, + "combined_sent": 200, + "combined_received": 420, + "combined_total": 620, + "combined_failed": 7, + "combined_failure_rate": 3.5, + "created": "2025-01-01T00:00:00Z", + "modified": "2026-01-15T12:00:00Z" + } + ] +} +``` + +**Response Fields:** + +| Field | Description | +|-------|-------------| +| `username` | Email address of the mailbox | +| `name` | Display name | +| `active` | Whether mailbox is active in Mailcow | +| `quota` / `quota_used` | Quota in bytes | +| `percent_in_use` | Quota usage percentage | +| `messages_in_mailbox` | Number of messages stored | +| `last_*_login` | Last login timestamps (null if never) | +| `rl_value` / `rl_frame` | Rate limiting (e.g., 100/m = 100 per minute) | +| `attributes` | Access permissions from Mailcow | +| `mailbox_counts` | Message statistics for mailbox only | +| `aliases` | Array of alias statistics | +| `combined_*` | Combined totals (mailbox + all aliases) | +| `created` / `modified` | Mailbox creation and last update timestamps | + +--- + +### GET /api/mailbox-stats/domains + +Get list of domains with mailbox counts for filter dropdown. + +**Response:** +```json +{ + "domains": [ + { + "domain": "example.com", + "mailbox_count": 15 + }, + { + "domain": "company.org", + "mailbox_count": 10 + } + ] +} +``` + +### Caching + +The Mailbox Statistics API uses in-memory caching to improve performance: + +| Setting | Value | +|---------|-------| +| **Cache TTL** | 5 minutes (300 seconds) | +| **Cache Scope** | Per unique query parameter combination | +| **Cached Parameters** | domain, active_only, hide_zero, search, date_range, start_date, end_date, sort_by, sort_order, page, page_size | + +**Cache Behavior:** +- First request with specific parameters fetches from database and caches result +- Subsequent requests with identical parameters return cached data +- Cache automatically expires after 5 minutes +- Changing any parameter results in a cache miss (new database query) + +**Cache Management:** +```python +from app.routers.mailbox_stats import clear_stats_cache + +# Clear all stats cache (e.g., after data import) +clear_stats_cache() +``` + +--- + ## Messages (Unified View) ### GET /messages @@ -2505,6 +2700,165 @@ Get history of IMAP sync operations. --- +## TLS-RPT (TLS Reporting) + +### Overview + +TLS-RPT (TLS Reporting) provides visibility into TLS connection failures when other mail servers attempt to deliver emails to your domain. This helps identify MTA-STS policy issues and certificate problems. + +--- + +### GET /api/dmarc/domains/{domain}/tls-reports + +Get TLS reports for a specific domain (individual reports). + +**Path Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `domain` | string | Domain name | + +**Query Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `days` | integer | 30 | Number of days to look back | + +**Response:** +```json +{ + "domain": "example.com", + "total": 15, + "data": [ + { + "id": 1, + "report_id": "2026-01-14T00:00:00Z!example.com!google.com", + "organization_name": "Google Inc.", + "start_datetime": "2026-01-14T00:00:00Z", + "end_datetime": "2026-01-15T00:00:00Z", + "total_success": 1250, + "total_fail": 5, + "success_rate": 99.6 + } + ] +} +``` + +--- + +### GET /api/dmarc/domains/{domain}/tls-reports/daily + +Get TLS reports aggregated by date (daily view). + +**Path Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `domain` | string | Domain name | + +**Query Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `days` | integer | 30 | Number of days to look back | +| `page` | integer | 1 | Page number | +| `page_size` | integer | 20 | Items per page | + +**Response:** +```json +{ + "domain": "example.com", + "totals": { + "total_days": 14, + "total_reports": 28, + "total_successful_sessions": 15000, + "total_failed_sessions": 25, + "overall_success_rate": 99.83 + }, + "data": [ + { + "date": "2026-01-17", + "report_count": 3, + "organization_count": 2, + "organizations": ["Google Inc.", "Microsoft Corporation"], + "total_success": 1500, + "total_fail": 2, + "success_rate": 99.87 + } + ] +} +``` + +--- + +### GET /api/dmarc/domains/{domain}/tls-reports/{report_date}/details + +Get detailed TLS reports for a specific date. + +**Path Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `domain` | string | Domain name | +| `report_date` | string | Date in YYYY-MM-DD format | + +**Response:** +```json +{ + "domain": "example.com", + "date": "2026-01-17", + "stats": { + "total_reports": 3, + "total_providers": 2, + "total_success": 1500, + "total_fail": 2, + "total_sessions": 1502, + "success_rate": 99.87 + }, + "providers": [ + { + "report_id": "2026-01-17T00:00:00Z!example.com!google.com", + "organization_name": "Google Inc.", + "contact_info": "smtp-tls-reporting@google.com", + "start_datetime": "2026-01-17T00:00:00Z", + "end_datetime": "2026-01-18T00:00:00Z", + "successful_sessions": 1200, + "failed_sessions": 1, + "total_sessions": 1201, + "success_rate": 99.92, + "policies": [ + { + "policy_type": "sts", + "policy_domain": "example.com", + "mx_host": "mail.example.com", + "successful_sessions": 1200, + "failed_sessions": 1, + "total_sessions": 1201, + "success_rate": 99.92, + "failure_details": null + } + ] + } + ] +} +``` + +--- + +### POST /api/dmarc/upload (TLS-RPT Support) + +The existing DMARC upload endpoint also accepts TLS-RPT reports. + +**Supported TLS-RPT Formats:** +- `.json.gz` - Gzip-compressed JSON (standard format) +- `.json` - Plain JSON + +**Detection:** +- File is identified as TLS-RPT if JSON contains `"policies"` array +- TLS-RPT reports use RFC 8460 JSON format + +--- + ## Error Responses All endpoints may return the following error responses: diff --git a/documentation/Email_Authentication_Monitoring.md b/documentation/Email_Authentication_Monitoring.md index 4bbce32..bddeeaa 100644 --- a/documentation/Email_Authentication_Monitoring.md +++ b/documentation/Email_Authentication_Monitoring.md @@ -2,7 +2,7 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewer** provides deep inspection and automated monitoring of the three core email authentication protocols: **SPF**, **DKIM**, and **DMARC**. -### The Authentication Stack +## The Authentication Stack | Protocol | Technical Purpose | System Validation Logic | | --- | --- | --- | @@ -12,7 +12,7 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe --- -### Advanced Monitoring & Intelligence +## Advanced Monitoring & Intelligence **mailcow-logs-viewer** goes beyond basic record checking by providing a comprehensive analysis of your mail flow: @@ -23,26 +23,48 @@ To maintain high deliverability and robust domain security, **mailcow-logs-viewe --- -### 🚀 Implementation: Enabling DMARC Reporting +## 🚀 Implementation: Enabling DMARC Reporting To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system. -#### 1. DNS Configuration +### 1. DNS Configuration Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`): ```text -v=DMARC1; p=none; rua=mailto:dmarc-reports@yourdomain.com; +v=DMARC1; p=none; rua=mailto:dmarc@example.net; ``` -#### 2. Parameter Details +### 2. Parameter Details * **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated. * **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer. * **`v=DMARC1`:** Required version prefix. -#### 3. Transitioning to Enforcement +### 3. External Domain Reporting (Verification) + +If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports. + +Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam. + +#### Option 1: Specific Domain Authorization (Recommended for security) +Add a TXT record to the DNS of the **receiving domain** (`example.net`): + +| Host / Name | Value | +| :--- | :--- | +| `example.com._report._dmarc.example.net` | `v=DMARC1;` | + +#### Option 2: Wildcard Authorization (Recommended for multiple domains) +If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once: + +| Host / Name | Value | +| :--- | :--- | +| `*._report._dmarc.example.net` | `v=DMARC1;` | + +*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.* + +### 4. Transitioning to Enforcement Once the dashboard confirms that your legitimate traffic (including third-party SaaS) is passing SPF/DKIM alignment, you should update your policy to `p=quarantine` or `p=reject` to fully secure your domain against spoofing. diff --git a/documentation/HelpDocs/DMARC.md b/documentation/HelpDocs/DMARC.md index df0a0ec..b8fb505 100644 --- a/documentation/HelpDocs/DMARC.md +++ b/documentation/HelpDocs/DMARC.md @@ -10,6 +10,8 @@ The DMARC Reports page provides detailed analysis of DMARC aggregate reports rec - Tells receiving servers what to do with emails that fail validation - Provides reports about email authentication results +--- + ## Report Types ### Aggregate Reports (XML) @@ -64,6 +66,8 @@ Click an IP address to see: - **Volume**: Number of messages from this source - **Reverse DNS**: Hostname associated with the IP +--- + ## Understanding Report Data ### DMARC Alignment @@ -81,6 +85,8 @@ What the receiving server did with the email: - **Policy**: What your DMARC record tells servers to do - **Disposition**: What servers actually did (they may override your policy) +--- + ## Key Features ### Geographic Visualization @@ -103,6 +109,8 @@ What the receiving server did with the email: - DMARC policy effectiveness - Recommendations for policy adjustments +--- + ## Common Scenarios ### Legitimate Sources Failing @@ -138,6 +146,51 @@ What the receiving server did with the email: - Verify DKIM is configured on all sending systems - Look for email forwarding issues +--- + +## 🚀 Implementation: Enabling DMARC Reporting + +To leverage the monitoring capabilities, you must publish a DMARC record in your DNS. This triggers global receivers (Google, Microsoft, etc.) to generate and send aggregate reports (`rua`) to your system. + +### 1. DNS Configuration + +Create a **TXT** record at the `_dmarc` subdomain (e.g., `_dmarc.example.com`): + +```text +v=DMARC1; p=none; rua=mailto:dmarc@example.net; + +``` + +### 2. Parameter Details + +* **`p=none` (Monitoring Mode):** The recommended starting point. It ensures no mail is blocked while you collect data to verify that all legitimate sources are correctly authenticated. +* **`rua=mailto:...`:** This is the feedback loop trigger. Ensure this address is the one configured in the **IMAP Settings** of Mailcow Logs Viewer. +* **`v=DMARC1`:** Required version prefix. + +### 3. External Domain Reporting (Verification) + +If you want to receive DMARC reports to a different domain from the one the record is set on (e.g., reports for `example.com` sent to `dmarc@example.net`), you must authorize (`example.net`) to receiving DMARC reports. + +Without this DNS record, major providers (like Google and Microsoft) will **not** send the reports to prevent spam. + +#### Option 1: Specific Domain Authorization (Recommended for security) +Add a TXT record to the DNS of the **receiving domain** (`example.net`): + +| Host / Name | Value | +| :--- | :--- | +| `example.com._report._dmarc.example.net` | `v=DMARC1;` | + +#### Option 2: Wildcard Authorization (Recommended for multiple domains) +If the receiving domain handles reports for many different domains, or if you prefer not to add a record for every single domain, you can use a wildcard record to authorize **all** domains at once: + +| Host / Name | Value | +| :--- | :--- | +| `*._report._dmarc.example.net` | `v=DMARC1;` | + +*Note: Not all DNS provider support wildcard records. use Cloudflare / Route53.* + +--- + ## Best Practices ### Policy Progression @@ -162,6 +215,8 @@ When using email services (marketing, support desk, etc.): - Test before going live - Monitor their authentication success +--- + ## Troubleshooting ### No Reports Appearing @@ -185,6 +240,8 @@ When using email services (marketing, support desk, etc.): - Older reports are automatically deleted to save space - Export reports before they're deleted if long-term analysis is needed +--- + ## Security Considerations ### Identifying Spoofing diff --git a/documentation/HelpDocs/Domains.md b/documentation/HelpDocs/Domains.md index cb72f54..21051a1 100644 --- a/documentation/HelpDocs/Domains.md +++ b/documentation/HelpDocs/Domains.md @@ -36,6 +36,8 @@ The system automatically validates three critical DNS records: - `none`: Monitoring only (weakest) - **Status**: Same indicators as SPF +--- + ## How to Use ### Viewing Domains @@ -59,6 +61,8 @@ When you expand a domain, the DNS Security section shows: - Specific warnings or recommendations - Time of last validation +--- + ## Best Practices 1. **Regular Monitoring**: Review DNS status regularly, especially after DNS changes @@ -67,6 +71,8 @@ When you expand a domain, the DNS Security section shows: 4. **SPF Optimization**: Keep SPF records concise (under 10 DNS lookups) 5. **DKIM Key Rotation**: Periodically rotate DKIM keys for security +--- + ## Troubleshooting ### DNS Changes Not Reflected diff --git a/env.example b/env.example index 644e090..4947748 100644 --- a/env.example +++ b/env.example @@ -69,11 +69,18 @@ CORRELATION_CHECK_INTERVAL=120 SMTP_ENABLED=false SMTP_HOST= SMTP_PORT= +# Use TLS instead of SSL (recommended) SMTP_USE_TLS=true +# Use SSL instead of TLS +SMTP_USE_SSL=false SMTP_USER= SMTP_PASSWORD= SMTP_FROM=noreply@yourdomain.com +# SMTP Relay Mode - Set to true for local relay servers that don't require authentication +# When enabled, username and password are not required +SMTP_RELAY_MODE=false + # ============================================================================= # Admin Email # ============================================================================= @@ -102,6 +109,7 @@ DMARC_IMAP_FOLDER=INBOX DMARC_IMAP_DELETE_AFTER=true DMARC_IMAP_INTERVAL=3600 DMARC_IMAP_RUN_ON_STARTUP=true +DMARC_IMAP_BATCH_SIZE=10 # DMARC Error Email Override (optional - uses ADMIN_EMAIL if not set) DMARC_ERROR_EMAIL= diff --git a/frontend/app.js b/frontend/app.js index 6c91b66..e4749c6 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -3,6 +3,174 @@ // Part 1: Core, Global State, Dashboard, Postfix, Rspamd, Netfilter // ============================================================================= +// ============================================================================= +// GLOBAL COLOR CONFIGURATION +// Edit these values to customize colors across the entire application +// ============================================================================= + +const APP_COLORS = { + // Email Direction Colors + directions: { + inbound: { + // Indigo + badge: 'bg-indigo-100 dark:bg-indigo-500/10 text-indigo-700 dark:text-indigo-300 border border-indigo-200 dark:border-indigo-500/20', + bg: 'bg-indigo-100 dark:bg-indigo-500/25', + text: 'text-indigo-700 dark:text-indigo-400' + }, + outbound: { + // Blue + badge: 'bg-blue-100 dark:bg-blue-500/10 text-blue-700 dark:text-blue-300 border border-blue-200 dark:border-blue-500/20', + bg: 'bg-blue-100 dark:bg-blue-500/25', + text: 'text-blue-700 dark:text-blue-400' + }, + internal: { + // Teal + badge: 'bg-teal-100 dark:bg-teal-500/10 text-teal-800 dark:text-teal-300 border border-teal-200 dark:border-teal-500/20', + bg: 'bg-teal-100 dark:bg-teal-500/25', + text: 'text-teal-700 dark:text-teal-400' + } + }, + statuses: { + delivered: { + // Emerald + badge: 'bg-emerald-100 dark:bg-emerald-500/10 text-emerald-700 dark:text-emerald-300 border border-emerald-200 dark:border-emerald-500/20', + bg: 'bg-emerald-100 dark:bg-emerald-500/25', + text: 'text-emerald-700 dark:text-emerald-400' + }, + sent: { + // Green + badge: 'bg-green-100 dark:bg-green-500/10 text-green-700 dark:text-green-300 border border-green-200 dark:border-green-500/20', + bg: 'bg-green-100 dark:bg-green-500/25', + text: 'text-green-700 dark:text-green-400' + }, + deferred: { + // Yellow (Fixed: Changed from Amber to Yellow) + badge: 'bg-yellow-100 dark:bg-yellow-500/10 text-yellow-700 dark:text-yellow-300 border border-yellow-200 dark:border-yellow-500/20', + bg: 'bg-yellow-100 dark:bg-yellow-500/25', + text: 'text-yellow-700 dark:text-yellow-400' + }, + bounced: { + // Orange + badge: 'bg-orange-100 dark:bg-orange-500/10 text-orange-700 dark:text-orange-300 border border-orange-200 dark:border-orange-500/20', + bg: 'bg-orange-100 dark:bg-orange-500/25', + text: 'text-orange-700 dark:text-orange-400' + }, + rejected: { + // Red + badge: 'bg-red-100 dark:bg-red-500/10 text-red-700 dark:text-red-300 border border-red-200 dark:border-red-500/20', + bg: 'bg-red-100 dark:bg-red-500/25', + text: 'text-red-700 dark:text-red-400' + }, + spam: { + // Fuchsia + badge: 'bg-fuchsia-100 dark:bg-fuchsia-500/10 text-fuchsia-700 dark:text-fuchsia-300 border border-fuchsia-200 dark:border-fuchsia-500/20', + bg: 'bg-fuchsia-100 dark:bg-fuchsia-500/25', + text: 'text-fuchsia-700 dark:text-fuchsia-400' + }, + expired: { + // Zinc + badge: 'bg-zinc-100 dark:bg-zinc-500/10 text-zinc-700 dark:text-zinc-300 border border-zinc-200 dark:border-zinc-500/20', + bg: 'bg-zinc-100 dark:bg-zinc-500/25', + text: 'text-zinc-700 dark:text-zinc-400' + } + }, + // Default color for unknown values + default: { + badge: 'bg-gray-100 dark:bg-gray-700 text-gray-800 dark:text-gray-300', + bg: 'bg-gray-100 dark:bg-gray-700', + text: 'text-gray-600 dark:text-gray-400' + } +}; + +// Helper functions for accessing colors +function getDirectionBadgeClass(direction) { + return APP_COLORS.directions[direction]?.badge || APP_COLORS.default.badge; +} + +function getDirectionBgClass(direction) { + return APP_COLORS.directions[direction]?.bg || APP_COLORS.default.bg; +} + +function getDirectionTextClass(direction) { + return APP_COLORS.directions[direction]?.text || APP_COLORS.default.text; +} + +function getStatusBadgeClass(status) { + return APP_COLORS.statuses[status]?.badge || APP_COLORS.default.badge; +} + +function getStatusBgClass(status) { + return APP_COLORS.statuses[status]?.bg || APP_COLORS.default.bg; +} + +function getStatusTextClass(status) { + return APP_COLORS.statuses[status]?.text || APP_COLORS.default.text; +} + +// ============================================================================= +// NAVIGATION HELPERS +// ============================================================================= + +/** + * Navigate to Messages page with pre-filled filters + * @param {Object} options - Filter options + * @param {string} options.email - Email address to filter by + * @param {string} options.filterType - 'sender' | 'recipient' | 'search' + * @param {string} options.direction - 'inbound' | 'outbound' | 'internal' + * @param {string} options.status - 'delivered' | 'bounced' | 'deferred' | 'rejected' + */ +function navigateToMessagesWithFilter(options) { + // Clear existing filters first + const filterSearch = document.getElementById('messages-filter-search'); + const filterSender = document.getElementById('messages-filter-sender'); + const filterRecipient = document.getElementById('messages-filter-recipient'); + const filterDirection = document.getElementById('messages-filter-direction'); + const filterStatus = document.getElementById('messages-filter-status'); + const filterUser = document.getElementById('messages-filter-user'); + const filterIp = document.getElementById('messages-filter-ip'); + + // Reset all filters + if (filterSearch) filterSearch.value = ''; + if (filterSender) filterSender.value = ''; + if (filterRecipient) filterRecipient.value = ''; + if (filterDirection) filterDirection.value = ''; + if (filterStatus) filterStatus.value = ''; + if (filterUser) filterUser.value = ''; + if (filterIp) filterIp.value = ''; + + // Set email filter based on type + if (options.email) { + if (options.filterType === 'sender') { + if (filterSender) filterSender.value = options.email; + } else if (options.filterType === 'recipient') { + if (filterRecipient) filterRecipient.value = options.email; + } else { + // Default: use search field + if (filterSearch) filterSearch.value = options.email; + } + } + + // Set direction filter + if (options.direction && filterDirection) { + filterDirection.value = options.direction; + } + + // Set status filter + if (options.status && filterStatus) { + filterStatus.value = options.status; + } + + // Navigate to Messages tab + navigateTo('messages'); + + // Apply filters after navigation + setTimeout(() => { + if (typeof applyMessagesFilters === 'function') { + applyMessagesFilters(); + } + }, 100); +} + // ============================================================================= // AUTHENTICATION SYSTEM // ============================================================================= @@ -61,26 +229,26 @@ async function authenticatedFetch(url, options = {}) { ...options.headers, ...getAuthHeader() }; - + const response = await fetch(url, { ...options, headers }); - + // Handle 401 Unauthorized if (response.status === 401) { clearAuthCredentials(); showLoginModal(); throw new Error('Authentication required'); } - + return response; } // Handle login form submission (not used in main app, only in login.html) async function handleLogin(event) { event.preventDefault(); - + const username = document.getElementById('login-username').value; const password = document.getElementById('login-password').value; const errorDiv = document.getElementById('login-error'); @@ -88,22 +256,22 @@ async function handleLogin(event) { const submitBtn = document.getElementById('login-submit'); const submitText = document.getElementById('login-submit-text'); const submitLoading = document.getElementById('login-submit-loading'); - + // Hide error if (errorDiv) errorDiv.classList.add('hidden'); - + // Show loading if (submitText) submitText.classList.add('hidden'); if (submitLoading) submitLoading.classList.remove('hidden'); if (submitBtn) submitBtn.disabled = true; - + try { // Save credentials saveAuthCredentials(username, password); - + // Test authentication with a simple API call const response = await authenticatedFetch('/api/info'); - + if (response.ok) { // Success - redirect to main app window.location.href = '/'; @@ -118,11 +286,11 @@ async function handleLogin(event) { errorText.textContent = error.message || 'Invalid username or password'; } } - + // Clear password field const passwordField = document.getElementById('login-password'); if (passwordField) passwordField.value = ''; - + // Clear credentials clearAuthCredentials(); } finally { @@ -156,16 +324,16 @@ async function checkAuthentication() { // If we can't check, assume auth is enabled for safety console.warn('Could not check auth status, assuming enabled'); } - + // Authentication is enabled, check credentials loadAuthCredentials(); - + if (!authCredentials) { // No credentials saved, redirect to login window.location.href = '/login'; return false; } - + try { // Test if credentials are still valid const response = await authenticatedFetch('/api/info'); @@ -218,14 +386,14 @@ let autoRefreshTimer = null; // Initialize on page load document.addEventListener('DOMContentLoaded', async () => { console.log('=== Mailcow Logs Viewer Initializing ==='); - + // Check authentication first const isAuthenticated = await checkAuthentication(); if (!isAuthenticated) { console.log('Authentication required - showing login modal'); return; } - + // Check if all required elements exist const requiredElements = [ 'app-title', @@ -238,20 +406,28 @@ document.addEventListener('DOMContentLoaded', async () => { 'content-settings', 'content-domains' ]; - + const missing = requiredElements.filter(id => !document.getElementById(id)); if (missing.length > 0) { console.error('Missing required elements:', missing); } else { console.log('[OK] All required DOM elements found'); } - + loadAppInfo(); - loadDashboard(); - + + // Initialize router and get initial route from URL + const routeInfo = typeof initRouter === 'function' ? initRouter() : { baseRoute: 'dashboard', params: {} }; + const initialTab = routeInfo.baseRoute || routeInfo; + const initialParams = routeInfo.params || {}; + console.log('Initial tab from URL:', initialTab, 'params:', initialParams); + + // Load the initial tab (use switchTab to ensure proper initialization) + switchTab(initialTab, initialParams); + // Start auto-refresh for all tabs startAutoRefresh(); - + console.log('=== Initialization Complete ==='); }); @@ -264,31 +440,31 @@ async function loadAppInfo() { // Use regular fetch since this is called after authentication check const response = await authenticatedFetch('/api/info'); const data = await response.json(); - + if (data.app_title) { document.getElementById('app-title').textContent = data.app_title; document.title = data.app_title; - + // Update footer app name const footerName = document.getElementById('app-name-footer'); if (footerName) { footerName.textContent = data.app_title; } } - + if (data.app_logo_url) { const logoImg = document.getElementById('app-logo'); logoImg.src = data.app_logo_url; logoImg.classList.remove('hidden'); document.getElementById('default-logo').classList.add('hidden'); } - + // Update footer version const footerVersion = document.getElementById('app-version-footer'); if (footerVersion && data.version) { footerVersion.textContent = `v${data.version}`; } - + // Show/hide logout button based on auth status const logoutBtn = document.getElementById('logout-btn'); if (logoutBtn) { @@ -298,7 +474,7 @@ async function loadAppInfo() { logoutBtn.classList.add('hidden'); } } - + // Store timezone for date formatting if (data.timezone) { appTimezone = data.timezone; @@ -306,10 +482,10 @@ async function loadAppInfo() { } else { console.warn('No timezone in API response, using default:', appTimezone); } - + // Load app version status for update check await loadAppVersionStatus(); - + // Load mailcow connection status await loadMailcowConnectionStatus(); } catch (error) { @@ -321,10 +497,10 @@ async function loadMailcowConnectionStatus() { try { const response = await authenticatedFetch('/api/status/mailcow-connection'); if (!response.ok) return; - + const data = await response.json(); const indicator = document.getElementById('mailcow-connection-indicator'); - + if (indicator) { indicator.classList.remove('hidden'); if (data.connected) { @@ -363,10 +539,10 @@ async function loadAppVersionStatus() { try { const response = await authenticatedFetch('/api/status/app-version'); if (!response.ok) return; - + const data = await response.json(); const updateBadge = document.getElementById('update-badge'); - + if (updateBadge && data.update_available) { updateBadge.classList.remove('hidden'); updateBadge.title = `Update available: v${data.latest_version}`; @@ -403,13 +579,13 @@ function startAutoRefresh() { if (autoRefreshTimer) { clearInterval(autoRefreshTimer); } - + // Set up auto-refresh interval autoRefreshTimer = setInterval(() => { smartRefreshCurrentTab(); }, AUTO_REFRESH_INTERVAL); - - console.log(`[OK] Auto-refresh started (every ${AUTO_REFRESH_INTERVAL/1000}s)`); + + console.log(`[OK] Auto-refresh started (every ${AUTO_REFRESH_INTERVAL / 1000}s)`); } function stopAutoRefresh() { @@ -427,7 +603,7 @@ async function smartRefreshCurrentTab() { if (modal && !modal.classList.contains('hidden')) { return; } - + try { switch (currentTab) { case 'dashboard': @@ -461,7 +637,7 @@ async function smartRefreshCurrentTab() { function hasDataChanged(newData, cacheKey) { const oldData = lastDataCache[cacheKey]; if (!oldData) return true; - + // Compare JSON strings for simple change detection const newJson = JSON.stringify(newData); const oldJson = JSON.stringify(oldData); @@ -472,21 +648,21 @@ function hasDataChanged(newData, cacheKey) { // Only refreshes if there are no active filters/search (to avoid disrupting user's view) async function smartRefreshMessages() { const filters = currentFilters.messages || {}; - + // Don't refresh if user has active search or filters - const hasActiveFilters = filters.search || filters.sender || filters.recipient || - filters.direction || filters.status || filters.user || filters.ip; - + const hasActiveFilters = filters.search || filters.sender || filters.recipient || + filters.direction || filters.status || filters.user || filters.ip; + // Don't refresh if user is not on first page if (hasActiveFilters || currentPage.messages > 1) { return; // Skip refresh to avoid disrupting user's view } - + const params = new URLSearchParams({ page: currentPage.messages, limit: 50 }); - + if (filters.search) params.append('search', filters.search); if (filters.sender) params.append('sender', filters.sender); if (filters.recipient) params.append('recipient', filters.recipient); @@ -494,12 +670,12 @@ async function smartRefreshMessages() { if (filters.status) params.append('status', filters.status); if (filters.user) params.append('user', filters.user); if (filters.ip) params.append('ip', filters.ip); - + const response = await authenticatedFetch(`/api/messages?${params}`); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'messages')) { console.log('[REFRESH] Messages data changed, updating UI'); lastDataCache.messages = data; @@ -511,12 +687,12 @@ async function smartRefreshMessages() { function renderMessagesData(data) { const container = document.getElementById('messages-logs'); if (!container) return; - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No messages found

'; return; } - + container.innerHTML = `
${data.data.map(msg => ` @@ -534,12 +710,12 @@ function renderMessagesData(data) {
${(() => { - const correlationStatus = getCorrelationStatusDisplay(msg); - if (correlationStatus) { - return `${correlationStatus.display}`; - } - return ''; - })()} + const correlationStatus = getCorrelationStatusDisplay(msg); + if (correlationStatus) { + return `${correlationStatus.display}`; + } + return ''; + })()} ${msg.direction ? `${msg.direction}` : ''} ${msg.is_spam !== null ? `${msg.is_spam ? 'SPAM' : 'CLEAN'}` : ''}
@@ -562,20 +738,20 @@ function renderMessagesData(data) { // Deduplicate netfilter logs based on message + time + priority function deduplicateNetfilterLogs(logs) { if (!logs || logs.length === 0) return []; - + const seen = new Set(); const uniqueLogs = []; - + for (const log of logs) { // Create unique key from message + time + priority const key = `${log.message || ''}|${log.time || ''}|${log.priority || ''}`; - + if (!seen.has(key)) { seen.add(key); uniqueLogs.push(log); } } - + return uniqueLogs; } @@ -583,21 +759,21 @@ function deduplicateNetfilterLogs(logs) { function renderNetfilterData(data) { const container = document.getElementById('netfilter-logs'); if (!container) return; - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No logs found

'; return; } - + // Deduplicate logs const uniqueLogs = deduplicateNetfilterLogs(data.data); - + // Update count display with total count from API (like Messages page) const countEl = document.getElementById('security-count'); if (countEl) { countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : ''; } - + container.innerHTML = `
${uniqueLogs.map(log => ` @@ -627,12 +803,12 @@ async function smartRefreshNetfilter() { limit: 50, ...filters }); - + const response = await authenticatedFetch(`/api/logs/netfilter?${params}`); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'netfilter')) { console.log('[REFRESH] Netfilter data changed, updating UI'); lastDataCache.netfilter = data; @@ -646,9 +822,9 @@ async function smartRefreshNetfilter() { async function smartRefreshQueue() { const response = await authenticatedFetch('/api/queue'); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'queue')) { console.log('[REFRESH] Queue data changed, updating UI'); lastDataCache.queue = data; @@ -661,9 +837,9 @@ async function smartRefreshQueue() { async function smartRefreshQuarantine() { const response = await authenticatedFetch('/api/quarantine'); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'quarantine')) { console.log('[REFRESH] Quarantine data changed, updating UI'); lastDataCache.quarantine = data; @@ -675,12 +851,12 @@ async function smartRefreshQuarantine() { function renderQuarantineData(data) { const container = document.getElementById('quarantine-logs'); if (!container) return; - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No quarantined messages

'; return; } - + container.innerHTML = `
${data.data.map(item => ` @@ -704,13 +880,13 @@ async function smartRefreshDashboard() { try { const response = await authenticatedFetch('/api/stats/dashboard'); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'dashboard')) { console.log('[REFRESH] Dashboard data changed, updating UI'); lastDataCache.dashboard = data; - + // Update stats without full reload document.getElementById('stat-messages-24h').textContent = data.messages['24h'].toLocaleString(); document.getElementById('stat-messages-7d').textContent = data.messages['7d'].toLocaleString(); @@ -722,7 +898,7 @@ async function smartRefreshDashboard() { document.getElementById('stat-auth-failures-24h').textContent = data.auth_failures['24h'].toLocaleString(); document.getElementById('stat-auth-failures-7d').textContent = data.auth_failures['7d'].toLocaleString(); } - + // Also refresh recent activity and status summary loadRecentActivity(); loadDashboardStatusSummary(); @@ -736,13 +912,13 @@ async function smartRefreshSettings() { try { const response = await authenticatedFetch('/api/settings/info'); if (!response.ok) return; - + const data = await response.json(); - + if (hasDataChanged(data, 'settings')) { console.log('[REFRESH] Settings data changed, updating UI'); lastDataCache.settings = data; - + const content = document.getElementById('settings-content'); if (content && !content.classList.contains('hidden')) { // Preserve version info from cache (don't reload it on smart refresh) @@ -752,7 +928,7 @@ async function smartRefreshSettings() { if (versionInfoCache.version_info) { data.version_info = versionInfoCache.version_info; } - + renderSettings(content, data); } } @@ -765,11 +941,11 @@ async function smartRefreshSettings() { // TAB SWITCHING // ============================================================================= -function switchTab(tab) { - console.log('Switching to tab:', tab); +function switchTab(tab, params = {}) { + console.log('Switching to tab:', tab, 'params:', params); currentTab = tab; - - // Update active tab button + + // Update active tab button (desktop) document.querySelectorAll('[id^="tab-"]').forEach(btn => { btn.classList.remove('tab-active'); btn.classList.add('text-gray-500', 'dark:text-gray-400'); @@ -779,12 +955,20 @@ function switchTab(tab) { activeBtn.classList.add('tab-active'); activeBtn.classList.remove('text-gray-500', 'dark:text-gray-400'); } - + + // Update mobile menu state and label + if (typeof updateMobileMenuActiveState === 'function') { + updateMobileMenuActiveState(tab); + } + if (typeof updateCurrentTabLabel === 'function') { + updateCurrentTabLabel(tab); + } + // Hide all tab contents document.querySelectorAll('.tab-content').forEach(content => { content.classList.add('hidden'); }); - + // Show current tab content const tabContent = document.getElementById(`content-${tab}`); if (tabContent) { @@ -818,7 +1002,10 @@ function switchTab(tab) { loadDomains(); break; case 'dmarc': - loadDmarc(); + handleDmarcRoute(params); + break; + case 'mailbox-stats': + loadMailboxStats(); break; case 'settings': loadSettings(); @@ -828,7 +1015,15 @@ function switchTab(tab) { } } -function refreshAllData() { +async function refreshAllData() { + if (currentTab === 'dmarc') { + try { + await authenticatedFetch('/api/dmarc/cache/clear', { method: 'POST' }); + console.log('DMARC cache cleared'); + } catch (e) { + console.error('Failed to clear DMARC cache:', e); + } + } switchTab(currentTab); } @@ -839,15 +1034,15 @@ function refreshAllData() { async function loadDashboard() { try { console.log('Loading Dashboard...'); - + const response = await authenticatedFetch('/api/stats/dashboard'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Dashboard data:', data); - + document.getElementById('stat-messages-24h').textContent = data.messages['24h'].toLocaleString(); document.getElementById('stat-messages-7d').textContent = data.messages['7d'].toLocaleString(); document.getElementById('stat-blocked-24h').textContent = data.blocked['24h'].toLocaleString(); @@ -857,7 +1052,7 @@ async function loadDashboard() { document.getElementById('stat-deferred-7d').textContent = data.deferred['7d'].toLocaleString(); document.getElementById('stat-auth-failures-24h').textContent = data.auth_failures['24h'].toLocaleString(); document.getElementById('stat-auth-failures-7d').textContent = data.auth_failures['7d'].toLocaleString(); - + loadRecentActivity(); loadDashboardStatusSummary(); } catch (error) { @@ -868,15 +1063,15 @@ async function loadDashboard() { async function loadDashboardStatusSummary() { try { console.log('Loading Dashboard Status Summary...'); - + const response = await authenticatedFetch('/api/status/summary'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Status summary data:', data); - + const containersDiv = document.getElementById('dashboard-containers-summary'); const containers = data.containers || {}; containersDiv.innerHTML = ` @@ -893,13 +1088,13 @@ async function loadDashboardStatusSummary() { ${containers.total || 0}
`; - + const storageDiv = document.getElementById('dashboard-storage-summary'); const storage = data.storage || {}; const usedPercent = parseInt(storage.used_percent) || 0; - const storageColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' : - usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' : - 'text-green-600 dark:text-green-400'; + const storageColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' : + usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' : + 'text-green-600 dark:text-green-400'; storageDiv.innerHTML = `
Used @@ -915,7 +1110,7 @@ async function loadDashboardStatusSummary() {
`; - + const systemDiv = document.getElementById('dashboard-system-summary'); const system = data.system || {}; systemDiv.innerHTML = ` @@ -939,23 +1134,23 @@ async function loadDashboardStatusSummary() { async function loadRecentActivity() { const container = document.getElementById('recent-activity'); - + try { console.log('Loading Recent Activity...'); - + const response = await authenticatedFetch('/api/stats/recent-activity?limit=10'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Recent Activity data:', data); - + if (data.activity.length === 0) { container.innerHTML = '

No recent activity

'; return; } - + container.innerHTML = data.activity.map(msg => `
@@ -986,7 +1181,7 @@ async function loadRecentActivity() { function performDashboardSearch() { const query = document.getElementById('dashboard-search-query').value; const status = document.getElementById('dashboard-search-status').value; - + // Set filters on Messages page document.getElementById('messages-filter-search').value = query; document.getElementById('messages-filter-sender').value = ''; @@ -994,14 +1189,14 @@ function performDashboardSearch() { document.getElementById('messages-filter-direction').value = ''; document.getElementById('messages-filter-status').value = status; document.getElementById('messages-filter-user').value = ''; - + // Apply filters currentFilters.messages = { search: query, status: status }; currentPage.messages = 1; - + // Switch to Messages tab and load switchTab('messages'); } @@ -1031,38 +1226,38 @@ function clearPostfixFilters() { async function loadPostfixLogs(page = 1) { const container = document.getElementById('postfix-logs'); - + // Show loading immediately container.innerHTML = '

Loading Postfix logs... This may take a few moments.

'; - + try { const filters = currentFilters.postfix || {}; const params = new URLSearchParams({ page: page, limit: 50 }); - + if (filters.search) params.append('search', filters.search); if (filters.sender) params.append('sender', filters.sender); if (filters.recipient) params.append('recipient', filters.recipient); - + console.log('Loading Postfix logs:', `/api/logs/postfix?${params}`); const startTime = performance.now(); - + const response = await authenticatedFetch(`/api/logs/postfix?${params}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); const loadTime = ((performance.now() - startTime) / 1000).toFixed(2); console.log(`Postfix data loaded in ${loadTime}s:`, data); - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No logs found

'; return; } - + container.innerHTML = `
@@ -1098,7 +1293,7 @@ async function loadPostfixLogs(page = 1) { ${renderPagination('postfix', data.page, data.pages)} `; - + currentPage.postfix = page; } catch (error) { console.error('Failed to load Postfix logs:', error); @@ -1133,37 +1328,37 @@ function clearRspamdFilters() { async function loadRspamdLogs(page = 1) { const container = document.getElementById('rspamd-logs'); - + try { container.innerHTML = '

Loading...

'; - + const filters = currentFilters.rspamd || {}; const params = new URLSearchParams({ page: page, limit: 50 }); - + if (filters.search) params.append('search', filters.search); if (filters.direction) params.append('direction', filters.direction); if (filters.is_spam === 'true') params.append('is_spam', 'true'); if (filters.is_spam === 'false') params.append('is_spam', 'false'); if (filters.min_score) params.append('min_score', filters.min_score); - + console.log('Loading Rspamd logs:', `/api/logs/rspamd?${params}`); - + const response = await authenticatedFetch(`/api/logs/rspamd?${params}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Rspamd data:', data); - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No logs found

'; return; } - + container.innerHTML = `
@@ -1200,7 +1395,7 @@ async function loadRspamdLogs(page = 1) { ${renderPagination('rspamd', data.page, data.pages)} `; - + currentPage.rspamd = page; } catch (error) { console.error('Failed to load Rspamd logs:', error); @@ -1233,43 +1428,43 @@ function clearNetfilterFilters() { async function loadNetfilterLogs(page = 1) { const container = document.getElementById('netfilter-logs'); - + try { container.innerHTML = '

Loading...

'; - + const filters = currentFilters.netfilter || {}; const params = new URLSearchParams({ page: page, limit: 50, ...filters }); - + console.log('Loading Netfilter logs:', `/api/logs/netfilter?${params}`); - + const response = await authenticatedFetch(`/api/logs/netfilter?${params}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Netfilter data:', data); - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No logs found

'; const countEl = document.getElementById('security-count'); if (countEl) countEl.textContent = ''; return; } - + // Deduplicate logs based on message + time + priority const uniqueLogs = deduplicateNetfilterLogs(data.data); - + // Update count display with total count from API (like Messages page) const countEl = document.getElementById('security-count'); if (countEl) { countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : ''; } - + container.innerHTML = `
${uniqueLogs.map(log => ` @@ -1289,7 +1484,7 @@ async function loadNetfilterLogs(page = 1) {
${renderPagination('netfilter', data.page, data.pages)} `; - + currentPage.netfilter = page; } catch (error) { console.error('Failed to load Netfilter logs:', error); @@ -1311,20 +1506,20 @@ let allQueueData = []; async function loadQueue() { const container = document.getElementById('queue-logs'); - + try { container.innerHTML = '

Loading...

'; - + console.log('Loading Queue...'); - + const response = await authenticatedFetch('/api/queue'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Queue data:', data); - + allQueueData = data.data || []; applyQueueFilters(); } catch (error) { @@ -1338,35 +1533,35 @@ async function loadQueue() { function applyQueueFilters() { const searchTerm = document.getElementById('queue-filter-search')?.value.toLowerCase() || ''; const queueId = document.getElementById('queue-filter-queue-id')?.value.toLowerCase() || ''; - + let filteredData = allQueueData; - + if (searchTerm) { - filteredData = filteredData.filter(item => + filteredData = filteredData.filter(item => item.sender.toLowerCase().includes(searchTerm) || item.recipients.some(r => r.toLowerCase().includes(searchTerm)) ); } - + if (queueId) { - filteredData = filteredData.filter(item => + filteredData = filteredData.filter(item => item.queue_id.toLowerCase().includes(queueId) ); } - + const container = document.getElementById('queue-logs'); - + // Update count display const countEl = document.getElementById('queue-count'); if (countEl) { countEl.textContent = `(${filteredData.length.toLocaleString()} items)`; } - + if (filteredData.length === 0) { container.innerHTML = '

No matching queue entries

'; return; } - + container.innerHTML = `
${filteredData.map(item => ` @@ -1403,31 +1598,31 @@ function clearQueueFilters() { async function loadQuarantine() { const container = document.getElementById('quarantine-logs'); - + try { container.innerHTML = '

Loading...

'; - + console.log('Loading Quarantine...'); - + const response = await authenticatedFetch('/api/quarantine'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Quarantine data:', data); - + // ⭐ NEW: Update counter display const countEl = document.getElementById('quarantine-count'); if (countEl) { countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : ''; } - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No quarantined messages

'; return; } - + // ⭐ NEW: Use separate render function renderQuarantineData(data); } catch (error) { @@ -1443,18 +1638,18 @@ async function loadQuarantine() { function renderQuarantineData(data) { const container = document.getElementById('quarantine-logs'); if (!container) return; - + // Update counter display const countEl = document.getElementById('quarantine-count'); if (countEl) { countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : ''; } - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No quarantined messages

'; return; } - + container.innerHTML = `
${data.data.map(item => ` @@ -1519,16 +1714,16 @@ function clearMessagesFilters() { async function loadMessages(page = 1) { const container = document.getElementById('messages-logs'); - + try { container.innerHTML = '

Loading...

'; - + const filters = currentFilters.messages || {}; const params = new URLSearchParams({ page: page, limit: 50 }); - + if (filters.search) params.append('search', filters.search); if (filters.sender) params.append('sender', filters.sender); if (filters.recipient) params.append('recipient', filters.recipient); @@ -1536,28 +1731,28 @@ async function loadMessages(page = 1) { if (filters.user) params.append('user', filters.user); if (filters.status) params.append('status', filters.status); if (filters.ip) params.append('ip', filters.ip); - + console.log('Loading Messages:', `/api/messages?${params}`); - + const response = await authenticatedFetch(`/api/messages?${params}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Messages data:', data); - + // Update count display const countEl = document.getElementById('messages-count'); if (countEl) { countEl.textContent = data.total ? `(${data.total.toLocaleString()} results)` : ''; } - + if (!data.data || data.data.length === 0) { container.innerHTML = '

No messages found

'; return; } - + container.innerHTML = `
${data.data.map(msg => ` @@ -1575,12 +1770,12 @@ async function loadMessages(page = 1) {
${(() => { - const correlationStatus = getCorrelationStatusDisplay(msg); - if (correlationStatus) { - return `${correlationStatus.display}`; - } - return ''; - })()} + const correlationStatus = getCorrelationStatusDisplay(msg); + if (correlationStatus) { + return `${correlationStatus.display}`; + } + return ''; + })()} ${msg.direction ? `${msg.direction}` : ''} ${msg.is_spam !== null ? `${msg.is_spam ? 'SPAM' : 'CLEAN'}` : ''}
@@ -1598,7 +1793,7 @@ async function loadMessages(page = 1) {
${renderPagination('messages', data.page, data.pages)} `; - + currentPage.messages = page; } catch (error) { console.error('Failed to load messages:', error); @@ -1629,15 +1824,15 @@ async function loadStatusContainers() { try { const response = await authenticatedFetch('/api/status/containers'); let data = await response.json(); - + const container = document.getElementById('status-containers'); - + let containersData = data.containers || data; - + if (Array.isArray(containersData) && containersData.length === 1 && typeof containersData[0] === 'object') { containersData = containersData[0]; } - + let containersList = []; if (Array.isArray(containersData)) { containersList = containersData; @@ -1649,12 +1844,12 @@ async function loadStatusContainers() { started_at: value.started_at || null })); } - + if (containersList.length > 0) { const running = containersList.filter(c => c.state === 'running').length; const stopped = containersList.filter(c => c.state !== 'running').length; const total = containersList.length; - + container.innerHTML = `
@@ -1682,7 +1877,7 @@ async function loadStatusContainers() {

${escapeHtml(c.name)}

-

${c.started_at ? new Date(c.started_at).toLocaleString('he-IL', {day: '2-digit', month: '2-digit', year: 'numeric', hour: '2-digit', minute: '2-digit'}) : 'Unknown'}

+

${c.started_at ? new Date(c.started_at).toLocaleString('he-IL', { day: '2-digit', month: '2-digit', year: 'numeric', hour: '2-digit', minute: '2-digit' }) : 'Unknown'}

${c.state} @@ -1701,18 +1896,18 @@ async function loadStatusContainers() { async function loadStatusSystem() { const container = document.getElementById('status-system'); - + try { console.log('Loading System Info...'); - + const response = await authenticatedFetch('/api/status/mailcow-info'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('System info data:', data); - + container.innerHTML = `
@@ -1742,31 +1937,31 @@ async function loadStatusSystem() { async function loadStatusStorage() { const container = document.getElementById('status-storage'); - + try { console.log('Loading Storage Info...'); - + const response = await authenticatedFetch('/api/status/storage'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + let rawData = await response.json(); console.log('Storage info data:', rawData); - + // Handle mailcow API format: [{ "type": "info", "disk": "/dev/sdb1", ... }] let data = rawData; if (Array.isArray(rawData) && rawData.length > 0) { data = rawData[0]; // Take first element } const usedPercent = parseInt(data.used_percent) || 0; - const storageColor = usedPercent > 90 ? 'bg-red-600' : - usedPercent > 75 ? 'bg-yellow-600' : - 'bg-green-600'; - const textColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' : - usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' : - 'text-green-600 dark:text-green-400'; - + const storageColor = usedPercent > 90 ? 'bg-red-600' : + usedPercent > 75 ? 'bg-yellow-600' : + 'bg-green-600'; + const textColor = usedPercent > 90 ? 'text-red-600 dark:text-red-400' : + usedPercent > 75 ? 'text-yellow-600 dark:text-yellow-400' : + 'text-green-600 dark:text-green-400'; + container.innerHTML = `
@@ -1811,19 +2006,19 @@ async function loadStatusExtended() { if (!response.ok) { throw new Error(`HTTP ${response.status}`); } - + const data = await response.json(); console.log('Extended status data loaded:', data); - + // Render Import Status renderStatusImport(data.import_status || {}); - + // Render Correlation Status renderStatusCorrelation(data.correlation_status || {}, data.recent_incomplete_correlations || []); - + // Render Background Jobs renderStatusJobs(data.background_jobs || {}); - + } catch (error) { console.error('Failed to load extended status:', error); document.getElementById('status-import').innerHTML = `

Failed to load: ${error.message}

`; @@ -1912,6 +2107,8 @@ function renderStatusJobs(jobs) { ${renderJobCard('Sync Active Domains', jobs.sync_local_domains)} ${renderJobCard('DMARC IMAP Import', jobs.dmarc_imap_sync)} ${renderJobCard('Update MaxMind Databases', jobs.update_geoip)} + ${renderJobCard('Mailbox Statistics', jobs.mailbox_stats)} + ${renderJobCard('Alias Statistics', jobs.alias_stats)}
`; } @@ -1925,43 +2122,43 @@ async function viewPostfixDetails(queueId) { console.error('No queue ID provided'); return; } - + console.log('Loading Postfix details for queue ID:', queueId); - + const modal = document.getElementById('message-modal'); const content = document.getElementById('message-modal-content'); - + if (!modal || !content) { console.error('Modal elements not found'); return; } - + // Block body scroll document.body.style.overflow = 'hidden'; - + modal.classList.remove('hidden'); content.innerHTML = '

Loading...

'; - + try { const response = await authenticatedFetch(`/api/logs/postfix/by-queue/${queueId}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Postfix details loaded:', data); - + if (data.logs && data.logs.length > 0) { // Sort logs by time const sortedLogs = data.logs.sort((a, b) => new Date(a.time) - new Date(b.time)); - + // Extract key information let sender = null, recipient = null; sortedLogs.forEach(log => { if (log.sender && !sender) sender = log.sender; if (log.recipient && !recipient) recipient = log.recipient; }); - + // CRITICAL: Store FULL data in currentModalData currentModalData = { queue_id: queueId, @@ -1975,12 +2172,12 @@ async function viewPostfixDetails(queueId) { rspamd: data.rspamd || null, netfilter: [] }; - + currentModalTab = 'overview'; // Start with Overview - + // Update Security tab indicator updateSecurityTabIndicator(currentModalData); - + // Reset modal tabs document.querySelectorAll('[id^="modal-tab-"]').forEach(btn => { btn.classList.remove('active'); @@ -1989,12 +2186,12 @@ async function viewPostfixDetails(queueId) { if (overviewTab) { overviewTab.classList.add('active'); } - + console.log('currentModalData set:', currentModalData); - + // Render the Overview tab renderModalTab('overview', currentModalData); - + } else { content.innerHTML = '

No logs found for this Queue ID

'; } @@ -2015,7 +2212,7 @@ async function viewPostfixDetails(queueId) { function switchModalTab(tab) { console.log('Switching modal tab to:', tab); currentModalTab = tab; - + // Update tab buttons document.querySelectorAll('[id^="modal-tab-"]').forEach(btn => { btn.classList.remove('active'); @@ -2026,7 +2223,7 @@ function switchModalTab(tab) { } else { console.error('Modal tab button not found:', `modal-tab-${tab}`); } - + // Render content if (currentModalData) { renderModalTab(tab, currentModalData); @@ -2040,38 +2237,38 @@ async function viewMessageDetails(correlationKey) { console.error('No correlation key provided'); return; } - + console.log('Loading message details for:', correlationKey); - + const modal = document.getElementById('message-modal'); const content = document.getElementById('message-modal-content'); - + if (!modal || !content) { console.error('Modal elements not found'); return; } - + // Block body scroll document.body.style.overflow = 'hidden'; - + modal.classList.remove('hidden'); content.innerHTML = '

Loading...

'; - + try { const response = await authenticatedFetch(`/api/message/${correlationKey}/details`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - + const data = await response.json(); console.log('Message details loaded:', data); - + currentModalData = data; currentModalTab = 'overview'; - + // Update Security tab indicator updateSecurityTabIndicator(data); - + document.querySelectorAll('[id^="modal-tab-"]').forEach(btn => { btn.classList.remove('active'); }); @@ -2079,7 +2276,7 @@ async function viewMessageDetails(correlationKey) { if (overviewTab) { overviewTab.classList.add('active'); } - + renderModalTab('overview', data); } catch (error) { console.error('Failed to load message details:', error); @@ -2089,7 +2286,7 @@ async function viewMessageDetails(correlationKey) { function renderModalTab(tab, data) { const content = document.getElementById('message-modal-content'); - + switch (tab) { case 'overview': renderOverviewTab(content, data); @@ -2116,12 +2313,12 @@ function renderOverviewTab(content, data) { } }); } - + // Use Postfix recipients if available, otherwise fall back to correlation recipients - const recipientsToDisplay = recipientsFromPostfix.size > 0 - ? Array.from(recipientsFromPostfix) + const recipientsToDisplay = recipientsFromPostfix.size > 0 + ? Array.from(recipientsFromPostfix) : (data.recipients || []); - + // Build recipients section for right column let recipientsRightColumn = ''; if (recipientsToDisplay.length > 0) { @@ -2157,7 +2354,7 @@ function renderOverviewTab(content, data) {
`; } - + content.innerHTML = `
@@ -2281,13 +2478,13 @@ function renderPostfixTab(content, data) { `; return; } - + // Extract key information from logs let sender = null, clientIp = null, relay = null; let messageId = null, finalStatus = null, totalDelay = null, queueId = null; let errorReasons = []; let recipientsFromPostfix = new Set(); // Collect all unique recipients from Postfix logs - + data.postfix.forEach(log => { if (log.queue_id && !queueId) queueId = log.queue_id; if (log.sender && !sender) sender = log.sender; @@ -2299,12 +2496,12 @@ function renderPostfixTab(content, data) { if (log.recipient) { recipientsFromPostfix.add(log.recipient); } - + if (!clientIp && log.message) { const ipMatch = log.message.match(/client=.*?\[(\d+\.\d+\.\d+\.\d+)\]/); if (ipMatch) clientIp = ipMatch[1]; } - + // Extract error reasons for non-sent statuses if (log.status && log.status !== 'sent' && log.message) { // Look for "said:" pattern (remote server response) @@ -2328,15 +2525,15 @@ function renderPostfixTab(content, data) { } } }); - + // Generate unique ID for accordion const accordionId = 'postfix-accordion-' + Date.now(); - + // Separate system logs from recipient logs const postfixByRecipient = data.postfix_by_recipient || {}; const systemLogs = postfixByRecipient['_system'] || []; const recipientEntries = Object.entries(postfixByRecipient).filter(([key]) => key !== '_system'); - + // Build error summary section const errorSummaryHtml = errorReasons.length > 0 ? `
@@ -2356,7 +2553,7 @@ function renderPostfixTab(content, data) {
` : ''; - + content.innerHTML = `
${errorSummaryHtml} @@ -2420,8 +2617,8 @@ function renderPostfixTab(content, data) {

Delivery Summary by Recipient

${recipientEntries.map(([recipient, logs]) => { - const statusLog = logs.find(l => l.status) || logs[0]; - return ` + const statusLog = logs.find(l => l.status) || logs[0]; + return `
${escapeHtml(recipient)} @@ -2430,7 +2627,7 @@ function renderPostfixTab(content, data) { ${statusLog.relay ? `

via ${escapeHtml(statusLog.relay)}

` : ''}
`; - }).join('')} + }).join('')}
` : ''} @@ -2467,7 +2664,7 @@ function renderPostfixTab(content, data) { function toggleAccordion(id) { const content = document.getElementById(id); const icon = document.getElementById(id + '-icon'); - + if (content.classList.contains('hidden')) { content.classList.remove('hidden'); icon.style.transform = 'rotate(180deg)'; @@ -2489,7 +2686,7 @@ function renderSpamTab(content, data) { `; return; } - + content.innerHTML = `
@@ -2519,21 +2716,21 @@ function renderSpamTab(content, data) {

Detection Symbols

${Object.entries(data.rspamd.symbols) - .sort((a, b) => { - const scoreA = a[1].score || a[1].metric_score || 0; - const scoreB = b[1].score || b[1].metric_score || 0; - if (scoreA === 0 && scoreB !== 0) return 1; - if (scoreA !== 0 && scoreB === 0) return -1; - return Math.abs(scoreB) - Math.abs(scoreA); - }) - .map(([name, details]) => { - const score = details.score || details.metric_score || 0; - const description = details.description || ''; - const options = details.options || []; - const scoreClass = score > 0 ? 'text-red-600 dark:text-red-400' : - score < 0 ? 'text-green-600 dark:text-green-400' : - 'text-gray-500 dark:text-gray-400'; - return ` + .sort((a, b) => { + const scoreA = a[1].score || a[1].metric_score || 0; + const scoreB = b[1].score || b[1].metric_score || 0; + if (scoreA === 0 && scoreB !== 0) return 1; + if (scoreA !== 0 && scoreB === 0) return -1; + return Math.abs(scoreB) - Math.abs(scoreA); + }) + .map(([name, details]) => { + const score = details.score || details.metric_score || 0; + const description = details.description || ''; + const options = details.options || []; + const scoreClass = score > 0 ? 'text-red-600 dark:text-red-400' : + score < 0 ? 'text-green-600 dark:text-green-400' : + 'text-gray-500 dark:text-gray-400'; + return `
${name} @@ -2543,7 +2740,7 @@ function renderSpamTab(content, data) { ${score > 0 ? '+' : ''}${score.toFixed(2)}
`; - }).join('')} + }).join('')}
` : ''} @@ -2564,7 +2761,7 @@ function renderNetfilterTab(content, data) { `; return; } - + content.innerHTML = `
@@ -2604,10 +2801,10 @@ function renderNetfilterTab(content, data) { function updateSecurityTabIndicator(data) { const securityTab = document.getElementById('modal-tab-netfilter'); if (!securityTab) return; - + const hasSecurityEvents = data.netfilter && data.netfilter.length > 0; const indicator = hasSecurityEvents ? '🔴' : '🟢'; - + securityTab.innerHTML = `Security ${indicator}`; } @@ -2630,7 +2827,7 @@ function showChangelogModal(changelog) { const modal = document.getElementById('changelog-modal'); const modalTitle = modal?.querySelector('h3'); const content = document.getElementById('changelog-content'); - + if (modal && content) { if (modalTitle) { modalTitle.textContent = 'Changelog'; @@ -2692,12 +2889,12 @@ function renderGeoIPInfo(rspamdData, size = '24x18') { if (rspamdData.country_name && flagUrl) { // Wrap image and country name in a span to keep them together and aligned - const countryPart = + const countryPart = `
` + - `${escapeHtml(rspamdData.country_name)}` + - `${escapeHtml(rspamdData.country_name)}` + + `${escapeHtml(rspamdData.country_name)}` + + `${escapeHtml(rspamdData.country_name)}` + ``; parts.push(countryPart); } @@ -2718,40 +2915,40 @@ function renderGeoIPForDMARC(record, size = '24x18') { if (!record || !record.source_ip) { return ''; } - + const ip = record.source_ip; const hasGeoIP = record.country_code; - + if (!hasGeoIP) { return escapeHtml(ip); } - + // Build flag URL const flagUrl = getFlagUrl(record.country_code, size); const [width, height] = size.split('x').map(Number); - + // Build location string let parts = []; - + if (record.country_name) { parts.push(escapeHtml(record.country_name)); } - + if (record.city) { parts.push(escapeHtml(record.city)); } - + if (record.asn_org) { parts.push(escapeHtml(record.asn_org)); } - + const locationText = parts.join(', '); - + // Return flag + location inline if (flagUrl && locationText) { return `${escapeHtml(record.country_name || '')}${locationText}`; } - + return locationText || escapeHtml(ip); } @@ -2764,7 +2961,7 @@ async function exportCSV(type) { try { const filters = currentFilters[type] || {}; const params = new URLSearchParams(filters); - + const response = await authenticatedFetch(`/api/export/${type}/csv?${params}`); const blob = await response.blob(); const url = window.URL.createObjectURL(blob); @@ -2787,7 +2984,7 @@ async function exportCSV(type) { function renderPagination(type, currentPage, totalPages) { if (totalPages <= 1) return ''; - + return `
`; } - + if (domains.length === 0) { container.innerHTML = `
@@ -3161,7 +3341,7 @@ function renderDomains(container, data) { `; return; } - + // Summary cards const summaryHTML = `
@@ -3194,7 +3374,7 @@ function renderDomains(container, data) {
`; - + // Search/Filter bar const filterHTML = `
@@ -3232,16 +3412,16 @@ function renderDomains(container, data) {
`; - + // Domains list with accordion style const domainsHTML = domains.map(domain => renderDomainAccordionRow(domain)).join(''); - + container.innerHTML = summaryHTML + filterHTML + `
${domainsHTML}
`; - + // Store domains data for filtering window.domainsData = domains; } @@ -3252,17 +3432,17 @@ function filterDomains() { const issuesCheckbox = document.getElementById('filter-issues-only'); const domainsList = document.getElementById('domains-list'); const countBadge = document.getElementById('domain-count-badge'); - + if (!searchInput || !domainsList || !window.domainsData) return; - + const searchTerm = searchInput.value.toLowerCase().trim(); const showIssuesOnly = issuesCheckbox ? issuesCheckbox.checked : false; - + // Filter domains let filteredDomains = window.domainsData.filter(domain => { // Search filter const matchesSearch = domain.domain_name.toLowerCase().includes(searchTerm); - + // Issues filter - check if domain has any DNS issues let hasIssues = false; if (showIssuesOnly) { @@ -3270,28 +3450,28 @@ function filterDomains() { const spf = dns.spf || {}; const dkim = dns.dkim || {}; const dmarc = dns.dmarc || {}; - + // Check if any DNS check has error or warning status - hasIssues = + hasIssues = spf.status === 'error' || spf.status === 'warning' || dkim.status === 'error' || dkim.status === 'warning' || dmarc.status === 'error' || dmarc.status === 'warning'; } - + return matchesSearch && (!showIssuesOnly || hasIssues); }); - + // Update count badge if (countBadge) { countBadge.textContent = `${filteredDomains.length} domain${filteredDomains.length !== 1 ? 's' : ''}`; } - + // Re-render filtered domains if (filteredDomains.length === 0) { - const noResultsMessage = showIssuesOnly && searchTerm === '' - ? 'No domains with DNS issues found' + const noResultsMessage = showIssuesOnly && searchTerm === '' + ? 'No domains with DNS issues found' : `No domains found matching "${escapeHtml(searchTerm)}"`; - + domainsList.innerHTML = `
@@ -3310,7 +3490,7 @@ function renderDomainAccordionRow(domain) { const spf = dns.spf || { status: 'unknown', message: 'Not checked' }; const dkim = dns.dkim || { status: 'unknown', message: 'Not checked' }; const dmarc = dns.dmarc || { status: 'unknown', message: 'Not checked' }; - + // Status icons for inline display const getStatusIcon = (status) => { if (status === 'success') return ''; @@ -3318,9 +3498,9 @@ function renderDomainAccordionRow(domain) { if (status === 'error') return ''; return '?'; }; - + const domainId = `domain-${escapeHtml(domain.domain_name).replace(/\./g, '-')}`; - + return `
@@ -3335,10 +3515,10 @@ function renderDomainAccordionRow(domain) {

${escapeHtml(domain.domain_name)}

- ${domain.active ? - 'Active' : - 'Inactive' - } + ${domain.active ? + 'Active' : + 'Inactive' + }
@@ -3389,10 +3569,10 @@ function renderDomainAccordionRow(domain) {

${escapeHtml(domain.domain_name)}

- ${domain.active ? - 'Active' : - 'Inactive' - } + ${domain.active ? + 'Active' : + 'Inactive' + }
@@ -3432,10 +3612,10 @@ function renderDomainAccordionRow(domain) {

Storage Used

${formatBytes(domain.bytes_total)}

- ${domain.max_quota_for_domain > 0 ? - `

${formatBytes(domain.max_quota_for_domain)} max

` : - '

Unlimited

' - } + ${domain.max_quota_for_domain > 0 ? + `

${formatBytes(domain.max_quota_for_domain)} max

` : + '

Unlimited

' + }

Total Messages

@@ -3507,7 +3687,7 @@ function toggleDomainDetails(domainId) { const details = document.getElementById(`${domainId}-details`); const iconDesktop = document.getElementById(`${domainId}-icon-desktop`); const iconMobile = document.getElementById(`${domainId}-icon-mobile`); - + if (details.classList.contains('hidden')) { details.classList.remove('hidden'); if (iconDesktop) iconDesktop.style.transform = 'rotate(90deg)'; @@ -3526,23 +3706,23 @@ function renderDNSCheck(type, check) { 'error': 'border-red-500 bg-red-50 dark:bg-red-900/20', 'unknown': 'border-gray-300 bg-gray-50 dark:bg-gray-800' }; - + const statusTextColors = { 'success': 'text-green-700 dark:text-green-400', 'warning': 'text-amber-700 dark:text-amber-400', 'error': 'text-red-700 dark:text-red-400', 'unknown': 'text-gray-600 dark:text-gray-400' }; - + const statusIcons = { 'success': '', 'warning': '', 'error': '', 'unknown': '' }; - + const status = check.status || 'unknown'; - + return `
@@ -3551,13 +3731,19 @@ function renderDNSCheck(type, check) {

${escapeHtml(check.message || 'No information')}

- ${check.record ? ` + ${check.record || check.actual_record ? `
View Record
- ${escapeHtml(check.record)} + ${check.dkim_domain ? ` +

+ Record Name: + ${escapeHtml(check.dkim_domain)} +

+ ` : ''} + ${escapeHtml(check.record || check.actual_record)}
` : ''} @@ -3606,22 +3792,22 @@ async function checkAllDomainsDNS() { showToast('DNS check already in progress', 'warning'); return; } - + const button = document.getElementById('check-all-dns-btn'); if (button) { button.disabled = true; button.innerHTML = ' Checking...'; } - + dnsCheckInProgress = true; - + try { const response = await authenticatedFetch('/api/domains/check-all-dns', { method: 'POST' }); - + const result = await response.json(); - + if (result.status === 'success') { showToast(`✓ Checked ${result.domains_checked} domains`, 'success'); setTimeout(() => loadDomains(), 1000); @@ -3633,7 +3819,7 @@ async function checkAllDomainsDNS() { showToast('Failed to check DNS', 'error'); } finally { dnsCheckInProgress = false; - + if (button) { button.disabled = false; button.innerHTML = ' Check Now'; @@ -3647,24 +3833,24 @@ async function checkSingleDomainDNS(domainName) { showToast('DNS check already in progress', 'warning'); return; } - + dnsCheckInProgress = true; showToast(`Checking DNS for ${domainName}...`, 'info'); - + // Find and update the button const domainId = `domain-${domainName.replace(/\./g, '-')}`; const detailsDiv = document.getElementById(`${domainId}-details`); - + try { const response = await authenticatedFetch(`/api/domains/${encodeURIComponent(domainName)}/check-dns`, { method: 'POST' }); - + const result = await response.json(); - + if (result.status === 'success') { showToast(`✓ DNS checked for ${domainName}`, 'success'); - + // Update only this domain's DNS section if (detailsDiv) { const dnsSection = detailsDiv.querySelector('.p-6:last-child'); @@ -3673,14 +3859,14 @@ async function checkSingleDomainDNS(domainName) { const domainsResponse = await authenticatedFetch('/api/domains/all'); const domainsData = await domainsResponse.json(); const updatedDomain = domainsData.domains.find(d => d.domain_name === domainName); - + if (updatedDomain) { // Re-render just the DNS section const dns = updatedDomain.dns_checks || {}; const spf = dns.spf || { status: 'unknown', message: 'Not checked' }; const dkim = dns.dkim || { status: 'unknown', message: 'Not checked' }; const dmarc = dns.dmarc || { status: 'unknown', message: 'Not checked' }; - + dnsSection.innerHTML = `

@@ -3714,7 +3900,7 @@ async function checkSingleDomainDNS(domainName) { ${renderDNSCheck('DMARC', dmarc)}

`; - + // Update inline badges in summary row const summaryRow = document.querySelector(`[onclick*="toggleDomainDetails('${domainId}')"]`); if (summaryRow) { @@ -3724,7 +3910,7 @@ async function checkSingleDomainDNS(domainName) { if (status === 'error') return ''; return '?'; }; - + const badgesContainer = summaryRow.querySelector('.flex.items-center.gap-2.text-base'); if (badgesContainer) { badgesContainer.innerHTML = ` @@ -3773,25 +3959,25 @@ function formatBytes(bytes) { async function loadSettings() { const loading = document.getElementById('settings-loading'); const content = document.getElementById('settings-content'); - + if (!loading || !content) { console.error('Settings elements not found'); return; } - + loading.classList.remove('hidden'); content.classList.add('hidden'); - + try { // Load settings info first (most important) const settingsResponse = await authenticatedFetch('/api/settings/info'); - + if (!settingsResponse.ok) { throw new Error(`HTTP ${settingsResponse.status}`); } - + const data = await settingsResponse.json(); - + // Use cached version info if available to show page immediately if (versionInfoCache.app_version) { data.app_version = versionInfoCache.app_version; @@ -3799,13 +3985,13 @@ async function loadSettings() { if (versionInfoCache.version_info) { data.version_info = versionInfoCache.version_info; } - + // Render settings immediately with cached or default data renderSettings(content, data); - + loading.classList.add('hidden'); content.classList.remove('hidden'); - + // Load app info and version status in parallel (non-blocking) (async () => { try { @@ -3813,10 +3999,10 @@ async function loadSettings() { authenticatedFetch('/api/info'), authenticatedFetch('/api/status/app-version') ]); - + const appInfo = appInfoResponse.ok ? await appInfoResponse.json() : null; const versionInfo = versionResponse.ok ? await versionResponse.json() : null; - + // Update cache if (appInfo) { versionInfoCache.app_version = appInfo.version; @@ -3824,7 +4010,7 @@ async function loadSettings() { if (versionInfo) { versionInfoCache.version_info = versionInfo; } - + // Update UI with fresh data if (appInfo || versionInfo) { const currentData = { ...data }; @@ -3841,7 +4027,7 @@ async function loadSettings() { // Page is already shown, so just log the error } })(); - + } catch (error) { console.error('Failed to load settings:', error); loading.innerHTML = ` @@ -3860,7 +4046,7 @@ function updateVersionInfoUI(versionInfo) { // Find the container with Latest Version by searching for the label const allContainers = document.querySelectorAll('#settings-content .p-4.bg-gray-50'); let latestVersionContainer = null; - + for (const container of allContainers) { const label = container.querySelector('.text-xs.uppercase'); if (label && label.textContent.trim() === 'LATEST VERSION') { @@ -3868,24 +4054,24 @@ function updateVersionInfoUI(versionInfo) { break; } } - + if (!latestVersionContainer) { return; } - + // Update version text const versionTextEl = latestVersionContainer.querySelector('.text-lg.font-semibold'); if (versionTextEl) { versionTextEl.textContent = versionInfo.latest_version ? `v${versionInfo.latest_version}` : 'Checking...'; } - + // Update last_checked date const badgeContainer = latestVersionContainer.querySelector('.flex.items-center'); if (badgeContainer) { // Find or create last_checked span let lastCheckedSpan = Array.from(badgeContainer.querySelectorAll('span.text-xs.text-gray-500, span.text-xs.text-gray-400')) .find(span => span.textContent.includes('Last checked')); - + if (versionInfo.last_checked) { if (!lastCheckedSpan) { lastCheckedSpan = document.createElement('span'); @@ -3901,13 +4087,13 @@ function updateVersionInfoUI(versionInfo) { } else if (lastCheckedSpan) { lastCheckedSpan.remove(); } - + // Remove existing badges (but keep the button and last_checked span) const existingBadges = Array.from(badgeContainer.querySelectorAll('span.px-2.py-1.rounded.text-xs')); existingBadges.forEach(badge => { badge.remove(); }); - + // Add new badge if needed if (versionInfo.update_available) { const badge = document.createElement('span'); @@ -3931,7 +4117,7 @@ function updateVersionInfoUI(versionInfo) { } } } - + // Update or create update message const versionSection = latestVersionContainer.closest('.bg-white, .dark\\:bg-gray-800'); if (versionSection) { @@ -3942,7 +4128,7 @@ function updateVersionInfoUI(versionInfo) { msg.remove(); } }); - + // Add new update message if update is available if (versionInfo.update_available) { const gridContainer = versionSection.querySelector('.grid.grid-cols-1'); @@ -3964,7 +4150,7 @@ function updateVersionInfoUI(versionInfo) { `; gridContainer.parentNode.insertBefore(messageDiv, gridContainer.nextSibling); - + // Render markdown in changelog if marked.js is available // Do this immediately after inserting to DOM if (typeof marked !== 'undefined' && versionInfo.changelog) { @@ -3987,7 +4173,7 @@ function renderSettings(content, data) { const config = data.configuration || {}; const appVersion = data.app_version || 'Unknown'; const versionInfo = data.version_info || {}; - + content.innerHTML = `
@@ -4076,47 +4262,47 @@ function renderSettings(content, data) {

Server IP

- ${config.server_ip ? - ` + ${config.server_ip ? + ` ${escapeHtml(config.server_ip)} - ` - : 'Not available' - } + ` + : 'Not available' + }

Authentication

- ${config.auth_enabled ? - ` + ${config.auth_enabled ? + ` Enabled - ` : - ` + ` : + ` Disabled ` - } + }

- ${config.auth_enabled && config.auth_username ? - `

Username: ${escapeHtml(config.auth_username)}

` : - '' - } + ${config.auth_enabled && config.auth_username ? + `

Username: ${escapeHtml(config.auth_username)}

` : + '' + }

Local Domains - ${config.local_domains && config.local_domains.length > 0 ? - `(${config.local_domains.length})` : - '' - } + ${config.local_domains && config.local_domains.length > 0 ? + `(${config.local_domains.length})` : + '' + }

- ${config.local_domains && config.local_domains.length > 0 ? - `
+ ${config.local_domains && config.local_domains.length > 0 ? + `
${config.local_domains.map(domain => `
@@ -4124,9 +4310,9 @@ function renderSettings(content, data) {
`).join('')}
-
` : - '

N/A

' - } +
` : + '

N/A

' + }

Fetch Interval

@@ -4197,15 +4383,15 @@ function renderSettings(content, data) {

SMTP Enabled

- ${data.smtp_configuration?.enabled ? - ` + ${data.smtp_configuration?.enabled ? + ` Enabled - ` : - `Disabled` - } + ` : + `Disabled` + }
`; - + // Add event listener for version number click (changelog popup) const currentVersionText = document.getElementById('current-version-text'); const currentVersionIcon = currentVersionText?.parentElement?.querySelector('svg'); - + const loadCurrentVersionChangelog = async () => { try { // Remove 'v' prefix if present for API call @@ -4305,14 +4491,14 @@ function renderSettings(content, data) { showChangelogModal('Failed to load changelog'); } }; - + if (currentVersionText) { currentVersionText.onclick = loadCurrentVersionChangelog; } if (currentVersionIcon) { currentVersionIcon.onclick = loadCurrentVersionChangelog; } - + // Render markdown in changelog sections if marked.js is available // Use versionInfo from the data object directly instead of data attributes if (typeof marked !== 'undefined' && versionInfo && versionInfo.changelog) { @@ -4329,7 +4515,7 @@ function renderSettings(content, data) { } }); } - + // Add event listener for version check button const checkVersionBtn = document.getElementById('check-version-btn'); if (checkVersionBtn) { @@ -4338,7 +4524,7 @@ function renderSettings(content, data) { const btn = checkVersionBtn; const icon = document.getElementById('check-version-icon'); const text = document.getElementById('check-version-text'); - + // Disable button and show loading state btn.disabled = true; if (icon) { @@ -4347,18 +4533,18 @@ function renderSettings(content, data) { if (text) { text.textContent = 'Checking...'; } - + try { // Force check for updates const response = await authenticatedFetch('/api/status/app-version?force=true'); const versionInfo = await response.json(); - + // Update cache versionInfoCache.version_info = versionInfo; - + // Update UI directly without reloading the page updateVersionInfoUI(versionInfo); - + // Show success state - green button with "Done" btn.classList.remove('bg-blue-500', 'hover:bg-blue-600'); btn.classList.add('bg-green-500', 'hover:bg-green-600'); @@ -4373,10 +4559,10 @@ function renderSettings(content, data) { path.setAttribute('d', 'M5 13l4 4L19 7'); } } - + // Re-enable button immediately after success (but keep green color) btn.disabled = false; - + // Reset button after 3 seconds setTimeout(() => { btn.classList.remove('bg-green-500', 'hover:bg-green-600'); @@ -4391,7 +4577,7 @@ function renderSettings(content, data) { } } }, 3000); - + } catch (error) { console.error('Failed to check version:', error); // Show error message @@ -4403,7 +4589,7 @@ function renderSettings(content, data) { if (icon) { icon.classList.remove('animate-spin'); } - + // Reset button after 2 seconds setTimeout(() => { btn.classList.remove('bg-red-500', 'hover:bg-red-600'); @@ -4432,7 +4618,7 @@ function renderMaxMindStatus(status) { `; } - + if (status.valid) { return ` @@ -4443,7 +4629,7 @@ function renderMaxMindStatus(status) { `; } - + return ` @@ -4461,13 +4647,13 @@ function renderImportCard(title, data, color) {

No data

`; } - + const colorClasses = { blue: 'border-blue-200 dark:border-blue-800 bg-blue-50 dark:bg-blue-900/20', purple: 'border-purple-200 dark:border-purple-800 bg-purple-50 dark:bg-purple-900/20', red: 'border-red-200 dark:border-red-800 bg-red-50 dark:bg-red-900/20' }; - + return `

${title}

@@ -4499,10 +4685,10 @@ function renderJobCard(name, job) { if (!job) { return ''; } - + let statusBadge = ''; - - switch(job.status) { + + switch (job.status) { case 'running': statusBadge = 'running'; break; @@ -4518,7 +4704,7 @@ function renderJobCard(name, job) { default: statusBadge = 'idle'; } - + return `
@@ -4561,21 +4747,21 @@ function showToast(message, type = 'info') { if (existingToast) { existingToast.remove(); } - + const colors = { 'success': 'bg-green-100 dark:bg-green-900/30 text-green-800 dark:text-green-200 border-green-500', 'error': 'bg-red-100 dark:bg-red-900/30 text-red-800 dark:text-red-200 border-red-500', 'warning': 'bg-yellow-100 dark:bg-yellow-900/30 text-yellow-800 dark:text-yellow-200 border-yellow-500', 'info': 'bg-blue-100 dark:bg-blue-900/30 text-blue-800 dark:text-blue-200 border-blue-500' }; - + const icons = { 'success': '✓', 'error': '✗', 'warning': '⚠', 'info': 'ℹ' }; - + const toast = document.createElement('div'); toast.id = 'toast-notification'; toast.className = `fixed bottom-4 right-4 z-50 ${colors[type]} border-l-4 p-4 rounded shadow-lg max-w-md animate-slide-in`; @@ -4586,9 +4772,9 @@ function showToast(message, type = 'info') {
`; - + document.body.appendChild(toast); - + // Auto-remove after 4 seconds setTimeout(() => { if (toast.parentElement) { @@ -4610,9 +4796,77 @@ let dmarcState = { currentSubTab: 'reports', currentReportDate: null, currentSourceIp: null, - chartInstance: null + chartInstance: null, + // Breadcrumb tracking: { label: string, action: function or null } + breadcrumb: [], + detailType: null // 'report', 'source', 'tls' }; +// Update breadcrumb display +function updateDmarcBreadcrumb() { + const container = document.getElementById('dmarc-breadcrumb'); + if (!container) return; + + if (dmarcState.breadcrumb.length === 0) { + container.innerHTML = ''; + container.classList.add('hidden'); + return; + } + + container.classList.remove('hidden'); + // Display as horizontal flex row + container.innerHTML = `
+ ${dmarcState.breadcrumb.map((item, idx) => { + const isLast = idx === dmarcState.breadcrumb.length - 1; + const separator = idx > 0 ? '' : ''; + + if (isLast) { + return `${separator}${escapeHtml(item.label)}`; + } else { + return `${separator}`; + } + }).join('')} +
`; +} + +// Set breadcrumb for different views (without "DMARC Reports" since title is static) +function setDmarcBreadcrumb(type, data = {}) { + switch (type) { + case 'domains': + // On domains list, no breadcrumb needed (we're at root) + dmarcState.breadcrumb = []; + break; + case 'domain': + // Just show domain name + dmarcState.breadcrumb = [ + { label: data.domain, action: null } + ]; + break; + case 'reportDetails': + dmarcState.breadcrumb = [ + { label: data.domain, action: `loadDomainOverview('${data.domain}')` }, + { label: 'Daily Reports', action: `loadDomainOverview('${data.domain}'); setTimeout(() => dmarcSwitchSubTab('reports'), 100)` }, + { label: data.date, action: null } + ]; + break; + case 'sourceDetails': + dmarcState.breadcrumb = [ + { label: data.domain, action: `loadDomainOverview('${data.domain}')` }, + { label: 'Source IPs', action: `loadDomainOverview('${data.domain}'); setTimeout(() => dmarcSwitchSubTab('sources'), 100)` }, + { label: data.ip, action: null } + ]; + break; + case 'tlsDetails': + dmarcState.breadcrumb = [ + { label: data.domain, action: `loadDomainOverview('${data.domain}')` }, + { label: 'TLS Reports', action: `loadDomainOverview('${data.domain}'); setTimeout(() => dmarcSwitchSubTab('tls'), 100)` }, + { label: data.date, action: null } + ]; + break; + } + updateDmarcBreadcrumb(); +} + async function loadDmarcSettings() { try { const response = await authenticatedFetch('/api/settings/info'); @@ -4620,11 +4874,11 @@ async function loadDmarcSettings() { dmarcConfiguration = null; return; } - + const data = await response.json(); dmarcConfiguration = data.dmarc_configuration || {}; console.log('DMARC settings loaded:', dmarcConfiguration); - + } catch (error) { console.error('Error loading DMARC settings:', error); dmarcConfiguration = null; @@ -4635,11 +4889,88 @@ async function loadDmarc() { console.log('Loading DMARC tab...'); dmarcState.currentView = 'domains'; dmarcState.currentDomain = null; + dmarcState.detailType = null; + dmarcState.currentReportDate = null; + dmarcState.currentSourceIp = null; + + // Destroy chart if exists + if (dmarcState.chartInstance) { + dmarcState.chartInstance.destroy(); + dmarcState.chartInstance = null; + } + + // Hide all sub-views and show main domains view + document.getElementById('dmarc-overview-view').classList.add('hidden'); + document.getElementById('dmarc-report-details-view').classList.add('hidden'); + document.getElementById('dmarc-source-details-view').classList.add('hidden'); + document.getElementById('dmarc-domains-view').classList.remove('hidden'); + document.getElementById('dmarc-page-title').textContent = 'DMARC Reports'; + + // Update breadcrumb + setDmarcBreadcrumb('domains'); + await loadDmarcSettings(); await loadDmarcImapStatus(); await loadDmarcDomains(); } +/** + * Handle DMARC route based on URL params + * Called from switchTab when navigating to DMARC + * @param {Object} params - Route params { domain, type, id } + */ +async function handleDmarcRoute(params = {}) { + console.log('handleDmarcRoute called with:', params); + + // If no domain specified, load domains list + if (!params.domain) { + await loadDmarc(); + return; + } + + // Load settings first if not loaded + if (!dmarcConfiguration) { + await loadDmarcSettings(); + } + + // Load IMAP status if not loaded + await loadDmarcImapStatus(); + + // If type is specified with an id, load that specific view + if (params.type && params.id) { + switch (params.type) { + case 'report': + // First load domain overview (don't update URL), then report details + await loadDomainOverview(params.domain, false); + await loadReportDetails(params.domain, params.id, false); + return; + case 'source': + // First load domain overview (don't update URL), then source details + await loadDomainOverview(params.domain, false); + await loadSourceDetails(params.domain, params.id, false); + return; + } + } + + // Load the domain overview (don't update URL since we came from router) + await loadDomainOverview(params.domain, false); + + // If type is specified (without id), navigate to sub-tab + if (params.type) { + switch (params.type) { + case 'reports': + dmarcSwitchSubTab('reports'); + break; + case 'sources': + dmarcSwitchSubTab('sources'); + break; + case 'tls': + dmarcSwitchSubTab('tls'); + break; + } + } +} + function getFlagEmoji(countryCode) { if (!countryCode || countryCode.length !== 2) return '🌍'; const codePoints = countryCode @@ -4669,7 +5000,7 @@ async function loadDmarcDomains() { try { const response = await authenticatedFetch('/api/dmarc/domains'); if (!response.ok) throw new Error('Failed to load domains'); - + const data = await response.json(); const domains = data.domains || []; @@ -4681,7 +5012,7 @@ async function loadDmarcDomains() { return sum + (msgs * pct / 100); }, 0); const overallPassPct = totalMessages > 0 ? Math.round((totalPass / totalMessages) * 100) : 0; - + const mainStatsContainer = document.getElementById('dmarc-main-stats-container'); if (mainStatsContainer) { mainStatsContainer.innerHTML = ` @@ -4731,73 +5062,94 @@ async function loadDmarcDomains() { domainsList.innerHTML = domains.map(domain => { const stats = domain.stats_30d || {}; const passRate = stats.dmarc_pass_pct || 0; - + // Status colors const passColor = passRate >= 95 ? 'text-green-500' : passRate >= 80 ? 'text-yellow-500' : 'text-red-500'; const barBg = passRate >= 95 ? 'bg-green-500' : passRate >= 80 ? 'bg-yellow-500' : 'bg-red-500'; const badgeBg = passRate >= 95 ? 'bg-green-900/30 text-green-400' : 'bg-red-900/30 text-red-400'; - const firstDate = domain.first_report ? new Date(domain.first_report * 1000).toLocaleDateString('en-US', {month: 'short', day: 'numeric'}) : '-'; - const lastDate = domain.last_report ? new Date(domain.last_report * 1000).toLocaleDateString('en-US', {month: 'short', day: 'numeric'}) : '-'; + const firstDate = domain.first_report ? new Date(domain.first_report * 1000).toLocaleDateString('en-US', { month: 'short', day: 'numeric' }) : '-'; + const lastDate = domain.last_report ? new Date(domain.last_report * 1000).toLocaleDateString('en-US', { month: 'short', day: 'numeric' }) : '-'; + // Badge for TLS-only domains + const hasTls = domain.has_tls; + const hasDmarc = domain.has_dmarc !== false; // default true for backwards compat + const tlsBadge = hasTls && !hasDmarc ? 'TLS' : ''; return ` -
- + - - - - - + -
-
${escapeHtml(domain.domain)}
- - ${passRate}% Pass +
${escapeHtml(domain.domain)}${tlsBadge}
+ + ${hasDmarc ? passRate + '% Pass' : 'TLS Only'}
-
+
-
Messages
-
${(stats.total_messages || 0).toLocaleString()}
+
Messages
+
${(stats.total_messages || 0).toLocaleString()}
-
Unique IPs
-
${stats.unique_ips || 0}
+
Unique IPs
+
${stats.unique_ips || 0}
-
Reports
-
${domain.report_count || 0}
+
Reports
+
+ ${domain.report_count || 0}${domain.tls_report_count > 0 ? ` +${domain.tls_report_count} TLS` : ''} +
-
Period
-
${firstDate} - ${lastDate}
+
Period
+
${firstDate} - ${lastDate}
@@ -4809,20 +5161,33 @@ async function loadDmarcDomains() { } } -async function loadDomainOverview(domain) { +async function loadDomainOverview(domain, updateUrl = true) { dmarcState.currentView = 'overview'; dmarcState.currentDomain = domain; - + dmarcState.detailType = null; + + // Update URL if requested (skip when called from handleDmarcRoute to avoid duplicate history) + if (updateUrl && typeof buildPath === 'function') { + const newPath = buildPath('dmarc', { domain }); + if (window.location.pathname !== newPath) { + history.pushState({ route: 'dmarc', params: { domain } }, '', newPath); + } + } + + // Update breadcrumb + setDmarcBreadcrumb('domain', { domain }); + document.getElementById('dmarc-domains-view').classList.add('hidden'); document.getElementById('dmarc-overview-view').classList.remove('hidden'); - document.getElementById('dmarc-back-btn').classList.remove('hidden'); - document.getElementById('dmarc-page-title').textContent = domain; - + document.getElementById('dmarc-report-details-view').classList.add('hidden'); + document.getElementById('dmarc-source-details-view').classList.add('hidden'); + // Title stays static as "DMARC Reports" + try { const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/overview?days=30`); const data = await response.json(); const totals = data.totals || {}; - + // Render the stats grid with 3 columns on mobile and icons // This replaces the old manual textContent updates const statsContainer = document.getElementById('dmarc-overview-stats-container'); @@ -4864,13 +5229,19 @@ async function loadDomainOverview(domain) {
`; } - + renderDmarcChart(data.daily_stats || []); - + + // Load initial sub-tab content based on current state if (dmarcState.currentSubTab === 'reports') { await loadDomainReports(domain); - } else { + } else if (dmarcState.currentSubTab === 'sources') { await loadDomainSources(domain); + } else if (dmarcState.currentSubTab === 'tls') { + await loadDomainTLSReports(domain); + } else { + // Default to reports + await loadDomainReports(domain); } } catch (error) { console.error('Error loading domain overview:', error); @@ -4881,17 +5252,17 @@ function renderDmarcChart(dailyStats) { const canvas = document.getElementById('dmarc-chart'); if (!canvas) return; const ctx = canvas.getContext('2d'); - + if (dmarcState.chartInstance) { dmarcState.chartInstance.destroy(); } - + // Fix: Remove * 1000 because d.date is an ISO string, not a timestamp const labels = dailyStats.map(d => { const date = new Date(d.date); return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); }); - + dmarcState.chartInstance = new Chart(ctx, { type: 'line', data: { @@ -4929,18 +5300,18 @@ async function loadDomainReports(domain) { const data = await response.json(); const reports = data.data || []; const reportsList = document.getElementById('dmarc-reports-list'); - + if (reports.length === 0) { reportsList.innerHTML = `

No daily reports available.

`; return; } - + reportsList.innerHTML = reports.map(report => { const date = new Date(report.date); const dateStr = date.toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' }); const passPct = report.dmarc_pass_pct || 0; const passColor = passPct >= 95 ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400' : 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; - + return `
@@ -4983,35 +5354,40 @@ async function loadDomainSources(domain) { try { const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/sources?days=30`); if (!response.ok) throw new Error('Failed to load sources'); - + const data = await response.json(); - const sources = data.data || []; + const sources = data.data || []; const sourcesList = document.getElementById('dmarc-sources-list'); - + if (sources.length === 0) { sourcesList.innerHTML = '

No sources found.

'; return; } - + sourcesList.innerHTML = `
${sources.map(s => { - const providerName = s.asn_org || 'Unknown Provider'; - const countryCode = s.country_code ? s.country_code.toLowerCase() : 'xx'; - const flagUrl = `/static/assets/flags/24x18/${countryCode}.png`; - - // Status Badge Logic - const passPct = s.dmarc_pass_pct || 0; - const passColor = passPct >= 95 ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400' : 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + const providerName = s.asn_org || 'Unknown Provider'; + const hasGeoData = s.country_code && s.country_code.length === 2; + const flagUrl = hasGeoData ? `/static/assets/flags/24x18/${s.country_code.toLowerCase()}.png` : null; - return ` + // Status Badge Logic + const passPct = s.dmarc_pass_pct || 0; + const passColor = passPct >= 95 ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400' : 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + + // Icon: show flag if available, otherwise show a generic server icon + const iconHtml = hasGeoData && flagUrl + ? `${s.country_name || 'Unknown'}` + : ``; + + return `
- ${s.country_name || 'Unknown'} + ${iconHtml}
${escapeHtml(providerName)}
@@ -5045,7 +5421,7 @@ async function loadDomainSources(domain) {
`; - }).join('')} + }).join('')}
`; } catch (error) { @@ -5053,41 +5429,340 @@ async function loadDomainSources(domain) { } } +// ============================================================================= +// TLS REPORTS TAB +// ============================================================================= + +function dmarcSwitchSubTab(tab) { + dmarcState.currentSubTab = tab; + + // Update tab buttons + document.getElementById('dmarc-subtab-reports').classList.remove('active'); + document.getElementById('dmarc-subtab-sources').classList.remove('active'); + document.getElementById('dmarc-subtab-tls')?.classList.remove('active'); + document.getElementById(`dmarc-subtab-${tab}`)?.classList.add('active'); + + // Update tab content + document.getElementById('dmarc-reports-content').classList.add('hidden'); + document.getElementById('dmarc-sources-content').classList.add('hidden'); + document.getElementById('dmarc-tls-content')?.classList.add('hidden'); + + // Show selected tab content + if (tab === 'reports') { + document.getElementById('dmarc-reports-content').classList.remove('hidden'); + loadDomainReports(dmarcState.currentDomain); + } else if (tab === 'sources') { + document.getElementById('dmarc-sources-content').classList.remove('hidden'); + loadDomainSources(dmarcState.currentDomain); + } else if (tab === 'tls') { + document.getElementById('dmarc-tls-content')?.classList.remove('hidden'); + loadDomainTLSReports(dmarcState.currentDomain); + } +} + +async function loadDomainTLSReports(domain) { + const tlsList = document.getElementById('dmarc-tls-list'); + if (!tlsList) return; + + try { + // Use daily aggregated API + const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/tls-reports/daily?days=30`); + if (!response.ok) throw new Error('Failed to load TLS reports'); + + const data = await response.json(); + const dailyReports = data.data || []; + const totals = data.totals || {}; + + if (dailyReports.length === 0) { + tlsList.innerHTML = ` +
+ + + +

No TLS-RPT reports found for this domain.

+

TLS reports will appear here once received from email providers.

+
`; + return; + } + + // Render summary stats + const successRate = totals.overall_success_rate || 100; + const successColor = successRate >= 95 ? 'text-green-500' : successRate >= 80 ? 'text-yellow-500' : 'text-red-500'; + + tlsList.innerHTML = ` + +
+
+
Days
+
${totals.total_days || 0}
+
+
+
Reports
+
${totals.total_reports || 0}
+
+
+
Success Rate
+
${successRate}%
+
+
+
Sessions
+
${((totals.total_successful_sessions || 0) + (totals.total_failed_sessions || 0)).toLocaleString()}
+
+
+ + +
+ ${dailyReports.map(day => { + const dateFormatted = new Date(day.date).toLocaleDateString('en-US', { weekday: 'short', month: 'short', day: 'numeric', year: 'numeric' }); + const rateColor = day.success_rate >= 95 ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400' : + day.success_rate >= 80 ? 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-400' : + 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + const barColor = day.success_rate >= 95 ? 'bg-green-500' : day.success_rate >= 80 ? 'bg-yellow-500' : 'bg-red-500'; + + return ` +
+
+
+
+ + + +
+
+
${dateFormatted}
+
+ ${day.report_count} report${day.report_count !== 1 ? 's' : ''} from ${day.organization_count} provider${day.organization_count !== 1 ? 's' : ''} +
+
+
+ + ${day.success_rate}% + +
+ + +
+
+
+ + +
+
+ + + + Success: + ${(day.total_success || 0).toLocaleString()} +
+
+ + + + Failed: + ${(day.total_fail || 0).toLocaleString()} +
+
+ Providers: + ${day.organizations.join(', ')} +
+
+
+ `; + }).join('')} +
+ `; + + } catch (error) { + console.error('Error loading TLS reports:', error); + tlsList.innerHTML = ` +
+ + + +

Failed to load TLS reports.

+
`; + } +} + +async function loadTLSReportDetails(domain, reportDate) { + const tlsList = document.getElementById('dmarc-tls-list'); + if (!tlsList) return; + + dmarcState.detailType = 'tls'; + const dateFormatted = new Date(reportDate).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' }); + setDmarcBreadcrumb('tlsDetails', { domain, date: dateFormatted }); + + try { + const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/tls-reports/${reportDate}/details`); + if (!response.ok) throw new Error('Failed to load TLS report details'); + + const data = await response.json(); + const stats = data.stats || {}; + const providers = data.providers || []; + + const dateFormatted = new Date(reportDate).toLocaleDateString('en-US', { weekday: 'long', month: 'long', day: 'numeric', year: 'numeric' }); + const successRate = stats.success_rate || 100; + const successColor = successRate >= 95 ? 'text-green-500' : successRate >= 80 ? 'text-yellow-500' : 'text-red-500'; + + tlsList.innerHTML = ` + +
+ +
+ + +
+
+

${dateFormatted}

+

TLS Report Details for ${escapeHtml(domain)}

+
+
+ + +
+
+
Sessions
+
${(stats.total_sessions || 0).toLocaleString()}
+
+
+
Success Rate
+
${successRate}%
+
+
+
Successful
+
${(stats.total_success || 0).toLocaleString()}
+
+
+
Failed
+
${(stats.total_fail || 0).toLocaleString()}
+
+
+ + +
+
+

Providers (${stats.total_providers || 0})

+
+ + +
+ + + + + + + + + + + ${providers.map(p => { + const rateColor = p.success_rate >= 95 ? 'text-green-600 dark:text-green-400' : p.success_rate >= 80 ? 'text-yellow-600 dark:text-yellow-400' : 'text-red-600 dark:text-red-400'; + return ` + + + + + + + `; + }).join('')} + +
ProviderSessionsSuccessFailedRate
+
${escapeHtml(p.organization_name || 'Unknown')}
+
${p.policies?.length || 0} policies
+
${(p.total_sessions || 0).toLocaleString()}${(p.successful_sessions || 0).toLocaleString()}${(p.failed_sessions || 0).toLocaleString()}${p.success_rate}%
+
+ + +
+ ${providers.map(p => { + const rateColor = p.success_rate >= 95 ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400' : p.success_rate >= 80 ? 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-400' : 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + return ` +
+
+
${escapeHtml(p.organization_name || 'Unknown')}
+ ${p.success_rate}% +
+
+
Sessions: ${p.total_sessions}
+
Success: ${p.successful_sessions}
+
Failed: ${p.failed_sessions}
+
+
`; + }).join('')} +
+
+ `; + + } catch (error) { + console.error('Error loading TLS report details:', error); + tlsList.innerHTML = ` +
+ + + +

Failed to load TLS report details.

+ +
`; + } +} + // ============================================================================= // REPORT DETAILS // ============================================================================= -async function loadReportDetails(domain, reportDate) { +async function loadReportDetails(domain, reportDate, updateUrl = true) { dmarcState.currentView = 'report_details'; dmarcState.currentReportDate = reportDate; - + dmarcState.detailType = 'report'; + + // Update URL if requested + if (updateUrl && typeof buildPath === 'function') { + const newPath = buildPath('dmarc', { domain, type: 'report', id: reportDate }); + if (window.location.pathname !== newPath) { + history.pushState({ route: 'dmarc', params: { domain, type: 'report', id: reportDate } }, '', newPath); + } + } + + document.getElementById('dmarc-domains-view').classList.add('hidden'); document.getElementById('dmarc-overview-view').classList.add('hidden'); document.getElementById('dmarc-report-details-view').classList.remove('hidden'); document.getElementById('dmarc-source-details-view').classList.add('hidden'); - + const dateObj = new Date(reportDate); const dateStr = dateObj.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }); - document.getElementById('dmarc-page-title').textContent = `${domain} - ${dateStr}`; - + const shortDate = dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' }); + // Title stays static as "DMARC Reports" + + // Update breadcrumb + setDmarcBreadcrumb('reportDetails', { domain, date: shortDate }); + try { const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/reports/${reportDate}/details`); const data = await response.json(); const totals = data.totals || {}; - + /* Inject icons and stats grid */ const statsContainer = document.getElementById('report-details-stats-container'); if (statsContainer) { statsContainer.innerHTML = generateDetailStatsGrid(totals); } - + const sources = data.sources || []; const sourcesList = document.getElementById('report-detail-sources-list'); - + if (sources.length === 0) { sourcesList.innerHTML = '

No sources found.

'; return; } - + sourcesList.innerHTML = ` @@ -5104,18 +5779,23 @@ async function loadReportDetails(domain, reportDate) { ${sources.map(s => { - const providerName = s.asn_org || s.source_name || 'Unknown'; - const countryCode = s.country_code ? s.country_code.toLowerCase() : 'xx'; - const flagUrl = `/static/assets/flags/48x36/${countryCode}.png`; - const dmarcColor = s.dmarc_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.dmarc_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - const spfColor = s.spf_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.spf_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - const dkimColor = s.dkim_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.dkim_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - - return ` + const providerName = s.asn_org || s.source_name || 'Unknown'; + const hasGeoData = s.country_code && s.country_code.length === 2; + const flagUrl = hasGeoData ? `/static/assets/flags/48x36/${s.country_code.toLowerCase()}.png` : null; + const dmarcColor = s.dmarc_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.dmarc_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + const spfColor = s.spf_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.spf_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + const dkimColor = s.dkim_pass_pct >= 95 ? 'text-green-600 dark:text-green-400' : s.dkim_pass_pct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + + // Icon: show flag if available, otherwise show a generic server icon + const iconHtml = hasGeoData && flagUrl + ? `${s.country_name || 'Unknown'}` + : ``; + + return ` `; - }).join('')} + }).join('')}
- ${s.country_name || 'Unknown'} + ${iconHtml}
${escapeHtml(providerName)}
${escapeHtml(s.source_ip)}
@@ -5130,7 +5810,7 @@ async function loadReportDetails(domain, reportDate) {
${s.dkim_pass_pct}% ${escapeHtml(s.reporter || '-')}
`; @@ -5144,45 +5824,65 @@ async function loadReportDetails(domain, reportDate) { // SOURCE DETAILS // ============================================================================= -async function loadSourceDetails(domain, sourceIp) { +async function loadSourceDetails(domain, sourceIp, updateUrl = true) { dmarcState.currentView = 'source_details'; dmarcState.currentSourceIp = sourceIp; - + dmarcState.detailType = 'source'; + + // Update URL if requested + if (updateUrl && typeof buildPath === 'function') { + const newPath = buildPath('dmarc', { domain, type: 'source', id: sourceIp }); + if (window.location.pathname !== newPath) { + history.pushState({ route: 'dmarc', params: { domain, type: 'source', id: sourceIp } }, '', newPath); + } + } + + document.getElementById('dmarc-domains-view').classList.add('hidden'); document.getElementById('dmarc-overview-view').classList.add('hidden'); document.getElementById('dmarc-report-details-view').classList.add('hidden'); document.getElementById('dmarc-source-details-view').classList.remove('hidden'); - document.getElementById('dmarc-page-title').textContent = `${domain} - ${sourceIp}`; - + // Title stays static as "DMARC Reports" + + // Update breadcrumb + setDmarcBreadcrumb('sourceDetails', { domain, ip: sourceIp }); + try { const response = await authenticatedFetch(`/api/dmarc/domains/${encodeURIComponent(domain)}/sources/${encodeURIComponent(sourceIp)}/details?days=30`); const data = await response.json(); - + /* Update Header Info */ - const countryCode = data.country_code ? data.country_code.toLowerCase() : 'xx'; - const flagUrl = `/static/assets/flags/48x36/${countryCode}.png`; - document.getElementById('source-detail-flag').src = flagUrl; + const hasGeoData = data.country_code && data.country_code.length === 2; + const flagImg = document.getElementById('source-detail-flag'); + if (hasGeoData) { + const flagUrl = `/static/assets/flags/48x36/${data.country_code.toLowerCase()}.png`; + flagImg.src = flagUrl; + flagImg.style.display = ''; + flagImg.onerror = function () { this.style.display = 'none'; }; + } else { + flagImg.style.display = 'none'; + } document.getElementById('source-detail-name').textContent = data.source_name || data.asn_org || 'Unknown Provider'; document.getElementById('source-detail-ip').textContent = sourceIp; - + const location = [data.city, data.country_name].filter(Boolean).join(', ') || 'Unknown location'; document.getElementById('source-detail-location').textContent = location; document.getElementById('source-detail-asn').textContent = data.asn ? `ASN ${data.asn}` : 'No ASN'; - + /* Inject icons and stats grid */ const totals = data.totals || {}; const statsContainer = document.getElementById('source-details-stats-container'); if (statsContainer) { statsContainer.innerHTML = generateDetailStatsGrid(totals); } - + const envelopes = data.envelope_from_groups || []; const envelopeList = document.getElementById('source-detail-envelope-list'); - + if (envelopes.length === 0) { envelopeList.innerHTML = '

No data found.

'; return; } - + envelopeList.innerHTML = ` @@ -5198,14 +5898,14 @@ async function loadSourceDetails(domain, sourceIp) { ${envelopes.map(env => { - const dmarcPct = env.volume > 0 ? Math.round((env.dmarc_pass / env.volume) * 100) : 0; - const spfPct = env.volume > 0 ? Math.round((env.spf_aligned / env.volume) * 100) : 0; - const dkimPct = env.volume > 0 ? Math.round((env.dkim_aligned / env.volume) * 100) : 0; - const dmarcColor = dmarcPct >= 95 ? 'text-green-600 dark:text-green-400' : dmarcPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - const spfColor = spfPct >= 95 ? 'text-green-600 dark:text-green-400' : spfPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - const dkimColor = dkimPct >= 95 ? 'text-green-600 dark:text-green-400' : dkimPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; - - return ` + const dmarcPct = env.volume > 0 ? Math.round((env.dmarc_pass / env.volume) * 100) : 0; + const spfPct = env.volume > 0 ? Math.round((env.spf_aligned / env.volume) * 100) : 0; + const dkimPct = env.volume > 0 ? Math.round((env.dkim_aligned / env.volume) * 100) : 0; + const dmarcColor = dmarcPct >= 95 ? 'text-green-600 dark:text-green-400' : dmarcPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + const spfColor = spfPct >= 95 ? 'text-green-600 dark:text-green-400' : spfPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + const dkimColor = dkimPct >= 95 ? 'text-green-600 dark:text-green-400' : dkimPct === 0 ? 'text-red-600 dark:text-red-400' : 'text-gray-900 dark:text-gray-100'; + + return ` @@ -5215,7 +5915,7 @@ async function loadSourceDetails(domain, sourceIp) { `; - }).join('')} + }).join('')}
${escapeHtml(env.header_from || '-')} ${escapeHtml(env.envelope_from || '-')}${dkimPct}% ${escapeHtml(env.reporter || '-')}
`; @@ -5267,72 +5967,7 @@ function generateDetailStatsGrid(totals) { } -// ============================================================================= -// NAVIGATION -// ============================================================================= -function dmarcGoBack() { - if (dmarcState.currentView === 'source_details' || dmarcState.currentView === 'report_details') { - dmarcState.currentView = 'overview'; - dmarcState.currentReportDate = null; - dmarcState.currentSourceIp = null; - - document.getElementById('dmarc-report-details-view').classList.add('hidden'); - document.getElementById('dmarc-source-details-view').classList.add('hidden'); - document.getElementById('dmarc-overview-view').classList.remove('hidden'); - document.getElementById('dmarc-page-title').textContent = dmarcState.currentDomain; - - if (dmarcState.currentSubTab === 'reports') { - loadDomainReports(dmarcState.currentDomain); - } else { - loadDomainSources(dmarcState.currentDomain); - } - - return; - } - - dmarcState.currentView = 'domains'; - dmarcState.currentDomain = null; - - if (dmarcState.chartInstance) { - dmarcState.chartInstance.destroy(); - dmarcState.chartInstance = null; - } - - document.getElementById('dmarc-overview-view').classList.add('hidden'); - document.getElementById('dmarc-report-details-view').classList.add('hidden'); - document.getElementById('dmarc-source-details-view').classList.add('hidden'); - document.getElementById('dmarc-domains-view').classList.remove('hidden'); - document.getElementById('dmarc-back-btn').classList.add('hidden'); - document.getElementById('dmarc-page-title').textContent = 'DMARC Reports'; - // document.getElementById('dmarc-breadcrumb').textContent = 'Domains'; - - loadDmarcDomains(); -} - -function dmarcSwitchSubTab(tab) { - dmarcState.currentSubTab = tab; - - // Update tab buttons - document.querySelectorAll('[id^="dmarc-subtab-"]').forEach(btn => { - btn.classList.remove('active'); - }); - document.getElementById(`dmarc-subtab-${tab}`).classList.add('active'); - - // Update content - if (tab === 'reports') { - document.getElementById('dmarc-reports-content').classList.remove('hidden'); - document.getElementById('dmarc-sources-content').classList.add('hidden'); - } else if (tab === 'sources') { - document.getElementById('dmarc-reports-content').classList.add('hidden'); - document.getElementById('dmarc-sources-content').classList.remove('hidden'); - - // Load sources if not loaded yet - if (dmarcState.currentDomain) { - loadDomainSources(dmarcState.currentDomain); - } - } -} // ============================================================================= // UPLOAD @@ -5341,42 +5976,50 @@ function dmarcSwitchSubTab(tab) { async function uploadDmarcReport(event) { const file = event.target.files[0]; if (!file) return; - + try { const formData = new FormData(); formData.append('file', file); - + const response = await authenticatedFetch('/api/dmarc/upload', { method: 'POST', body: formData }); - + if (response.status === 403) { showToast('Manual upload is disabled', 'error'); event.target.value = ''; return; } - + if (!response.ok) throw new Error('Upload failed'); - + const result = await response.json(); - + const reportType = result.report_type === 'tls-rpt' ? 'TLS-RPT' : 'DMARC'; + if (result.status === 'success') { - showToast(`Report uploaded: ${result.records_count} records`, 'success'); + const count = result.records_count || result.policies_count || 0; + const countLabel = result.report_type === 'tls-rpt' ? 'policies' : 'records'; + showToast(`${reportType} report uploaded: ${count} ${countLabel}`, 'success'); + if (dmarcState.currentView === 'domains') { loadDmarcDomains(); } else if (dmarcState.currentDomain) { loadDomainOverview(dmarcState.currentDomain); + // If TLS report was uploaded and we're on TLS tab, refresh it + if (result.report_type === 'tls-rpt' && dmarcState.currentSubTab === 'tls') { + loadDomainTLSReports(dmarcState.currentDomain); + } } } else if (result.status === 'duplicate') { - showToast('Report already exists', 'warning'); + showToast(`${reportType} report already exists`, 'warning'); } - + } catch (error) { console.error('Upload error:', error); showToast('Failed to upload report', 'error'); } - + event.target.value = ''; } @@ -5391,10 +6034,10 @@ async function loadDmarcImapStatus() { dmarcImapStatus = null; return; } - + dmarcImapStatus = await response.json(); updateDmarcControls(); - + } catch (error) { console.error('Error loading DMARC IMAP status:', error); dmarcImapStatus = null; @@ -5405,7 +6048,7 @@ function updateDmarcControls() { const uploadBtn = document.getElementById('dmarc-upload-btn'); const syncContainer = document.getElementById('dmarc-sync-container'); const lastSyncInfo = document.getElementById('dmarc-last-sync-info'); - + // Toggle upload button if (uploadBtn) { if (dmarcConfiguration?.manual_upload_enabled === true) { @@ -5414,21 +6057,21 @@ function updateDmarcControls() { uploadBtn.classList.add('hidden'); } } - + // Toggle sync container if (dmarcImapStatus && dmarcImapStatus.enabled) { syncContainer.classList.remove('hidden'); - + // Update last sync info to match Domains Overview style if (dmarcImapStatus.latest_sync) { const sync = dmarcImapStatus.latest_sync; const timeStr = formatTime(sync.started_at); - + let statusPrefix = ''; if (sync.status === 'success') statusPrefix = '✓ '; if (sync.status === 'error') statusPrefix = '✗ '; if (sync.status === 'running') statusPrefix = '⟳ '; - + lastSyncInfo.innerHTML = `
@@ -5450,37 +6093,37 @@ function updateDmarcControls() { async function triggerDmarcSync() { const btn = document.getElementById('dmarc-sync-btn'); const btnText = document.getElementById('dmarc-sync-btn-text'); - + if (!dmarcImapStatus || !dmarcImapStatus.enabled) { showToast('IMAP sync is not enabled', 'error'); return; } - + btn.disabled = true; btnText.textContent = 'Syncing...'; - + try { const response = await authenticatedFetch('/api/dmarc/imap/sync', { method: 'POST' }); - + const result = await response.json(); - + if (result.status === 'already_running') { showToast('Sync is already in progress', 'info'); } else if (result.status === 'started') { showToast('IMAP sync started', 'success'); - + // Immediate UI update to show "Running" state - await loadDmarcImapStatus(); - + await loadDmarcImapStatus(); + // Delayed update to catch the final result (success/fail) setTimeout(async () => { await loadDmarcImapStatus(); await loadDmarcDomains(); }, 5000); // Increased to 5s to give the sync time to work } - + } catch (error) { console.error('Error triggering sync:', error); showToast('Failed to start sync', 'error'); @@ -5494,9 +6137,9 @@ async function triggerDmarcSync() { async function showDmarcSyncHistory() { const modal = document.getElementById('dmarc-sync-history-modal'); const content = document.getElementById('dmarc-sync-history-content'); - + modal.classList.remove('hidden'); - + const closeOnBackdrop = (e) => { if (e.target === modal) { closeDmarcSyncHistoryModal(); @@ -5504,16 +6147,16 @@ async function showDmarcSyncHistory() { } }; modal.addEventListener('click', closeOnBackdrop); - + try { const response = await authenticatedFetch('/api/dmarc/imap/history?limit=20'); const data = await response.json(); - + if (data.data.length === 0) { content.innerHTML = '

No sync history yet

'; return; } - + content.innerHTML = `
@@ -5531,12 +6174,12 @@ async function showDmarcSyncHistory() { ${data.data.map(sync => { - const statusClass = sync.status === 'success' ? 'text-green-600' : - sync.status === 'error' ? 'text-red-600' : 'text-blue-600'; - const date = formatDate(sync.started_at); - const duration = sync.duration_seconds ? `${Math.round(sync.duration_seconds)}s` : '-'; - - return ` + const statusClass = sync.status === 'success' ? 'text-green-600' : + sync.status === 'error' ? 'text-red-600' : 'text-blue-600'; + const date = formatDate(sync.started_at); + const duration = sync.duration_seconds ? `${Math.round(sync.duration_seconds)}s` : '-'; + + return ` `; - }).join('')} + }).join('')}
${date} @@ -5552,12 +6195,12 @@ async function showDmarcSyncHistory() { ${duration}
`; - + } catch (error) { console.error('Error loading sync history:', error); content.innerHTML = '

Failed to load sync history

'; @@ -5574,22 +6217,22 @@ function closeDmarcSyncHistoryModal() { async function testSmtpConnection() { showConnectionTestModal('SMTP Connection Test', 'Testing SMTP connection...'); - + try { const response = await authenticatedFetch('/api/settings/test/smtp', { method: 'POST' }); - + if (!response.ok) { throw new Error(`HTTP ${response.status}`); } - + const result = await response.json(); - + // Ensure logs is an array const logs = result.logs || ['No logs available']; updateConnectionTestModal(result.success ? 'success' : 'error', logs); - + } catch (error) { updateConnectionTestModal('error', [ 'Failed to test SMTP connection', @@ -5600,22 +6243,22 @@ async function testSmtpConnection() { async function testImapConnection() { showConnectionTestModal('IMAP Connection Test', 'Testing IMAP connection...'); - + try { const response = await authenticatedFetch('/api/settings/test/imap', { method: 'POST' }); - + if (!response.ok) { throw new Error(`HTTP ${response.status}`); } - + const result = await response.json(); - + // Ensure logs is an array const logs = result.logs || ['No logs available']; updateConnectionTestModal(result.success ? 'success' : 'error', logs); - + } catch (error) { updateConnectionTestModal('error', [ 'Failed to test IMAP connection', @@ -5653,31 +6296,31 @@ function showConnectionTestModal(title, message) {
`; - + // Close on backdrop click modal.addEventListener('click', (e) => { if (e.target === modal) { closeConnectionTestModal(); } }); - + document.body.appendChild(modal); } function updateConnectionTestModal(status, logs) { const content = document.getElementById('connection-test-content'); if (!content) return; - + // Ensure logs is an array if (!Array.isArray(logs)) { logs = ['Error: Invalid response format']; } - + const statusColor = status === 'success' ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'; - const statusIcon = status === 'success' ? + const statusIcon = status === 'success' ? '' : ''; - + content.innerHTML = `
@@ -5689,12 +6332,12 @@ function updateConnectionTestModal(status, logs) {
${logs.map(log => { - let color = 'text-gray-300'; - if (log.includes('✓')) color = 'text-green-400'; - if (log.includes('✗') || log.includes('ERROR')) color = 'text-red-400'; - if (log.includes('WARNING')) color = 'text-yellow-400'; - return `
${escapeHtml(log)}
`; - }).join('')} + let color = 'text-gray-300'; + if (log.includes('✓')) color = 'text-green-400'; + if (log.includes('✗') || log.includes('ERROR')) color = 'text-red-400'; + if (log.includes('WARNING')) color = 'text-yellow-400'; + return `
${escapeHtml(log)}
`; + }).join('')}
`; } @@ -5713,13 +6356,13 @@ function closeConnectionTestModal() { async function showHelpModal(docName) { try { const response = await authenticatedFetch(`/api/docs/${docName}`); - + if (!response.ok) { throw new Error(`Failed to load documentation: ${response.statusText}`); } - + const markdown = await response.text(); - + let htmlContent = markdown; if (typeof marked !== 'undefined') { marked.setOptions({ @@ -5728,27 +6371,27 @@ async function showHelpModal(docName) { }); htmlContent = marked.parse(markdown); } - + const modal = document.getElementById('changelog-modal'); const modalTitle = modal?.querySelector('h3'); const content = document.getElementById('changelog-content'); - + if (modal && content) { if (modalTitle) { modalTitle.textContent = `Help - ${docName}`; } - + content.innerHTML = htmlContent; modal.classList.remove('hidden'); document.body.style.overflow = 'hidden'; } } catch (error) { console.error('Failed to load help documentation:', error); - + const modal = document.getElementById('changelog-modal'); const modalTitle = modal?.querySelector('h3'); const content = document.getElementById('changelog-content'); - + if (modal && content) { if (modalTitle) { modalTitle.textContent = 'Help'; @@ -5765,5 +6408,759 @@ async function showHelpModal(docName) { // ============================================================================= console.log('[OK] Mailcow Logs Viewer - Complete Frontend Loaded'); -console.log('Features: Dashboard, Messages, Postfix, Rspamd, Netfilter, Queue, Quarantine, Status, Settings'); -console.log('UI: Dark mode, Modals with tabs, Responsive design'); \ No newline at end of file +console.log('Features: Dashboard, Messages, Postfix, Rspamd, Netfilter, Queue, Quarantine, Status, Mailbox Stats, Settings'); +console.log('UI: Dark mode, Modals with tabs, Responsive design'); + +// ============================================================================= +// MAILBOX STATISTICS - REDESIGNED WITH MESSAGE COUNTS +// ============================================================================= + +// Cached mailbox stats data +let mailboxStatsCache = { + summary: null, + mailboxes: null, + domains: null, + lastLoad: null, + expandedMailboxes: new Set() // Track expanded accordion states +}; + +async function loadMailboxStats() { + console.log('Loading mailbox statistics...'); + + // Show loading state + const loading = document.getElementById('mailbox-stats-loading'); + const content = document.getElementById('mailbox-stats-content'); + + if (loading) loading.classList.remove('hidden'); + if (content) content.classList.add('hidden'); + + try { + const dateRange = document.getElementById('mailbox-stats-date-range')?.value || '30days'; + const customStartDate = document.getElementById('mailbox-stats-start-date')?.value || ''; + const customEndDate = document.getElementById('mailbox-stats-end-date')?.value || ''; + + // Build summary URL with optional custom date range + let summaryUrl = `/api/mailbox-stats/summary?date_range=${dateRange}`; + if (dateRange === 'custom' && customStartDate && customEndDate) { + summaryUrl += `&start_date=${encodeURIComponent(customStartDate)}&end_date=${encodeURIComponent(customEndDate)}`; + } + + // Load summary and domains in parallel + const [summaryRes, domainsRes] = await Promise.all([ + authenticatedFetch(summaryUrl), + authenticatedFetch('/api/mailbox-stats/domains') + ]); + + if (!summaryRes.ok || !domainsRes.ok) { + throw new Error('Failed to fetch mailbox statistics'); + } + + const summary = await summaryRes.json(); + const domains = await domainsRes.json(); + + mailboxStatsCache.summary = summary; + mailboxStatsCache.domains = domains.domains || []; + + // Render summary cards + renderMailboxStatsSummary(summary); + + // Populate domain filter + populateMailboxStatsDomainFilter(mailboxStatsCache.domains); + + // Load all mailboxes + await loadMailboxStatsList(); + + // Update last update time + const lastUpdateEl = document.getElementById('mailbox-stats-last-update'); + if (lastUpdateEl && summary.last_update) { + lastUpdateEl.textContent = `Last updated: ${formatTime(summary.last_update)}`; + } + + // Show content, hide loading + if (loading) loading.classList.add('hidden'); + if (content) content.classList.remove('hidden'); + + mailboxStatsCache.lastLoad = new Date(); + + } catch (error) { + console.error('Error loading mailbox stats:', error); + if (loading) { + loading.innerHTML = ` +
+ + + +

Failed to load mailbox statistics

+

${error.message}

+ +
+ `; + } + } +} + +function refreshMailboxStats() { + loadMailboxStats(); +} + +function renderMailboxStatsSummary(summary) { + // Update summary cards (new 4-card design: Sent, Received, Failed, Failure Rate) + const sentEl = document.getElementById('mailbox-stats-sent'); + const receivedEl = document.getElementById('mailbox-stats-received'); + const failedEl = document.getElementById('mailbox-stats-failed'); + const failureRateEl = document.getElementById('mailbox-stats-failure-rate'); + + if (sentEl) sentEl.textContent = (summary.total_sent || 0).toLocaleString(); + if (receivedEl) receivedEl.textContent = (summary.total_received || 0).toLocaleString(); + if (failedEl) failedEl.textContent = (summary.sent_failed || 0).toLocaleString(); + if (failureRateEl) failureRateEl.textContent = `${summary.failure_rate || 0}%`; + + // Update date labels based on selected range + const dateRange = document.getElementById('mailbox-stats-date-range')?.value || '30days'; + let dateLabel; + + if (dateRange === 'custom') { + const startDate = document.getElementById('mailbox-stats-start-date')?.value; + const endDate = document.getElementById('mailbox-stats-end-date')?.value; + if (startDate && endDate) { + dateLabel = `${formatDateShort(startDate)} - ${formatDateShort(endDate)}`; + } else { + dateLabel = 'Custom Range'; + } + } else { + dateLabel = dateRange === 'today' ? 'Today' : + dateRange === '7days' ? 'Last 7 days' : + dateRange === '90days' ? 'Last 90 days' : 'Last 30 days'; + } + + ['sent', 'recv', 'failed', 'rate'].forEach(s => { + const el = document.getElementById(`mailbox-stats-date-label-${s}`); + if (el) el.textContent = dateLabel; + }); +} + +function populateMailboxStatsDomainFilter(domains) { + const select = document.getElementById('mailbox-stats-domain-filter'); + if (!select) return; + + // Clear existing options except "All Domains" + select.innerHTML = ''; + + // Add domain options + domains.forEach(d => { + const option = document.createElement('option'); + option.value = d.domain; + option.textContent = `${d.domain} (${d.mailbox_count})`; + select.appendChild(option); + }); +} + +// Current page for pagination +let mailboxStatsPage = 1; + +async function loadMailboxStatsList(page = 1) { + mailboxStatsPage = page; + const dateRange = document.getElementById('mailbox-stats-date-range')?.value || '30days'; + const customStartDate = document.getElementById('mailbox-stats-start-date')?.value || ''; + const customEndDate = document.getElementById('mailbox-stats-end-date')?.value || ''; + const domainFilter = document.getElementById('mailbox-stats-domain-filter')?.value || ''; + const sortValue = document.getElementById('mailbox-stats-sort')?.value || 'sent_total-desc'; + const activeOnly = document.getElementById('mailbox-stats-active-only')?.checked ?? true; + const hideZero = document.getElementById('mailbox-stats-hide-zero')?.checked ?? false; + const search = document.getElementById('mailbox-stats-search')?.value || ''; + + const [sortBy, sortOrder] = sortValue.split('-'); + + let url = `/api/mailbox-stats/all?date_range=${dateRange}&sort_by=${sortBy}&sort_order=${sortOrder}&page=${page}&page_size=50`; + + // Add custom date range parameters if using custom mode + if (dateRange === 'custom' && customStartDate && customEndDate) { + url += `&start_date=${encodeURIComponent(customStartDate)}&end_date=${encodeURIComponent(customEndDate)}`; + } + + if (domainFilter) url += `&domain=${encodeURIComponent(domainFilter)}`; + if (activeOnly) url += '&active_only=true'; + else url += '&active_only=false'; + if (hideZero) url += '&hide_zero=true'; + if (search) url += `&search=${encodeURIComponent(search)}`; + + try { + const response = await authenticatedFetch(url); + if (!response.ok) throw new Error('Failed to fetch mailboxes'); + + const data = await response.json(); + mailboxStatsCache.mailboxes = data.mailboxes || []; + + // Update count + const countEl = document.getElementById('mailbox-stats-count'); + if (countEl) countEl.textContent = `${data.total || 0} mailboxes`; + + // Update pagination info + const pageInfoEl = document.getElementById('mailbox-stats-page-info'); + if (pageInfoEl && data.total_pages > 1) { + pageInfoEl.textContent = `Page ${data.page} of ${data.total_pages}`; + } else if (pageInfoEl) { + pageInfoEl.textContent = ''; + } + + renderMailboxStatsAccordion(data.mailboxes || [], data.page, data.total_pages); + + } catch (error) { + console.error('Error loading mailbox list:', error); + } +} + +function renderMailboxStatsAccordion(mailboxes, page = 1, totalPages = 1) { + const container = document.getElementById('mailbox-stats-list'); + if (!container) return; + + if (mailboxes.length === 0) { + container.innerHTML = ` +
+ + + +

No mailboxes found

+
+ `; + return; + } + + // Build mailbox rows first + let html = mailboxes.map((mb, index) => { + const isExpanded = mailboxStatsCache.expandedMailboxes.has(mb.username); + const statusClass = mb.active + ? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-300' + : 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300'; + + // Failure rate color + const failureColor = mb.combined_failure_rate >= 10 ? 'text-red-600 dark:text-red-400' + : mb.combined_failure_rate >= 5 ? 'text-yellow-600 dark:text-yellow-400' + : 'text-green-600 dark:text-green-400'; + + // Quota bar + const quotaPercent = mb.percent_in_use || 0; + const quotaColor = quotaPercent >= 90 ? 'bg-red-500' : quotaPercent >= 75 ? 'bg-yellow-500' : 'bg-blue-500'; + + return ` +
+ +
+
+ + + + +
+ +
+ + + +
+
${escapeHtml(mb.username)}
+
+ +
+ + ${mb.active ? 'Active' : 'Inactive'} +
+
+ + +
+ + ↑ ${mb.combined_sent.toLocaleString()} Sent + + + ↓ ${mb.combined_received.toLocaleString()} Received + +
+ + +
+ + ✓ ${(mb.combined_delivered || 0).toLocaleString()} Delivered + + + ${mb.combined_failure_rate}% Failed + +
+
+
+
+ + +
+ +
+
+
+

Quota Used

+

${mb.quota_used_formatted} / ${mb.quota_formatted}

+

${mb.percent_in_use || 0}% used

+
+
+

Messages in Mailbox

+

${(mb.messages_in_mailbox || 0).toLocaleString()}

+
+
+

Created / Modified

+

${mb.created ? formatTime(mb.created) : 'N/A'}

+

${mb.modified ? formatTime(mb.modified) : 'N/A'}

+
+
+

Rate Limit

+

${mb.rl_value ? mb.rl_value + '/' + (mb.rl_frame === 's' ? 'sec' : mb.rl_frame === 'm' ? 'min' : mb.rl_frame === 'h' ? 'hour' : mb.rl_frame === 'd' ? 'day' : mb.rl_frame || 'min') : 'None'}

+
+
+ + +
+
+
+ + IMAP +
+ ${mb.last_imap_login ? formatTime(mb.last_imap_login) : 'Never'} +
+
+
+ + POP3 +
+ ${mb.last_pop3_login ? formatTime(mb.last_pop3_login) : 'Never'} +
+
+
+ + SMTP +
+ ${mb.last_smtp_login ? formatTime(mb.last_smtp_login) : 'Never'} +
+
+
+ + Sieve +
+
+
+
+ + TLS Enforce +
+
+
+
+ + +
+

Message Statistics

+ + +
+
+
${mb.combined_sent || 0}
+
Sent
+
+
+
${mb.combined_received || 0}
+
Received
+
+
+
${mb.combined_internal || 0}
+
Internal
+
+
+ + +
+
+
${mb.combined_delivered || 0}
+
Delivered
+
+
+
${(mb.mailbox_counts?.sent_deferred || 0) + (mb.aliases || []).reduce((sum, a) => sum + (a.sent_deferred || 0), 0)}
+
Deferred
+
+
+
${(mb.mailbox_counts?.sent_bounced || 0) + (mb.aliases || []).reduce((sum, a) => sum + (a.sent_bounced || 0), 0)}
+
Bounced
+
+
+
${(mb.mailbox_counts?.sent_rejected || 0) + (mb.aliases || []).reduce((sum, a) => sum + (a.sent_rejected || 0), 0)}
+
Rejected
+
+
+
+ + + ${mb.aliases && mb.aliases.length > 0 ? ` +
+

Aliases (${mb.aliases.length})

+
+ + + + + + + + + + + + + + + + ${(() => { + const hideZero = document.getElementById('mailbox-stats-hide-zero')?.checked ?? true; + const filteredAliases = hideZero + ? mb.aliases.filter(a => (a.sent_total || 0) + (a.received_total || 0) > 0) + : mb.aliases; + return filteredAliases.map(alias => ` + + + + + + + + + + + + `).join(''); + })()} + +
AliasSentReceivedInternalDeliveredDeferredBouncedRejectedFail %
+
+ ${escapeHtml(alias.alias_address)} + ${alias.is_catch_all ? 'catch-all' : ''} + ${!alias.active ? 'inactive' : ''} +
+
${alias.sent_total || 0}${alias.received_total || 0}${alias.direction_internal || 0}${alias.sent_delivered || 0}${alias.sent_deferred || 0}${alias.sent_bounced || 0}${alias.sent_rejected || 0}${alias.failure_rate || 0}%
+
+
+ ` : ''} +
+
+ `; + }).join(''); + + // Add pagination controls if there are multiple pages + if (totalPages > 1) { + html += ` +
+ + + + Page ${page} of ${totalPages} + + + +
+ `; + } + + container.innerHTML = html; +} + +function toggleMailboxAccordion(username) { + const mailboxes = mailboxStatsCache.mailboxes || []; + const index = mailboxes.findIndex(m => m.username === username); + if (index === -1) return; + + const content = document.getElementById(`accordion-content-${index}`); + const icon = document.getElementById(`accordion-icon-${index}`); + + if (content) { + const isHidden = content.classList.contains('hidden'); + content.classList.toggle('hidden'); + + if (isHidden) { + mailboxStatsCache.expandedMailboxes.add(username); + } else { + mailboxStatsCache.expandedMailboxes.delete(username); + } + } + + if (icon) { + icon.classList.toggle('rotate-90'); + } +} + +// ============================================================================= +// DATE RANGE PICKER +// ============================================================================= + +// Date range picker state +let dateRangePickerOpen = false; + +function toggleDateRangePicker() { + const dropdown = document.getElementById('date-range-dropdown'); + const arrow = document.getElementById('date-range-arrow'); + + if (!dropdown) return; + + dateRangePickerOpen = !dateRangePickerOpen; + + if (dateRangePickerOpen) { + dropdown.classList.remove('hidden'); + arrow?.classList.add('rotate-180'); + + // Set default dates for custom range inputs + const today = new Date(); + const thirtyDaysAgo = new Date(today); + thirtyDaysAgo.setDate(today.getDate() - 30); + + const startInput = document.getElementById('date-range-start'); + const endInput = document.getElementById('date-range-end'); + + if (startInput && !startInput.value) { + startInput.value = thirtyDaysAgo.toISOString().split('T')[0]; + } + if (endInput && !endInput.value) { + endInput.value = today.toISOString().split('T')[0]; + } + + // Add click outside listener + setTimeout(() => { + document.addEventListener('click', closeDateRangePickerOnClickOutside); + }, 0); + } else { + closeDateRangePicker(); + } +} + +function closeDateRangePicker() { + const dropdown = document.getElementById('date-range-dropdown'); + const arrow = document.getElementById('date-range-arrow'); + + if (dropdown) dropdown.classList.add('hidden'); + if (arrow) arrow.classList.remove('rotate-180'); + dateRangePickerOpen = false; + + document.removeEventListener('click', closeDateRangePickerOnClickOutside); +} + +function closeDateRangePickerOnClickOutside(e) { + const container = document.getElementById('date-range-picker-container'); + if (container && !container.contains(e.target)) { + closeDateRangePicker(); + } +} + +function selectDatePreset(preset) { + // Update hidden input + const hiddenInput = document.getElementById('mailbox-stats-date-range'); + if (hiddenInput) hiddenInput.value = preset; + + // Clear custom date inputs + document.getElementById('mailbox-stats-start-date').value = ''; + document.getElementById('mailbox-stats-end-date').value = ''; + + // Update label + const labelMap = { + 'today': 'Today', + '7days': 'Last 7 Days', + '30days': 'Last 30 Days', + '90days': 'Last 90 Days' + }; + const label = document.getElementById('date-range-label'); + if (label) label.textContent = labelMap[preset] || preset; + + // Update active state on buttons + updateDatePresetButtons(preset); + + // Close dropdown and reload data + closeDateRangePicker(); + loadMailboxStats(); +} + +function updateDatePresetButtons(activePreset) { + const buttons = document.querySelectorAll('.date-preset-btn'); + buttons.forEach(btn => { + const preset = btn.getAttribute('data-preset'); + if (preset === activePreset) { + btn.className = 'date-preset-btn px-3 py-1.5 text-xs font-medium rounded-md border border-blue-500 bg-blue-500 text-white transition-colors'; + } else { + btn.className = 'date-preset-btn px-3 py-1.5 text-xs font-medium rounded-md border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-100 dark:hover:bg-gray-700 transition-colors'; + } + }); +} + +function applyCustomDateRange() { + const startInput = document.getElementById('date-range-start'); + const endInput = document.getElementById('date-range-end'); + + if (!startInput?.value || !endInput?.value) { + showToast('Please select both start and end dates', 'error'); + return; + } + + const startDate = new Date(startInput.value); + const endDate = new Date(endInput.value); + + if (startDate > endDate) { + showToast('Start date must be before end date', 'error'); + return; + } + + // Set to custom mode + const hiddenInput = document.getElementById('mailbox-stats-date-range'); + if (hiddenInput) hiddenInput.value = 'custom'; + + // Store custom dates + document.getElementById('mailbox-stats-start-date').value = startInput.value; + document.getElementById('mailbox-stats-end-date').value = endInput.value; + + // Update label with date range + const label = document.getElementById('date-range-label'); + if (label) { + const startFormatted = formatDateShort(startInput.value); + const endFormatted = formatDateShort(endInput.value); + label.textContent = `${startFormatted} - ${endFormatted}`; + } + + // Clear active state on preset buttons (none active for custom) + updateDatePresetButtons('custom'); + + // Close dropdown and reload data + closeDateRangePicker(); + loadMailboxStats(); +} + +function formatDateShort(dateStr) { + const date = new Date(dateStr); + const month = (date.getMonth() + 1).toString().padStart(2, '0'); + const day = date.getDate().toString().padStart(2, '0'); + return `${day}/${month}`; +} + +function applyMailboxStatsFilters() { + loadMailboxStatsList(1); // Reset to page 1 when filters change +} + +function resetMailboxStatsFilters() { + // Reset search + const searchEl = document.getElementById('mailbox-stats-search'); + if (searchEl) searchEl.value = ''; + + // Reset date range to 30 days + const dateRangeEl = document.getElementById('mailbox-stats-date-range'); + if (dateRangeEl) dateRangeEl.value = '30days'; + + // Reset custom date inputs + const startDateEl = document.getElementById('mailbox-stats-start-date'); + if (startDateEl) startDateEl.value = ''; + const endDateEl = document.getElementById('mailbox-stats-end-date'); + if (endDateEl) endDateEl.value = ''; + + // Reset date range label + const labelEl = document.getElementById('date-range-label'); + if (labelEl) labelEl.textContent = 'Last 30 Days'; + + // Update preset buttons + updateDatePresetButtons('30days'); + + // Reset the date picker inputs as well + const startInput = document.getElementById('date-range-start'); + const endInput = document.getElementById('date-range-end'); + if (startInput) startInput.value = ''; + if (endInput) endInput.value = ''; + + // Reset domain filter + const domainEl = document.getElementById('mailbox-stats-domain-filter'); + if (domainEl) domainEl.value = ''; + + // Reset sort + const sortEl = document.getElementById('mailbox-stats-sort'); + if (sortEl) sortEl.value = 'sent_total-desc'; + + // Set active only to checked (default) + const activeOnlyEl = document.getElementById('mailbox-stats-active-only'); + if (activeOnlyEl) activeOnlyEl.checked = true; + + // Set hide zero to unchecked (default) + const hideZeroEl = document.getElementById('mailbox-stats-hide-zero'); + if (hideZeroEl) hideZeroEl.checked = true; + + // Reload everything + loadMailboxStats(); +} + +function loadMailboxStatsPage(page) { + loadMailboxStatsList(page); +} + +// Helper function to escape HTML +function escapeHtml(text) { + if (!text) return ''; + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} \ No newline at end of file diff --git a/frontend/index.html b/frontend/index.html index ffa4a23..49954b7 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,17 +1,20 @@ + Mailcow Logs Viewer - + - - + + - + -