"""Application observability: metrics and request correlation. This module owns Prometheus metrics and request-id generation so that the rest of the backend can stay focused on business logic. """ from __future__ import annotations import uuid from contextvars import ContextVar from typing import Any from fastapi import Request, Response from prometheus_client import ( CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest, ) # Context-local request id for code paths that cannot receive a Request object. _current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None) # Request metrics REQUESTS_TOTAL = Counter( "manage_api_requests_total", "Total API requests", ["method", "path", "status_code"], ) REQUEST_DURATION = Histogram( "manage_api_request_duration_seconds", "API request duration", ["method", "path"], buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0), ) # Operation metrics SSH_COMMANDS_TOTAL = Counter( "manage_ssh_commands_total", "Total SSH/local commands executed", ["machine_id", "action", "status"], ) SSH_COMMAND_DURATION = Histogram( "manage_ssh_command_duration_seconds", "SSH/local command duration", ["machine_id", "action"], buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0), ) MEDIA_INDEX_BUILDS_TOTAL = Counter( "manage_media_index_builds_total", "Total media index build attempts", ["status"], ) MEDIA_INDEX_BUILD_DURATION = Histogram( "manage_media_index_build_duration_seconds", "Media index build duration", buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0), ) BACKUP_RUNS_TOTAL = Counter( "manage_backup_runs_total", "Total backup runs", ["job_name", "status"], ) BACKUP_RUNS_LAST_SUCCESS = Gauge( "manage_backup_runs_last_success_timestamp", "Unix timestamp of the last successful backup run per job", ["job_name"], ) MAIL_QUEUE_SIZE = Counter( "manage_mail_queue_messages_total", "Total messages enqueued", ["status"], ) def set_current_request_id(request_id: str | None) -> None: """Set the context-local request id.""" _current_request_id.set(request_id) def get_current_request_id() -> str | None: """Return the current context-local request id or None.""" return _current_request_id.get() def generate_request_id() -> str: """Return a short unique request id.""" return uuid.uuid4().hex[:16] def get_request_id(request: Request | None = None) -> str: """Resolve a request id from the request header, context, or a new value.""" if request is not None: header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id") if header: return header.strip() existing = _current_request_id.get() if existing: return existing new_id = generate_request_id() _current_request_id.set(new_id) return new_id def metrics_payload() -> tuple[bytes, str]: """Return the Prometheus metrics payload and content type.""" return generate_latest(), CONTENT_TYPE_LATEST def record_request(request: Request, response: Response, duration_seconds: float) -> None: """Record Prometheus metrics for a completed request.""" status = str(response.status_code) path = request.url.path method = request.method REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc() REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds) def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None: """Record metrics for an SSH/local command.""" SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc() SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds) def record_media_index_build(status: str, duration_seconds: float | None = None) -> None: """Record metrics for a media index build.""" MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc() if duration_seconds is not None: MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds) def record_backup_run(job_name: str, status: str, success: bool = False) -> None: """Record metrics for a backup run.""" job_name = job_name or "unknown" BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc() if success: BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time() def record_mail_queue(status: str) -> None: """Record metrics for a mail queue message outcome.""" MAIL_QUEUE_SIZE.labels(status=status).inc() def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]: """Build a standard extra dict for structured logging.""" extra: dict[str, Any] = {"request_id": get_request_id(request)} extra.update(kwargs) return extra