feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

backend/src/media_library_viewer_api/observability.py

@@ -0,0 +1,154 @@
+"""Application observability: metrics and request correlation.
+
+This module owns Prometheus metrics and request-id generation so that the
+rest of the backend can stay focused on business logic.
+"""
+
+from __future__ import annotations
+
+import uuid
+from contextvars import ContextVar
+from typing import Any
+
+from fastapi import Request, Response
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+
+# Context-local request id for code paths that cannot receive a Request object.
+_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+# Request metrics
+REQUESTS_TOTAL = Counter(
+    "manage_api_requests_total",
+    "Total API requests",
+    ["method", "path", "status_code"],
+)
+REQUEST_DURATION = Histogram(
+    "manage_api_request_duration_seconds",
+    "API request duration",
+    ["method", "path"],
+    buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
+)
+
+# Operation metrics
+SSH_COMMANDS_TOTAL = Counter(
+    "manage_ssh_commands_total",
+    "Total SSH/local commands executed",
+    ["machine_id", "action", "status"],
+)
+SSH_COMMAND_DURATION = Histogram(
+    "manage_ssh_command_duration_seconds",
+    "SSH/local command duration",
+    ["machine_id", "action"],
+    buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0),
+)
+
+MEDIA_INDEX_BUILDS_TOTAL = Counter(
+    "manage_media_index_builds_total",
+    "Total media index build attempts",
+    ["status"],
+)
+MEDIA_INDEX_BUILD_DURATION = Histogram(
+    "manage_media_index_build_duration_seconds",
+    "Media index build duration",
+    buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0),
+)
+
+BACKUP_RUNS_TOTAL = Counter(
+    "manage_backup_runs_total",
+    "Total backup runs",
+    ["job_name", "status"],
+)
+
+BACKUP_RUNS_LAST_SUCCESS = Gauge(
+    "manage_backup_runs_last_success_timestamp",
+    "Unix timestamp of the last successful backup run per job",
+    ["job_name"],
+)
+
+MAIL_QUEUE_SIZE = Counter(
+    "manage_mail_queue_messages_total",
+    "Total messages enqueued",
+    ["status"],
+)
+
+
+def set_current_request_id(request_id: str | None) -> None:
+    """Set the context-local request id."""
+    _current_request_id.set(request_id)
+
+
+def get_current_request_id() -> str | None:
+    """Return the current context-local request id or None."""
+    return _current_request_id.get()
+
+
+def generate_request_id() -> str:
+    """Return a short unique request id."""
+    return uuid.uuid4().hex[:16]
+
+
+def get_request_id(request: Request | None = None) -> str:
+    """Resolve a request id from the request header, context, or a new value."""
+    if request is not None:
+        header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id")
+        if header:
+            return header.strip()
+    existing = _current_request_id.get()
+    if existing:
+        return existing
+    new_id = generate_request_id()
+    _current_request_id.set(new_id)
+    return new_id
+
+
+def metrics_payload() -> tuple[bytes, str]:
+    """Return the Prometheus metrics payload and content type."""
+    return generate_latest(), CONTENT_TYPE_LATEST
+
+
+def record_request(request: Request, response: Response, duration_seconds: float) -> None:
+    """Record Prometheus metrics for a completed request."""
+    status = str(response.status_code)
+    path = request.url.path
+    method = request.method
+    REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc()
+    REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds)
+
+
+def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None:
+    """Record metrics for an SSH/local command."""
+    SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc()
+    SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds)
+
+
+def record_media_index_build(status: str, duration_seconds: float | None = None) -> None:
+    """Record metrics for a media index build."""
+    MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc()
+    if duration_seconds is not None:
+        MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds)
+
+
+def record_backup_run(job_name: str, status: str, success: bool = False) -> None:
+    """Record metrics for a backup run."""
+    job_name = job_name or "unknown"
+    BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc()
+    if success:
+        BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time()
+
+
+def record_mail_queue(status: str) -> None:
+    """Record metrics for a mail queue message outcome."""
+    MAIL_QUEUE_SIZE.labels(status=status).inc()
+
+
+def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]:
+    """Build a standard extra dict for structured logging."""
+    extra: dict[str, Any] = {"request_id": get_request_id(request)}
+    extra.update(kwargs)
+    return extra