feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

backend/src/media_library_viewer_api/main.py

@@ -7,13 +7,20 @@ import time
 from contextlib import asynccontextmanager
 
 import uvicorn
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import Response as FastAPIResponse
 
 from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
 from media_library_viewer_api.config import get_settings
-from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
-from media_library_viewer_api.logging_utils import configure_logging, describe_settings
+from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller, get_settings_store
+from media_library_viewer_api.logging_utils import configure_logging, describe_settings, sanitize_log_extra
+from media_library_viewer_api.observability import (
+    get_request_id,
+    metrics_payload,
+    record_request,
+    set_current_request_id,
+)
 from media_library_viewer_api.routers import backups as backups_router
 from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
 from media_library_viewer_api.routers.settings import router as settings_router
@@ -28,10 +35,16 @@ logger = logging.getLogger(__name__)
 async def lifespan(app: FastAPI):
     """Application lifespan — startup/shutdown."""
     settings = get_settings()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, settings.log_format)
     validate_auth_settings(settings)
     logger.info("Backend startup complete: %s", describe_settings(settings))
     logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
+    try:
+        from media_library_viewer_api.services.targets import write_prometheus_targets
+
+        write_prometheus_targets(get_settings_store())
+    except Exception:
+        logger.exception("Failed to write Prometheus file-SD targets during startup")
     mail_queue = get_mail_queue()
     monitoring_poller = get_monitoring_poller()
     backup_poller = get_backup_poller()
@@ -49,8 +62,7 @@ app = FastAPI(
     title="Manage API",
     version=get_backend_version(),
     description=(
-        "Manage API for Jellyfin media browsing, SSH file inspection, "
-        "server monitoring, and JWT-protected access."
+        "Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access."
     ),
     lifespan=lifespan,
 )
@@ -69,30 +81,51 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
 @app.middleware("http")
 async def enforce_jwt_auth(request: Request, call_next):
-    if request.url.path in {"/api/health", "/api/version"}:
+    if request.url.path in {"/api/health", "/api/version", "/metrics"}:
         return await call_next(request)
     return await require_jwt_auth(request, call_next)
 
+
 @app.middleware("http")
 async def log_requests(request: Request, call_next):
-    """Log every API request with timing and outcome."""
+    """Log every API request with timing, outcome, and request id."""
     start = time.perf_counter()
+    request_id = get_request_id(request)
+    set_current_request_id(request_id)
+    request.state.request_id = request_id
     client_host = request.client.host if request.client else "unknown"
-    logger.info("request start %s %s client=%s", request.method, request.url.path, client_host)
+    extra = sanitize_log_extra({"request_id": request_id, "client_host": client_host})
+    logger.info("request start %s %s", request.method, request.url.path, extra=extra)
     try:
         response = await call_next(request)
     except Exception:
-        logger.exception("request error %s %s client=%s", request.method, request.url.path, client_host)
+        logger.exception(
+            "request error %s %s",
+            request.method,
+            request.url.path,
+            extra=sanitize_log_extra({"request_id": request_id, "client_host": client_host}),
+        )
         raise
     elapsed_ms = (time.perf_counter() - start) * 1000.0
+    response.headers["X-Request-Id"] = request_id
+    record_request(request, response, elapsed_ms / 1000.0)
     logger.info(
         "request end %s %s status=%s elapsed_ms=%.1f",
         request.method,
         request.url.path,
         response.status_code,
         elapsed_ms,
+        extra=sanitize_log_extra(
+            {
+                "request_id": request_id,
+                "client_host": client_host,
+                "status_code": response.status_code,
+                "elapsed_ms": round(elapsed_ms, 1),
+            }
+        ),
     )
     return response
 
@@ -123,5 +156,12 @@ def version_info() -> dict[str, str]:
     return get_version_info()
 
 
+@app.get("/metrics")
+def metrics() -> Response:
+    """Expose Prometheus metrics."""
+    data, content_type = metrics_payload()
+    return FastAPIResponse(content=data, media_type=content_type)
+
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=8000)