Developer
155 lines
4.8 KiB
Python
155 lines
4.8 KiB
Python
"""Application observability: metrics and request correlation.
|
|
|
|
This module owns Prometheus metrics and request-id generation so that the
|
|
rest of the backend can stay focused on business logic.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from contextvars import ContextVar
|
|
from typing import Any
|
|
|
|
from fastapi import Request, Response
|
|
from prometheus_client import (
|
|
CONTENT_TYPE_LATEST,
|
|
Counter,
|
|
Gauge,
|
|
Histogram,
|
|
generate_latest,
|
|
)
|
|
|
|
# Context-local request id for code paths that cannot receive a Request object.
|
|
_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
|
|
|
|
# Request metrics
|
|
REQUESTS_TOTAL = Counter(
|
|
"manage_api_requests_total",
|
|
"Total API requests",
|
|
["method", "path", "status_code"],
|
|
)
|
|
REQUEST_DURATION = Histogram(
|
|
"manage_api_request_duration_seconds",
|
|
"API request duration",
|
|
["method", "path"],
|
|
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
|
|
)
|
|
|
|
# Operation metrics
|
|
SSH_COMMANDS_TOTAL = Counter(
|
|
"manage_ssh_commands_total",
|
|
"Total SSH/local commands executed",
|
|
["machine_id", "action", "status"],
|
|
)
|
|
SSH_COMMAND_DURATION = Histogram(
|
|
"manage_ssh_command_duration_seconds",
|
|
"SSH/local command duration",
|
|
["machine_id", "action"],
|
|
buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0),
|
|
)
|
|
|
|
MEDIA_INDEX_BUILDS_TOTAL = Counter(
|
|
"manage_media_index_builds_total",
|
|
"Total media index build attempts",
|
|
["status"],
|
|
)
|
|
MEDIA_INDEX_BUILD_DURATION = Histogram(
|
|
"manage_media_index_build_duration_seconds",
|
|
"Media index build duration",
|
|
buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0),
|
|
)
|
|
|
|
BACKUP_RUNS_TOTAL = Counter(
|
|
"manage_backup_runs_total",
|
|
"Total backup runs",
|
|
["job_name", "status"],
|
|
)
|
|
|
|
BACKUP_RUNS_LAST_SUCCESS = Gauge(
|
|
"manage_backup_runs_last_success_timestamp",
|
|
"Unix timestamp of the last successful backup run per job",
|
|
["job_name"],
|
|
)
|
|
|
|
MAIL_QUEUE_SIZE = Counter(
|
|
"manage_mail_queue_messages_total",
|
|
"Total messages enqueued",
|
|
["status"],
|
|
)
|
|
|
|
|
|
def set_current_request_id(request_id: str | None) -> None:
|
|
"""Set the context-local request id."""
|
|
_current_request_id.set(request_id)
|
|
|
|
|
|
def get_current_request_id() -> str | None:
|
|
"""Return the current context-local request id or None."""
|
|
return _current_request_id.get()
|
|
|
|
|
|
def generate_request_id() -> str:
|
|
"""Return a short unique request id."""
|
|
return uuid.uuid4().hex[:16]
|
|
|
|
|
|
def get_request_id(request: Request | None = None) -> str:
|
|
"""Resolve a request id from the request header, context, or a new value."""
|
|
if request is not None:
|
|
header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id")
|
|
if header:
|
|
return header.strip()
|
|
existing = _current_request_id.get()
|
|
if existing:
|
|
return existing
|
|
new_id = generate_request_id()
|
|
_current_request_id.set(new_id)
|
|
return new_id
|
|
|
|
|
|
def metrics_payload() -> tuple[bytes, str]:
|
|
"""Return the Prometheus metrics payload and content type."""
|
|
return generate_latest(), CONTENT_TYPE_LATEST
|
|
|
|
|
|
def record_request(request: Request, response: Response, duration_seconds: float) -> None:
|
|
"""Record Prometheus metrics for a completed request."""
|
|
status = str(response.status_code)
|
|
path = request.url.path
|
|
method = request.method
|
|
REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc()
|
|
REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds)
|
|
|
|
|
|
def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None:
|
|
"""Record metrics for an SSH/local command."""
|
|
SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc()
|
|
SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds)
|
|
|
|
|
|
def record_media_index_build(status: str, duration_seconds: float | None = None) -> None:
|
|
"""Record metrics for a media index build."""
|
|
MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc()
|
|
if duration_seconds is not None:
|
|
MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds)
|
|
|
|
|
|
def record_backup_run(job_name: str, status: str, success: bool = False) -> None:
|
|
"""Record metrics for a backup run."""
|
|
job_name = job_name or "unknown"
|
|
BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc()
|
|
if success:
|
|
BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time()
|
|
|
|
|
|
def record_mail_queue(status: str) -> None:
|
|
"""Record metrics for a mail queue message outcome."""
|
|
MAIL_QUEUE_SIZE.labels(status=status).inc()
|
|
|
|
|
|
def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]:
|
|
"""Build a standard extra dict for structured logging."""
|
|
extra: dict[str, Any] = {"request_id": get_request_id(request)}
|
|
extra.update(kwargs)
|
|
return extra
|