feat(v2): complete v2 reimplementation

backend/src/backup_tool/observability/metrics.py

@@ -0,0 +1,102 @@
+"""Small dependency-free Prometheus exposition for the single-node appliance."""
+
+from __future__ import annotations
+
+import os
+import threading
+from collections import defaultdict
+from collections.abc import Iterable
+from datetime import UTC, datetime
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.config import Settings
+from backup_tool.db.models import Backup, Execution, Repository, Schedule
+from backup_tool.execution import ACTIVE_STATES
+
+
+class Metrics:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
+        self._durations: dict[tuple[str, str], tuple[int, float]] = {}
+
+    def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None:
+        route = path if path in {"/livez", "/readyz", "/metrics"} else "/api"
+        with self._lock:
+            self._requests[(method, route, status)] += 1
+            count, total = self._durations.get((method, route), (0, 0.0))
+            self._durations[(method, route)] = (count + 1, total + duration_seconds)
+
+    def render(self, operational: Iterable[tuple[str, float]]) -> str:
+        lines = [
+            "# HELP backup_tool_http_requests_total HTTP requests handled by the web role.",
+            "# TYPE backup_tool_http_requests_total counter",
+        ]
+        with self._lock:
+            for (method, path, status), value in sorted(self._requests.items()):
+                labels = f'method="{method}",path="{path}",status="{status}"'
+                lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}")
+            lines.extend(
+                [
+                    "# HELP backup_tool_http_request_duration_seconds HTTP request duration.",
+                    "# TYPE backup_tool_http_request_duration_seconds summary",
+                ]
+            )
+            for (method, path), (count, total) in sorted(self._durations.items()):
+                labels = f'method="{method}",path="{path}"'
+                lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}")
+                lines.append(
+                    f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}"
+                )
+        lines.extend(f"{name} {value}" for name, value in operational)
+        return "\n".join(lines) + "\n"
+
+
+async def collect_operational_metrics(
+    settings: Settings, db: AsyncSession
+) -> list[tuple[str, float]]:
+    now = datetime.now(UTC)
+    active = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES))
+    )
+    stale = await db.scalar(
+        select(func.count())
+        .select_from(Execution)
+        .where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now)
+    )
+    failed = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state == "failed")
+    )
+    corrupt = await db.scalar(
+        select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt")
+    )
+    schedule_lag = await db.scalar(
+        select(func.min(Schedule.next_nominal_at)).where(
+            Schedule.enabled, Schedule.next_nominal_at.is_not(None)
+        )
+    )
+    values = [
+        ("backup_tool_active_executions", active or 0),
+        ("backup_tool_stale_execution_leases", stale or 0),
+        ("backup_tool_failed_executions", failed or 0),
+        ("backup_tool_corrupt_backups", corrupt or 0),
+        (
+            "backup_tool_schedule_lag_seconds",
+            max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0,
+        ),
+    ]
+    roots = list(settings.repository_roots) + list(settings.restore_roots)
+    for index, root in enumerate(roots):
+        try:
+            stats = os.statvfs(root)
+        except OSError:
+            continue
+        name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}'
+        values.append((name, stats.f_bavail * stats.f_frsize))
+    unavailable = await db.scalar(
+        select(func.count()).select_from(Repository).where(Repository.state == "unavailable")
+    )
+    values.append(("backup_tool_unavailable_repositories", unavailable or 0))
+    return values