feat(observability): persist standalone stack data and expose ports via env

- Switch docker-compose.observability.yml from named volumes to host
  bind mounts under OBSERVABILITY_DATA_ROOT, defaulting to
  ./observability-data.
- Make all service ports configurable via environment variables
  (PROMETHEUS_PORT, LOKI_PORT, ALLOY_PORT, GRAFANA_PORT,
  ALERTMANAGER_PORT, NODE_EXPORTER_PORT).
- Add VITE_GRAFANA_URL handling to ObservabilityPage so Grafana links
  point to the configured standalone instance.
- Update docs/observability-runbooks.md with the env variable table,
  reachable-web-UI table, and backup/restore instructions for the new
  host-directory layout.

docker-compose.observability.yml

@@ -4,6 +4,8 @@
 # Alloy and Node Exporter without the rest of Manage. It exposes the usual
 # ports directly; no Traefik or OIDC integration is required.
 #
+#   cp .env.example .env
+#   # optionally edit OBSERVABILITY_* paths and ports
 #   docker compose -f docker-compose.observability.yml up -d
 #
 # Then open Grafana at http://localhost:3000 (admin / admin by default).
@@ -27,9 +29,9 @@ services:
       - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro
       - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
       - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro
-      - prometheus_data:/prometheus
+      - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/prometheus:/prometheus
     ports:
-      - "9090:9090"
+      - "${PROMETHEUS_PORT:-9090}:9090"
     networks:
       - monitoring
     restart: unless-stopped
@@ -54,9 +56,9 @@ services:
     command: -config.file=/etc/loki/loki.yml
     volumes:
       - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
-      - loki_data:/loki
+      - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/loki:/loki
     ports:
-      - "3100:3100"
+      - "${LOKI_PORT:-3100}:3100"
     networks:
       - monitoring
     restart: unless-stopped
@@ -86,6 +88,9 @@ services:
       - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alloy:/var/lib/alloy
+    ports:
+      - "${ALLOY_PORT:-12345}:12345"
     networks:
       - monitoring
     depends_on:
@@ -123,9 +128,9 @@ services:
     volumes:
       - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
       - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
-      - grafana_data:/var/lib/grafana
+      - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/grafana:/var/lib/grafana
     ports:
-      - "3000:3000"
+      - "${GRAFANA_PORT:-3000}:3000"
     networks:
       - monitoring
     depends_on:
@@ -164,9 +169,9 @@ services:
       ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
     volumes:
       - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - alertmanager_data:/alertmanager
+      - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alertmanager:/alertmanager
     ports:
-      - "9093:9093"
+      - "${ALERTMANAGER_PORT:-9093}:9093"
     networks:
       - monitoring
     restart: unless-stopped
@@ -193,7 +198,7 @@ services:
     volumes:
       - /:/host:ro,rslave
     ports:
-      - "9100:9100"
+      - "${NODE_EXPORTER_PORT:-9100}:9100"
     networks:
       - monitoring
     restart: unless-stopped
@@ -214,9 +219,3 @@ services:
 
 networks:
   monitoring:
-
-volumes:
-  prometheus_data:
-  loki_data:
-  grafana_data:
-  alertmanager_data:

docs/observability-runbooks.md

@@ -141,72 +141,102 @@ docker compose exec grafana du -sh /var/lib/grafana
 
 ## Backup and Disaster Recovery
 
-The observability data lives in named volumes:
+The observability data lives on the host under `OBSERVABILITY_DATA_ROOT` (`./observability-data` by default). Subdirectories are created for each service:
 
-- `prometheus_data`
+- `prometheus`
-- `loki_data`
+- `loki`
-- `grafana_data`
+- `grafana`
-- `alertmanager_data`
+- `alertmanager`
+- `alloy`
 
-### Backup volumes
+### Backup data
 
 ```bash
 # Stop the stack to ensure consistency
-docker compose down
+docker compose -f docker-compose.observability.yml down
 
-# Back up each volume to a tarball
+# Back up the whole data directory
-docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
+rsync -aP --delete "$OBSERVABILITY_DATA_ROOT" /mnt/backups/observability-data/
-  tar czf /backups/prometheus-$(date +%F).tar.gz -C /data .
-docker run --rm -v manage_loki_data:/data -v $(pwd)/backups:/backups alpine \
-  tar czf /backups/loki-$(date +%F).tar.gz -C /data .
-docker run --rm -v manage_grafana_data:/data -v $(pwd)/backups:/backups alpine \
-  tar czf /backups/grafana-$(date +%F).tar.gz -C /data .
-docker run --rm -v manage_alertmanager_data:/data -v $(pwd)/backups:/backups alpine \
-  tar czf /backups/alertmanager-$(date +%F).tar.gz -C /data .
 
 # Start the stack again
-docker compose up -d
+docker compose -f docker-compose.observability.yml up -d
 ```
 
-> Replace `manage_` with your actual Docker Compose project name if different.
+### Restore data
-
-### Restore a volume
 
 ```bash
-docker compose down
+docker compose -f docker-compose.observability.yml down
-docker volume rm manage_prometheus_data
+rm -rf "$OBSERVABILITY_DATA_ROOT"
-docker volume create manage_prometheus_data
+rsync -aP /mnt/backups/observability-data/ "$OBSERVABILITY_DATA_ROOT"
-docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
+docker compose -f docker-compose.observability.yml up -d
-  tar xzf /backups/prometheus-YYYY-MM-DD.tar.gz -C /data
-docker compose up -d
 ```
 
 ---
 
 ## Standalone Observability Stack
 
-You can run the observability services on their own without the Manage backend or frontend:
+Run the observability services without the Manage backend or frontend:
 
 ```bash
 cd /path/to/manage
+cp .env.example .env
+# edit .env as needed
 docker compose -f docker-compose.observability.yml up -d
 ```
 
-This starts Prometheus, Grafana, Loki, Alertmanager, Alloy and Node Exporter. Exposed ports:
+### Reachable web UIs
 
-| Service | URL |
+Only three services expose a human-facing web interface:
-|---------|-----|
+
-| Grafana | <http://localhost:3000> |
+| Service | Has UI | Default URL | Notes |
-| Prometheus | <http://localhost:9090> |
+|---------|--------|-------------|-------|
-| Alertmanager | <http://localhost:9093> |
+| Grafana | yes | `http://localhost:3000` | Dashboards, log explore, alert management |
-| Loki | <http://localhost:3100> |
+| Prometheus | yes | `http://localhost:9090` | Query, targets, alerts, config status |
-| Node Exporter | <http://localhost:9100> |
+| Alertmanager | yes | `http://localhost:9093` | Alerts, silences, routing status |
-| Alloy | <http://localhost:12345> |
+| Loki | no | `http://localhost:3100` | Log API only; browse logs through Grafana |
+| Alloy | partial | `http://localhost:12345` | Agent debug UI for pipeline inspection |
+| Node Exporter | no | `http://localhost:9100` | Metrics endpoint only (`/metrics`) |
 
 Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically.
 
+### Environment variables
+
+| Variable | Default | Purpose |
+|----------|---------|---------|
+| `OBSERVABILITY_DATA_ROOT` | `./observability-data` | Host directory where all service data is stored persistently. Each service gets a subdirectory inside it. |
+| `PROMETHEUS_PORT` | `9090` | Host port for Prometheus web UI and API. |
+| `LOKI_PORT` | `3100` | Host port for Loki API. |
+| `ALLOY_PORT` | `12345` | Host port for Alloy debug UI. |
+| `GRAFANA_PORT` | `3000` | Host port for Grafana web UI. |
+| `ALERTMANAGER_PORT` | `9093` | Host port for Alertmanager web UI. |
+| `NODE_EXPORTER_PORT` | `9100` | Host port for Node Exporter metrics endpoint. |
+| `GRAFANA_ADMIN_USER` | `admin` | Grafana admin username. |
+| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin password. Change this in production. |
+| `GF_AUTH_GENERIC_OAUTH_CLIENT_ID` | empty | Generic OAuth client ID for Authentik or another provider. |
+| `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET` | empty | Generic OAuth client secret. |
+| `GF_AUTH_GENERIC_OAUTH_AUTH_URL` | empty | OAuth authorization endpoint. |
+| `GF_AUTH_GENERIC_OAUTH_TOKEN_URL` | empty | OAuth token endpoint. |
+| `GF_AUTH_GENERIC_OAUTH_API_URL` | empty | OAuth userinfo endpoint. |
+| `LOG_LEVEL` | `INFO` | Grafana log level. |
+| `SMTP_HOST` | `smtp.example.com` | SMTP host for Alertmanager email notifications. |
+| `SMTP_PORT` | `587` | SMTP port for Alertmanager. |
+| `SMTP_USERNAME` | empty | SMTP username. |
+| `SMTP_PASSWORD` | empty | SMTP password. |
+| `SMTP_FROM_ADDRESS` | `no-reply@example.com` | From address for alert emails. |
+| `ALERT_EMAIL_TO` | `admin@example.com` | Default recipient for alert emails. |
+
 To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`.
 
+### Backing up standalone data
+
+Because data is stored on the host under `OBSERVABILITY_DATA_ROOT`, you can back it up with normal filesystem tools:
+
+```bash
+rsync -aP --delete "$OBSERVABILITY_DATA_ROOT" /mnt/backups/observability-data/
+```
+
+Stop the stack first if you need a consistent snapshot.
+
 ## Scaling Notes
 
 - The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days.

frontend/src/components/ObservabilityPage.tsx

@@ -37,9 +37,14 @@ import {
 	CollapsibleContent,
 	CollapsibleTrigger,
 } from "@/components/ui/collapsible";
-import type { AlertmanagerAlert, MonitoringMachine, PrometheusTarget } from "../types";
+import type {
+	AlertmanagerAlert,
+	MonitoringMachine,
+	PrometheusTarget,
+} from "../types";
 
-const GRAFANA_BASE_URL = import.meta.env.VITE_GRAFANA_URL || "http://localhost:3000";
+const GRAFANA_BASE_URL =
+	import.meta.env.VITE_GRAFANA_URL || "http://localhost:3000";
 
 function severityVariant(
 	severity: string,
@@ -112,7 +117,9 @@ function EmptyState({
 		<div className="flex h-full min-h-[160px] flex-col items-center justify-center gap-2 rounded-md border p-6 text-center">
 			<Icon className="h-8 w-8 text-muted-foreground" />
 			<div className="font-medium">{title}</div>
-			<div className="max-w-md text-sm text-muted-foreground">{description}</div>
+			<div className="max-w-md text-sm text-muted-foreground">
+				{description}
+			</div>
 			{action ? <div className="mt-2">{action}</div> : null}
 		</div>
 	);
@@ -150,7 +157,9 @@ function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
 					<div className="flex items-start justify-between gap-2">
 						<div className="font-medium text-sm">{alert.name}</div>
 						<div className="flex items-center gap-1">
-							<Badge variant={severityVariant(alert.severity)}>{alert.severity}</Badge>
+							<Badge variant={severityVariant(alert.severity)}>
+								{alert.severity}
+							</Badge>
 							<ChevronDown className="h-4 w-4 text-muted-foreground transition-transform group-data-[state=open]:rotate-180" />
 						</div>
 					</div>
@@ -168,7 +177,8 @@ function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
 				<div className="space-y-2 rounded-b-lg border-x border-b p-3 text-sm">
 					{alert.description && (
 						<div>
-							<span className="font-medium">Description:</span> {alert.description}
+							<span className="font-medium">Description:</span>{" "}
+							{alert.description}
 						</div>
 					)}
 					<div className="grid grid-cols-2 gap-2 text-xs">
@@ -245,7 +255,12 @@ function GrafanaLinkCard({
 					<div className="text-sm text-muted-foreground">{description}</div>
 				</div>
 				<Button variant="outline" size="sm" asChild>
-					<a href={href} target="_blank" rel="noopener noreferrer" className="gap-1">
+					<a
+						href={href}
+						target="_blank"
+						rel="noopener noreferrer"
+						className="gap-1"
+					>
 						Open in Grafana
 						<ExternalLink className="h-3 w-3" />
 					</a>
@@ -283,7 +298,8 @@ export function ObservabilityPage() {
 	const [selectedMachineId, setSelectedMachineId] = useState<string>("");
 
 	const selectedMachine = useMemo<MonitoringMachine | null>(
-		() => machines.find((m) => m.id === selectedMachineId) ?? machines[0] ?? null,
+		() =>
+			machines.find((m) => m.id === selectedMachineId) ?? machines[0] ?? null,
 		[machines, selectedMachineId],
 	);
 
@@ -295,7 +311,8 @@ export function ObservabilityPage() {
 
 	const logsUrl = useMemo(() => {
 		if (!selectedMachine) return "";
-		const container = selectedMachine.mode === "local" ? "backend" : selectedMachine.name;
+		const container =
+			selectedMachine.mode === "local" ? "backend" : selectedMachine.name;
 		return `${GRAFANA_BASE_URL}/explore?orgId=1&left=${encodeURIComponent(
 			JSON.stringify({
 				datasource: "Loki",
@@ -352,7 +369,13 @@ export function ObservabilityPage() {
 				<HealthCard
 					title="Alertmanager"
 					status={
-						statusError ? "error" : alertmanagerStatus?.up ? "ok" : statusLoading ? "unknown" : "error"
+						statusError
+							? "error"
+							: alertmanagerStatus?.up
+								? "ok"
+								: statusLoading
+									? "unknown"
+									: "error"
 					}
 					detail={alertmanagerStatusDetail}
 					icon={Bell}
@@ -383,16 +406,32 @@ export function ObservabilityPage() {
 
 			<div className="space-y-3">
 				{statusError && (
-					<QueryError label="Alertmanager status" error={statusError} refetch={refetchStatus} />
+					<QueryError
+						label="Alertmanager status"
+						error={statusError}
+						refetch={refetchStatus}
+					/>
 				)}
 				{alertsError && (
-					<QueryError label="Active alerts" error={alertsError} refetch={refetchAlerts} />
+					<QueryError
+						label="Active alerts"
+						error={alertsError}
+						refetch={refetchAlerts}
+					/>
 				)}
 				{targetsError && (
-					<QueryError label="Prometheus targets" error={targetsError} refetch={refetchTargets} />
+					<QueryError
+						label="Prometheus targets"
+						error={targetsError}
+						refetch={refetchTargets}
+					/>
 				)}
 				{machinesError && (
-					<QueryError label="Monitoring machines" error={machinesError} refetch={refetchMachines} />
+					<QueryError
+						label="Monitoring machines"
+						error={machinesError}
+						refetch={refetchMachines}
+					/>
 				)}
 			</div>
 
@@ -400,7 +439,8 @@ export function ObservabilityPage() {
 				<Alert variant="destructive">
 					<AlertTitle>Alertmanager unreachable</AlertTitle>
 					<AlertDescription>
-						The UI cannot reach Alertmanager right now. Alerts shown here may be stale.
+						The UI cannot reach Alertmanager right now. Alerts shown here may be
+						stale.
 					</AlertDescription>
 				</Alert>
 			)}
@@ -434,8 +474,12 @@ export function ObservabilityPage() {
 									))}
 									{alertsSummary.total > alertsSummary.alerts.length && (
 										<div className="text-center text-xs text-muted-foreground">
-											{alertsSummary.total - alertsSummary.alerts.length} more alert
+											{alertsSummary.total - alertsSummary.alerts.length} more
-											{alertsSummary.total - alertsSummary.alerts.length === 1 ? "" : "s"} in Alertmanager
+											alert
+											{alertsSummary.total - alertsSummary.alerts.length === 1
+												? ""
+												: "s"}{" "}
+											in Alertmanager
 										</div>
 									)}
 								</>