diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index 0252342..1cf48f9 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -4,6 +4,8 @@ # Alloy and Node Exporter without the rest of Manage. It exposes the usual # ports directly; no Traefik or OIDC integration is required. # +# cp .env.example .env +# # optionally edit OBSERVABILITY_* paths and ports # docker compose -f docker-compose.observability.yml up -d # # Then open Grafana at http://localhost:3000 (admin / admin by default). @@ -27,9 +29,9 @@ services: - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro - - prometheus_data:/prometheus + - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/prometheus:/prometheus ports: - - "9090:9090" + - "${PROMETHEUS_PORT:-9090}:9090" networks: - monitoring restart: unless-stopped @@ -54,9 +56,9 @@ services: command: -config.file=/etc/loki/loki.yml volumes: - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro - - loki_data:/loki + - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/loki:/loki ports: - - "3100:3100" + - "${LOKI_PORT:-3100}:3100" networks: - monitoring restart: unless-stopped @@ -86,6 +88,9 @@ services: - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alloy:/var/lib/alloy + ports: + - "${ALLOY_PORT:-12345}:12345" networks: - monitoring depends_on: @@ -123,9 +128,9 @@ services: volumes: - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro - - grafana_data:/var/lib/grafana + - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/grafana:/var/lib/grafana ports: - - "3000:3000" + - "${GRAFANA_PORT:-3000}:3000" networks: - monitoring depends_on: @@ -164,9 +169,9 @@ services: ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com} volumes: - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro - - alertmanager_data:/alertmanager + - ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alertmanager:/alertmanager ports: - - "9093:9093" + - "${ALERTMANAGER_PORT:-9093}:9093" networks: - monitoring restart: unless-stopped @@ -193,7 +198,7 @@ services: volumes: - /:/host:ro,rslave ports: - - "9100:9100" + - "${NODE_EXPORTER_PORT:-9100}:9100" networks: - monitoring restart: unless-stopped @@ -214,9 +219,3 @@ services: networks: monitoring: - -volumes: - prometheus_data: - loki_data: - grafana_data: - alertmanager_data: diff --git a/docs/observability-runbooks.md b/docs/observability-runbooks.md index 1b288ef..d911b7c 100644 --- a/docs/observability-runbooks.md +++ b/docs/observability-runbooks.md @@ -141,72 +141,102 @@ docker compose exec grafana du -sh /var/lib/grafana ## Backup and Disaster Recovery -The observability data lives in named volumes: +The observability data lives on the host under `OBSERVABILITY_DATA_ROOT` (`./observability-data` by default). Subdirectories are created for each service: -- `prometheus_data` -- `loki_data` -- `grafana_data` -- `alertmanager_data` +- `prometheus` +- `loki` +- `grafana` +- `alertmanager` +- `alloy` -### Backup volumes +### Backup data ```bash # Stop the stack to ensure consistency -docker compose down +docker compose -f docker-compose.observability.yml down -# Back up each volume to a tarball -docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \ - tar czf /backups/prometheus-$(date +%F).tar.gz -C /data . -docker run --rm -v manage_loki_data:/data -v $(pwd)/backups:/backups alpine \ - tar czf /backups/loki-$(date +%F).tar.gz -C /data . -docker run --rm -v manage_grafana_data:/data -v $(pwd)/backups:/backups alpine \ - tar czf /backups/grafana-$(date +%F).tar.gz -C /data . -docker run --rm -v manage_alertmanager_data:/data -v $(pwd)/backups:/backups alpine \ - tar czf /backups/alertmanager-$(date +%F).tar.gz -C /data . +# Back up the whole data directory +rsync -aP --delete "$OBSERVABILITY_DATA_ROOT" /mnt/backups/observability-data/ # Start the stack again -docker compose up -d +docker compose -f docker-compose.observability.yml up -d ``` -> Replace `manage_` with your actual Docker Compose project name if different. - -### Restore a volume +### Restore data ```bash -docker compose down -docker volume rm manage_prometheus_data -docker volume create manage_prometheus_data -docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \ - tar xzf /backups/prometheus-YYYY-MM-DD.tar.gz -C /data -docker compose up -d +docker compose -f docker-compose.observability.yml down +rm -rf "$OBSERVABILITY_DATA_ROOT" +rsync -aP /mnt/backups/observability-data/ "$OBSERVABILITY_DATA_ROOT" +docker compose -f docker-compose.observability.yml up -d ``` --- ## Standalone Observability Stack -You can run the observability services on their own without the Manage backend or frontend: +Run the observability services without the Manage backend or frontend: ```bash cd /path/to/manage +cp .env.example .env +# edit .env as needed docker compose -f docker-compose.observability.yml up -d ``` -This starts Prometheus, Grafana, Loki, Alertmanager, Alloy and Node Exporter. Exposed ports: +### Reachable web UIs -| Service | URL | -|---------|-----| -| Grafana | | -| Prometheus | | -| Alertmanager | | -| Loki | | -| Node Exporter | | -| Alloy | | +Only three services expose a human-facing web interface: + +| Service | Has UI | Default URL | Notes | +|---------|--------|-------------|-------| +| Grafana | yes | `http://localhost:3000` | Dashboards, log explore, alert management | +| Prometheus | yes | `http://localhost:9090` | Query, targets, alerts, config status | +| Alertmanager | yes | `http://localhost:9093` | Alerts, silences, routing status | +| Loki | no | `http://localhost:3100` | Log API only; browse logs through Grafana | +| Alloy | partial | `http://localhost:12345` | Agent debug UI for pipeline inspection | +| Node Exporter | no | `http://localhost:9100` | Metrics endpoint only (`/metrics`) | Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically. +### Environment variables + +| Variable | Default | Purpose | +|----------|---------|---------| +| `OBSERVABILITY_DATA_ROOT` | `./observability-data` | Host directory where all service data is stored persistently. Each service gets a subdirectory inside it. | +| `PROMETHEUS_PORT` | `9090` | Host port for Prometheus web UI and API. | +| `LOKI_PORT` | `3100` | Host port for Loki API. | +| `ALLOY_PORT` | `12345` | Host port for Alloy debug UI. | +| `GRAFANA_PORT` | `3000` | Host port for Grafana web UI. | +| `ALERTMANAGER_PORT` | `9093` | Host port for Alertmanager web UI. | +| `NODE_EXPORTER_PORT` | `9100` | Host port for Node Exporter metrics endpoint. | +| `GRAFANA_ADMIN_USER` | `admin` | Grafana admin username. | +| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin password. Change this in production. | +| `GF_AUTH_GENERIC_OAUTH_CLIENT_ID` | empty | Generic OAuth client ID for Authentik or another provider. | +| `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET` | empty | Generic OAuth client secret. | +| `GF_AUTH_GENERIC_OAUTH_AUTH_URL` | empty | OAuth authorization endpoint. | +| `GF_AUTH_GENERIC_OAUTH_TOKEN_URL` | empty | OAuth token endpoint. | +| `GF_AUTH_GENERIC_OAUTH_API_URL` | empty | OAuth userinfo endpoint. | +| `LOG_LEVEL` | `INFO` | Grafana log level. | +| `SMTP_HOST` | `smtp.example.com` | SMTP host for Alertmanager email notifications. | +| `SMTP_PORT` | `587` | SMTP port for Alertmanager. | +| `SMTP_USERNAME` | empty | SMTP username. | +| `SMTP_PASSWORD` | empty | SMTP password. | +| `SMTP_FROM_ADDRESS` | `no-reply@example.com` | From address for alert emails. | +| `ALERT_EMAIL_TO` | `admin@example.com` | Default recipient for alert emails. | + To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`. +### Backing up standalone data + +Because data is stored on the host under `OBSERVABILITY_DATA_ROOT`, you can back it up with normal filesystem tools: + +```bash +rsync -aP --delete "$OBSERVABILITY_DATA_ROOT" /mnt/backups/observability-data/ +``` + +Stop the stack first if you need a consistent snapshot. + ## Scaling Notes - The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days. diff --git a/frontend/src/components/ObservabilityPage.tsx b/frontend/src/components/ObservabilityPage.tsx index 60e845a..3afebb7 100644 --- a/frontend/src/components/ObservabilityPage.tsx +++ b/frontend/src/components/ObservabilityPage.tsx @@ -37,9 +37,14 @@ import { CollapsibleContent, CollapsibleTrigger, } from "@/components/ui/collapsible"; -import type { AlertmanagerAlert, MonitoringMachine, PrometheusTarget } from "../types"; +import type { + AlertmanagerAlert, + MonitoringMachine, + PrometheusTarget, +} from "../types"; -const GRAFANA_BASE_URL = import.meta.env.VITE_GRAFANA_URL || "http://localhost:3000"; +const GRAFANA_BASE_URL = + import.meta.env.VITE_GRAFANA_URL || "http://localhost:3000"; function severityVariant( severity: string, @@ -112,7 +117,9 @@ function EmptyState({
{title}
-
{description}
+
+ {description} +
{action ?
{action}
: null}
); @@ -150,7 +157,9 @@ function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
{alert.name}
- {alert.severity} + + {alert.severity} +
@@ -168,7 +177,8 @@ function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
{alert.description && (
- Description: {alert.description} + Description:{" "} + {alert.description}
)}
@@ -245,7 +255,12 @@ function GrafanaLinkCard({
{description}
@@ -400,7 +439,8 @@ export function ObservabilityPage() { Alertmanager unreachable - The UI cannot reach Alertmanager right now. Alerts shown here may be stale. + The UI cannot reach Alertmanager right now. Alerts shown here may be + stale. )} @@ -434,8 +474,12 @@ export function ObservabilityPage() { ))} {alertsSummary.total > alertsSummary.alerts.length && (
- {alertsSummary.total - alertsSummary.alerts.length} more alert - {alertsSummary.total - alertsSummary.alerts.length === 1 ? "" : "s"} in Alertmanager + {alertsSummary.total - alertsSummary.alerts.length} more + alert + {alertsSummary.total - alertsSummary.alerts.length === 1 + ? "" + : "s"}{" "} + in Alertmanager
)}