feat(observability): per-service root directories for config and data

- Replace OBSERVABILITY_DATA_ROOT with per-service *_ROOT variables in
  docker-compose.observability.yml.
- Each service root must contain config/ (mounted read-only) and data/
  (mounted read-write), grouping config and state together for Portainer.
- Update docs/observability-runbooks.md with the new variables, a setup
  script that copies repo configs into each service root, and updated
  backup/restore examples.

docker-compose.observability.yml

@@ -4,10 +4,19 @@
 # Alloy and Node Exporter without the rest of Manage.
 #
 # Required environment variables:
-#   - OBSERVABILITY_DATA_ROOT (absolute path, e.g. /var/lib/manage/observability)
+#   - *_ROOT directories (absolute paths) for services you want to persist
 #   - CERT_RESOLVER (Traefik cert resolver name)
 #   - GRAFANA_APP_HOST, PROMETHEUS_APP_HOST, ALERTMANAGER_APP_HOST
 #
+# Each service root should contain a config/ and data/ subdirectory, e.g.:
+#
+#   /var/lib/manage/observability/prometheus/
+#   ├── config/
+#   │   ├── prometheus.yml
+#   │   ├── rules/
+#   │   └── file-sd/
+#   └── data/
+#
 # Optional environment variables:
 #   - TRAEFIK_ENTRYPOINT (default: websecure)
 #   - *_APP_NAME router/service names (defaults below)
@@ -17,10 +26,6 @@
 #   docker compose -f docker-compose.observability.yml up -d
 #
 # Then open Grafana at the configured host (default admin / admin).
-#
-# To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml
-# and add a static target for it, or drop a file-SD JSON file into
-# monitoring/prometheus/file-sd/.
 
 services:
   prometheus:
@@ -34,10 +39,10 @@ services:
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--web.enable-lifecycle"
     volumes:
-      - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/rules:/etc/prometheus/rules:ro
-      - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/file-sd:/etc/prometheus/file-sd:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/prometheus:/prometheus
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/data:/prometheus
     ports:
       - "${PROMETHEUS_PORT:-9090}:9090"
     networks:
@@ -70,8 +75,8 @@ services:
     container_name: loki
     command: -config.file=/etc/loki/loki.yml
     volumes:
-      - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
+      - ${LOKI_ROOT:?set LOKI_ROOT}/config/loki.yml:/etc/loki/loki.yml:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/loki:/loki
+      - ${LOKI_ROOT:?set LOKI_ROOT}/data:/loki
     ports:
       - "${LOKI_PORT:-3100}:3100"
     networks:
@@ -100,10 +105,10 @@ services:
       - /etc/alloy/config.alloy
       - --storage.path=/var/lib/alloy
     volumes:
-      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - ${ALLOY_ROOT:?set ALLOY_ROOT}/config/config.alloy:/etc/alloy/config.alloy:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alloy:/var/lib/alloy
+      - ${ALLOY_ROOT:?set ALLOY_ROOT}/data:/var/lib/alloy
     ports:
       - "${ALLOY_PORT:-12345}:12345"
     networks:
@@ -141,9 +146,9 @@ services:
       GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
       LOG_LEVEL: ${LOG_LEVEL:-INFO}
     volumes:
-      - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/config/grafana.ini:/etc/grafana/grafana.ini:ro
-      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/config/provisioning:/etc/grafana/provisioning:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/grafana:/var/lib/grafana
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/data:/var/lib/grafana
     ports:
       - "${GRAFANA_PORT:-3000}:3000"
     networks:
@@ -190,8 +195,8 @@ services:
       SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
       ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
     volumes:
-      - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - ${ALERTMANAGER_ROOT:?set ALERTMANAGER_ROOT}/config/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alertmanager:/alertmanager
+      - ${ALERTMANAGER_ROOT:?set ALERTMANAGER_ROOT}/data:/alertmanager
     ports:
       - "${ALERTMANAGER_PORT:-9093}:9093"
     networks:

docs/observability-runbooks.md

@@ -181,17 +181,34 @@ Run the observability services without the Manage backend or frontend:
 cd /path/to/manage
 # create an env file with at least the required variables
 cat > .env.observability <<EOF
-OBSERVABILITY_DATA_ROOT=/var/lib/manage/observability
 CERT_RESOLVER=myresolver
+
+PROMETHEUS_ROOT=/var/lib/manage/observability/prometheus
+LOKI_ROOT=/var/lib/manage/observability/loki
+ALLOY_ROOT=/var/lib/manage/observability/alloy
+GRAFANA_ROOT=/var/lib/manage/observability/grafana
+ALERTMANAGER_ROOT=/var/lib/manage/observability/alertmanager
+
 GRAFANA_APP_HOST=grafana.example.com
 PROMETHEUS_APP_HOST=prometheus.example.com
 ALERTMANAGER_APP_HOST=alertmanager.example.com
 EOF
 
+# prepare config directories from the repository defaults
+mkdir -p "$PROMETHEUS_ROOT"/{config,data} "$LOKI_ROOT"/{config,data} "$ALLOY_ROOT"/{config,data} "$GRAFANA_ROOT"/{config,data} "$ALERTMANAGER_ROOT"/{config,data}
+cp monitoring/prometheus/prometheus.standalone.yml "$PROMETHEUS_ROOT/config/prometheus.yml"
+cp -r monitoring/prometheus/rules "$PROMETHEUS_ROOT/config/rules"
+cp -r monitoring/prometheus/file-sd "$PROMETHEUS_ROOT/config/file-sd"
+cp monitoring/loki/loki.yml "$LOKI_ROOT/config/loki.yml"
+cp monitoring/alloy/config.alloy "$ALLOY_ROOT/config/config.alloy"
+cp monitoring/grafana/grafana.ini "$GRAFANA_ROOT/config/grafana.ini"
+cp -r monitoring/grafana/provisioning "$GRAFANA_ROOT/config/provisioning"
+cp monitoring/alertmanager/alertmanager.yml "$ALERTMANAGER_ROOT/config/alertmanager.yml"
+
 docker compose -f docker-compose.observability.yml --env-file .env.observability up -d
 ```
 
-If you do not use Traefik, set `CERT_RESOLVER` to any non-empty value and do not attach the services to a `web` network. The direct host ports still work without Traefik.
+Each service root must contain `config/` (read-only config files) and `data/` (runtime state). If you do not use Traefik, set `CERT_RESOLVER` to any non-empty value and do not attach the services to a `web` network. The direct host ports still work without Traefik.
 
 ### Reachable web UIs
 
@@ -212,7 +229,11 @@ Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisione
 
 | Variable | Default | Purpose |
 |----------|---------|---------|
-| `OBSERVABILITY_DATA_ROOT` | required | Absolute host directory where all service data is stored. Each service gets a subdirectory inside it. Must be set before deploy. |
+| `PROMETHEUS_ROOT` | required | Absolute host directory for Prometheus config and data. Must contain `config/` and `data/`. |
+| `LOKI_ROOT` | required | Absolute host directory for Loki config and data. Must contain `config/` and `data/`. |
+| `ALLOY_ROOT` | required | Absolute host directory for Alloy config and data. Must contain `config/` and `data/`. |
+| `GRAFANA_ROOT` | required | Absolute host directory for Grafana config and data. Must contain `config/` and `data/`. |
+| `ALERTMANAGER_ROOT` | required | Absolute host directory for Alertmanager config and data. Must contain `config/` and `data/`. |
 | `CERT_RESOLVER` | required | Traefik certificate resolver name (for example `letsencrypt` or `cloudflare`). Must be set before deploy. |
 | `TRAEFIK_ENTRYPOINT` | `websecure` | Traefik entrypoint to use for the web UIs. |
 | `GRAFANA_APP_HOST` | required | Public hostname for Grafana (for example `grafana.example.com`). |
@@ -251,10 +272,12 @@ To scrape a Manage backend from this standalone stack, edit `monitoring/promethe
 
 ### Backing up standalone data
 
-Because data is stored on the host under `OBSERVABILITY_DATA_ROOT`, you can back it up with normal filesystem tools:
+Because config and data are stored on the host under each service root, you can back them up with normal filesystem tools:
 
 ```bash
-rsync -aP --delete "$OBSERVABILITY_DATA_ROOT" /mnt/backups/observability-data/
+for svc in prometheus loki alloy grafana alertmanager; do
+  rsync -aP --delete "/var/lib/manage/observability/$svc" "/mnt/backups/observability/$svc"
+done
 ```
 
 Stop the stack first if you need a consistent snapshot.