feat(observability): per-service root directories for config and data

- Replace OBSERVABILITY_DATA_ROOT with per-service *_ROOT variables in
  docker-compose.observability.yml.
- Each service root must contain config/ (mounted read-only) and data/
  (mounted read-write), grouping config and state together for Portainer.
- Update docs/observability-runbooks.md with the new variables, a setup
  script that copies repo configs into each service root, and updated
  backup/restore examples.

docker-compose.observability.yml

@@ -4,10 +4,19 @@
 # Alloy and Node Exporter without the rest of Manage.
 #
 # Required environment variables:
-#   - OBSERVABILITY_DATA_ROOT (absolute path, e.g. /var/lib/manage/observability)
+#   - *_ROOT directories (absolute paths) for services you want to persist
 #   - CERT_RESOLVER (Traefik cert resolver name)
 #   - GRAFANA_APP_HOST, PROMETHEUS_APP_HOST, ALERTMANAGER_APP_HOST
 #
+# Each service root should contain a config/ and data/ subdirectory, e.g.:
+#
+#   /var/lib/manage/observability/prometheus/
+#   ├── config/
+#   │   ├── prometheus.yml
+#   │   ├── rules/
+#   │   └── file-sd/
+#   └── data/
+#
 # Optional environment variables:
 #   - TRAEFIK_ENTRYPOINT (default: websecure)
 #   - *_APP_NAME router/service names (defaults below)
@@ -17,10 +26,6 @@
 #   docker compose -f docker-compose.observability.yml up -d
 #
 # Then open Grafana at the configured host (default admin / admin).
-#
-# To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml
-# and add a static target for it, or drop a file-SD JSON file into
-# monitoring/prometheus/file-sd/.
 
 services:
   prometheus:
@@ -34,10 +39,10 @@ services:
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--web.enable-lifecycle"
     volumes:
-      - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro
-      - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
-      - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/prometheus:/prometheus
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/rules:/etc/prometheus/rules:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/config/file-sd:/etc/prometheus/file-sd:ro
+      - ${PROMETHEUS_ROOT:?set PROMETHEUS_ROOT}/data:/prometheus
     ports:
       - "${PROMETHEUS_PORT:-9090}:9090"
     networks:
@@ -70,8 +75,8 @@ services:
     container_name: loki
     command: -config.file=/etc/loki/loki.yml
     volumes:
-      - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/loki:/loki
+      - ${LOKI_ROOT:?set LOKI_ROOT}/config/loki.yml:/etc/loki/loki.yml:ro
+      - ${LOKI_ROOT:?set LOKI_ROOT}/data:/loki
     ports:
       - "${LOKI_PORT:-3100}:3100"
     networks:
@@ -100,10 +105,10 @@ services:
       - /etc/alloy/config.alloy
       - --storage.path=/var/lib/alloy
     volumes:
-      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - ${ALLOY_ROOT:?set ALLOY_ROOT}/config/config.alloy:/etc/alloy/config.alloy:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alloy:/var/lib/alloy
+      - ${ALLOY_ROOT:?set ALLOY_ROOT}/data:/var/lib/alloy
     ports:
       - "${ALLOY_PORT:-12345}:12345"
     networks:
@@ -141,9 +146,9 @@ services:
       GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
       LOG_LEVEL: ${LOG_LEVEL:-INFO}
     volumes:
-      - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
-      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/grafana:/var/lib/grafana
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/config/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/config/provisioning:/etc/grafana/provisioning:ro
+      - ${GRAFANA_ROOT:?set GRAFANA_ROOT}/data:/var/lib/grafana
     ports:
       - "${GRAFANA_PORT:-3000}:3000"
     networks:
@@ -190,8 +195,8 @@ services:
       SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
       ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
     volumes:
-      - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alertmanager:/alertmanager
+      - ${ALERTMANAGER_ROOT:?set ALERTMANAGER_ROOT}/config/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - ${ALERTMANAGER_ROOT:?set ALERTMANAGER_ROOT}/data:/alertmanager
     ports:
       - "${ALERTMANAGER_PORT:-9093}:9093"
     networks: