alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add Traefik routing and required absolute data root

Browse Source 
- Make OBSERVABILITY_DATA_ROOT required in docker-compose.observability.yml
  so Portainer deployments must set an absolute host path.
- Add Traefik labels and the web external network to Grafana, Prometheus,
  and Alertmanager with env-driven hostnames, router/service names,
  internal ports, entrypoint, and cert resolver.
- Keep direct host ports as a fallback for local debugging.
- Update docs/observability-runbooks.md with required variables,
  Traefik hostname column, and a .env.observability example.
This commit is contained in:
Developer
2026-06-16 16:23:54 +00:00
parent 53dcd16735
commit 2d1674cb08
2 changed files with 82 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.observability.yml
+41 -10
View File
@@ -1,14 +1,22 @@
# Standalone observability stack. # Standalone observability stack.
# #
# Run this on a host where you want Grafana, Prometheus, Loki, Alertmanager, # Run this on a host where you want Grafana, Prometheus, Loki, Alertmanager,
# Alloy and Node Exporter without the rest of Manage. It exposes the usual # Alloy and Node Exporter without the rest of Manage.
# ports directly; no Traefik or OIDC integration is required.
# #
# cp .env.example .env # Required environment variables:
# # optionally edit OBSERVABILITY_* paths and ports # - OBSERVABILITY_DATA_ROOT (absolute path, e.g. /var/lib/manage/observability)
# - CERT_RESOLVER (Traefik cert resolver name)
# - GRAFANA_APP_HOST, PROMETHEUS_APP_HOST, ALERTMANAGER_APP_HOST
#
# Optional environment variables:
# - TRAEFIK_ENTRYPOINT (default: websecure)
# - *_APP_NAME router/service names (defaults below)
# - all *_PORT variables for direct host ports
#
# Deploy:
# docker compose -f docker-compose.observability.yml up -d # docker compose -f docker-compose.observability.yml up -d
# #
# Then open Grafana at http://localhost:3000 (admin / admin by default). # Then open Grafana at the configured host (default admin / admin).
# #
# To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml # To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml
# and add a static target for it, or drop a file-SD JSON file into # and add a static target for it, or drop a file-SD JSON file into
@@ -29,11 +37,12 @@ services:
- ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro
- ./monitoring/prometheus/rules:/etc/prometheus/rules:ro - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
- ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro
- ${OBSERVABILITY_DATA_ROOT:-./observability-data}/prometheus:/prometheus - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/prometheus:/prometheus
ports: ports:
- "${PROMETHEUS_PORT:-9090}:9090" - "${PROMETHEUS_PORT:-9090}:9090"
networks: networks:
- monitoring - monitoring
- web
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"] test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
@@ -41,6 +50,12 @@ services:
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
labels:
- "traefik.enable=true"
- "traefik.http.routers.${PROMETHEUS_APP_NAME:-prometheus}.rule=Host(`${PROMETHEUS_APP_HOST:?set PROMETHEUS_APP_HOST}`)"
- "traefik.http.routers.${PROMETHEUS_APP_NAME:-prometheus}.entrypoints=${TRAEFIK_ENTRYPOINT:-websecure}"
- "traefik.http.routers.${PROMETHEUS_APP_NAME:-prometheus}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}"
- "traefik.http.services.${PROMETHEUS_APP_NAME:-prometheus}.loadbalancer.server.port=${PROMETHEUS_APP_PORT:-9090}"
deploy: deploy:
resources: resources:
limits: limits:
@@ -56,7 +71,7 @@ services:
command: -config.file=/etc/loki/loki.yml command: -config.file=/etc/loki/loki.yml
volumes: volumes:
- ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
- ${OBSERVABILITY_DATA_ROOT:-./observability-data}/loki:/loki - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/loki:/loki
ports: ports:
- "${LOKI_PORT:-3100}:3100" - "${LOKI_PORT:-3100}:3100"
networks: networks:
@@ -88,7 +103,7 @@ services:
- ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
- /var/run/docker.sock:/var/run/docker.sock:ro - /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro
- ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alloy:/var/lib/alloy - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alloy:/var/lib/alloy
ports: ports:
- "${ALLOY_PORT:-12345}:12345" - "${ALLOY_PORT:-12345}:12345"
networks: networks:
@@ -128,11 +143,12 @@ services:
volumes: volumes:
- ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
- ${OBSERVABILITY_DATA_ROOT:-./observability-data}/grafana:/var/lib/grafana - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/grafana:/var/lib/grafana
ports: ports:
- "${GRAFANA_PORT:-3000}:3000" - "${GRAFANA_PORT:-3000}:3000"
networks: networks:
- monitoring - monitoring
- web
depends_on: depends_on:
prometheus: prometheus:
condition: service_healthy condition: service_healthy
@@ -145,6 +161,12 @@ services:
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
labels:
- "traefik.enable=true"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.rule=Host(`${GRAFANA_APP_HOST:?set GRAFANA_APP_HOST}`)"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.entrypoints=${TRAEFIK_ENTRYPOINT:-websecure}"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}"
- "traefik.http.services.${GRAFANA_APP_NAME:-grafana}.loadbalancer.server.port=${GRAFANA_APP_PORT:-3000}"
deploy: deploy:
resources: resources:
limits: limits:
@@ -169,11 +191,12 @@ services:
ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com} ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
volumes: volumes:
- ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
- ${OBSERVABILITY_DATA_ROOT:-./observability-data}/alertmanager:/alertmanager - ${OBSERVABILITY_DATA_ROOT:?set OBSERVABILITY_DATA_ROOT}/alertmanager:/alertmanager
ports: ports:
- "${ALERTMANAGER_PORT:-9093}:9093" - "${ALERTMANAGER_PORT:-9093}:9093"
networks: networks:
- monitoring - monitoring
- web
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"] test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
@@ -181,6 +204,12 @@ services:
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
labels:
- "traefik.enable=true"
- "traefik.http.routers.${ALERTMANAGER_APP_NAME:-alertmanager}.rule=Host(`${ALERTMANAGER_APP_HOST:?set ALERTMANAGER_APP_HOST}`)"
- "traefik.http.routers.${ALERTMANAGER_APP_NAME:-alertmanager}.entrypoints=${TRAEFIK_ENTRYPOINT:-websecure}"
- "traefik.http.routers.${ALERTMANAGER_APP_NAME:-alertmanager}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}"
- "traefik.http.services.${ALERTMANAGER_APP_NAME:-alertmanager}.loadbalancer.server.port=${ALERTMANAGER_APP_PORT:-9093}"
deploy: deploy:
resources: resources:
limits: limits:
@@ -219,3 +248,5 @@ services:
networks: networks:
monitoring: monitoring:
web:
external: true
docs/observability-runbooks.md
+41 -19
View File
@@ -179,23 +179,32 @@ Run the observability services without the Manage backend or frontend:
```bash ```bash
cd /path/to/manage cd /path/to/manage
cp .env.example .env # create an env file with at least the required variables
# edit .env as needed cat > .env.observability <<EOF
docker compose -f docker-compose.observability.yml up -d OBSERVABILITY_DATA_ROOT=/var/lib/manage/observability
CERT_RESOLVER=myresolver
GRAFANA_APP_HOST=grafana.example.com
PROMETHEUS_APP_HOST=prometheus.example.com
ALERTMANAGER_APP_HOST=alertmanager.example.com
EOF
docker compose -f docker-compose.observability.yml --env-file .env.observability up -d
``` ```
If you do not use Traefik, set `CERT_RESOLVER` to any non-empty value and do not attach the services to a `web` network. The direct host ports still work without Traefik.
### Reachable web UIs ### Reachable web UIs
Only three services expose a human-facing web interface: Only three services expose a human-facing web interface. With Traefik they are served on their public hostnames; direct ports are still open on localhost for debugging.
| Service | Has UI | Default URL | Notes | | Service | Has UI | Direct URL | Traefik hostname variable | Notes |
|---------|--------|-------------|-------| |---------|--------|------------|---------------------------|-------|
| Grafana | yes | `http://localhost:3000` | Dashboards, log explore, alert management | | Grafana | yes | `http://localhost:3000` | `GRAFANA_APP_HOST` | Dashboards, log explore, alert management |
| Prometheus | yes | `http://localhost:9090` | Query, targets, alerts, config status | | Prometheus | yes | `http://localhost:9090` | `PROMETHEUS_APP_HOST` | Query, targets, alerts, config status |
| Alertmanager | yes | `http://localhost:9093` | Alerts, silences, routing status | | Alertmanager | yes | `http://localhost:9093` | `ALERTMANAGER_APP_HOST` | Alerts, silences, routing status |
| Loki | no | `http://localhost:3100` | Log API only; browse logs through Grafana | | Loki | no | `http://localhost:3100` | none | Log API only; browse logs through Grafana |
| Alloy | partial | `http://localhost:12345` | Agent debug UI for pipeline inspection | | Alloy | partial | `http://localhost:12345` | none | Agent debug UI for pipeline inspection |
| Node Exporter | no | `http://localhost:9100` | Metrics endpoint only (`/metrics`) | | Node Exporter | no | `http://localhost:9100` | none | Metrics endpoint only (`/metrics`) |
Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically. Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically.
@@ -203,13 +212,24 @@ Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisione
| Variable | Default | Purpose | | Variable | Default | Purpose |
|----------|---------|---------| |----------|---------|---------|
| `OBSERVABILITY_DATA_ROOT` | `./observability-data` | Host directory where all service data is stored persistently. Each service gets a subdirectory inside it. | | `OBSERVABILITY_DATA_ROOT` | required | Absolute host directory where all service data is stored. Each service gets a subdirectory inside it. Must be set before deploy. |
| `PROMETHEUS_PORT` | `9090` | Host port for Prometheus web UI and API. | | `CERT_RESOLVER` | required | Traefik certificate resolver name (for example `letsencrypt` or `cloudflare`). Must be set before deploy. |
| `LOKI_PORT` | `3100` | Host port for Loki API. | | `TRAEFIK_ENTRYPOINT` | `websecure` | Traefik entrypoint to use for the web UIs. |
| `ALLOY_PORT` | `12345` | Host port for Alloy debug UI. | | `GRAFANA_APP_HOST` | required | Public hostname for Grafana (for example `grafana.example.com`). |
| `GRAFANA_PORT` | `3000` | Host port for Grafana web UI. | | `GRAFANA_APP_NAME` | `grafana` | Traefik router/service name for Grafana. |
| `ALERTMANAGER_PORT` | `9093` | Host port for Alertmanager web UI. | | `GRAFANA_APP_PORT` | `3000` | Internal port Traefik forwards to for Grafana. |
| `NODE_EXPORTER_PORT` | `9100` | Host port for Node Exporter metrics endpoint. | | `PROMETHEUS_APP_HOST` | required | Public hostname for Prometheus (for example `prometheus.example.com`). |
| `PROMETHEUS_APP_NAME` | `prometheus` | Traefik router/service name for Prometheus. |
| `PROMETHEUS_APP_PORT` | `9090` | Internal port Traefik forwards to for Prometheus. |
| `ALERTMANAGER_APP_HOST` | required | Public hostname for Alertmanager (for example `alertmanager.example.com`). |
| `ALERTMANAGER_APP_NAME` | `alertmanager` | Traefik router/service name for Alertmanager. |
| `ALERTMANAGER_APP_PORT` | `9093` | Internal port Traefik forwards to for Alertmanager. |
| `PROMETHEUS_PORT` | `9090` | Direct host port for Prometheus web UI and API. |
| `LOKI_PORT` | `3100` | Direct host port for Loki API. |
| `ALLOY_PORT` | `12345` | Direct host port for Alloy debug UI. |
| `GRAFANA_PORT` | `3000` | Direct host port for Grafana web UI. |
| `ALERTMANAGER_PORT` | `9093` | Direct host port for Alertmanager web UI. |
| `NODE_EXPORTER_PORT` | `9100` | Direct host port for Node Exporter metrics endpoint. |
| `GRAFANA_ADMIN_USER` | `admin` | Grafana admin username. | | `GRAFANA_ADMIN_USER` | `admin` | Grafana admin username. |
| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin password. Change this in production. | | `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin password. Change this in production. |
| `GF_AUTH_GENERIC_OAUTH_CLIENT_ID` | empty | Generic OAuth client ID for Authentik or another provider. | | `GF_AUTH_GENERIC_OAUTH_CLIENT_ID` | empty | Generic OAuth client ID for Authentik or another provider. |
@@ -225,6 +245,8 @@ Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisione
| `SMTP_FROM_ADDRESS` | `no-reply@example.com` | From address for alert emails. | | `SMTP_FROM_ADDRESS` | `no-reply@example.com` | From address for alert emails. |
| `ALERT_EMAIL_TO` | `admin@example.com` | Default recipient for alert emails. | | `ALERT_EMAIL_TO` | `admin@example.com` | Default recipient for alert emails. |
Prometheus and Alertmanager do not have authentication. When exposing them through Traefik, add a basic-auth middleware or restrict access by network.
To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`. To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`.
### Backing up standalone data ### Backing up standalone data