feat(observability): add standalone observability compose stack
Add docker-compose.observability.yml to run Grafana, Prometheus, Loki, Alertmanager, Alloy and Node Exporter independently of Manage. Includes a standalone Prometheus config, empty file-SD placeholder, and runbook documentation.
This commit is contained in:
Developer
@@ -0,0 +1,222 @@
|
||||
# Standalone observability stack.
|
||||
#
|
||||
# Run this on a host where you want Grafana, Prometheus, Loki, Alertmanager,
|
||||
# Alloy and Node Exporter without the rest of Manage. It exposes the usual
|
||||
# ports directly; no Traefik or OIDC integration is required.
|
||||
#
|
||||
# docker compose -f docker-compose.observability.yml up -d
|
||||
#
|
||||
# Then open Grafana at http://localhost:3000 (admin / admin by default).
|
||||
#
|
||||
# To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml
|
||||
# and add a static target for it, or drop a file-SD JSON file into
|
||||
# monitoring/prometheus/file-sd/.
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
container_name: prometheus
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=30d"
|
||||
- "--web.console.libraries=/usr/share/prometheus/console_libraries"
|
||||
- "--web.console.templates=/usr/share/prometheus/consoles"
|
||||
- "--web.enable-lifecycle"
|
||||
volumes:
|
||||
- ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.1.1
|
||||
container_name: loki
|
||||
command: -config.file=/etc/loki/loki.yml
|
||||
volumes:
|
||||
- ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki_data:/loki
|
||||
ports:
|
||||
- "3100:3100"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
alloy:
|
||||
image: grafana/alloy:v1.5.0
|
||||
container_name: alloy
|
||||
command:
|
||||
- run
|
||||
- /etc/alloy/config.alloy
|
||||
- --storage.path=/var/lib/alloy
|
||||
volumes:
|
||||
- ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
user: root
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.10"
|
||||
memory: 128M
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.3.1
|
||||
container_name: grafana
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-}
|
||||
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-}
|
||||
GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-}
|
||||
GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-}
|
||||
GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
volumes:
|
||||
- ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
|
||||
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.27.0
|
||||
container_name: alertmanager
|
||||
command:
|
||||
- "--config.file=/etc/alertmanager/alertmanager.yml"
|
||||
- "--storage.path=/alertmanager"
|
||||
environment:
|
||||
SMTP_HOST: ${SMTP_HOST:-smtp.example.com}
|
||||
SMTP_PORT: ${SMTP_PORT:-587}
|
||||
SMTP_USERNAME: ${SMTP_USERNAME:-}
|
||||
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
|
||||
SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
|
||||
ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
|
||||
volumes:
|
||||
- ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- alertmanager_data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.10"
|
||||
memory: 64M
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.8.2
|
||||
container_name: node-exporter
|
||||
command:
|
||||
- "--path.rootfs=/host"
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
ports:
|
||||
- "9100:9100"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
reservations:
|
||||
cpus: "0.05"
|
||||
memory: 32M
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
grafana_data:
|
||||
alertmanager_data:
|
||||
@@ -183,6 +183,30 @@ docker compose up -d
|
||||
|
||||
---
|
||||
|
||||
## Standalone Observability Stack
|
||||
|
||||
You can run the observability services on their own without the Manage backend or frontend:
|
||||
|
||||
```bash
|
||||
cd /path/to/manage
|
||||
docker compose -f docker-compose.observability.yml up -d
|
||||
```
|
||||
|
||||
This starts Prometheus, Grafana, Loki, Alertmanager, Alloy and Node Exporter. Exposed ports:
|
||||
|
||||
| Service | URL |
|
||||
|---------|-----|
|
||||
| Grafana | <http://localhost:3000> |
|
||||
| Prometheus | <http://localhost:9090> |
|
||||
| Alertmanager | <http://localhost:9093> |
|
||||
| Loki | <http://localhost:3100> |
|
||||
| Node Exporter | <http://localhost:9100> |
|
||||
| Alloy | <http://localhost:12345> |
|
||||
|
||||
Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically.
|
||||
|
||||
To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`.
|
||||
|
||||
## Scaling Notes
|
||||
|
||||
- The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1,46 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
monitor: manage-observability
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost:9090
|
||||
|
||||
- job_name: node-exporter
|
||||
static_configs:
|
||||
- targets:
|
||||
- node-exporter:9100
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets:
|
||||
- loki:3100
|
||||
|
||||
- job_name: alertmanager
|
||||
static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
- job_name: grafana
|
||||
static_configs:
|
||||
- targets:
|
||||
- grafana:3000
|
||||
|
||||
- job_name: node-exporter-remote
|
||||
file_sd_configs:
|
||||
- files:
|
||||
- /etc/prometheus/file-sd/node_exporter_targets.json
|
||||
refresh_interval: 30s
|
||||
Reference in New Issue
Block a user