feat(v2): release v2.0.0
# Conflicts: # README.md
This commit is contained in:
@@ -22,6 +22,7 @@ wheels/
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
.venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
@@ -64,6 +65,9 @@ logs/
|
||||
|
||||
# Tests
|
||||
tests/
|
||||
!tests/ssh-fixture/
|
||||
!tests/ssh-fixture/Dockerfile
|
||||
!tests/ssh-fixture/sshd_config
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
|
||||
|
||||
+4
-1
@@ -50,7 +50,7 @@ Thumbs.db
|
||||
# Frontend
|
||||
frontend/node_modules/
|
||||
frontend/dist/
|
||||
frontend/package-lock.json
|
||||
frontend/test-results/
|
||||
frontend/yarn.lock
|
||||
frontend/pnpm-lock.yaml
|
||||
|
||||
@@ -60,8 +60,11 @@ logs/
|
||||
|
||||
# Testing
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Backup tool specific
|
||||
backups/
|
||||
.pi-subagents/
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.12.11
|
||||
@@ -0,0 +1,25 @@
|
||||
# Agent Operating Instructions
|
||||
|
||||
## Continuous milestone execution
|
||||
|
||||
When the user asks to continue, work autonomously through the active milestone
|
||||
and its verification. Do **not** send progress-only, acknowledgement, empty, or
|
||||
status final responses. Reply only when:
|
||||
|
||||
1. the user asks for status;
|
||||
2. a real product, security, credentials, or destructive-action decision needs
|
||||
the user's input; or
|
||||
3. the requested milestone is fully implemented and verified.
|
||||
|
||||
Use one foreground milestone batch where possible. If work must run in a
|
||||
background subagent, avoid notifying the user manually; inspect and verify the
|
||||
result before replying at the same completion boundary.
|
||||
|
||||
A chat turn still necessarily ends after a model response. This file prevents
|
||||
unnecessary model-generated completion messages; it cannot suppress
|
||||
harness-generated tool or subagent notifications.
|
||||
|
||||
## Verification
|
||||
|
||||
Before claiming a milestone boundary, run `make check` and record focused
|
||||
acceptance evidence under `docs/release/`.
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Pinned by manifest digest; update tag and digest together.
|
||||
FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS builder
|
||||
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
WORKDIR /build
|
||||
COPY backend/pyproject.toml ./pyproject.toml
|
||||
COPY backend/src ./src
|
||||
RUN python -m pip install --upgrade pip==25.3 && \
|
||||
python -m pip wheel --wheel-dir /wheels .
|
||||
|
||||
FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS runtime
|
||||
|
||||
ENV PATH="/opt/venv/bin:${PATH}" \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
BACKUP_TOOL_ALEMBIC_ROOT=/app
|
||||
|
||||
RUN groupadd --gid 10001 backup-tool && \
|
||||
useradd --uid 10001 --gid backup-tool --create-home --home-dir /home/backup-tool \
|
||||
--shell /usr/sbin/nologin backup-tool && \
|
||||
python -m venv /opt/venv && \
|
||||
install --directory --owner=backup-tool --group=backup-tool --mode=0750 \
|
||||
/var/lib/backup-tool \
|
||||
/var/lib/backup-tool/repositories \
|
||||
/var/lib/backup-tool/restores \
|
||||
/run/backup-tool
|
||||
|
||||
COPY --from=builder /wheels /wheels
|
||||
RUN python -m pip install --no-index --find-links=/wheels backup-tool==2.0.0.dev0 && \
|
||||
rm -rf /wheels
|
||||
COPY --chown=backup-tool:backup-tool backend/alembic.ini /app/alembic.ini
|
||||
COPY --chown=backup-tool:backup-tool backend/alembic /app/alembic
|
||||
|
||||
WORKDIR /app
|
||||
USER backup-tool:backup-tool
|
||||
|
||||
ENTRYPOINT ["backup-tool"]
|
||||
CMD ["web"]
|
||||
@@ -0,0 +1,52 @@
|
||||
PYTHON ?= .venv/bin/python
|
||||
BOOTSTRAP_PYTHON ?= python3
|
||||
NPM ?= npm
|
||||
|
||||
.PHONY: setup install test-fast test-integration test-fault test-security test-e2e test-ssh-integration lint typecheck frontend-build check-v1-absent check-openapi check
|
||||
|
||||
setup:
|
||||
test -x $(PYTHON) || $(BOOTSTRAP_PYTHON) -m venv .venv
|
||||
$(PYTHON) -m pip install --upgrade pip
|
||||
$(PYTHON) -m pip install -e 'backend[dev]'
|
||||
$(NPM) --prefix frontend ci
|
||||
|
||||
install: setup
|
||||
|
||||
test-fast:
|
||||
$(PYTHON) -m pytest tests/unit tests/contract -q
|
||||
|
||||
test-integration:
|
||||
$(PYTHON) -m pytest tests/integration -q
|
||||
|
||||
test-fault:
|
||||
$(PYTHON) -m pytest tests/fault -q
|
||||
|
||||
test-security:
|
||||
$(PYTHON) -m pytest tests/security -q
|
||||
|
||||
test-e2e:
|
||||
BACKUP_TOOL_COMPOSE_E2E=1 $(PYTHON) -m pytest tests/e2e/test_compose_v2.py -q
|
||||
|
||||
test-ssh-integration:
|
||||
BACKUP_TOOL_SSH_INTEGRATION=1 $(PYTHON) -m pytest tests/integration/test_ssh_backup_restore.py -q
|
||||
|
||||
lint:
|
||||
$(PYTHON) -m ruff check --config backend/pyproject.toml backend/src backend/alembic tests tools
|
||||
$(PYTHON) -m ruff format --check --config backend/pyproject.toml backend/src backend/alembic tests tools
|
||||
|
||||
typecheck:
|
||||
$(PYTHON) -m mypy --config-file backend/pyproject.toml
|
||||
$(NPM) --prefix frontend run typecheck
|
||||
|
||||
frontend-build:
|
||||
$(NPM) --prefix frontend run build
|
||||
|
||||
check-openapi:
|
||||
$(PYTHON) tools/export_openapi.py
|
||||
$(NPM) --prefix frontend run api:generate
|
||||
git diff --exit-code -- openapi/v2.json frontend/src/api/generated
|
||||
|
||||
check-v1-absent:
|
||||
$(PYTHON) tools/forbidden_v1_scan.py .
|
||||
|
||||
check: check-v1-absent check-openapi test-fast test-integration test-fault test-security lint typecheck frontend-build
|
||||
@@ -1,121 +1,119 @@
|
||||
# Backup Tool
|
||||
# Backup Tool v2
|
||||
|
||||
Backup Tool is a FastAPI and React application for managing local, SSH/SFTP, PostgreSQL, and MySQL backup sources. It provides backup jobs and executions, scheduling, retention policies, a web dashboard, and a REST API.
|
||||
> A self-hosted, integrity-first backup appliance for local and hardened SSH sources.
|
||||
|
||||
## Status and limitations
|
||||
Backup Tool creates signed, content-addressed backups, supports encrypted repositories and offline recovery, and runs as isolated web, scheduler, worker, migration, and admin roles. v2.0 supports **local** and **SSH** sources, local repositories, restore, encrypted recovery, email, and signed webhooks.
|
||||
|
||||
The backend package is versioned `0.1.0` and classified as beta. Runtime verification is outside this README; review the configuration and test commands below before using the application for production backups.
|
||||
## What is supported
|
||||
|
||||
The Compose configuration sets `BACKUP_STORAGE_PATH=/app/backups`, but the current Python application does not read that environment variable. The `/app/backups` volume is still mounted by Compose; the application's effective use of that path is not verified here.
|
||||
- Signed full and incremental backups, restore, retention, verification, and recovery.
|
||||
- AES-256-GCM repository encryption with key epochs and offline Argon2id recovery bundles.
|
||||
- Local sources and SSH sources using **private-key-only, pinned-host-key, forced-SFTP chroot** access.
|
||||
- Durable, at-least-once webhook and STARTTLS SMTP notifications.
|
||||
- A generated OpenAPI TypeScript client and operator UI.
|
||||
|
||||
## Prerequisites
|
||||
Never use an SSH source with a shell, password, agent forwarding, remote commands, or an unchrooted account. See [SSH source requirements](docs/runbooks/ssh-sources.md).
|
||||
|
||||
- Docker with Docker Compose for the containerized setup.
|
||||
- Python 3.11 or newer for manual backend development.
|
||||
- Node.js for manual frontend development. The development Compose service uses Node 20 Alpine; no standalone Node version is declared by the frontend package.
|
||||
## Quick local verification
|
||||
|
||||
## Run with Docker Compose
|
||||
Prerequisites: Docker Compose, Python 3.12–3.14, Node/npm, and a working virtual environment.
|
||||
|
||||
From the repository root:
|
||||
```sh
|
||||
make setup
|
||||
make check
|
||||
make test-e2e
|
||||
make test-ssh-integration
|
||||
```
|
||||
|
||||
```bash
|
||||
# Backend only
|
||||
`make test-e2e` creates temporary test-only keys and source data, starts the complete Compose stack, checks readiness/metrics/restart behavior, and tears it down. `make test-ssh-integration` builds a separate forced-SFTP fixture and verifies key-authenticated SSH probe, backup, verification, and restore.
|
||||
|
||||
## Run locally with Docker Compose
|
||||
|
||||
The production Compose topology binds the proxy to `127.0.0.1:8080`; put a TLS reverse proxy in front of it for remote access.
|
||||
|
||||
```sh
|
||||
mkdir -p secrets sources
|
||||
head -c 32 /dev/urandom > secrets/master.key
|
||||
chmod 600 secrets/master.key
|
||||
# The container runs as UID 10001; grant that UID read access to the key.
|
||||
chown 10001:10001 secrets/master.key
|
||||
|
||||
# Put test source data below ./sources, then start the isolated roles.
|
||||
docker compose build --pull
|
||||
docker compose run --rm migrate upgrade
|
||||
docker compose up -d
|
||||
|
||||
# Backend plus the production frontend
|
||||
docker compose --profile prod up -d
|
||||
|
||||
# Backend plus the Vite development frontend
|
||||
docker compose --profile dev up -d
|
||||
curl -fsS http://127.0.0.1:8080/readyz
|
||||
curl -fsS http://127.0.0.1:8080/metrics
|
||||
```
|
||||
|
||||
The backend is published at <http://localhost:8000>; its OpenAPI documentation is at <http://localhost:8000/docs>. Either frontend profile publishes the frontend at <http://localhost:3000>.
|
||||
On a fresh non-loopback deployment, set `BACKUP_TOOL_PUBLIC_BASE_URL` to the externally reachable HTTPS URL and set `BACKUP_TOOL_BOOTSTRAP_SECRET` before first setup. Never expose a fresh setup endpoint without bootstrap protection.
|
||||
|
||||
Compose persists the SQLite data directory in `backup-data` and the backup directory in `backup-storage`. The backend health check requests `/api/health`.
|
||||
Stop the local stack with:
|
||||
|
||||
## Manual development
|
||||
|
||||
### Backend
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
python -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -e ".[dev]"
|
||||
uvicorn app.main:app --reload --port 8000
|
||||
```sh
|
||||
docker compose down
|
||||
# Add --volumes only when intentionally discarding local metadata and repositories.
|
||||
```
|
||||
|
||||
The backend requires Python 3.11+. Its default `DATABASE_URL` is `sqlite+aiosqlite:///./backup_tool.db`.
|
||||
## Operator workflow
|
||||
|
||||
### Frontend
|
||||
1. Open `http://127.0.0.1:8080` and create the first administrator.
|
||||
2. Create a local or encrypted repository.
|
||||
3. Create a local source, or configure a hardened SSH source as documented in [SSH sources](docs/runbooks/ssh-sources.md).
|
||||
4. Create a job, run a probe, enqueue a backup, verify it, then perform a test restore.
|
||||
5. Configure notification subscriptions and test them before relying on delivery.
|
||||
6. Export and validate a recovery bundle after encrypted repository creation and each key rotation.
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
To build the frontend for the backend to serve:
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run build
|
||||
```
|
||||
|
||||
When `frontend/dist` exists, the backend mounts its assets and serves the SPA fallback. For a manually built frontend and backend, start the backend as above (or bind explicitly with `uvicorn app.main:app --host 0.0.0.0 --port 8000`).
|
||||
The browser exposes recovery status and the CLI procedure only; it never transfers recovery bundles or passphrases.
|
||||
|
||||
## Configuration
|
||||
|
||||
The backend reads these environment variables:
|
||||
Compose supplies the core runtime variables. The important host paths are:
|
||||
|
||||
| Variable | Purpose | Default |
|
||||
| Setting | Compose value | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `DATABASE_URL` | SQLAlchemy database URL | `sqlite+aiosqlite:///./backup_tool.db` |
|
||||
| `SQL_ECHO` | Enable SQL query logging when `true` | `false` |
|
||||
| `CORS_ORIGINS` | Comma-separated allowed origins | `http://localhost:3000` |
|
||||
| `BACKUP_TOOL_MASTER_KEY_FILE` | `/run/backup-tool-secrets/master.key` | Service-owned `0600` master key |
|
||||
| `BACKUP_TOOL_SOURCES_DIR` | `./sources` | Read-only local-source bind mount |
|
||||
| `BACKUP_TOOL_PUBLIC_BASE_URL` | `http://localhost:8080` | External URL and bootstrap policy |
|
||||
| `BACKUP_TOOL_PORT` | `8080` | Loopback proxy port |
|
||||
|
||||
Compose supplies a SQLite URL under `/app/data`, `CORS_ORIGINS=http://localhost:3000`, and mounts persistent data and backup volumes. Configure credentials for backup sources through the application rather than committing them to the repository.
|
||||
Use absolute, allowlisted paths for repositories, sources, and restores. The service rejects unsafe paths, symlinks where they violate the contract, missing key files, and non-current schemas.
|
||||
|
||||
## Development and database operations
|
||||
## Operations and security
|
||||
|
||||
Run the backend tests:
|
||||
- [Observability](docs/runbooks/observability.md): readiness, liveness, metrics, and alert response.
|
||||
- [Upgrade and rollback](docs/runbooks/upgrade.md)
|
||||
- [Disaster recovery](docs/runbooks/disaster-recovery.md)
|
||||
- [Recovery bundles](docs/runbooks/recovery-bundle.md)
|
||||
- [Repository keys](docs/runbooks/keys.md)
|
||||
- [Notifications](docs/runbooks/notifications.md)
|
||||
- [SSH sources](docs/runbooks/ssh-sources.md)
|
||||
- [Security policies](docs/README.md#security-boundaries)
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
pytest
|
||||
## Development
|
||||
|
||||
```sh
|
||||
make setup
|
||||
make test-fast
|
||||
make test-integration
|
||||
make test-fault
|
||||
make test-security
|
||||
npm --prefix frontend test -- --run
|
||||
npm --prefix frontend exec playwright test
|
||||
```
|
||||
|
||||
The frontend package defines `dev`, `build`, and `preview` scripts; it does not currently define a test script.
|
||||
OpenAPI and the generated browser client are committed artifacts:
|
||||
|
||||
Alembic commands are available from `backend/`:
|
||||
|
||||
```bash
|
||||
alembic revision --autogenerate -m "Description"
|
||||
alembic upgrade head
|
||||
```sh
|
||||
.venv/bin/python tools/export_openapi.py --check openapi/v2.json
|
||||
npm --prefix frontend run api:generate
|
||||
git diff --exit-code -- openapi/v2.json frontend/src/api/generated
|
||||
```
|
||||
|
||||
## Deployment and operations
|
||||
## Release evidence
|
||||
|
||||
For the provided production frontend container, use:
|
||||
|
||||
```bash
|
||||
docker compose --profile prod up -d
|
||||
```
|
||||
|
||||
This starts the backend and an nginx-served frontend. For manual deployment, build the frontend and run the backend; the backend can serve `frontend/dist` when that directory exists. Choose and protect backup-source credentials, retention policies, and persistent storage for the environment.
|
||||
|
||||
## Repository layout
|
||||
|
||||
```text
|
||||
.
|
||||
├── backend/ # FastAPI application, backup engine, Alembic, and tests
|
||||
│ └── pyproject.toml # Python dependencies and pytest configuration
|
||||
├── frontend/ # React/Vite frontend
|
||||
│ └── package.json # Frontend scripts and dependencies
|
||||
├── docker-compose.yml # Backend and optional frontend profiles
|
||||
└── docs/ # Project documentation
|
||||
```
|
||||
Milestone evidence, the SBOM, provenance, and synthetic scale report live in [docs/release](docs/release/). M15 certification is synthetic metadata certification on the reference CI host; it does not claim a physical 10 TiB transfer.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# Stage 1: Builder
|
||||
FROM python:3.14 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install build dependencies
|
||||
RUN pip install --no-cache-dir setuptools wheel
|
||||
|
||||
# Copy pyproject.toml first for better layer caching
|
||||
COPY pyproject.toml ./
|
||||
|
||||
# Copy source code
|
||||
COPY app/ ./app/
|
||||
COPY backup/ ./backup/
|
||||
COPY alembic/ ./alembic/
|
||||
COPY alembic.ini ./
|
||||
|
||||
# Build the package with dev dependencies
|
||||
RUN pip install --no-cache-dir -e ".[dev]"
|
||||
|
||||
# Stage 2: Runtime
|
||||
FROM python:3.14-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Create non-root user
|
||||
RUN groupadd -r backup-tool && useradd -r -g backup-tool backup-tool
|
||||
|
||||
# Copy installed packages from builder
|
||||
COPY --from=builder /usr/local/lib/python3.14/site-packages/ /usr/local/lib/python3.14/site-packages/
|
||||
COPY --from=builder /usr/local/bin/ /usr/local/bin/
|
||||
|
||||
# Copy application code
|
||||
COPY --from=builder /app/ ./
|
||||
|
||||
# Create data directory for SQLite and backups
|
||||
RUN mkdir -p /app/data /app/backups && \
|
||||
chown -R backup-tool:backup-tool /app
|
||||
|
||||
USER backup-tool
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')" || exit 1
|
||||
|
||||
# Default command
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
+38
-4
@@ -1,5 +1,39 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
sqlalchemy.url = sqlite+aiosqlite:///./backup_tool.db
|
||||
script_location = %(here)s/alembic
|
||||
prepend_sys_path = %(here)s/src
|
||||
path_separator = os
|
||||
sqlalchemy.url = sqlite+aiosqlite:////var/lib/backup-tool/metadata.db
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
Generic single-database configuration.
|
||||
+59
-23
@@ -1,45 +1,81 @@
|
||||
import asyncio
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
from typing import Any
|
||||
|
||||
from alembic import context
|
||||
from app.models import Base
|
||||
from sqlalchemy import create_engine, event, pool
|
||||
from sqlalchemy.engine import URL, make_url
|
||||
|
||||
models = importlib.import_module("backup_tool.db.models")
|
||||
sqlite_runtime = importlib.import_module("backup_tool.db.sqlite")
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
target_metadata = models.Base.metadata
|
||||
|
||||
|
||||
def configured_url() -> str:
|
||||
raw_url = os.environ.get("BACKUP_TOOL_DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||
if raw_url is None:
|
||||
raise RuntimeError("sqlalchemy.url is required")
|
||||
return raw_url
|
||||
|
||||
|
||||
def busy_timeout_ms() -> int:
|
||||
raw_value = os.environ.get(
|
||||
"BACKUP_TOOL_SQLITE_BUSY_TIMEOUT_MS", str(sqlite_runtime.DEFAULT_BUSY_TIMEOUT_MS)
|
||||
)
|
||||
try:
|
||||
value = int(raw_value)
|
||||
except ValueError as error:
|
||||
raise RuntimeError("SQLite busy timeout must be an integer") from error
|
||||
if value < 1_000 or value > 120_000:
|
||||
raise RuntimeError("SQLite busy timeout must be between 1000 and 120000 ms")
|
||||
return value
|
||||
|
||||
|
||||
def synchronous_url(raw_url: str) -> URL:
|
||||
url = make_url(raw_url)
|
||||
if url.drivername != "sqlite+aiosqlite":
|
||||
raise RuntimeError("v2 migrations require sqlite+aiosqlite")
|
||||
return url.set(drivername="sqlite")
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
url = synchronous_url(configured_url())
|
||||
context.configure(
|
||||
url=url,
|
||||
url=url.render_as_string(hide_password=False),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
connectable = async_engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(run_async_migrations())
|
||||
timeout = busy_timeout_ms()
|
||||
engine = create_engine(
|
||||
synchronous_url(configured_url()),
|
||||
poolclass=pool.NullPool,
|
||||
connect_args=sqlite_runtime.sqlite_connect_args(timeout),
|
||||
)
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def configure_sqlite(dbapi_connection: Any, _connection_record: Any) -> None:
|
||||
sqlite_runtime.configure_sqlite_connection(dbapi_connection)
|
||||
|
||||
with engine.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
@@ -11,7 +10,6 @@ from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
@@ -19,10 +17,8 @@ depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
|
||||
@@ -0,0 +1,548 @@
|
||||
"""v2 baseline
|
||||
|
||||
Revision ID: 0001_v2_baseline
|
||||
Revises:
|
||||
Create Date: 2026-07-27 19:15:52.319338
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0001_v2_baseline"
|
||||
down_revision: str | Sequence[str] | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"repositories",
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("root", sa.Text(), nullable=False),
|
||||
sa.Column("format_version", sa.Integer(), nullable=False),
|
||||
sa.Column("compression", sa.String(length=32), nullable=False),
|
||||
sa.Column("encryption", sa.String(length=32), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('active','archived','unavailable')", name=op.f("ck_repositories_state")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"format_version > 0", name=op.f("ck_repositories_format_version_positive")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_repositories")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_repositories_name")),
|
||||
sa.UniqueConstraint("root", name=op.f("uq_repositories_root")),
|
||||
)
|
||||
op.create_table(
|
||||
"secrets",
|
||||
sa.Column("ciphertext", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("key_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("purpose", sa.String(length=64), nullable=False),
|
||||
sa.Column("version", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("version > 0", name=op.f("ck_secrets_version_positive")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_secrets")),
|
||||
)
|
||||
op.create_table(
|
||||
"sources",
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("public_config", sa.JSON(), nullable=False),
|
||||
sa.Column("secret_refs", sa.JSON(), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("last_probe", sa.JSON(), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"kind IN ('local','sftp','postgresql','mysql')", name=op.f("ck_sources_kind")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('active','archived','unavailable')", name=op.f("ck_sources_state")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_sources_name")),
|
||||
)
|
||||
op.create_table(
|
||||
"users",
|
||||
sa.Column("username", sa.String(length=255), nullable=False),
|
||||
sa.Column("password_hash", sa.Text(), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("state IN ('active','disabled')", name=op.f("ck_users_state")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
|
||||
sa.UniqueConstraint("username", name=op.f("uq_users_username")),
|
||||
)
|
||||
op.create_table(
|
||||
"api_tokens",
|
||||
sa.Column("owner_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("token_hash", sa.Text(), nullable=False),
|
||||
sa.Column("scopes", sa.JSON(), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["owner_id"],
|
||||
["users.id"],
|
||||
name=op.f("fk_api_tokens_owner_id_users"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_api_tokens")),
|
||||
sa.UniqueConstraint("token_hash", name=op.f("uq_api_tokens_token_hash")),
|
||||
)
|
||||
op.create_index("ix_api_tokens_owner_id", "api_tokens", ["owner_id"], unique=False)
|
||||
op.create_table(
|
||||
"audit_events",
|
||||
sa.Column("actor_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("action", sa.String(length=255), nullable=False),
|
||||
sa.Column("resource_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("resource_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("outcome", sa.String(length=32), nullable=False),
|
||||
sa.Column("request_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"outcome IN ('success','failure','denied')", name=op.f("ck_audit_events_outcome")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["actor_id"],
|
||||
["users.id"],
|
||||
name=op.f("fk_audit_events_actor_id_users"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_events")),
|
||||
)
|
||||
op.create_index("ix_audit_events_actor_id", "audit_events", ["actor_id"], unique=False)
|
||||
op.create_index("ix_audit_events_created_at", "audit_events", ["created_at"], unique=False)
|
||||
op.create_table(
|
||||
"idempotency_records",
|
||||
sa.Column("actor_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("key", sa.String(length=255), nullable=False),
|
||||
sa.Column("operation", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("response_resource_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("response_resource_id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["actor_id"],
|
||||
["users.id"],
|
||||
name=op.f("fk_idempotency_records_actor_id_users"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_idempotency_records")),
|
||||
sa.UniqueConstraint(
|
||||
"actor_id", "key", "operation", name="uq_idempotency_actor_key_operation"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_idempotency_created_at", "idempotency_records", ["created_at"], unique=False
|
||||
)
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("repository_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("requested_mode", sa.String(length=32), nullable=False),
|
||||
sa.Column("exclusions", sa.JSON(), nullable=False),
|
||||
sa.Column("retention", sa.JSON(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||
sa.Column("allow_empty", sa.Boolean(), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"requested_mode IN ('full','incremental')", name=op.f("ck_jobs_requested_mode")
|
||||
),
|
||||
sa.CheckConstraint("state IN ('active','archived')", name=op.f("ck_jobs_state")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["repository_id"],
|
||||
["repositories.id"],
|
||||
name=op.f("fk_jobs_repository_id_repositories"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
["sources.id"],
|
||||
name=op.f("fk_jobs_source_id_sources"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_jobs")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_jobs_name")),
|
||||
)
|
||||
op.create_index("ix_jobs_repository_id", "jobs", ["repository_id"], unique=False)
|
||||
op.create_index("ix_jobs_source_id", "jobs", ["source_id"], unique=False)
|
||||
op.create_table(
|
||||
"notification_subscriptions",
|
||||
sa.Column("channel", sa.String(length=32), nullable=False),
|
||||
sa.Column("event_filters", sa.JSON(), nullable=False),
|
||||
sa.Column("destination_config", sa.JSON(), nullable=False),
|
||||
sa.Column("secret_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"channel IN ('webhook','email')", name=op.f("ck_notification_subscriptions_channel")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('active','disabled','archived')",
|
||||
name=op.f("ck_notification_subscriptions_state"),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["secret_id"],
|
||||
["secrets.id"],
|
||||
name=op.f("fk_notification_subscriptions_secret_id_secrets"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_notification_subscriptions")),
|
||||
)
|
||||
op.create_table(
|
||||
"notification_deliveries",
|
||||
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("subscription_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("response_class", sa.String(length=64), nullable=True),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('pending','delivered','retry','failed')",
|
||||
name=op.f("ck_notification_deliveries_state"),
|
||||
),
|
||||
sa.CheckConstraint("attempt > 0", name=op.f("ck_notification_deliveries_attempt_positive")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["subscription_id"],
|
||||
["notification_subscriptions.id"],
|
||||
name=op.f("fk_notification_deliveries_subscription_id_notification_subscriptions"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_notification_deliveries")),
|
||||
sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_deliveries_state_next",
|
||||
"notification_deliveries",
|
||||
["state", "next_attempt_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"schedules",
|
||||
sa.Column("job_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("cron", sa.String(length=255), nullable=False),
|
||||
sa.Column("timezone", sa.String(length=255), nullable=False),
|
||||
sa.Column("misfire_grace_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("overlap_policy", sa.String(length=32), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||
sa.Column("next_nominal_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_enqueue_outcome", sa.String(length=64), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"overlap_policy IN ('prohibit')", name=op.f("ck_schedules_overlap_policy")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"misfire_grace_seconds >= 0", name=op.f("ck_schedules_misfire_nonnegative")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"], ["jobs.id"], name=op.f("fk_schedules_job_id_jobs"), ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_schedules")),
|
||||
sa.UniqueConstraint("job_id", name=op.f("uq_schedules_job_id")),
|
||||
)
|
||||
op.create_table(
|
||||
"executions",
|
||||
sa.Column("job_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("schedule_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("nominal_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("trigger", sa.String(length=32), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("lease_owner", sa.String(length=255), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("progress", sa.JSON(), nullable=False),
|
||||
sa.Column("reason_code", sa.String(length=64), nullable=True),
|
||||
sa.Column("operator_message", sa.Text(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('queued','preparing','running','verifying','committed',"
|
||||
"'cancelling','cancelled','failed')",
|
||||
name=op.f("ck_executions_state"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"trigger IN ('manual','schedule','retry')", name=op.f("ck_executions_trigger")
|
||||
),
|
||||
sa.CheckConstraint("attempt > 0", name=op.f("ck_executions_attempt_positive")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"], ["jobs.id"], name=op.f("fk_executions_job_id_jobs"), ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["schedule_id"],
|
||||
["schedules.id"],
|
||||
name=op.f("fk_executions_schedule_id_schedules"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_executions")),
|
||||
sa.UniqueConstraint(
|
||||
"schedule_id", "nominal_run_at", name="uq_execution_schedule_occurrence"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_executions_job_id", "executions", ["job_id"], unique=False)
|
||||
op.create_index(
|
||||
"ix_executions_state_created", "executions", ["state", "created_at"], unique=False
|
||||
)
|
||||
op.create_index(
|
||||
"uq_executions_active_job",
|
||||
"executions",
|
||||
["job_id"],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||
)
|
||||
op.create_table(
|
||||
"backups",
|
||||
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("parent_backup_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("manifest_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("manifest_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("logical_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("stored_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("integrity", sa.String(length=32), nullable=False),
|
||||
sa.Column("pinned", sa.Boolean(), nullable=False),
|
||||
sa.Column("tombstoned_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"integrity IN ('unverified','verified','degraded','corrupt')",
|
||||
name=op.f("ck_backups_integrity"),
|
||||
),
|
||||
sa.CheckConstraint("logical_bytes >= 0", name=op.f("ck_backups_logical_bytes_nonnegative")),
|
||||
sa.CheckConstraint("stored_bytes >= 0", name=op.f("ck_backups_stored_bytes_nonnegative")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["execution_id"],
|
||||
["executions.id"],
|
||||
name=op.f("fk_backups_execution_id_executions"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["parent_backup_id"],
|
||||
["backups.id"],
|
||||
name=op.f("fk_backups_parent_backup_id_backups"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_backups")),
|
||||
sa.UniqueConstraint("execution_id", name=op.f("uq_backups_execution_id")),
|
||||
sa.UniqueConstraint("manifest_digest", name=op.f("uq_backups_manifest_digest")),
|
||||
sa.UniqueConstraint("manifest_id", name=op.f("uq_backups_manifest_id")),
|
||||
)
|
||||
op.create_index("ix_backups_created_at", "backups", ["created_at"], unique=False)
|
||||
op.create_table(
|
||||
"restores",
|
||||
sa.Column("backup_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("destination", sa.Text(), nullable=False),
|
||||
sa.Column("selection", sa.JSON(), nullable=False),
|
||||
sa.Column("overwrite_policy", sa.String(length=32), nullable=False),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column("result", sa.JSON(), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"overwrite_policy IN ('fail','skip','replace')",
|
||||
name=op.f("ck_restores_overwrite_policy"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('queued','running','committed','cancelled','failed')",
|
||||
name=op.f("ck_restores_state"),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["backup_id"],
|
||||
["backups.id"],
|
||||
name=op.f("fk_restores_backup_id_backups"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_restores")),
|
||||
)
|
||||
op.create_index("ix_restores_backup_id", "restores", ["backup_id"], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("ix_restores_backup_id", table_name="restores")
|
||||
op.drop_table("restores")
|
||||
op.drop_index("ix_backups_created_at", table_name="backups")
|
||||
op.drop_table("backups")
|
||||
op.drop_index(
|
||||
"uq_executions_active_job",
|
||||
table_name="executions",
|
||||
sqlite_where=sa.text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||
)
|
||||
op.drop_index("ix_executions_state_created", table_name="executions")
|
||||
op.drop_index("ix_executions_job_id", table_name="executions")
|
||||
op.drop_table("executions")
|
||||
op.drop_table("schedules")
|
||||
op.drop_index("ix_notification_deliveries_state_next", table_name="notification_deliveries")
|
||||
op.drop_table("notification_deliveries")
|
||||
op.drop_table("notification_subscriptions")
|
||||
op.drop_index("ix_jobs_source_id", table_name="jobs")
|
||||
op.drop_index("ix_jobs_repository_id", table_name="jobs")
|
||||
op.drop_table("jobs")
|
||||
op.drop_index("ix_idempotency_created_at", table_name="idempotency_records")
|
||||
op.drop_table("idempotency_records")
|
||||
op.drop_index("ix_audit_events_created_at", table_name="audit_events")
|
||||
op.drop_index("ix_audit_events_actor_id", table_name="audit_events")
|
||||
op.drop_table("audit_events")
|
||||
op.drop_index("ix_api_tokens_owner_id", table_name="api_tokens")
|
||||
op.drop_table("api_tokens")
|
||||
op.drop_table("users")
|
||||
op.drop_table("sources")
|
||||
op.drop_table("secrets")
|
||||
op.drop_table("repositories")
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,50 @@
|
||||
"""persist authenticated sessions
|
||||
|
||||
Revision ID: 0002_sessions
|
||||
Revises: 0001_v2_baseline
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002_sessions"
|
||||
down_revision: str | Sequence[str] | None = "0001_v2_baseline"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"sessions",
|
||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["user_id"],
|
||||
["users.id"],
|
||||
name=op.f("fk_sessions_user_id_users"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_sessions")),
|
||||
)
|
||||
op.create_index(op.f("ix_sessions_user_id"), "sessions", ["user_id"], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f("ix_sessions_user_id"), table_name="sessions")
|
||||
op.drop_table("sessions")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""add durable execution event revision
|
||||
|
||||
Revision ID: 0003_execution_events
|
||||
Revises: 0002_sessions
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0003_execution_events"
|
||||
down_revision = "0002_sessions"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("executions") as batch:
|
||||
batch.add_column(sa.Column("revision", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.create_table(
|
||||
"execution_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"execution_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("executions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.UniqueConstraint("execution_id", "revision", name="uq_execution_event_revision"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("execution_events")
|
||||
with op.batch_alter_table("executions") as batch:
|
||||
batch.drop_column("revision")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""bind repositories to manifest signing public keys
|
||||
|
||||
Revision ID: 0004_repository_signing_keys
|
||||
Revises: 0003_execution_events
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0004_repository_signing_keys"
|
||||
down_revision = "0003_execution_events"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("repositories") as batch:
|
||||
batch.add_column(
|
||||
sa.Column("signing_key_id", sa.String(length=64), nullable=False, server_default="")
|
||||
)
|
||||
batch.add_column(
|
||||
sa.Column("signing_public_key", sa.String(length=64), nullable=False, server_default="")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("repositories") as batch:
|
||||
batch.drop_column("signing_public_key")
|
||||
batch.drop_column("signing_key_id")
|
||||
@@ -0,0 +1,23 @@
|
||||
"""persist restore dry-run intent
|
||||
|
||||
Revision ID: 0005_restore_dry_run
|
||||
Revises: 0004_repository_signing_keys
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0005_restore_dry_run"
|
||||
down_revision = "0004_repository_signing_keys"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("restores") as batch:
|
||||
batch.add_column(sa.Column("dry_run", sa.Boolean(), nullable=False, server_default="0"))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("restores") as batch:
|
||||
batch.drop_column("dry_run")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""restrict persisted sources to local
|
||||
|
||||
Revision ID: 0006_local_sources_only
|
||||
Revises: 0005_restore_dry_run
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0006_local_sources_only"
|
||||
down_revision = "0005_restore_dry_run"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _reject_nonlocal_sources() -> None:
|
||||
connection = op.get_bind()
|
||||
sources = sa.table("sources", sa.column("kind"))
|
||||
count = connection.scalar(
|
||||
sa.select(sa.func.count()).select_from(sources).where(sources.c.kind != "local")
|
||||
)
|
||||
if count is None:
|
||||
raise RuntimeError("Cannot inspect persisted source kinds before migration.")
|
||||
if count:
|
||||
raise RuntimeError(
|
||||
"Cannot restrict sources to local: "
|
||||
f"found {count} non-local source row(s). Remove or migrate them before upgrading."
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_reject_nonlocal_sources()
|
||||
with op.batch_alter_table("sources") as batch:
|
||||
batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
|
||||
batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("sources") as batch:
|
||||
batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
|
||||
batch.create_check_constraint(
|
||||
op.f("ck_sources_kind"), "kind IN ('local','sftp','postgresql','mysql')"
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
"""add repository data key epochs
|
||||
|
||||
Revision ID: 0007_repository_data_key_epochs
|
||||
Revises: 0006_local_sources_only
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0007_repository_data_key_epochs"
|
||||
down_revision = "0006_local_sources_only"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("repositories", sa.Column("active_data_key_id", sa.String(36), nullable=True))
|
||||
op.add_column("backups", sa.Column("data_key_id", sa.String(36), nullable=True))
|
||||
op.create_table(
|
||||
"repository_data_key_epochs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.Column(
|
||||
"repository_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("repositories.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("key_id", sa.String(36), nullable=False),
|
||||
sa.Column("state", sa.String(16), nullable=False),
|
||||
sa.Column("retired_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint(
|
||||
"repository_id", "key_id", name="uq_repository_data_key_epochs_repository_key_epoch"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('active','retired')",
|
||||
name="ck_repository_data_key_epochs_repository_data_key_epoch_state",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_repository_data_key_epochs_repository_id",
|
||||
"repository_data_key_epochs",
|
||||
["repository_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"uq_repository_data_key_epochs_active",
|
||||
"repository_data_key_epochs",
|
||||
["repository_id"],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("state = 'active'"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
epochs = sa.table("repository_data_key_epochs")
|
||||
repositories = sa.table("repositories", sa.column("active_data_key_id"))
|
||||
backups = sa.table("backups", sa.column("data_key_id"))
|
||||
epoch_count = connection.scalar(sa.select(sa.func.count()).select_from(epochs))
|
||||
active_key_count = connection.scalar(
|
||||
sa.select(sa.func.count())
|
||||
.select_from(repositories)
|
||||
.where(repositories.c.active_data_key_id.is_not(None))
|
||||
)
|
||||
backup_key_count = connection.scalar(
|
||||
sa.select(sa.func.count()).select_from(backups).where(backups.c.data_key_id.is_not(None))
|
||||
)
|
||||
if epoch_count or active_key_count or backup_key_count:
|
||||
raise RuntimeError("cannot downgrade while repository data key metadata exists")
|
||||
op.drop_index("uq_repository_data_key_epochs_active", table_name="repository_data_key_epochs")
|
||||
op.drop_index(
|
||||
"ix_repository_data_key_epochs_repository_id", table_name="repository_data_key_epochs"
|
||||
)
|
||||
op.drop_table("repository_data_key_epochs")
|
||||
op.drop_column("backups", "data_key_id")
|
||||
op.drop_column("repositories", "active_data_key_id")
|
||||
@@ -0,0 +1,445 @@
|
||||
"""replace notification attempt stub with a durable M12 outbox
|
||||
|
||||
Revision ID: 0008_notification_outbox
|
||||
Revises: 0007_repository_data_key_epochs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from backup_tool.ids import new_uuid7
|
||||
|
||||
revision = "0008_notification_outbox"
|
||||
down_revision = "0007_repository_data_key_epochs"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _parameterized_execute(connection: sa.Connection, statement: object) -> Any:
|
||||
"""Execute only SQLAlchemy Core statements, never dynamic SQL strings."""
|
||||
database = connection.execution_options()
|
||||
return database.execute(statement) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
# SQLite batch-rebuilds notification_subscriptions. Its old delivery table
|
||||
# references this table, so enforcement must be suspended for this migration
|
||||
# only while the legacy rows are copied into the replacement outbox shape.
|
||||
if connection.dialect.name == "sqlite":
|
||||
connection.exec_driver_sql("PRAGMA foreign_keys=OFF")
|
||||
# Extend the existing subscription rows first: pre-M12 rows stay disabled until
|
||||
# an operator explicitly configures a new credential/key.
|
||||
with op.batch_alter_table("notification_subscriptions") as batch:
|
||||
batch.add_column(
|
||||
sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60")
|
||||
)
|
||||
batch.add_column(sa.Column("rate_tokens", sa.Float(), nullable=False, server_default="60"))
|
||||
batch.add_column(sa.Column("rate_updated_at", sa.DateTime(timezone=True), nullable=True))
|
||||
batch.add_column(sa.Column("revision", sa.Integer(), nullable=False, server_default="1"))
|
||||
batch.create_check_constraint("rate_positive", "rate_limit_per_minute > 0")
|
||||
batch.create_check_constraint("rate_tokens_nonnegative", "rate_tokens >= 0")
|
||||
batch.create_check_constraint("revision_positive", "revision > 0")
|
||||
|
||||
op.create_table(
|
||||
"notification_events",
|
||||
sa.Column("type", sa.String(96), nullable=False),
|
||||
sa.Column("schema_version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("correlation_id", sa.String(36), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False),
|
||||
sa.Column("resource_refs", sa.JSON(), nullable=False),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.Column("canonical_envelope", sa.Text(), nullable=False),
|
||||
sa.Column("deduplication_key", sa.String(255), nullable=True),
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.CheckConstraint("schema_version = 1", name="ck_notification_events_schema_version"),
|
||||
sa.CheckConstraint(
|
||||
"severity IN ('info','warning','error','security')",
|
||||
name="ck_notification_events_severity",
|
||||
),
|
||||
sa.UniqueConstraint("deduplication_key", name="uq_notification_events_deduplication_key"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_events_type_occurred", "notification_events", ["type", "occurred_at"]
|
||||
)
|
||||
|
||||
# Preserve unexpected rows from the unused baseline shape. Renaming first
|
||||
# keeps the original data intact if an upgrade is interrupted before copy.
|
||||
op.rename_table("notification_deliveries", "notification_deliveries_legacy")
|
||||
op.create_table(
|
||||
"notification_deliveries",
|
||||
sa.Column(
|
||||
"event_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("notification_events.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"subscription_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("due_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("lease_owner", sa.String(255), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("terminal_reason", sa.String(96), nullable=True),
|
||||
sa.Column("response_class", sa.String(64), nullable=True),
|
||||
sa.Column("response_summary", sa.String(512), nullable=True),
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"attempt_count >= 0", name="ck_notification_deliveries_attempt_count_nonnegative"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('pending','leased','delivered','retry','failed')",
|
||||
name="ck_notification_deliveries_state",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"event_id", "subscription_id", name="uq_notification_deliveries_event_subscription"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_deliveries_due", "notification_deliveries", ["state", "due_at"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_deliveries_lease", "notification_deliveries", ["state", "lease_expires_at"]
|
||||
)
|
||||
op.create_table(
|
||||
"notification_delivery_attempts",
|
||||
sa.Column(
|
||||
"delivery_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("notification_deliveries.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("outcome", sa.String(32), nullable=False, server_default="started"),
|
||||
sa.Column("response_class", sa.String(64), nullable=True),
|
||||
sa.Column("diagnostic", sa.String(512), nullable=True),
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.CheckConstraint("number > 0", name="ck_notification_delivery_attempts_number_positive"),
|
||||
sa.CheckConstraint(
|
||||
"outcome IN ('started','delivered','retry','failed')",
|
||||
name="ck_notification_delivery_attempts_outcome",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"delivery_id", "number", name="uq_notification_delivery_attempts_delivery_number"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_attempts_delivery",
|
||||
"notification_delivery_attempts",
|
||||
["delivery_id", "number"],
|
||||
)
|
||||
|
||||
legacy = sa.table(
|
||||
"notification_deliveries_legacy",
|
||||
sa.column("id"),
|
||||
sa.column("event_id"),
|
||||
sa.column("subscription_id"),
|
||||
sa.column("attempt"),
|
||||
sa.column("state"),
|
||||
sa.column("response_class"),
|
||||
sa.column("next_attempt_at"),
|
||||
sa.column("created_at"),
|
||||
sa.column("updated_at"),
|
||||
)
|
||||
rows = _parameterized_execute(connection, sa.select(legacy)).mappings().all()
|
||||
events = sa.table(
|
||||
"notification_events",
|
||||
sa.column("id"),
|
||||
sa.column("type"),
|
||||
sa.column("schema_version"),
|
||||
sa.column("occurred_at"),
|
||||
sa.column("correlation_id"),
|
||||
sa.column("severity"),
|
||||
sa.column("resource_refs", sa.JSON()),
|
||||
sa.column("payload", sa.JSON()),
|
||||
sa.column("canonical_envelope"),
|
||||
sa.column("deduplication_key"),
|
||||
)
|
||||
deliveries = sa.table(
|
||||
"notification_deliveries",
|
||||
*[
|
||||
sa.column(name)
|
||||
for name in (
|
||||
"id",
|
||||
"event_id",
|
||||
"subscription_id",
|
||||
"state",
|
||||
"due_at",
|
||||
"attempt_count",
|
||||
"terminal_reason",
|
||||
"response_class",
|
||||
"response_summary",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
)
|
||||
],
|
||||
)
|
||||
attempts = sa.table(
|
||||
"notification_delivery_attempts",
|
||||
*[
|
||||
sa.column(name)
|
||||
for name in (
|
||||
"id",
|
||||
"delivery_id",
|
||||
"number",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"outcome",
|
||||
"response_class",
|
||||
"diagnostic",
|
||||
)
|
||||
],
|
||||
)
|
||||
for row in rows:
|
||||
occurred = row["created_at"] or _now()
|
||||
event_id, delivery_id, attempt_id = (str(new_uuid7()), str(new_uuid7()), str(new_uuid7()))
|
||||
payload = {"legacy_event_id": str(row["event_id"]), "legacy_delivery_id": str(row["id"])}
|
||||
envelope = {
|
||||
"event_schema_version": 1,
|
||||
"id": event_id,
|
||||
"type": "notification.legacy",
|
||||
"occurred_at": occurred.isoformat()
|
||||
if hasattr(occurred, "isoformat")
|
||||
else str(occurred),
|
||||
"correlation_id": event_id,
|
||||
"severity": "warning",
|
||||
"resource": {"subscription_id": str(row["subscription_id"])},
|
||||
"payload": payload,
|
||||
}
|
||||
_parameterized_execute(
|
||||
connection,
|
||||
events.insert().values(
|
||||
id=event_id,
|
||||
type="notification.legacy",
|
||||
schema_version=1,
|
||||
occurred_at=occurred,
|
||||
correlation_id=event_id,
|
||||
severity="warning",
|
||||
resource_refs=envelope["resource"],
|
||||
payload=payload,
|
||||
canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
|
||||
deduplication_key=f"legacy:{row['id']}",
|
||||
),
|
||||
)
|
||||
old_state = str(row["state"])
|
||||
new_state = "retry" if old_state == "pending" else old_state
|
||||
try:
|
||||
legacy_attempt = max(1, int(row["attempt"]))
|
||||
except (TypeError, ValueError) as error:
|
||||
raise RuntimeError("legacy notification delivery has an invalid attempt") from error
|
||||
_parameterized_execute(
|
||||
connection,
|
||||
deliveries.insert().values(
|
||||
id=delivery_id,
|
||||
event_id=event_id,
|
||||
subscription_id=row["subscription_id"],
|
||||
state=new_state,
|
||||
due_at=row["next_attempt_at"] or occurred,
|
||||
attempt_count=legacy_attempt,
|
||||
terminal_reason="legacy_migrated" if new_state == "failed" else None,
|
||||
response_class=row["response_class"],
|
||||
response_summary="legacy delivery migrated",
|
||||
created_at=occurred,
|
||||
updated_at=row["updated_at"] or occurred,
|
||||
),
|
||||
)
|
||||
_parameterized_execute(
|
||||
connection,
|
||||
attempts.insert().values(
|
||||
id=attempt_id,
|
||||
delivery_id=delivery_id,
|
||||
number=legacy_attempt,
|
||||
started_at=occurred,
|
||||
completed_at=row["updated_at"] if new_state in {"delivered", "failed"} else None,
|
||||
outcome="delivered"
|
||||
if new_state == "delivered"
|
||||
else ("failed" if new_state == "failed" else "retry"),
|
||||
response_class=row["response_class"],
|
||||
diagnostic="legacy delivery migrated",
|
||||
),
|
||||
)
|
||||
op.drop_table("notification_deliveries_legacy")
|
||||
|
||||
op.create_table(
|
||||
"notification_signing_keys",
|
||||
sa.Column(
|
||||
"subscription_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("version", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"secret_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("state", sa.String(16), nullable=False, server_default="active"),
|
||||
sa.Column("overlap_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.CheckConstraint("version > 0", name="ck_notification_signing_keys_version_positive"),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('active','overlap','retired')", name="ck_notification_signing_keys_state"
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"subscription_id", "version", name="uq_notification_signing_keys_subscription_version"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_signing_keys_subscription",
|
||||
"notification_signing_keys",
|
||||
["subscription_id", "state"],
|
||||
)
|
||||
op.create_index(
|
||||
"uq_notification_signing_keys_active",
|
||||
"notification_signing_keys",
|
||||
["subscription_id"],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("state = 'active'"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_notification_signing_keys_overlap",
|
||||
"notification_signing_keys",
|
||||
["subscription_id"],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("state = 'overlap'"),
|
||||
)
|
||||
op.create_table(
|
||||
"notification_email_settings",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("host", sa.String(255), nullable=False),
|
||||
sa.Column("port", sa.Integer(), nullable=False, server_default="587"),
|
||||
sa.Column("username", sa.String(255), nullable=False),
|
||||
sa.Column(
|
||||
"password_secret_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("sender", sa.String(320), nullable=False),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="5"),
|
||||
sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60"),
|
||||
sa.CheckConstraint("id = 1", name="ck_notification_email_settings_singleton"),
|
||||
sa.CheckConstraint("port > 0 AND port < 65536", name="ck_notification_email_settings_port"),
|
||||
sa.CheckConstraint("max_attempts > 0", name="ck_notification_email_settings_max_attempts"),
|
||||
sa.CheckConstraint(
|
||||
"rate_limit_per_minute > 0", name="ck_notification_email_settings_rate_positive"
|
||||
),
|
||||
)
|
||||
if connection.dialect.name == "sqlite":
|
||||
connection.exec_driver_sql("PRAGMA foreign_keys=ON")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
for table in (
|
||||
"notification_events",
|
||||
"notification_delivery_attempts",
|
||||
"notification_signing_keys",
|
||||
"notification_email_settings",
|
||||
):
|
||||
if bind.scalar(sa.text(f"SELECT count(*) FROM {table}")):
|
||||
raise RuntimeError(
|
||||
"cannot downgrade while M12 notification history or configuration exists"
|
||||
)
|
||||
# A fresh M12 schema can safely return to the historical stub shape.
|
||||
op.drop_table("notification_email_settings")
|
||||
op.drop_index("uq_notification_signing_keys_overlap", table_name="notification_signing_keys")
|
||||
op.drop_index("uq_notification_signing_keys_active", table_name="notification_signing_keys")
|
||||
op.drop_index(
|
||||
"ix_notification_signing_keys_subscription", table_name="notification_signing_keys"
|
||||
)
|
||||
op.drop_table("notification_signing_keys")
|
||||
op.drop_index("ix_notification_attempts_delivery", table_name="notification_delivery_attempts")
|
||||
op.drop_table("notification_delivery_attempts")
|
||||
op.drop_index("ix_notification_deliveries_lease", table_name="notification_deliveries")
|
||||
op.drop_index("ix_notification_deliveries_due", table_name="notification_deliveries")
|
||||
op.drop_table("notification_deliveries")
|
||||
op.drop_index("ix_notification_events_type_occurred", table_name="notification_events")
|
||||
op.drop_table("notification_events")
|
||||
op.create_table(
|
||||
"notification_deliveries",
|
||||
sa.Column("event_id", sa.String(36), nullable=False),
|
||||
sa.Column("subscription_id", sa.String(36), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("response_class", sa.String(64)),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
),
|
||||
sa.CheckConstraint("attempt > 0", name="ck_notification_deliveries_attempt_positive"),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('pending','delivered','retry','failed')",
|
||||
name="ck_notification_deliveries_state",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["subscription_id"], ["notification_subscriptions.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_notification_deliveries_state_next",
|
||||
"notification_deliveries",
|
||||
["state", "next_attempt_at"],
|
||||
)
|
||||
with op.batch_alter_table("notification_subscriptions") as batch:
|
||||
batch.drop_constraint("revision_positive", type_="check")
|
||||
batch.drop_constraint("rate_tokens_nonnegative", type_="check")
|
||||
batch.drop_constraint("rate_positive", type_="check")
|
||||
batch.drop_column("revision")
|
||||
batch.drop_column("rate_updated_at")
|
||||
batch.drop_column("rate_tokens")
|
||||
batch.drop_column("rate_limit_per_minute")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""allow staged SSH source definitions
|
||||
|
||||
Revision ID: 0009_allow_ssh_sources
|
||||
Revises: 0008_notification_outbox
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0009_allow_ssh_sources"
|
||||
down_revision = "0008_notification_outbox"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _reject_ssh_sources() -> None:
|
||||
connection = op.get_bind()
|
||||
sources = sa.table("sources", sa.column("kind"))
|
||||
count = connection.scalar(
|
||||
sa.select(sa.func.count()).select_from(sources).where(sources.c.kind == "ssh")
|
||||
)
|
||||
if count is None:
|
||||
raise RuntimeError("Cannot inspect persisted SSH sources before migration.")
|
||||
if count:
|
||||
raise RuntimeError(
|
||||
"Cannot restrict sources to local: "
|
||||
f"found {count} SSH source row(s). Remove them before downgrading."
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("sources") as batch:
|
||||
batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
|
||||
batch.create_check_constraint(op.f("ck_sources_kind"), "kind IN ('local','ssh')")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
_reject_ssh_sources()
|
||||
with op.batch_alter_table("sources") as batch:
|
||||
batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
|
||||
batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,17 +0,0 @@
|
||||
import os
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///./backup_tool.db")
|
||||
|
||||
engine = create_async_engine(DATABASE_URL, echo=os.environ.get("SQL_ECHO", "false").lower() == "true")
|
||||
AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
async def get_db():
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
await session.close()
|
||||
@@ -1,60 +0,0 @@
|
||||
import os
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
from contextlib import asynccontextmanager
|
||||
from app.database import engine, Base
|
||||
from app import models # noqa: F401 - registers models with Base.metadata
|
||||
from app.routers import sources, jobs, executions, backups, settings, dashboard
|
||||
from backup.scheduler import backup_scheduler
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
backup_scheduler.start()
|
||||
await backup_scheduler.sync_schedules()
|
||||
yield
|
||||
backup_scheduler.shutdown()
|
||||
|
||||
app = FastAPI(
|
||||
title="Backup Tool API",
|
||||
version="0.1.0",
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=os.environ.get("CORS_ORIGINS", "http://localhost:3000").split(","),
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(sources.router)
|
||||
app.include_router(jobs.router)
|
||||
app.include_router(executions.router)
|
||||
app.include_router(backups.router)
|
||||
app.include_router(settings.router)
|
||||
app.include_router(dashboard.router)
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health_check():
|
||||
return {"status": "healthy"}
|
||||
|
||||
def main():
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True)
|
||||
|
||||
# Static files for production
|
||||
frontend_dist = os.path.join(os.path.dirname(__file__), "../../frontend/dist")
|
||||
if os.path.exists(frontend_dist):
|
||||
app.mount("/assets", StaticFiles(directory=os.path.join(frontend_dist, "assets")), name="assets")
|
||||
|
||||
@app.get("/{path:path}")
|
||||
async def serve_frontend(path: str):
|
||||
index_file = os.path.join(frontend_dist, "index.html")
|
||||
if os.path.exists(index_file):
|
||||
return FileResponse(index_file)
|
||||
return {"detail": "Frontend not built"}
|
||||
@@ -1,137 +0,0 @@
|
||||
from sqlalchemy import Column, Integer, String, DateTime, Boolean, Text, ForeignKey, JSON
|
||||
from sqlalchemy.orm import relationship
|
||||
from datetime import datetime, timezone
|
||||
from .database import Base
|
||||
|
||||
|
||||
class Source(Base):
|
||||
__tablename__ = "sources"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String, nullable=False)
|
||||
type = Column(String, nullable=False) # local, ssh, database
|
||||
config = Column(JSON, default=lambda: {})
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
jobs = relationship("Job", back_populates="source", cascade="all, delete-orphan")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Source(id={self.id}, name='{self.name}', type='{self.type}')>"
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String, nullable=False)
|
||||
source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
|
||||
strategy = Column(String, nullable=False, default="full") # full, incremental
|
||||
destination_path = Column(String, nullable=False)
|
||||
exclude_patterns = Column(JSON, default=lambda: [])
|
||||
enabled = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
source = relationship("Source", back_populates="jobs")
|
||||
schedule = relationship(
|
||||
"Schedule", back_populates="job", uselist=False, cascade="all, delete-orphan"
|
||||
)
|
||||
executions = relationship(
|
||||
"JobExecution", back_populates="job", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Job(id={self.id}, name='{self.name}', source_id={self.source_id})>"
|
||||
|
||||
|
||||
class Schedule(Base):
|
||||
__tablename__ = "schedules"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
job_id = Column(
|
||||
Integer, ForeignKey("jobs.id"), unique=True, nullable=False, index=True
|
||||
)
|
||||
cron_expression = Column(String, nullable=False)
|
||||
enabled = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
job = relationship("Job", back_populates="schedule")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Schedule(id={self.id}, job_id={self.job_id}, cron='{self.cron_expression}')>"
|
||||
|
||||
|
||||
class JobExecution(Base):
|
||||
__tablename__ = "job_executions"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
job_id = Column(Integer, ForeignKey("jobs.id"), nullable=False, index=True)
|
||||
status = Column(
|
||||
String, nullable=False, default="pending"
|
||||
) # pending, running, success, failed, cancelled
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
bytes_processed = Column(Integer, default=0)
|
||||
bytes_backed_up = Column(Integer, default=0)
|
||||
error_message = Column(Text, nullable=True)
|
||||
triggered_by = Column(String, nullable=False) # manual, schedule
|
||||
|
||||
job = relationship("Job", back_populates="executions")
|
||||
backups = relationship(
|
||||
"Backup", back_populates="execution", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<JobExecution(id={self.id}, job_id={self.job_id}, status='{self.status}')>"
|
||||
|
||||
|
||||
class Backup(Base):
|
||||
__tablename__ = "backups"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
execution_id = Column(
|
||||
Integer, ForeignKey("job_executions.id"), nullable=False, index=True
|
||||
)
|
||||
storage_path = Column(String, nullable=False)
|
||||
size_bytes = Column(Integer, default=0)
|
||||
checksum = Column(String, nullable=True)
|
||||
type = Column(String, nullable=False) # full, incremental
|
||||
parent_backup_id = Column(
|
||||
Integer, ForeignKey("backups.id"), nullable=True, index=True
|
||||
)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
execution = relationship("JobExecution", back_populates="backups")
|
||||
parent_backup = relationship("Backup", remote_side=[id])
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Backup(id={self.id}, execution_id={self.execution_id}, type='{self.type}')>"
|
||||
|
||||
|
||||
class Setting(Base):
|
||||
__tablename__ = "settings"
|
||||
|
||||
key = Column(String, primary_key=True)
|
||||
value = Column(Text, nullable=True)
|
||||
updated_at = Column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Setting(key='{self.key}', value='{self.value}')>"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,34 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List
|
||||
from app.database import get_db
|
||||
from app.models import Backup
|
||||
from app.schemas import Backup as BackupSchema
|
||||
|
||||
router = APIRouter(prefix="/api/backups", tags=["backups"])
|
||||
|
||||
@router.get("/", response_model=List[BackupSchema])
|
||||
async def list_backups(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Backup).order_by(Backup.created_at.desc()))
|
||||
backups = result.scalars().all()
|
||||
return backups
|
||||
|
||||
@router.get("/{backup_id}", response_model=BackupSchema)
|
||||
async def get_backup(backup_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Backup).where(Backup.id == backup_id))
|
||||
backup = result.scalar_one_or_none()
|
||||
if not backup:
|
||||
raise HTTPException(status_code=404, detail="Backup not found")
|
||||
return backup
|
||||
|
||||
@router.delete("/{backup_id}")
|
||||
async def delete_backup(backup_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Backup).where(Backup.id == backup_id))
|
||||
backup = result.scalar_one_or_none()
|
||||
if not backup:
|
||||
raise HTTPException(status_code=404, detail="Backup not found")
|
||||
|
||||
await db.delete(backup)
|
||||
await db.commit()
|
||||
return {"message": "Backup deleted"}
|
||||
@@ -1,54 +0,0 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, and_
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Job, JobExecution, Backup
|
||||
from app.schemas import DashboardStats, JobExecution as JobExecutionSchema
|
||||
|
||||
router = APIRouter(prefix="/api/dashboard", tags=["dashboard"])
|
||||
|
||||
|
||||
@router.get("/", response_model=DashboardStats)
|
||||
async def get_dashboard_stats(db: AsyncSession = Depends(get_db)):
|
||||
# Active jobs count (enabled jobs)
|
||||
active_jobs_result = await db.execute(
|
||||
select(func.count(Job.id)).where(Job.enabled == True)
|
||||
)
|
||||
active_jobs = active_jobs_result.scalar() or 0
|
||||
|
||||
# Total backups count
|
||||
total_backups_result = await db.execute(select(func.count(Backup.id)))
|
||||
total_backups = total_backups_result.scalar() or 0
|
||||
|
||||
# Storage used bytes (sum of all backup sizes)
|
||||
storage_used_result = await db.execute(select(func.sum(Backup.size_bytes)))
|
||||
storage_used_bytes = storage_used_result.scalar() or 0
|
||||
|
||||
# Recent failures count (last 24 hours)
|
||||
twenty_four_hours_ago = datetime.now(timezone.utc) - timedelta(hours=24)
|
||||
recent_failures_result = await db.execute(
|
||||
select(func.count(JobExecution.id)).where(
|
||||
and_(
|
||||
JobExecution.status == "failed",
|
||||
JobExecution.completed_at >= twenty_four_hours_ago,
|
||||
)
|
||||
)
|
||||
)
|
||||
recent_failures = recent_failures_result.scalar() or 0
|
||||
|
||||
# Recent executions (last 10, ordered by started_at desc)
|
||||
recent_executions_result = await db.execute(
|
||||
select(JobExecution).order_by(JobExecution.started_at.desc()).limit(10)
|
||||
)
|
||||
recent_executions = recent_executions_result.scalars().all()
|
||||
|
||||
return DashboardStats(
|
||||
active_jobs=active_jobs,
|
||||
total_backups=total_backups,
|
||||
storage_used_bytes=storage_used_bytes,
|
||||
recent_failures=recent_failures,
|
||||
recent_executions=list(recent_executions),
|
||||
)
|
||||
@@ -1,23 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List
|
||||
from app.database import get_db
|
||||
from app.models import JobExecution
|
||||
from app.schemas import JobExecution as JobExecutionSchema
|
||||
|
||||
router = APIRouter(prefix="/api/executions", tags=["executions"])
|
||||
|
||||
@router.get("/", response_model=List[JobExecutionSchema])
|
||||
async def list_executions(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(JobExecution).order_by(JobExecution.started_at.desc()))
|
||||
executions = result.scalars().all()
|
||||
return executions
|
||||
|
||||
@router.get("/{execution_id}", response_model=JobExecutionSchema)
|
||||
async def get_execution(execution_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(JobExecution).where(JobExecution.id == execution_id))
|
||||
execution = result.scalar_one_or_none()
|
||||
if not execution:
|
||||
raise HTTPException(status_code=404, detail="Execution not found")
|
||||
return execution
|
||||
@@ -1,101 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List
|
||||
from app.database import get_db, AsyncSessionLocal
|
||||
from app.models import Job, Schedule
|
||||
from app.schemas import JobCreate, JobUpdate, Job as JobSchema, ScheduleCreate, Schedule as ScheduleSchema
|
||||
from backup.engine import BackupEngine
|
||||
|
||||
router = APIRouter(prefix="/api/jobs", tags=["jobs"])
|
||||
|
||||
@router.get("/", response_model=List[JobSchema])
|
||||
async def list_jobs(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Job))
|
||||
jobs = result.scalars().all()
|
||||
return jobs
|
||||
|
||||
@router.post("/", response_model=JobSchema)
|
||||
async def create_job(job: JobCreate, db: AsyncSession = Depends(get_db)):
|
||||
db_job = Job(**job.model_dump())
|
||||
db.add(db_job)
|
||||
await db.commit()
|
||||
await db.refresh(db_job)
|
||||
return db_job
|
||||
|
||||
@router.get("/{job_id}", response_model=JobSchema)
|
||||
async def get_job(job_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Job).where(Job.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return job
|
||||
|
||||
@router.put("/{job_id}", response_model=JobSchema)
|
||||
async def update_job(
|
||||
job_id: int,
|
||||
job_update: JobUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
result = await db.execute(select(Job).where(Job.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
update_data = job_update.model_dump(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
setattr(job, field, value)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
return job
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
async def delete_job(job_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Job).where(Job.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
await db.delete(job)
|
||||
await db.commit()
|
||||
return {"message": "Job deleted"}
|
||||
|
||||
@router.post("/{job_id}/run")
|
||||
async def run_job(
|
||||
job_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
result = await db.execute(select(Job).where(Job.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Run in background
|
||||
async def execute():
|
||||
async with AsyncSessionLocal() as session:
|
||||
engine = BackupEngine(session)
|
||||
await engine.execute_job(job_id, triggered_by="manual")
|
||||
|
||||
background_tasks.add_task(execute)
|
||||
return {"message": "Job execution started"}
|
||||
|
||||
@router.post("/{job_id}/schedule", response_model=ScheduleSchema)
|
||||
async def create_schedule(
|
||||
job_id: int,
|
||||
schedule: ScheduleCreate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
result = await db.execute(select(Job).where(Job.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
schedule_data = schedule.model_dump()
|
||||
schedule_data["job_id"] = job_id
|
||||
db_schedule = Schedule(**schedule_data)
|
||||
db.add(db_schedule)
|
||||
await db.commit()
|
||||
await db.refresh(db_schedule)
|
||||
return ScheduleSchema.model_validate(db_schedule)
|
||||
@@ -1,43 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List
|
||||
from app.database import get_db
|
||||
from app.models import Setting
|
||||
from app.schemas import Setting as SettingSchema, SettingUpdate
|
||||
|
||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
|
||||
@router.get("/", response_model=List[SettingSchema])
|
||||
async def list_settings(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Setting))
|
||||
settings = result.scalars().all()
|
||||
return settings
|
||||
|
||||
@router.get("/{key}", response_model=SettingSchema)
|
||||
async def get_setting(key: str, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Setting).where(Setting.key == key))
|
||||
setting = result.scalar_one_or_none()
|
||||
if not setting:
|
||||
raise HTTPException(status_code=404, detail="Setting not found")
|
||||
return setting
|
||||
|
||||
@router.put("/{key}", response_model=SettingSchema)
|
||||
async def update_setting(
|
||||
key: str,
|
||||
setting_update: SettingUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
result = await db.execute(select(Setting).where(Setting.key == key))
|
||||
setting = result.scalar_one_or_none()
|
||||
|
||||
if not setting:
|
||||
# Create if not exists
|
||||
setting = Setting(key=key, value=setting_update.value)
|
||||
db.add(setting)
|
||||
else:
|
||||
setting.value = setting_update.value
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(setting)
|
||||
return setting
|
||||
@@ -1,61 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List
|
||||
from app.database import get_db
|
||||
from app.models import Source
|
||||
from app.schemas import SourceCreate, SourceUpdate, Source as SourceSchema
|
||||
|
||||
router = APIRouter(prefix="/api/sources", tags=["sources"])
|
||||
|
||||
@router.get("/", response_model=List[SourceSchema])
|
||||
async def list_sources(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Source))
|
||||
sources = result.scalars().all()
|
||||
return sources
|
||||
|
||||
@router.post("/", response_model=SourceSchema)
|
||||
async def create_source(source: SourceCreate, db: AsyncSession = Depends(get_db)):
|
||||
db_source = Source(**source.model_dump())
|
||||
db.add(db_source)
|
||||
await db.commit()
|
||||
await db.refresh(db_source)
|
||||
return db_source
|
||||
|
||||
@router.get("/{source_id}", response_model=SourceSchema)
|
||||
async def get_source(source_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Source).where(Source.id == source_id))
|
||||
source = result.scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source not found")
|
||||
return source
|
||||
|
||||
@router.put("/{source_id}", response_model=SourceSchema)
|
||||
async def update_source(
|
||||
source_id: int,
|
||||
source_update: SourceUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
result = await db.execute(select(Source).where(Source.id == source_id))
|
||||
source = result.scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source not found")
|
||||
|
||||
update_data = source_update.model_dump(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
setattr(source, field, value)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(source)
|
||||
return source
|
||||
|
||||
@router.delete("/{source_id}")
|
||||
async def delete_source(source_id: int, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Source).where(Source.id == source_id))
|
||||
source = result.scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source not found")
|
||||
|
||||
await db.delete(source)
|
||||
await db.commit()
|
||||
return {"message": "Source deleted"}
|
||||
@@ -1,143 +0,0 @@
|
||||
from pydantic import BaseModel, Field, ConfigDict, field_validator
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
# Source schemas
|
||||
class SourceBase(BaseModel):
|
||||
name: str
|
||||
type: str = Field(..., pattern="^(local|ssh|database)$")
|
||||
config: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
class SourceCreate(SourceBase):
|
||||
pass
|
||||
|
||||
class SourceUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
|
||||
class Source(SourceBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# Job schemas
|
||||
class JobBase(BaseModel):
|
||||
name: str
|
||||
source_id: int
|
||||
strategy: str = Field(default="full", pattern="^(full|incremental)$")
|
||||
destination_path: str
|
||||
exclude_patterns: List[str] = Field(default_factory=list)
|
||||
enabled: bool = True
|
||||
|
||||
class JobCreate(JobBase):
|
||||
pass
|
||||
|
||||
class JobUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
strategy: Optional[str] = None
|
||||
destination_path: Optional[str] = None
|
||||
exclude_patterns: Optional[List[str]] = None
|
||||
enabled: Optional[bool] = None
|
||||
|
||||
class Job(JobBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# Schedule schemas
|
||||
class ScheduleBase(BaseModel):
|
||||
job_id: int
|
||||
cron_expression: str
|
||||
enabled: bool = True
|
||||
|
||||
@field_validator('cron_expression')
|
||||
@classmethod
|
||||
def validate_cron(cls, v: str) -> str:
|
||||
pattern = r'^([0-9*,/-]+)\s+([0-9*,/-]+)\s+([0-9*,/-]+)\s+([0-9*,/-]+)\s+([0-9*,/-]+)$'
|
||||
if not re.match(pattern, v):
|
||||
raise ValueError('Invalid cron expression format')
|
||||
return v
|
||||
|
||||
class ScheduleCreate(ScheduleBase):
|
||||
pass
|
||||
|
||||
class ScheduleUpdate(BaseModel):
|
||||
cron_expression: Optional[str] = None
|
||||
enabled: Optional[bool] = None
|
||||
|
||||
class Schedule(ScheduleBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# Execution schemas
|
||||
class JobExecutionBase(BaseModel):
|
||||
job_id: int
|
||||
status: str = Field(default="pending", pattern="^(pending|running|success|failed|cancelled)$")
|
||||
triggered_by: str = Field(..., pattern="^(manual|schedule)$")
|
||||
|
||||
class JobExecutionCreate(JobExecutionBase):
|
||||
pass
|
||||
|
||||
class JobExecution(JobExecutionBase):
|
||||
id: int
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
bytes_processed: int = 0
|
||||
bytes_backed_up: int = 0
|
||||
error_message: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
class JobExecutionUpdate(BaseModel):
|
||||
status: Optional[str] = Field(None, pattern="^(pending|running|success|failed|cancelled)$")
|
||||
error_message: Optional[str] = None
|
||||
|
||||
# Backup schemas
|
||||
class BackupBase(BaseModel):
|
||||
execution_id: int
|
||||
storage_path: str
|
||||
size_bytes: int = 0
|
||||
checksum: Optional[str] = None
|
||||
type: str = Field(..., pattern="^(full|incremental)$")
|
||||
parent_backup_id: Optional[int] = None
|
||||
|
||||
class BackupCreate(BackupBase):
|
||||
pass
|
||||
|
||||
class Backup(BackupBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# Settings schemas
|
||||
class SettingBase(BaseModel):
|
||||
key: str
|
||||
value: Optional[str] = None
|
||||
|
||||
class SettingCreate(SettingBase):
|
||||
pass
|
||||
|
||||
class SettingUpdate(BaseModel):
|
||||
value: Optional[str] = None
|
||||
|
||||
class Setting(SettingBase):
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# Dashboard schemas
|
||||
class DashboardStats(BaseModel):
|
||||
active_jobs: int
|
||||
total_backups: int
|
||||
storage_used_bytes: int
|
||||
recent_failures: int
|
||||
recent_executions: List[JobExecution]
|
||||
Binary file not shown.
@@ -1,17 +0,0 @@
|
||||
from typing import Dict, Any
|
||||
from .base import SourceAdapter
|
||||
from .local import LocalAdapter
|
||||
from .ssh import SSHAdapter
|
||||
from .database import DatabaseAdapter
|
||||
|
||||
ADAPTER_MAP = {
|
||||
"local": LocalAdapter,
|
||||
"ssh": SSHAdapter,
|
||||
"database": DatabaseAdapter,
|
||||
}
|
||||
|
||||
def get_adapter(source_type: str, config: Dict[str, Any]) -> SourceAdapter:
|
||||
adapter_class = ADAPTER_MAP.get(source_type)
|
||||
if not adapter_class:
|
||||
raise ValueError(f"Unknown source type: {source_type}")
|
||||
return adapter_class(config)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,39 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Any, AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class FileInfo:
|
||||
path: str
|
||||
size: int
|
||||
modified_time: float
|
||||
is_directory: bool
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
async def connect(self) -> None:
|
||||
"""Establish connection to source."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def disconnect(self) -> None:
|
||||
"""Close connection to source."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def list_files(self, path: str = "") -> List[FileInfo]:
|
||||
"""List files at given path."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def read_file(self, path: str) -> AsyncIterator[bytes]:
|
||||
"""Read file in chunks."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_database_dump(self, config: Dict[str, Any]) -> AsyncIterator[bytes]:
|
||||
"""Get database dump. Only implemented for database adapters."""
|
||||
pass
|
||||
@@ -1,74 +0,0 @@
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import List, AsyncIterator, Dict, Any
|
||||
from .base import SourceAdapter, FileInfo
|
||||
|
||||
|
||||
class DatabaseAdapter(SourceAdapter):
|
||||
async def connect(self) -> None:
|
||||
pass
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
pass
|
||||
|
||||
async def list_files(self, path: str = "") -> List[FileInfo]:
|
||||
return []
|
||||
|
||||
async def read_file(self, path: str) -> AsyncIterator[bytes]:
|
||||
raise NotImplementedError("Database adapter does not support file reading")
|
||||
|
||||
async def get_database_dump(self, config: Dict[str, Any]) -> AsyncIterator[bytes]:
|
||||
db_type = config.get("db_type", "postgresql")
|
||||
host = config.get("host", "localhost")
|
||||
port = config.get("port", 5432 if db_type == "postgresql" else 3306)
|
||||
database = config.get("database")
|
||||
username = config.get("username")
|
||||
password = config.get("password")
|
||||
|
||||
if not database:
|
||||
raise ValueError("Database name is required")
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".sql", delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
if db_type == "postgresql":
|
||||
env = os.environ.copy()
|
||||
if password:
|
||||
env["PGPASSWORD"] = password
|
||||
|
||||
cmd = [
|
||||
"pg_dump",
|
||||
"-h", host,
|
||||
"-p", str(port),
|
||||
"-U", username or "postgres",
|
||||
"-f", tmp_path,
|
||||
database
|
||||
]
|
||||
elif db_type == "mysql":
|
||||
env = os.environ.copy()
|
||||
if password:
|
||||
env["MYSQL_PWD"] = password
|
||||
|
||||
cmd = [
|
||||
"mysqldump",
|
||||
"-h", host,
|
||||
"-P", str(port),
|
||||
"-u", username or "root",
|
||||
"--result-file", tmp_path,
|
||||
database
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Unsupported database type: {db_type}")
|
||||
|
||||
result = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Database dump failed: {result.stderr}")
|
||||
|
||||
with open(tmp_path, "rb") as f:
|
||||
while chunk := f.read(8192):
|
||||
yield chunk
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
@@ -1,47 +0,0 @@
|
||||
import os
|
||||
import aiofiles
|
||||
from pathlib import Path
|
||||
from typing import List, AsyncIterator, Dict, Any
|
||||
from .base import SourceAdapter, FileInfo
|
||||
|
||||
class LocalAdapter(SourceAdapter):
|
||||
async def connect(self) -> None:
|
||||
base_path = self.config.get("path", ".")
|
||||
if not os.path.exists(base_path):
|
||||
raise FileNotFoundError(f"Path not found: {base_path}")
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
pass
|
||||
|
||||
async def list_files(self, path: str = "") -> List[FileInfo]:
|
||||
base_path = Path(self.config.get("path", "."))
|
||||
target_path = base_path / path if path else base_path
|
||||
|
||||
files = []
|
||||
exclude_patterns = self.config.get("exclude", [])
|
||||
|
||||
for item in target_path.iterdir():
|
||||
# Check exclude patterns
|
||||
if any(item.match(pattern) for pattern in exclude_patterns):
|
||||
continue
|
||||
|
||||
stat = item.stat()
|
||||
files.append(FileInfo(
|
||||
path=str(item.relative_to(base_path)),
|
||||
size=stat.st_size,
|
||||
modified_time=stat.st_mtime,
|
||||
is_directory=item.is_dir()
|
||||
))
|
||||
|
||||
return files
|
||||
|
||||
async def read_file(self, path: str) -> AsyncIterator[bytes]:
|
||||
base_path = Path(self.config.get("path", "."))
|
||||
file_path = base_path / path
|
||||
|
||||
async with aiofiles.open(file_path, "rb") as f:
|
||||
while chunk := await f.read(8192):
|
||||
yield chunk
|
||||
|
||||
async def get_database_dump(self, config: Dict[str, Any]) -> AsyncIterator[bytes]:
|
||||
raise NotImplementedError("Local adapter does not support database dumps")
|
||||
@@ -1,92 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import List, AsyncIterator, Dict, Any
|
||||
import paramiko
|
||||
from .base import SourceAdapter, FileInfo
|
||||
|
||||
|
||||
class SSHAdapter(SourceAdapter):
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(config)
|
||||
self.client = None
|
||||
self.sftp = None
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.client = paramiko.SSHClient()
|
||||
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
host = self.config.get("host", "localhost")
|
||||
port = self.config.get("port", 22)
|
||||
username = self.config.get("username")
|
||||
password = self.config.get("password")
|
||||
key_path = self.config.get("key_path")
|
||||
|
||||
connect_kwargs = {
|
||||
"hostname": host,
|
||||
"port": port,
|
||||
"username": username,
|
||||
}
|
||||
|
||||
if password:
|
||||
connect_kwargs["password"] = password
|
||||
elif key_path and os.path.exists(key_path):
|
||||
connect_kwargs["key_filename"] = key_path
|
||||
|
||||
self.client.connect(**connect_kwargs)
|
||||
self.sftp = self.client.open_sftp()
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.sftp:
|
||||
self.sftp.close()
|
||||
self.sftp = None
|
||||
if self.client:
|
||||
self.client.close()
|
||||
self.client = None
|
||||
|
||||
async def list_files(self, path: str = "") -> List[FileInfo]:
|
||||
remote_path = self.config.get("path", ".")
|
||||
target_path = f"{remote_path}/{path}" if path else remote_path
|
||||
|
||||
files = []
|
||||
exclude_patterns = self.config.get("exclude", [])
|
||||
|
||||
try:
|
||||
for entry in self.sftp.listdir_attr(target_path):
|
||||
entry_path = f"{target_path}/{entry.filename}"
|
||||
rel_path = entry_path.replace(remote_path + "/", "", 1) if remote_path != "." else entry_path
|
||||
|
||||
if any(pattern in rel_path for pattern in exclude_patterns):
|
||||
continue
|
||||
|
||||
is_dir = entry.st_mode & 0o40000 == 0o40000 if hasattr(entry, 'st_mode') else False
|
||||
|
||||
files.append(FileInfo(
|
||||
path=rel_path,
|
||||
size=entry.st_size,
|
||||
modified_time=entry.st_mtime,
|
||||
is_directory=is_dir
|
||||
))
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
return files
|
||||
|
||||
async def read_file(self, path: str) -> AsyncIterator[bytes]:
|
||||
remote_path = self.config.get("path", ".")
|
||||
file_path = f"{remote_path}/{path}" if not path.startswith("/") else path
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
self.sftp.get(file_path, tmp_path)
|
||||
with open(tmp_path, "rb") as f:
|
||||
while chunk := f.read(8192):
|
||||
yield chunk
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
async def get_database_dump(self, config: Dict[str, Any]) -> AsyncIterator[bytes]:
|
||||
raise NotImplementedError("SSH adapter does not support database dumps directly")
|
||||
@@ -1,138 +0,0 @@
|
||||
import os
|
||||
import hashlib
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from app.models import Job, JobExecution, Backup
|
||||
from backup.adapters import get_adapter
|
||||
from backup.retention import RetentionPolicy
|
||||
|
||||
class BackupEngine:
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def execute_job(self, job_id: int, triggered_by: str = "manual") -> JobExecution:
|
||||
# Create execution record
|
||||
execution = JobExecution(
|
||||
job_id=job_id,
|
||||
status="pending",
|
||||
triggered_by=triggered_by
|
||||
)
|
||||
self.db.add(execution)
|
||||
await self.db.commit()
|
||||
await self.db.refresh(execution)
|
||||
|
||||
try:
|
||||
# Load job with source
|
||||
result = await self.db.execute(
|
||||
select(Job).where(Job.id == job_id)
|
||||
)
|
||||
job = result.scalar_one()
|
||||
|
||||
# Update status to running
|
||||
execution.status = "running"
|
||||
execution.started_at = datetime.now(timezone.utc)
|
||||
await self.db.commit()
|
||||
|
||||
# Determine strategy
|
||||
strategy = job.strategy
|
||||
parent_backup_id = None
|
||||
|
||||
if strategy == "incremental":
|
||||
# Find last successful full backup
|
||||
result = await self.db.execute(
|
||||
select(Backup)
|
||||
.join(JobExecution)
|
||||
.where(
|
||||
JobExecution.job_id == job_id,
|
||||
JobExecution.status == "success",
|
||||
Backup.type == "full"
|
||||
)
|
||||
.order_by(Backup.created_at.desc())
|
||||
)
|
||||
last_full = result.scalar_one_or_none()
|
||||
|
||||
if last_full:
|
||||
parent_backup_id = last_full.id
|
||||
else:
|
||||
# No full backup exists, do full instead
|
||||
strategy = "full"
|
||||
|
||||
# Create backup directory
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H%M%S")
|
||||
backup_dir = Path(job.destination_path) / str(job_id) / f"{timestamp}_{strategy}"
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get adapter and connect
|
||||
adapter = get_adapter(job.source.type, job.source.config)
|
||||
await adapter.connect()
|
||||
|
||||
try:
|
||||
# Copy files
|
||||
total_processed = 0
|
||||
total_backed_up = 0
|
||||
|
||||
source_path = Path(job.source.config.get("path", "."))
|
||||
|
||||
for item in source_path.rglob("*"):
|
||||
if item.is_file():
|
||||
rel_path = item.relative_to(source_path)
|
||||
dest_path = backup_dir / "data" / rel_path
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Copy file
|
||||
shutil.copy2(item, dest_path)
|
||||
|
||||
size = item.stat().st_size
|
||||
total_processed += size
|
||||
total_backed_up += size
|
||||
|
||||
# Calculate checksum
|
||||
checksum = await self._calculate_checksum(backup_dir)
|
||||
|
||||
# Create backup record
|
||||
backup = Backup(
|
||||
execution_id=execution.id,
|
||||
storage_path=str(backup_dir),
|
||||
size_bytes=total_backed_up,
|
||||
checksum=checksum,
|
||||
type=strategy,
|
||||
parent_backup_id=parent_backup_id
|
||||
)
|
||||
self.db.add(backup)
|
||||
|
||||
# Update execution
|
||||
execution.status = "success"
|
||||
execution.completed_at = datetime.now(timezone.utc)
|
||||
execution.bytes_processed = total_processed
|
||||
execution.bytes_backed_up = total_backed_up
|
||||
|
||||
# Apply retention policy
|
||||
retention = RetentionPolicy(self.db)
|
||||
keep_count = getattr(job, 'retention_count', None)
|
||||
keep_days = getattr(job, 'retention_days', None)
|
||||
if keep_count or keep_days:
|
||||
await retention.apply_retention_for_job(job_id, keep_count, keep_days)
|
||||
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
except Exception as e:
|
||||
execution.status = "failed"
|
||||
execution.completed_at = datetime.now(timezone.utc)
|
||||
execution.error_message = str(e)
|
||||
|
||||
await self.db.commit()
|
||||
return execution
|
||||
|
||||
async def _calculate_checksum(self, path: Path) -> str:
|
||||
hasher = hashlib.sha256()
|
||||
for item in sorted(path.rglob("*")):
|
||||
if item.is_file():
|
||||
with open(item, "rb") as f:
|
||||
while chunk := f.read(8192):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
@@ -1,90 +0,0 @@
|
||||
import shutil
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import Backup
|
||||
|
||||
|
||||
class RetentionPolicy:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def apply_retention_for_job(
|
||||
self,
|
||||
job_id: int,
|
||||
keep_count: Optional[int] = None,
|
||||
keep_days: Optional[int] = None
|
||||
) -> List[Backup]:
|
||||
"""
|
||||
Apply retention policy for a job's backups.
|
||||
|
||||
Args:
|
||||
job_id: The job ID to apply retention for
|
||||
keep_count: Maximum number of backups to keep (oldest deleted first)
|
||||
keep_days: Delete backups older than this many days
|
||||
|
||||
Returns:
|
||||
List of deleted backups
|
||||
"""
|
||||
deleted_backups = []
|
||||
|
||||
result = await self.db.execute(
|
||||
select(Backup)
|
||||
.where(Backup.execution.has(job_id=job_id))
|
||||
.order_by(Backup.created_at.asc())
|
||||
)
|
||||
backups = result.scalars().all()
|
||||
|
||||
if not backups:
|
||||
return deleted_backups
|
||||
|
||||
backups_to_delete = set()
|
||||
|
||||
if keep_count is not None and len(backups) > keep_count:
|
||||
backups_to_delete.update(backups[:-keep_count])
|
||||
|
||||
if keep_days is not None:
|
||||
cutoff_date = datetime.now(timezone.utc) - timedelta(days=keep_days)
|
||||
for backup in backups:
|
||||
if backup.created_at < cutoff_date:
|
||||
backups_to_delete.add(backup)
|
||||
|
||||
for backup in list(backups_to_delete):
|
||||
await self._delete_backup(backup)
|
||||
deleted_backups.append(backup)
|
||||
|
||||
await self.db.commit()
|
||||
return deleted_backups
|
||||
|
||||
async def _delete_backup(self, backup: Backup):
|
||||
"""Delete a backup and its storage."""
|
||||
try:
|
||||
storage_path = Path(backup.storage_path)
|
||||
if storage_path.exists():
|
||||
shutil.rmtree(storage_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await self.db.delete(backup)
|
||||
|
||||
async def cleanup_orphaned_backups(self) -> int:
|
||||
"""
|
||||
Remove backup records whose storage no longer exists.
|
||||
|
||||
Returns:
|
||||
Number of orphaned backups removed
|
||||
"""
|
||||
result = await self.db.execute(select(Backup))
|
||||
backups = result.scalars().all()
|
||||
|
||||
removed_count = 0
|
||||
for backup in backups:
|
||||
storage_path = Path(backup.storage_path)
|
||||
if not storage_path.exists():
|
||||
await self.db.delete(backup)
|
||||
removed_count += 1
|
||||
|
||||
await self.db.commit()
|
||||
return removed_count
|
||||
@@ -1,81 +0,0 @@
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from sqlalchemy import select
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Schedule
|
||||
from backup.engine import BackupEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BackupScheduler:
|
||||
def __init__(self):
|
||||
self.scheduler = AsyncIOScheduler()
|
||||
self._job_map = {}
|
||||
|
||||
def start(self):
|
||||
"""Start the scheduler."""
|
||||
self.scheduler.start()
|
||||
logger.info("Backup scheduler started")
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the scheduler."""
|
||||
self.scheduler.shutdown()
|
||||
logger.info("Backup scheduler shutdown")
|
||||
|
||||
async def sync_schedules(self):
|
||||
"""Sync all enabled schedules from database."""
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(Schedule).where(Schedule.enabled == True)
|
||||
)
|
||||
schedules = result.scalars().all()
|
||||
|
||||
# Clear existing jobs
|
||||
for schedule_id, job_id in list(self._job_map.items()):
|
||||
self.scheduler.remove_job(job_id)
|
||||
del self._job_map[schedule_id]
|
||||
|
||||
# Add new jobs
|
||||
for schedule in schedules:
|
||||
await self._add_schedule_job(schedule)
|
||||
|
||||
async def _add_schedule_job(self, schedule: Schedule):
|
||||
"""Add a single schedule job to the scheduler."""
|
||||
try:
|
||||
trigger = CronTrigger.from_crontab(schedule.cron_expression)
|
||||
job = self.scheduler.add_job(
|
||||
self._run_backup_job,
|
||||
trigger=trigger,
|
||||
args=[schedule.job_id],
|
||||
id=f"backup_job_{schedule.job_id}",
|
||||
replace_existing=True
|
||||
)
|
||||
self._job_map[schedule.id] = job.id
|
||||
logger.info(f"Scheduled backup job {schedule.job_id} with cron: {schedule.cron_expression}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to schedule job {schedule.job_id}: {e}")
|
||||
|
||||
async def _run_backup_job(self, job_id: int):
|
||||
"""Execute a backup job."""
|
||||
logger.info(f"Running scheduled backup job {job_id}")
|
||||
async with AsyncSessionLocal() as db:
|
||||
engine = BackupEngine(db)
|
||||
await engine.execute_job(job_id, triggered_by="schedule")
|
||||
|
||||
async def add_schedule(self, schedule: Schedule):
|
||||
"""Add a new schedule to the scheduler."""
|
||||
await self._add_schedule_job(schedule)
|
||||
|
||||
def remove_schedule(self, schedule_id: int):
|
||||
"""Remove a schedule from the scheduler."""
|
||||
if schedule_id in self._job_map:
|
||||
self.scheduler.remove_job(self._job_map[schedule_id])
|
||||
del self._job_map[schedule_id]
|
||||
|
||||
|
||||
# Global scheduler instance
|
||||
backup_scheduler = BackupScheduler()
|
||||
Binary file not shown.
+45
-48
@@ -1,68 +1,65 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
requires = ["setuptools==80.9.0", "wheel==0.46.3"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-tool"
|
||||
version = "0.1.0"
|
||||
description = "Web-based backup management tool for small teams and SMBs"
|
||||
readme = "README.md"
|
||||
license = {text = "MIT"}
|
||||
requires-python = ">=3.11"
|
||||
authors = [
|
||||
{name = "Backup Tool Team"}
|
||||
]
|
||||
keywords = ["backup", "restore", "scheduler", "web"]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: System Administrators",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: System :: Archiving :: Backup",
|
||||
]
|
||||
version = "2.0.0.dev0"
|
||||
description = "Self-hosted, integrity-first backup appliance"
|
||||
license = "MIT"
|
||||
requires-python = ">=3.12,<3.15"
|
||||
authors = [{ name = "Backup Tool Team" }]
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0",
|
||||
"uvicorn[standard]>=0.34.0",
|
||||
"sqlalchemy[asyncio]>=2.0.0",
|
||||
"aiosqlite>=0.21.0",
|
||||
"alembic>=1.15.0",
|
||||
"pydantic>=2.13.0",
|
||||
"pydantic-settings>=2.9.0",
|
||||
"apscheduler>=3.11.0",
|
||||
"paramiko>=3.5.0",
|
||||
"aiofiles>=23.2.0",
|
||||
"httpx>=0.28.0",
|
||||
"aiofiles==25.1.0",
|
||||
"aiosqlite==0.22.1",
|
||||
"alembic==1.18.5",
|
||||
"argon2-cffi==25.1.0",
|
||||
"apscheduler==3.11.3",
|
||||
"cryptography==49.0.0",
|
||||
"fastapi==0.136.1",
|
||||
"httpx==0.28.1",
|
||||
"pydantic==2.13.4",
|
||||
"pydantic-settings==2.14.2",
|
||||
"paramiko==5.0.0",
|
||||
"sqlalchemy[asyncio]==2.0.49",
|
||||
"uvicorn[standard]==0.51.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3.0",
|
||||
"pytest-asyncio>=0.26.0",
|
||||
]
|
||||
prod = [
|
||||
"gunicorn>=23.0.0",
|
||||
"jsonschema==4.26.0",
|
||||
"mypy==2.3.0",
|
||||
"pytest==9.0.3",
|
||||
"pytest-asyncio==1.3.0",
|
||||
"ruff==0.16.0",
|
||||
]
|
||||
prod = ["gunicorn==23.0.0"]
|
||||
|
||||
[project.scripts]
|
||||
backup-tool = "app.main:main"
|
||||
backup-tool = "backup_tool.cli:main"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/backup-tool/backup-tool"
|
||||
Documentation = "https://github.com/backup-tool/backup-tool#readme"
|
||||
Repository = "https://github.com/backup-tool/backup-tool.git"
|
||||
[tool.setuptools]
|
||||
package-dir = { "" = "src" }
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["app*", "backup*", "alembic*"]
|
||||
where = ["src"]
|
||||
include = ["backup_tool*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
backup_tool = ["py.typed"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
pythonpath = [".", "app", "backup"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
alembic = ["*.ini", "*.py", "*.mako"]
|
||||
[tool.ruff]
|
||||
target-version = "py312"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B", "SIM"]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
strict = true
|
||||
packages = ["backup_tool"]
|
||||
mypy_path = "src"
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Backup Tool v2 package."""
|
||||
|
||||
__version__ = "2.0.0.dev0"
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from backup_tool.config import Settings
|
||||
|
||||
|
||||
class SourceError(ValueError):
|
||||
"""A redacted source failure with a stable execution reason."""
|
||||
|
||||
def __init__(self, message: str, *, reason_code: str = "source_invalid") -> None:
|
||||
super().__init__(message)
|
||||
self.reason_code = reason_code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Entry:
|
||||
path: str
|
||||
kind: str
|
||||
size: int
|
||||
mode: int
|
||||
mtime_ns: int
|
||||
link_target: str | None = None
|
||||
|
||||
|
||||
class SourceReader(Protocol):
|
||||
def validate_config(self) -> None: ...
|
||||
|
||||
async def probe(self) -> dict[str, int]: ...
|
||||
|
||||
def enumerate_entries(self) -> AsyncIterator[Entry]: ...
|
||||
|
||||
def open_content(self, path: str) -> AsyncIterator[bytes]: ...
|
||||
|
||||
async def close(self) -> None: ...
|
||||
|
||||
|
||||
class LocalAdapter:
|
||||
def __init__(self, root: Path, settings: Settings):
|
||||
self.root = root.expanduser().resolve()
|
||||
self.settings = settings
|
||||
|
||||
def validate_config(self) -> None:
|
||||
if not self.root.is_dir():
|
||||
raise SourceError("local source root must be an existing directory")
|
||||
if not any(self.root.is_relative_to(root) for root in self.settings.local_source_roots):
|
||||
raise SourceError("local source root is outside configured allowlists")
|
||||
|
||||
async def probe(self) -> dict[str, int]:
|
||||
self.validate_config()
|
||||
count = sum(1 for item in self.root.rglob("*") if item.is_file() and not item.is_symlink())
|
||||
return {"entry_count": count}
|
||||
|
||||
async def enumerate_entries(self) -> AsyncIterator[Entry]:
|
||||
self.validate_config()
|
||||
for item in sorted(self.root.rglob("*"), key=lambda candidate: candidate.as_posix()):
|
||||
relative = item.relative_to(self.root).as_posix()
|
||||
metadata = item.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode):
|
||||
target = os.readlink(item)
|
||||
target_path = Path(target)
|
||||
if (
|
||||
target_path.is_absolute()
|
||||
or "\\" in target
|
||||
or ".." in target_path.parts
|
||||
or not target
|
||||
):
|
||||
raise SourceError(f"symlink target for {relative!r} is unsafe")
|
||||
yield Entry(
|
||||
relative,
|
||||
"symlink",
|
||||
0,
|
||||
stat.S_IMODE(metadata.st_mode),
|
||||
metadata.st_mtime_ns,
|
||||
target,
|
||||
)
|
||||
elif stat.S_ISREG(metadata.st_mode):
|
||||
yield Entry(
|
||||
relative,
|
||||
"file",
|
||||
metadata.st_size,
|
||||
stat.S_IMODE(metadata.st_mode),
|
||||
metadata.st_mtime_ns,
|
||||
)
|
||||
elif stat.S_ISDIR(metadata.st_mode):
|
||||
yield Entry(
|
||||
relative,
|
||||
"directory",
|
||||
0,
|
||||
stat.S_IMODE(metadata.st_mode),
|
||||
metadata.st_mtime_ns,
|
||||
)
|
||||
else:
|
||||
raise SourceError(f"local source entry {relative!r} has an unsupported type")
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
async def open_content(self, path: str) -> AsyncIterator[bytes]:
|
||||
requested = Path(path)
|
||||
if not path or requested.is_absolute() or "\\" in path or ".." in requested.parts:
|
||||
raise SourceError("invalid local source entry")
|
||||
candidate = self.root / requested
|
||||
metadata = candidate.lstat()
|
||||
if not stat.S_ISREG(metadata.st_mode) or candidate.is_symlink():
|
||||
raise SourceError("invalid local source entry")
|
||||
resolved = candidate.resolve()
|
||||
if not resolved.is_relative_to(self.root):
|
||||
raise SourceError("invalid local source entry")
|
||||
with candidate.open("rb") as handle:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
yield chunk
|
||||
@@ -0,0 +1 @@
|
||||
"""HTTP boundary for Backup Tool v2."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,953 @@
|
||||
"""Backup Tool v2 process-role command line."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from collections.abc import Callable, Sequence
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from backup_tool import __version__
|
||||
from backup_tool.config import Settings
|
||||
|
||||
from .db.engine import assert_schema_current, create_engine
|
||||
from .db.models import (
|
||||
Backup,
|
||||
Base,
|
||||
Execution,
|
||||
Job,
|
||||
Repository,
|
||||
RepositoryDataKeyEpoch,
|
||||
Source,
|
||||
)
|
||||
from .repository import (
|
||||
begin_key_rotation,
|
||||
finish_key_rotation,
|
||||
inspect_repository,
|
||||
install_signing_key,
|
||||
load_signing_key,
|
||||
reconcile_key_rotations,
|
||||
replace_active_data_key,
|
||||
)
|
||||
from .scheduler import run_scheduler
|
||||
from .security.recovery_bundle import (
|
||||
RecoveryBundleError,
|
||||
decrypt_bundle,
|
||||
encrypt_bundle,
|
||||
read_bundle_file,
|
||||
read_passphrase_fd,
|
||||
write_bundle_exclusive,
|
||||
)
|
||||
from .security.repository_crypto import (
|
||||
RepositoryKeyError,
|
||||
create_data_key,
|
||||
install_data_key,
|
||||
load_data_key,
|
||||
)
|
||||
from .web import run_web
|
||||
from .worker import run_worker
|
||||
|
||||
RoleHandler = Callable[[Settings], int]
|
||||
DATABASE_ROLES = ("web", "scheduler", "worker", "admin")
|
||||
|
||||
|
||||
def _placeholder_role(_settings: Settings) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
ROLE_HANDLERS: dict[str, RoleHandler] = {
|
||||
"web": run_web,
|
||||
"scheduler": run_scheduler,
|
||||
"worker": run_worker,
|
||||
"admin": _placeholder_role,
|
||||
}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="backup-tool")
|
||||
parser.add_argument("--version", action="version", version=__version__)
|
||||
subparsers = parser.add_subparsers(dest="role", required=True)
|
||||
for role in DATABASE_ROLES:
|
||||
role_parser = subparsers.add_parser(role, help=f"run the {role} role")
|
||||
if role == "admin":
|
||||
admin_parsers = role_parser.add_subparsers(dest="admin_command")
|
||||
repository_key = admin_parsers.add_parser("repository-key")
|
||||
repository_key_parsers = repository_key.add_subparsers(dest="repository_key_command")
|
||||
rotate = repository_key_parsers.add_parser("rotate")
|
||||
rotate.add_argument("--repository-id", required=True)
|
||||
recovery = admin_parsers.add_parser("recovery")
|
||||
recovery_parsers = recovery.add_subparsers(dest="recovery_command", required=True)
|
||||
export = recovery_parsers.add_parser("export")
|
||||
export.add_argument("--output", required=True, type=Path)
|
||||
export.add_argument("--passphrase-fd", required=True, type=int)
|
||||
validate = recovery_parsers.add_parser("validate")
|
||||
validate.add_argument("--input", required=True, type=Path)
|
||||
validate.add_argument("--passphrase-fd", required=True, type=int)
|
||||
import_ = recovery_parsers.add_parser("import")
|
||||
import_.add_argument("--input", required=True, type=Path)
|
||||
import_.add_argument("--passphrase-fd", required=True, type=int)
|
||||
migrate = subparsers.add_parser("migrate", help="manage the metadata schema")
|
||||
migrate.add_argument("action", choices=("upgrade", "downgrade", "current"))
|
||||
health = subparsers.add_parser("health", help="check runtime role readiness")
|
||||
health.add_argument("health_role", choices=("web", "scheduler", "worker"))
|
||||
return parser
|
||||
|
||||
|
||||
def build_alembic_config(settings: Settings) -> Config:
|
||||
default_root = Path(__file__).resolve().parents[2]
|
||||
backend_root = Path(os.environ.get("BACKUP_TOOL_ALEMBIC_ROOT", default_root))
|
||||
migration = Config(backend_root / "alembic.ini")
|
||||
migration.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
return migration
|
||||
|
||||
|
||||
def require_current_schema(settings: Settings) -> None:
|
||||
async def check() -> None:
|
||||
engine = create_engine(settings)
|
||||
try:
|
||||
await assert_schema_current(engine, build_alembic_config(settings))
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
asyncio.run(check())
|
||||
|
||||
|
||||
def run_role_healthcheck(settings: Settings, role: str) -> int:
|
||||
"""Verify the selected isolated runtime role is ready to serve work."""
|
||||
if role != "web":
|
||||
from backup_tool.observability.health import check_role_readiness
|
||||
|
||||
asyncio.run(check_role_readiness(settings, role))
|
||||
return 0
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
||||
client.settimeout(2)
|
||||
client.connect(str(settings.web_socket_path))
|
||||
request = b"GET /readyz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
|
||||
client.sendall(request)
|
||||
status_line = client.recv(64).split(b"\r\n", 1)[0]
|
||||
if status_line != b"HTTP/1.1 200 OK":
|
||||
raise RuntimeError("web role is not ready")
|
||||
return 0
|
||||
|
||||
|
||||
def run_migration(action: str, settings: Settings) -> int:
|
||||
migration = build_alembic_config(settings)
|
||||
if action == "upgrade":
|
||||
command.upgrade(migration, "head")
|
||||
elif action == "downgrade":
|
||||
command.downgrade(migration, "base")
|
||||
else:
|
||||
command.current(migration)
|
||||
return 0
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings() # type: ignore[call-arg]
|
||||
|
||||
|
||||
async def rotate_repository_key(
|
||||
settings: Settings,
|
||||
repository_id: str,
|
||||
*,
|
||||
after_database_commit: Callable[[], None] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Journal a key rotation so a crash always has a deterministic recovery path."""
|
||||
engine = create_engine(settings)
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
new_key_path: Path | None = None
|
||||
database_committed = False
|
||||
journal_started = False
|
||||
root: Path | None = None
|
||||
old_key_id = ""
|
||||
new_key_id = ""
|
||||
try:
|
||||
async with sessions() as db:
|
||||
await reconcile_key_rotations(settings, db)
|
||||
await db.rollback()
|
||||
async with db.begin():
|
||||
repository = await db.scalar(
|
||||
select(Repository).where(Repository.id == repository_id).with_for_update()
|
||||
)
|
||||
if repository is None:
|
||||
raise ValueError("repository was not found")
|
||||
if repository.encryption != "aes-256-gcm" or repository.active_data_key_id is None:
|
||||
raise ValueError("repository encryption is not enabled")
|
||||
inspected = inspect_repository(settings, Path(repository.root))
|
||||
root = inspected.root
|
||||
if (
|
||||
inspected.encryption != "aes-256-gcm"
|
||||
or inspected.data_key_id != repository.active_data_key_id
|
||||
):
|
||||
raise ValueError("repository encryption metadata is invalid")
|
||||
epochs = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(RepositoryDataKeyEpoch).where(
|
||||
RepositoryDataKeyEpoch.repository_id == repository.id
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
active = [epoch for epoch in epochs if epoch.state == "active"]
|
||||
if len(active) != 1 or active[0].key_id != repository.active_data_key_id:
|
||||
raise ValueError("repository key epochs are invalid")
|
||||
old_key_id = active[0].key_id
|
||||
new_key_id, new_key_path = create_data_key(settings, inspected.repository_id)
|
||||
begin_key_rotation(
|
||||
inspected.root,
|
||||
repository.id,
|
||||
inspected.repository_id,
|
||||
old_key_id,
|
||||
new_key_id,
|
||||
)
|
||||
journal_started = True
|
||||
active[0].state = "retired"
|
||||
active[0].retired_at = datetime.now(UTC)
|
||||
db.add(
|
||||
RepositoryDataKeyEpoch(
|
||||
repository_id=repository.id,
|
||||
key_id=new_key_id,
|
||||
state="active",
|
||||
)
|
||||
)
|
||||
repository.active_data_key_id = new_key_id
|
||||
database_committed = True
|
||||
if after_database_commit is not None:
|
||||
after_database_commit()
|
||||
if root is None:
|
||||
raise ValueError("repository rotation metadata is invalid")
|
||||
replace_active_data_key(root, old_key_id, new_key_id)
|
||||
finish_key_rotation(root)
|
||||
journal_started = False
|
||||
return {
|
||||
"repository_id": repository_id,
|
||||
"retired_key_id": old_key_id,
|
||||
"active_key_id": new_key_id,
|
||||
}
|
||||
finally:
|
||||
if not database_committed:
|
||||
if journal_started and root is not None:
|
||||
with suppress(ValueError):
|
||||
finish_key_rotation(root)
|
||||
if new_key_path is not None:
|
||||
new_key_path.unlink(missing_ok=True)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def recovery_export_payload(settings: Settings) -> dict[str, Any]:
|
||||
"""Collect the current repository trust catalog and offline key material."""
|
||||
engine = create_engine(settings)
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
async with sessions() as db:
|
||||
await reconcile_key_rotations(settings, db)
|
||||
await db.rollback()
|
||||
repositories = list(
|
||||
(await db.scalars(select(Repository).order_by(Repository.id))).all()
|
||||
)
|
||||
catalog_repositories: list[dict[str, Any]] = []
|
||||
key_records: list[dict[str, Any]] = []
|
||||
for repository in repositories:
|
||||
inspected = inspect_repository(settings, Path(repository.root))
|
||||
if inspected.encryption != repository.encryption or (
|
||||
repository.encryption == "aes-256-gcm"
|
||||
and repository.active_data_key_id != inspected.data_key_id
|
||||
):
|
||||
raise RecoveryBundleError("recovery bundle export is unavailable")
|
||||
private_key = load_signing_key(
|
||||
settings,
|
||||
inspected.repository_id,
|
||||
repository.signing_key_id,
|
||||
repository.signing_public_key,
|
||||
)
|
||||
epochs = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(RepositoryDataKeyEpoch)
|
||||
.where(RepositoryDataKeyEpoch.repository_id == repository.id)
|
||||
.order_by(RepositoryDataKeyEpoch.key_id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
active = [epoch for epoch in epochs if epoch.state == "active"]
|
||||
if repository.encryption == "aes-256-gcm":
|
||||
if (
|
||||
repository.active_data_key_id is None
|
||||
or len(active) != 1
|
||||
or active[0].key_id != repository.active_data_key_id
|
||||
):
|
||||
raise RecoveryBundleError("recovery bundle export is unavailable")
|
||||
elif epochs:
|
||||
raise RecoveryBundleError("recovery bundle export is unavailable")
|
||||
data_keys: list[dict[str, str]] = []
|
||||
for epoch in epochs:
|
||||
data_keys.append(
|
||||
{
|
||||
"key_id": epoch.key_id,
|
||||
"key": base64.b64encode(
|
||||
load_data_key(settings, inspected.repository_id, epoch.key_id)
|
||||
).decode("ascii"),
|
||||
}
|
||||
)
|
||||
catalog_repositories.append(
|
||||
{
|
||||
"active_data_key_id": repository.active_data_key_id,
|
||||
"compression": repository.compression,
|
||||
"data_key_epochs": [
|
||||
{
|
||||
"key_id": epoch.key_id,
|
||||
"retired_at": (
|
||||
epoch.retired_at.isoformat()
|
||||
if epoch.retired_at is not None
|
||||
else None
|
||||
),
|
||||
"state": epoch.state,
|
||||
}
|
||||
for epoch in epochs
|
||||
],
|
||||
"database_id": repository.id,
|
||||
"encryption": repository.encryption,
|
||||
"format_version": repository.format_version,
|
||||
"name": repository.name,
|
||||
"repository_id": inspected.repository_id,
|
||||
"root": repository.root,
|
||||
"signing_key_id": repository.signing_key_id,
|
||||
"signing_public_key": repository.signing_public_key,
|
||||
"state": repository.state,
|
||||
}
|
||||
)
|
||||
key_records.append(
|
||||
{
|
||||
"data_keys": data_keys,
|
||||
"repository_id": inspected.repository_id,
|
||||
"signing_private_key": base64.b64encode(
|
||||
private_key.private_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PrivateFormat.Raw,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
).decode("ascii"),
|
||||
}
|
||||
)
|
||||
sources = list((await db.scalars(select(Source).order_by(Source.id))).all())
|
||||
jobs = list((await db.scalars(select(Job).order_by(Job.id))).all())
|
||||
executions = list((await db.scalars(select(Execution).order_by(Execution.id))).all())
|
||||
backups = list((await db.scalars(select(Backup).order_by(Backup.id))).all())
|
||||
if any(source.secret_refs for source in sources):
|
||||
raise RecoveryBundleError("recovery bundle export is unavailable")
|
||||
return {
|
||||
"catalog": {
|
||||
"backups": [
|
||||
{
|
||||
"created_at": backup.created_at.isoformat(),
|
||||
"data_key_id": backup.data_key_id,
|
||||
"execution_id": backup.execution_id,
|
||||
"id": backup.id,
|
||||
"integrity": backup.integrity,
|
||||
"logical_bytes": backup.logical_bytes,
|
||||
"manifest_digest": backup.manifest_digest,
|
||||
"manifest_id": backup.manifest_id,
|
||||
"parent_backup_id": backup.parent_backup_id,
|
||||
"pinned": backup.pinned,
|
||||
"stored_bytes": backup.stored_bytes,
|
||||
"tombstoned_at": (
|
||||
backup.tombstoned_at.isoformat()
|
||||
if backup.tombstoned_at is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
for backup in backups
|
||||
],
|
||||
"executions": [
|
||||
{
|
||||
"attempt": execution.attempt,
|
||||
"id": execution.id,
|
||||
"job_id": execution.job_id,
|
||||
"progress": execution.progress,
|
||||
"revision": execution.revision,
|
||||
"trigger": execution.trigger,
|
||||
}
|
||||
for execution in executions
|
||||
],
|
||||
"format": "backup-tool-recovery-catalog",
|
||||
"jobs": [
|
||||
{
|
||||
"allow_empty": job.allow_empty,
|
||||
"exclusions": job.exclusions,
|
||||
"id": job.id,
|
||||
"name": job.name,
|
||||
"repository_id": job.repository_id,
|
||||
"requested_mode": job.requested_mode,
|
||||
"retention": job.retention,
|
||||
"source_id": job.source_id,
|
||||
}
|
||||
for job in jobs
|
||||
],
|
||||
"repositories": catalog_repositories,
|
||||
"sources": [
|
||||
{
|
||||
"id": source.id,
|
||||
"kind": source.kind,
|
||||
"name": source.name,
|
||||
"public_config": source.public_config,
|
||||
}
|
||||
for source in sources
|
||||
],
|
||||
"version": 2,
|
||||
},
|
||||
"keys": key_records,
|
||||
}
|
||||
except (OSError, RepositoryKeyError, ValueError) as error:
|
||||
if isinstance(error, RecoveryBundleError):
|
||||
raise
|
||||
raise RecoveryBundleError("recovery bundle export is unavailable") from error
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _validate_legacy_recovery_payload(payload: dict[str, Any]) -> int:
|
||||
"""Validate the authenticated export shape without exposing key material."""
|
||||
try:
|
||||
if set(payload) != {"catalog", "keys"}:
|
||||
raise ValueError
|
||||
catalog = payload["catalog"]
|
||||
keys = payload["keys"]
|
||||
if (
|
||||
not isinstance(catalog, dict)
|
||||
or set(catalog) != {"format", "repositories", "version"}
|
||||
or catalog["format"] != "backup-tool-recovery-catalog"
|
||||
or catalog["version"] != 1
|
||||
or not isinstance(catalog["repositories"], list)
|
||||
or not isinstance(keys, list)
|
||||
or len(catalog["repositories"]) != len(keys)
|
||||
):
|
||||
raise ValueError
|
||||
seen: set[str] = set()
|
||||
for repository, key_record in zip(catalog["repositories"], keys, strict=True):
|
||||
if not isinstance(repository, dict) or not isinstance(key_record, dict):
|
||||
raise ValueError
|
||||
required_repository = {
|
||||
"active_data_key_id",
|
||||
"compression",
|
||||
"data_key_epochs",
|
||||
"database_id",
|
||||
"encryption",
|
||||
"format_version",
|
||||
"name",
|
||||
"repository_id",
|
||||
"root",
|
||||
"signing_key_id",
|
||||
"signing_public_key",
|
||||
"state",
|
||||
}
|
||||
if set(repository) != required_repository or set(key_record) != {
|
||||
"data_keys",
|
||||
"repository_id",
|
||||
"signing_private_key",
|
||||
}:
|
||||
raise ValueError
|
||||
repository_id = repository["repository_id"]
|
||||
if (
|
||||
not isinstance(repository_id, str)
|
||||
or not repository_id
|
||||
or repository_id in seen
|
||||
or key_record["repository_id"] != repository_id
|
||||
or not isinstance(repository["data_key_epochs"], list)
|
||||
or not isinstance(key_record["data_keys"], list)
|
||||
):
|
||||
raise ValueError
|
||||
seen.add(repository_id)
|
||||
private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
|
||||
public_key = bytes.fromhex(repository["signing_public_key"])
|
||||
if len(private_key) != 32 or len(public_key) != 32:
|
||||
raise ValueError
|
||||
signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
|
||||
derived_public = signing_key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PublicFormat.Raw,
|
||||
)
|
||||
if derived_public != public_key:
|
||||
raise ValueError
|
||||
epochs = repository["data_key_epochs"]
|
||||
data_keys = key_record["data_keys"]
|
||||
if len(epochs) != len(data_keys):
|
||||
raise ValueError
|
||||
epoch_ids: set[str] = set()
|
||||
for epoch, data_key in zip(epochs, data_keys, strict=True):
|
||||
if (
|
||||
not isinstance(epoch, dict)
|
||||
or set(epoch) != {"key_id", "retired_at", "state"}
|
||||
or not isinstance(data_key, dict)
|
||||
or set(data_key) != {"key", "key_id"}
|
||||
or not isinstance(epoch["key_id"], str)
|
||||
or epoch["key_id"] in epoch_ids
|
||||
or data_key["key_id"] != epoch["key_id"]
|
||||
or len(base64.b64decode(data_key["key"], validate=True)) != 32
|
||||
):
|
||||
raise ValueError
|
||||
epoch_ids.add(epoch["key_id"])
|
||||
active = [epoch for epoch in epochs if epoch["state"] == "active"]
|
||||
if repository["encryption"] == "aes-256-gcm":
|
||||
if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
|
||||
raise ValueError
|
||||
elif (
|
||||
repository["encryption"] != "none"
|
||||
or epochs
|
||||
or repository["active_data_key_id"] is not None
|
||||
):
|
||||
raise ValueError
|
||||
return len(seen)
|
||||
except (binascii.Error, KeyError, TypeError, ValueError) as error:
|
||||
raise RecoveryBundleError("recovery bundle is invalid") from error
|
||||
|
||||
|
||||
def _validated_recovery_catalog(
|
||||
payload: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
"""Validate a v2 catalog before any filesystem or metadata mutation."""
|
||||
try:
|
||||
if set(payload) != {"catalog", "keys"}:
|
||||
raise ValueError
|
||||
catalog = payload["catalog"]
|
||||
keys = payload["keys"]
|
||||
required_catalog = {
|
||||
"backups",
|
||||
"executions",
|
||||
"format",
|
||||
"jobs",
|
||||
"repositories",
|
||||
"sources",
|
||||
"version",
|
||||
}
|
||||
if (
|
||||
not isinstance(catalog, dict)
|
||||
or set(catalog) != required_catalog
|
||||
or catalog["format"] != "backup-tool-recovery-catalog"
|
||||
or catalog["version"] != 2
|
||||
or not isinstance(keys, list)
|
||||
or any(
|
||||
not isinstance(catalog[name], list)
|
||||
for name in required_catalog - {"format", "version"}
|
||||
)
|
||||
or len(catalog["repositories"]) != len(keys)
|
||||
):
|
||||
raise ValueError
|
||||
repositories = catalog["repositories"]
|
||||
key_by_repository: dict[str, dict[str, Any]] = {}
|
||||
repository_by_id: dict[str, dict[str, Any]] = {}
|
||||
canonical_ids: set[str] = set()
|
||||
for repository, key_record in zip(repositories, keys, strict=True):
|
||||
if not isinstance(repository, dict) or not isinstance(key_record, dict):
|
||||
raise ValueError
|
||||
required_repository = {
|
||||
"active_data_key_id",
|
||||
"compression",
|
||||
"data_key_epochs",
|
||||
"database_id",
|
||||
"encryption",
|
||||
"format_version",
|
||||
"name",
|
||||
"repository_id",
|
||||
"root",
|
||||
"signing_key_id",
|
||||
"signing_public_key",
|
||||
"state",
|
||||
}
|
||||
if set(repository) != required_repository or set(key_record) != {
|
||||
"data_keys",
|
||||
"repository_id",
|
||||
"signing_private_key",
|
||||
}:
|
||||
raise ValueError
|
||||
database_id = repository["database_id"]
|
||||
canonical_id = repository["repository_id"]
|
||||
if (
|
||||
not isinstance(database_id, str)
|
||||
or not isinstance(canonical_id, str)
|
||||
or not database_id
|
||||
or not canonical_id
|
||||
or database_id in repository_by_id
|
||||
or canonical_id in canonical_ids
|
||||
or key_record["repository_id"] != canonical_id
|
||||
or not isinstance(repository["data_key_epochs"], list)
|
||||
or not isinstance(key_record["data_keys"], list)
|
||||
):
|
||||
raise ValueError
|
||||
private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
|
||||
public_key = bytes.fromhex(repository["signing_public_key"])
|
||||
signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
|
||||
if (
|
||||
len(public_key) != 32
|
||||
or signing_key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw, serialization.PublicFormat.Raw
|
||||
)
|
||||
!= public_key
|
||||
):
|
||||
raise ValueError
|
||||
epochs = repository["data_key_epochs"]
|
||||
data_keys = key_record["data_keys"]
|
||||
if len(epochs) != len(data_keys):
|
||||
raise ValueError
|
||||
epoch_ids: set[str] = set()
|
||||
for epoch, data_key in zip(epochs, data_keys, strict=True):
|
||||
if (
|
||||
not isinstance(epoch, dict)
|
||||
or set(epoch) != {"key_id", "retired_at", "state"}
|
||||
or epoch.get("state") not in {"active", "retired"}
|
||||
or not isinstance(epoch.get("key_id"), str)
|
||||
or epoch["key_id"] in epoch_ids
|
||||
or not isinstance(data_key, dict)
|
||||
or set(data_key) != {"key", "key_id"}
|
||||
or data_key["key_id"] != epoch["key_id"]
|
||||
or len(base64.b64decode(data_key["key"], validate=True)) != 32
|
||||
):
|
||||
raise ValueError
|
||||
epoch_ids.add(epoch["key_id"])
|
||||
active = [epoch for epoch in epochs if epoch["state"] == "active"]
|
||||
if repository["encryption"] == "aes-256-gcm":
|
||||
if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
|
||||
raise ValueError
|
||||
elif (
|
||||
repository["encryption"] != "none"
|
||||
or epochs
|
||||
or repository["active_data_key_id"] is not None
|
||||
):
|
||||
raise ValueError
|
||||
repository_by_id[database_id] = repository
|
||||
canonical_ids.add(canonical_id)
|
||||
key_by_repository[canonical_id] = key_record
|
||||
source_ids = _catalog_ids(catalog["sources"], {"id", "kind", "name", "public_config"})
|
||||
job_ids = _catalog_ids(
|
||||
catalog["jobs"],
|
||||
{
|
||||
"allow_empty",
|
||||
"exclusions",
|
||||
"id",
|
||||
"name",
|
||||
"repository_id",
|
||||
"requested_mode",
|
||||
"retention",
|
||||
"source_id",
|
||||
},
|
||||
)
|
||||
for job in catalog["jobs"]:
|
||||
if job["source_id"] not in source_ids or job["repository_id"] not in repository_by_id:
|
||||
raise ValueError
|
||||
execution_ids = _catalog_ids(
|
||||
catalog["executions"], {"attempt", "id", "job_id", "progress", "revision", "trigger"}
|
||||
)
|
||||
for execution in catalog["executions"]:
|
||||
if execution["job_id"] not in job_ids:
|
||||
raise ValueError
|
||||
backup_ids = _catalog_ids(
|
||||
catalog["backups"],
|
||||
{
|
||||
"created_at",
|
||||
"data_key_id",
|
||||
"execution_id",
|
||||
"id",
|
||||
"integrity",
|
||||
"logical_bytes",
|
||||
"manifest_digest",
|
||||
"manifest_id",
|
||||
"parent_backup_id",
|
||||
"pinned",
|
||||
"stored_bytes",
|
||||
"tombstoned_at",
|
||||
},
|
||||
)
|
||||
for backup in catalog["backups"]:
|
||||
if backup["execution_id"] not in execution_ids:
|
||||
raise ValueError
|
||||
_catalog_datetime(backup["created_at"])
|
||||
_catalog_datetime(backup["tombstoned_at"], allow_none=True)
|
||||
if (
|
||||
backup["parent_backup_id"] is not None
|
||||
and backup["parent_backup_id"] not in backup_ids
|
||||
):
|
||||
raise ValueError
|
||||
return (
|
||||
catalog,
|
||||
[key_by_repository[repository["repository_id"]] for repository in repositories],
|
||||
)
|
||||
except (binascii.Error, KeyError, TypeError, ValueError) as error:
|
||||
raise RecoveryBundleError("recovery bundle is invalid") from error
|
||||
|
||||
|
||||
def _catalog_ids(rows: list[Any], required: set[str]) -> set[str]:
|
||||
identifiers: set[str] = set()
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or set(row) != required or not isinstance(row.get("id"), str):
|
||||
raise ValueError
|
||||
if not row["id"] or row["id"] in identifiers:
|
||||
raise ValueError
|
||||
identifiers.add(row["id"])
|
||||
return identifiers
|
||||
|
||||
|
||||
def _catalog_datetime(value: object, *, allow_none: bool = False) -> datetime | None:
|
||||
if value is None and allow_none:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00"))
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _validate_recovery_payload(payload: dict[str, Any]) -> int:
|
||||
catalog = payload.get("catalog")
|
||||
if isinstance(catalog, dict) and catalog.get("version") == 1:
|
||||
return _validate_legacy_recovery_payload(payload)
|
||||
validated_catalog, _keys = _validated_recovery_catalog(payload)
|
||||
return len(validated_catalog["repositories"])
|
||||
|
||||
|
||||
def _install_or_verify_recovery_keys(
|
||||
settings: Settings,
|
||||
repository: dict[str, Any],
|
||||
key_record: dict[str, Any],
|
||||
installed_paths: list[Path],
|
||||
) -> None:
|
||||
inspected = inspect_repository(settings, Path(repository["root"]))
|
||||
if (
|
||||
inspected.repository_id != repository["repository_id"]
|
||||
or inspected.encryption != repository["encryption"]
|
||||
or inspected.data_key_id != repository["active_data_key_id"]
|
||||
):
|
||||
raise RecoveryBundleError("recovery bundle is invalid")
|
||||
private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
|
||||
signing_path = settings.data_dir / "repository-keys" / f"{inspected.repository_id}.ed25519"
|
||||
if signing_path.exists() or signing_path.is_symlink():
|
||||
loaded = load_signing_key(
|
||||
settings,
|
||||
inspected.repository_id,
|
||||
repository["signing_key_id"],
|
||||
repository["signing_public_key"],
|
||||
)
|
||||
if (
|
||||
loaded.private_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PrivateFormat.Raw,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
!= private_key
|
||||
):
|
||||
raise RecoveryBundleError("recovery destination has conflicting keys")
|
||||
else:
|
||||
installed_paths.append(
|
||||
install_signing_key(
|
||||
settings,
|
||||
inspected.repository_id,
|
||||
repository["signing_key_id"],
|
||||
repository["signing_public_key"],
|
||||
private_key,
|
||||
)
|
||||
)
|
||||
for data_key in key_record["data_keys"]:
|
||||
key_value = base64.b64decode(data_key["key"], validate=True)
|
||||
key_path = (
|
||||
settings.data_dir
|
||||
/ "repository-data-keys"
|
||||
/ f"{inspected.repository_id}.{data_key['key_id']}.key"
|
||||
)
|
||||
if key_path.exists() or key_path.is_symlink():
|
||||
if load_data_key(settings, inspected.repository_id, data_key["key_id"]) != key_value:
|
||||
raise RecoveryBundleError("recovery destination has conflicting keys")
|
||||
else:
|
||||
installed_paths.append(
|
||||
install_data_key(settings, inspected.repository_id, data_key["key_id"], key_value)
|
||||
)
|
||||
|
||||
|
||||
async def import_recovery_payload(
|
||||
settings: Settings,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
after_key_install: Callable[[], None] | None = None,
|
||||
) -> int:
|
||||
"""Install a complete authenticated catalog into an empty, migrated database."""
|
||||
catalog, key_records = _validated_recovery_catalog(payload)
|
||||
engine = create_engine(settings)
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
installed_paths: list[Path] = []
|
||||
completed = False
|
||||
try:
|
||||
async with sessions() as db:
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if await db.scalar(select(func.count()).select_from(table)):
|
||||
raise RecoveryBundleError("recovery destination is not empty")
|
||||
await db.rollback()
|
||||
for repository, key_record in zip(catalog["repositories"], key_records, strict=True):
|
||||
_install_or_verify_recovery_keys(settings, repository, key_record, installed_paths)
|
||||
if after_key_install is not None:
|
||||
after_key_install()
|
||||
async with db.begin():
|
||||
for repository in catalog["repositories"]:
|
||||
db.add(
|
||||
Repository(
|
||||
id=repository["database_id"],
|
||||
name=repository["name"],
|
||||
root=repository["root"],
|
||||
format_version=repository["format_version"],
|
||||
compression=repository["compression"],
|
||||
encryption=repository["encryption"],
|
||||
signing_key_id=repository["signing_key_id"],
|
||||
signing_public_key=repository["signing_public_key"],
|
||||
active_data_key_id=repository["active_data_key_id"],
|
||||
state=repository["state"],
|
||||
)
|
||||
)
|
||||
for repository in catalog["repositories"]:
|
||||
for epoch in repository["data_key_epochs"]:
|
||||
db.add(
|
||||
RepositoryDataKeyEpoch(
|
||||
repository_id=repository["database_id"],
|
||||
key_id=epoch["key_id"],
|
||||
state=epoch["state"],
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
for source in catalog["sources"]:
|
||||
db.add(
|
||||
Source(
|
||||
id=source["id"],
|
||||
name=source["name"],
|
||||
kind=source["kind"],
|
||||
public_config=source["public_config"],
|
||||
secret_refs=[],
|
||||
state="unavailable",
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
for job in catalog["jobs"]:
|
||||
db.add(
|
||||
Job(
|
||||
id=job["id"],
|
||||
name=job["name"],
|
||||
source_id=job["source_id"],
|
||||
repository_id=job["repository_id"],
|
||||
requested_mode=job["requested_mode"],
|
||||
exclusions=job["exclusions"],
|
||||
retention=job["retention"],
|
||||
enabled=False,
|
||||
allow_empty=job["allow_empty"],
|
||||
state="archived",
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
for execution in catalog["executions"]:
|
||||
db.add(
|
||||
Execution(
|
||||
id=execution["id"],
|
||||
job_id=execution["job_id"],
|
||||
schedule_id=None,
|
||||
trigger=execution["trigger"],
|
||||
state="committed",
|
||||
attempt=execution["attempt"],
|
||||
progress=execution["progress"],
|
||||
revision=execution["revision"],
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
for backup in catalog["backups"]:
|
||||
db.add(
|
||||
Backup(
|
||||
id=backup["id"],
|
||||
execution_id=backup["execution_id"],
|
||||
parent_backup_id=None,
|
||||
manifest_id=backup["manifest_id"],
|
||||
manifest_digest=backup["manifest_digest"],
|
||||
logical_bytes=backup["logical_bytes"],
|
||||
stored_bytes=backup["stored_bytes"],
|
||||
integrity=backup["integrity"],
|
||||
data_key_id=backup["data_key_id"],
|
||||
pinned=backup["pinned"],
|
||||
created_at=_catalog_datetime(backup["created_at"]),
|
||||
tombstoned_at=_catalog_datetime(
|
||||
backup["tombstoned_at"], allow_none=True
|
||||
),
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
for backup in catalog["backups"]:
|
||||
if backup["parent_backup_id"] is not None:
|
||||
imported = await db.get(Backup, backup["id"])
|
||||
if imported is None:
|
||||
raise RecoveryBundleError("recovery bundle is invalid")
|
||||
imported.parent_backup_id = backup["parent_backup_id"]
|
||||
completed = True
|
||||
return len(catalog["repositories"])
|
||||
except (OSError, RepositoryKeyError, ValueError) as error:
|
||||
if isinstance(error, RecoveryBundleError):
|
||||
raise
|
||||
raise RecoveryBundleError("recovery import failed") from error
|
||||
finally:
|
||||
await engine.dispose()
|
||||
if not completed:
|
||||
for path in installed_paths:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def run_admin(settings: Settings, args: argparse.Namespace) -> int:
|
||||
if (
|
||||
getattr(args, "admin_command", None) == "repository-key"
|
||||
and getattr(args, "repository_key_command", None) == "rotate"
|
||||
):
|
||||
result = asyncio.run(rotate_repository_key(settings, args.repository_id))
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
return 0
|
||||
if getattr(args, "admin_command", None) == "recovery":
|
||||
passphrase = read_passphrase_fd(args.passphrase_fd)
|
||||
if args.recovery_command == "export":
|
||||
payload = asyncio.run(recovery_export_payload(settings))
|
||||
write_bundle_exclusive(args.output, encrypt_bundle(payload, passphrase))
|
||||
print(json.dumps({"repositories": len(payload["keys"]), "status": "exported"}))
|
||||
return 0
|
||||
if args.recovery_command == "validate":
|
||||
payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
|
||||
validation_result: dict[str, object] = {
|
||||
"repositories": _validate_recovery_payload(payload),
|
||||
"status": "valid",
|
||||
}
|
||||
print(json.dumps(validation_result))
|
||||
return 0
|
||||
if args.recovery_command == "import":
|
||||
payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
|
||||
imported = asyncio.run(import_recovery_payload(settings, payload))
|
||||
print(json.dumps({"repositories": imported, "status": "imported"}))
|
||||
return 0
|
||||
raise ValueError("admin command is invalid")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None, *, settings: Settings | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
runtime_settings = settings or load_settings()
|
||||
if args.role == "migrate":
|
||||
return run_migration(args.action, runtime_settings)
|
||||
if args.role == "health":
|
||||
return run_role_healthcheck(runtime_settings, args.health_role)
|
||||
require_current_schema(runtime_settings)
|
||||
if args.role == "admin":
|
||||
return run_admin(runtime_settings, args)
|
||||
return ROLE_HANDLERS[args.role](runtime_settings)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class Clock(Protocol):
|
||||
def now(self) -> datetime: ...
|
||||
|
||||
|
||||
class SystemClock:
|
||||
def now(self) -> datetime:
|
||||
return datetime.now(UTC)
|
||||
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Literal, Self
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="BACKUP_TOOL_",
|
||||
extra="forbid",
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
data_dir: Path = Path("/var/lib/backup-tool")
|
||||
web_socket_path: Path = Path("/tmp/backup-tool-web.sock")
|
||||
database_url: str = "sqlite+aiosqlite:////var/lib/backup-tool/metadata.db"
|
||||
repository_roots: tuple[Path, ...]
|
||||
local_source_roots: tuple[Path, ...]
|
||||
restore_roots: tuple[Path, ...]
|
||||
master_key_file: Path
|
||||
public_base_url: str = "http://127.0.0.1:8000"
|
||||
bootstrap_secret: str | None = Field(default=None, min_length=16)
|
||||
session_ttl_seconds: int = Field(default=28_800, ge=60, le=2_592_000)
|
||||
cors_origins: tuple[str, ...] = ()
|
||||
worker_concurrency: Literal[1] = 1
|
||||
notification_delivery_lease_seconds: int = Field(default=60, ge=5, le=3600)
|
||||
notification_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
|
||||
notification_read_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
|
||||
notification_max_attempts: int = Field(default=5, ge=1, le=20)
|
||||
notification_retry_cap_seconds: int = Field(default=3600, ge=1, le=86_400)
|
||||
notification_global_rate_per_minute: int = Field(default=120, ge=1, le=10_000)
|
||||
notification_default_rate_per_minute: int = Field(default=60, ge=1, le=10_000)
|
||||
notification_max_webhook_body_bytes: int = Field(default=65_536, ge=256, le=1_048_576)
|
||||
notification_max_response_bytes: int = Field(default=16_384, ge=256, le=1_048_576)
|
||||
ssh_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
|
||||
ssh_operation_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
|
||||
ssh_read_chunk_bytes: int = Field(default=1_048_576, ge=4096, le=8_388_608)
|
||||
ssh_list_read_aheads: int = Field(default=32, ge=1, le=256)
|
||||
ssh_max_entries: int = Field(default=1_000_000, ge=1, le=10_000_000)
|
||||
ssh_max_traversal_depth: int = Field(default=128, ge=1, le=1024)
|
||||
min_free_bytes: int = Field(default=1_073_741_824, ge=0)
|
||||
min_free_percent: int = Field(default=5, gt=0, lt=100)
|
||||
sqlite_busy_timeout_ms: int = Field(default=5_000, ge=1_000, le=120_000)
|
||||
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
||||
|
||||
@field_validator(
|
||||
"data_dir",
|
||||
"master_key_file",
|
||||
"web_socket_path",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def validate_absolute_path(cls, value: object) -> Path:
|
||||
path = Path(str(value)).expanduser()
|
||||
if not path.is_absolute():
|
||||
raise ValueError("path must be absolute")
|
||||
return path.resolve()
|
||||
|
||||
@field_validator("repository_roots", "local_source_roots", "restore_roots", mode="before")
|
||||
@classmethod
|
||||
def validate_roots(cls, value: object) -> tuple[Path, ...]:
|
||||
if isinstance(value, (str, Path)):
|
||||
raw_roots = [value]
|
||||
elif isinstance(value, (list, tuple, set)):
|
||||
raw_roots = list(value)
|
||||
else:
|
||||
raise ValueError("allowlist roots must be a sequence of paths")
|
||||
roots: list[Path] = []
|
||||
for raw_root in raw_roots:
|
||||
root = Path(str(raw_root)).expanduser()
|
||||
if not root.is_absolute():
|
||||
raise ValueError("allowlist roots must be absolute")
|
||||
roots.append(root.resolve())
|
||||
if not roots:
|
||||
raise ValueError("at least one allowlist root is required")
|
||||
if len(set(roots)) != len(roots):
|
||||
raise ValueError("allowlist roots must be unique")
|
||||
return tuple(roots)
|
||||
|
||||
@field_validator("database_url")
|
||||
@classmethod
|
||||
def validate_database_url(cls, value: str) -> str:
|
||||
url = make_url(value)
|
||||
if url.drivername != "sqlite+aiosqlite":
|
||||
raise ValueError("v2 requires sqlite+aiosqlite")
|
||||
if not url.database or not Path(url.database).is_absolute():
|
||||
raise ValueError("SQLite database path must be absolute")
|
||||
return value
|
||||
|
||||
@field_validator("public_base_url")
|
||||
@classmethod
|
||||
def validate_public_base_url(cls, value: str) -> str:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("public base URL must be an absolute HTTP(S) URL")
|
||||
if parsed.query or parsed.fragment:
|
||||
raise ValueError("public base URL must not contain query or fragment")
|
||||
return value.rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_master_key(self) -> Self:
|
||||
key = self.master_key_file
|
||||
if not key.is_file():
|
||||
raise ValueError("master key file must exist and be a regular file")
|
||||
key_stat = key.stat()
|
||||
if hasattr(os, "getuid") and key_stat.st_uid != os.getuid():
|
||||
raise ValueError("master key file must be owned by the service user")
|
||||
mode = stat.S_IMODE(key_stat.st_mode)
|
||||
if mode != 0o600:
|
||||
raise ValueError("master key file permissions must be 0600")
|
||||
if key_stat.st_size < 32:
|
||||
raise ValueError("master key file must contain at least 32 bytes")
|
||||
return self
|
||||
|
||||
@property
|
||||
def setup_requires_bootstrap(self) -> bool:
|
||||
hostname = urlsplit(self.public_base_url).hostname
|
||||
return hostname not in {"127.0.0.1", "::1", "localhost"}
|
||||
|
||||
@property
|
||||
def database_path(self) -> Path:
|
||||
database = make_url(self.database_url).database
|
||||
if database is None: # pragma: no cover - guarded by validation
|
||||
raise RuntimeError("database path is unavailable")
|
||||
return Path(database).resolve()
|
||||
@@ -0,0 +1,4 @@
|
||||
from .engine import SchemaNotCurrentError, assert_schema_current, create_engine
|
||||
from .models import Base
|
||||
|
||||
__all__ = ["Base", "SchemaNotCurrentError", "assert_schema_current", "create_engine"]
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
from backup_tool.config import Settings
|
||||
|
||||
from . import sqlite as sqlite_runtime
|
||||
|
||||
|
||||
class SchemaNotCurrentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def create_engine(settings: Settings) -> AsyncEngine:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
connect_args=sqlite_runtime.sqlite_connect_args(settings.sqlite_busy_timeout_ms),
|
||||
)
|
||||
|
||||
@event.listens_for(engine.sync_engine, "connect")
|
||||
def configure_sqlite(dbapi_connection: Any, _connection_record: Any) -> None:
|
||||
sqlite_runtime.configure_sqlite_connection(dbapi_connection)
|
||||
|
||||
return engine
|
||||
|
||||
|
||||
async def assert_schema_current(engine: AsyncEngine, alembic_config: Config) -> None:
|
||||
expected = ScriptDirectory.from_config(alembic_config).get_current_head()
|
||||
|
||||
def get_current_revision(connection: Connection) -> str | None:
|
||||
return MigrationContext.configure(connection).get_current_revision()
|
||||
|
||||
async with engine.connect() as connection:
|
||||
current = await connection.run_sync(get_current_revision)
|
||||
if current != expected:
|
||||
raise SchemaNotCurrentError(
|
||||
f"database schema is not current: expected {expected!r}, found {current!r}"
|
||||
)
|
||||
@@ -0,0 +1,451 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
LargeBinary,
|
||||
MetaData,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
from backup_tool.ids import new_uuid7
|
||||
|
||||
from .types import UTCDateTime
|
||||
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
|
||||
|
||||
class IdentityMixin:
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(new_uuid7()))
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
UTCDateTime(), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
UTCDateTime(),
|
||||
nullable=False,
|
||||
server_default=func.current_timestamp(),
|
||||
onupdate=func.current_timestamp(),
|
||||
)
|
||||
|
||||
|
||||
class User(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "users"
|
||||
username: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (CheckConstraint("state IN ('active','disabled')", name="state"),)
|
||||
|
||||
|
||||
class ApiToken(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "api_tokens"
|
||||
owner_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
scopes: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
__table_args__ = (Index("ix_api_tokens_owner_id", "owner_id"),)
|
||||
|
||||
|
||||
class Session(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "sessions"
|
||||
user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
expires_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
__table_args__ = (Index("ix_sessions_user_id", "user_id"),)
|
||||
|
||||
|
||||
class Secret(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "secrets"
|
||||
ciphertext: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
key_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
purpose: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
__table_args__ = (CheckConstraint("version > 0", name="version_positive"),)
|
||||
|
||||
|
||||
class Repository(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "repositories"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
root: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
format_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
compression: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
encryption: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
signing_key_id: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
|
||||
signing_public_key: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
|
||||
active_data_key_id: Mapped[str | None] = mapped_column(String(36))
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (
|
||||
CheckConstraint("format_version > 0", name="format_version_positive"),
|
||||
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||
)
|
||||
|
||||
|
||||
class RepositoryDataKeyEpoch(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "repository_data_key_epochs"
|
||||
repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
|
||||
key_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
retired_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
__table_args__ = (
|
||||
UniqueConstraint("repository_id", "key_id", name="repository_key_epoch"),
|
||||
CheckConstraint("state IN ('active','retired')", name="repository_data_key_epoch_state"),
|
||||
Index("ix_repository_data_key_epochs_repository_id", "repository_id"),
|
||||
Index(
|
||||
"uq_repository_data_key_epochs_active",
|
||||
"repository_id",
|
||||
unique=True,
|
||||
sqlite_where=text("state = 'active'"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Source(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "sources"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
public_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
secret_refs: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||
__table_args__ = (
|
||||
CheckConstraint("kind IN ('local','ssh')", name="kind"),
|
||||
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||
)
|
||||
|
||||
|
||||
class Job(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "jobs"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id", ondelete="RESTRICT"))
|
||||
repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
|
||||
requested_mode: Mapped[str] = mapped_column(String(32), nullable=False, default="incremental")
|
||||
exclusions: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
retention: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
allow_empty: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (
|
||||
CheckConstraint("requested_mode IN ('full','incremental')", name="requested_mode"),
|
||||
CheckConstraint("state IN ('active','archived')", name="state"),
|
||||
Index("ix_jobs_source_id", "source_id"),
|
||||
Index("ix_jobs_repository_id", "repository_id"),
|
||||
)
|
||||
|
||||
|
||||
class Schedule(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "schedules"
|
||||
job_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("jobs.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||
)
|
||||
cron: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
timezone: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
misfire_grace_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=900)
|
||||
overlap_policy: Mapped[str] = mapped_column(String(32), nullable=False, default="prohibit")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
next_nominal_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
last_enqueue_outcome: Mapped[str | None] = mapped_column(String(64))
|
||||
__table_args__ = (
|
||||
CheckConstraint("misfire_grace_seconds >= 0", name="misfire_nonnegative"),
|
||||
CheckConstraint("overlap_policy IN ('prohibit')", name="overlap_policy"),
|
||||
)
|
||||
|
||||
|
||||
class Execution(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "executions"
|
||||
job_id: Mapped[str] = mapped_column(ForeignKey("jobs.id", ondelete="RESTRICT"))
|
||||
schedule_id: Mapped[str | None] = mapped_column(ForeignKey("schedules.id", ondelete="RESTRICT"))
|
||||
nominal_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
trigger: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(255))
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
progress: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
reason_code: Mapped[str | None] = mapped_column(String(64))
|
||||
operator_message: Mapped[str | None] = mapped_column(Text)
|
||||
started_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt > 0", name="attempt_positive"),
|
||||
CheckConstraint(
|
||||
"state IN ('queued','preparing','running','verifying','committed',"
|
||||
"'cancelling','cancelled','failed')",
|
||||
name="state",
|
||||
),
|
||||
CheckConstraint("trigger IN ('manual','schedule','retry')", name="trigger"),
|
||||
UniqueConstraint("schedule_id", "nominal_run_at", name="uq_execution_schedule_occurrence"),
|
||||
Index("ix_executions_job_id", "job_id"),
|
||||
Index("ix_executions_state_created", "state", "created_at"),
|
||||
Index(
|
||||
"uq_executions_active_job",
|
||||
"job_id",
|
||||
unique=True,
|
||||
sqlite_where=text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ExecutionEvent(Base):
|
||||
__tablename__ = "execution_events"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
execution_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("executions.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("execution_id", "revision", name="uq_execution_event_revision"),
|
||||
)
|
||||
|
||||
|
||||
class Backup(IdentityMixin, Base):
|
||||
__tablename__ = "backups"
|
||||
execution_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("executions.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||
)
|
||||
parent_backup_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("backups.id", ondelete="RESTRICT")
|
||||
)
|
||||
manifest_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True)
|
||||
manifest_digest: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified")
|
||||
data_key_id: Mapped[str | None] = mapped_column(String(36))
|
||||
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
tombstoned_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
UTCDateTime(), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
CheckConstraint("logical_bytes >= 0", name="logical_bytes_nonnegative"),
|
||||
CheckConstraint("stored_bytes >= 0", name="stored_bytes_nonnegative"),
|
||||
CheckConstraint(
|
||||
"integrity IN ('unverified','verified','degraded','corrupt')", name="integrity"
|
||||
),
|
||||
Index("ix_backups_created_at", "created_at"),
|
||||
)
|
||||
|
||||
|
||||
class Restore(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "restores"
|
||||
backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT"))
|
||||
destination: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
selection: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
dry_run: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0"
|
||||
)
|
||||
overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
result: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||
__table_args__ = (
|
||||
CheckConstraint("overwrite_policy IN ('fail','skip','replace')", name="overwrite_policy"),
|
||||
CheckConstraint(
|
||||
"state IN ('queued','running','committed','cancelled','failed')", name="state"
|
||||
),
|
||||
Index("ix_restores_backup_id", "backup_id"),
|
||||
)
|
||||
|
||||
|
||||
class AuditEvent(IdentityMixin, Base):
|
||||
__tablename__ = "audit_events"
|
||||
actor_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
action: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
resource_id: Mapped[str | None] = mapped_column(String(36))
|
||||
outcome: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
request_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
details: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
UTCDateTime(), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
CheckConstraint("outcome IN ('success','failure','denied')", name="outcome"),
|
||||
Index("ix_audit_events_created_at", "created_at"),
|
||||
Index("ix_audit_events_actor_id", "actor_id"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationSubscription(IdentityMixin, TimestampMixin, Base):
|
||||
"""A typed, mutable outbound channel configuration; credential material is never here."""
|
||||
|
||||
__tablename__ = "notification_subscriptions"
|
||||
channel: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
# Retained only to make the migration from the unused v1-shaped table lossless.
|
||||
# New webhook secrets live in NotificationSigningKey and SMTP secrets in settings.
|
||||
secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
|
||||
rate_tokens: Mapped[float] = mapped_column(nullable=False, default=60.0)
|
||||
rate_updated_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
__table_args__ = (
|
||||
CheckConstraint("channel IN ('webhook','email')", name="channel"),
|
||||
CheckConstraint("state IN ('active','disabled','archived')", name="state"),
|
||||
CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
|
||||
CheckConstraint("rate_tokens >= 0", name="rate_tokens_nonnegative"),
|
||||
CheckConstraint("revision > 0", name="revision_positive"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationEvent(IdentityMixin, Base):
|
||||
"""Immutable canonical outbox envelope, created in the source mutation transaction."""
|
||||
|
||||
__tablename__ = "notification_events"
|
||||
type: Mapped[str] = mapped_column(String(96), nullable=False)
|
||||
schema_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
occurred_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
|
||||
correlation_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
severity: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
resource_refs: Mapped[dict[str, str]] = mapped_column(JSON, nullable=False)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
canonical_envelope: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
deduplication_key: Mapped[str | None] = mapped_column(String(255), unique=True)
|
||||
__table_args__ = (
|
||||
CheckConstraint("schema_version = 1", name="schema_version"),
|
||||
CheckConstraint("severity IN ('info','warning','error','security')", name="severity"),
|
||||
Index("ix_notification_events_type_occurred", "type", "occurred_at"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationDelivery(IdentityMixin, TimestampMixin, Base):
|
||||
"""One durable delivery per matching subscription, with a lease-owned lifecycle."""
|
||||
|
||||
__tablename__ = "notification_deliveries"
|
||||
event_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("notification_events.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
subscription_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
|
||||
due_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(255))
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
terminal_reason: Mapped[str | None] = mapped_column(String(96))
|
||||
response_class: Mapped[str | None] = mapped_column(String(64))
|
||||
response_summary: Mapped[str | None] = mapped_column(String(512))
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt_count >= 0", name="attempt_count_nonnegative"),
|
||||
CheckConstraint("state IN ('pending','leased','delivered','retry','failed')", name="state"),
|
||||
UniqueConstraint("event_id", "subscription_id", name="event_subscription"),
|
||||
Index("ix_notification_deliveries_due", "state", "due_at"),
|
||||
Index("ix_notification_deliveries_lease", "state", "lease_expires_at"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationDeliveryAttempt(IdentityMixin, Base):
|
||||
"""Append-only history. Diagnostics are bounded/redacted before persistence."""
|
||||
|
||||
__tablename__ = "notification_delivery_attempts"
|
||||
delivery_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("notification_deliveries.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
started_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
outcome: Mapped[str] = mapped_column(String(32), nullable=False, default="started")
|
||||
response_class: Mapped[str | None] = mapped_column(String(64))
|
||||
diagnostic: Mapped[str | None] = mapped_column(String(512))
|
||||
__table_args__ = (
|
||||
CheckConstraint("number > 0", name="number_positive"),
|
||||
CheckConstraint("outcome IN ('started','delivered','retry','failed')", name="outcome"),
|
||||
UniqueConstraint("delivery_id", "number", name="delivery_number"),
|
||||
Index("ix_notification_attempts_delivery", "delivery_id", "number"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationSigningKey(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "notification_signing_keys"
|
||||
subscription_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
secret_id: Mapped[str] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
|
||||
overlap_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
|
||||
__table_args__ = (
|
||||
CheckConstraint("version > 0", name="version_positive"),
|
||||
CheckConstraint("state IN ('active','overlap','retired')", name="state"),
|
||||
UniqueConstraint("subscription_id", "version", name="subscription_version"),
|
||||
Index("ix_notification_signing_keys_subscription", "subscription_id", "state"),
|
||||
Index(
|
||||
"uq_notification_signing_keys_active",
|
||||
"subscription_id",
|
||||
unique=True,
|
||||
sqlite_where=text("state = 'active'"),
|
||||
),
|
||||
Index(
|
||||
"uq_notification_signing_keys_overlap",
|
||||
"subscription_id",
|
||||
unique=True,
|
||||
sqlite_where=text("state = 'overlap'"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class NotificationEmailSettings(Base):
|
||||
__tablename__ = "notification_email_settings"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
|
||||
host: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
port: Mapped[int] = mapped_column(Integer, nullable=False, default=587)
|
||||
username: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
password_secret_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("secrets.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
sender: Mapped[str] = mapped_column(String(320), nullable=False)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
||||
rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
|
||||
__table_args__ = (
|
||||
CheckConstraint("id = 1", name="singleton"),
|
||||
CheckConstraint("port > 0 AND port < 65536", name="port"),
|
||||
CheckConstraint("max_attempts > 0", name="max_attempts"),
|
||||
CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
|
||||
)
|
||||
|
||||
|
||||
class IdempotencyRecord(IdentityMixin, Base):
|
||||
__tablename__ = "idempotency_records"
|
||||
actor_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
operation: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_digest: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
response_resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
response_resource_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
UTCDateTime(), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("actor_id", "key", "operation", name="uq_idempotency_actor_key_operation"),
|
||||
Index("ix_idempotency_created_at", "created_at"),
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_BUSY_TIMEOUT_MS = 5_000
|
||||
|
||||
|
||||
def sqlite_connect_args(busy_timeout_ms: int) -> dict[str, float]:
|
||||
"""Configure SQLite lock waiting without dynamic PRAGMA SQL."""
|
||||
return {"timeout": busy_timeout_ms / 1_000}
|
||||
|
||||
|
||||
def configure_sqlite_connection(dbapi_connection: Any) -> None:
|
||||
"""Apply connection-local safety and persistent WAL mode."""
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
finally:
|
||||
cursor.close()
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime
|
||||
from sqlalchemy.engine import Dialect
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
|
||||
class UTCDateTime(TypeDecorator[datetime]):
|
||||
"""Store instants and always return timezone-aware UTC datetimes."""
|
||||
|
||||
impl = DateTime
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect: Dialect) -> Any:
|
||||
return dialect.type_descriptor(DateTime(timezone=True))
|
||||
|
||||
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||
del dialect
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("datetime values must be timezone-aware")
|
||||
return value.astimezone(UTC)
|
||||
|
||||
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||
del dialect
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC)
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
|
||||
class ExclusionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _validate(value: str) -> None:
|
||||
path = PurePosixPath(value)
|
||||
if not value or "\\" in value or path.is_absolute() or ".." in path.parts:
|
||||
raise ExclusionError("exclusion paths must be normalized relative POSIX paths")
|
||||
|
||||
|
||||
def matches(path: str, patterns: list[str]) -> bool:
|
||||
"""Return whether normalized relative `path` is excluded by ordered gitignore-like rules."""
|
||||
_validate(path)
|
||||
excluded = False
|
||||
candidate = PurePosixPath(path)
|
||||
for pattern in patterns:
|
||||
negated = pattern.startswith("!")
|
||||
raw = pattern[1:] if negated else pattern
|
||||
if not raw or raw.startswith("/") or "\\" in raw or ".." in PurePosixPath(raw).parts:
|
||||
raise ExclusionError("exclusion pattern is invalid")
|
||||
directory = raw.endswith("/")
|
||||
raw = raw.rstrip("/")
|
||||
if not raw:
|
||||
raise ExclusionError("exclusion pattern is invalid")
|
||||
matched = candidate.match(raw) or candidate.match(f"**/{raw}")
|
||||
if directory:
|
||||
matched = matched or any(
|
||||
parent.match(raw) or parent.match(f"**/{raw}") for parent in candidate.parents
|
||||
)
|
||||
if matched:
|
||||
excluded = not negated
|
||||
return excluded
|
||||
@@ -0,0 +1,306 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.db.models import Execution, ExecutionEvent, Job
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.security.redaction import redact
|
||||
|
||||
ACTIVE_STATES = frozenset({"queued", "preparing", "running", "verifying", "cancelling"})
|
||||
TERMINAL_STATES = frozenset({"committed", "cancelled", "failed"})
|
||||
TRANSIENT_REASONS = frozenset({"worker_lost", "timeout", "unavailable", "transient_io"})
|
||||
|
||||
_ALLOWED: Mapping[str, frozenset[str]] = {
|
||||
"queued": frozenset({"preparing", "cancelled", "failed"}),
|
||||
"preparing": frozenset({"running", "cancelling", "failed"}),
|
||||
"running": frozenset({"verifying", "cancelling", "failed"}),
|
||||
"verifying": frozenset({"committed", "failed"}),
|
||||
"cancelling": frozenset({"cancelled", "failed"}),
|
||||
"committed": frozenset(),
|
||||
"cancelled": frozenset(),
|
||||
"failed": frozenset(),
|
||||
}
|
||||
|
||||
|
||||
class TransitionError(ValueError):
|
||||
code = "invalid_transition"
|
||||
|
||||
|
||||
class EnqueueError(ValueError):
|
||||
def __init__(self, code: str, detail: str, active_execution_id: str | None = None):
|
||||
super().__init__(detail)
|
||||
self.code = code
|
||||
self.active_execution_id = active_execution_id
|
||||
|
||||
|
||||
def transition(current: str, target: str) -> str:
|
||||
"""Validate and return one durable execution-state transition."""
|
||||
if target not in _ALLOWED.get(current, frozenset()):
|
||||
raise TransitionError(f"Cannot transition execution from {current!r} to {target!r}.")
|
||||
return target
|
||||
|
||||
|
||||
async def enqueue(
|
||||
db: AsyncSession,
|
||||
job_id: str,
|
||||
trigger: str = "manual",
|
||||
*,
|
||||
schedule_id: str | None = None,
|
||||
nominal_run_at: datetime | None = None,
|
||||
) -> Execution:
|
||||
"""Create exactly one active execution for an enabled active job."""
|
||||
job = await db.get(Job, job_id)
|
||||
if job is None:
|
||||
raise EnqueueError("resource_not_found", "Job was not found.")
|
||||
if job.state != "active" or not job.enabled:
|
||||
raise EnqueueError("job_disabled", "Job is disabled or archived.")
|
||||
job_identifier = job.id
|
||||
execution = Execution(
|
||||
job_id=job_identifier,
|
||||
trigger=trigger,
|
||||
schedule_id=schedule_id,
|
||||
nominal_run_at=nominal_run_at,
|
||||
progress={},
|
||||
)
|
||||
db.add(execution)
|
||||
try:
|
||||
await db.flush()
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
except IntegrityError as error:
|
||||
await db.rollback()
|
||||
existing = await db.scalar(
|
||||
select(Execution.id).where(
|
||||
Execution.job_id == job_identifier, Execution.state.in_(ACTIVE_STATES)
|
||||
)
|
||||
)
|
||||
raise EnqueueError(
|
||||
"execution_active", "Job already has an active execution.", existing
|
||||
) from error
|
||||
await db.refresh(execution)
|
||||
return execution
|
||||
|
||||
|
||||
async def claim(
|
||||
db: AsyncSession, execution_id: str, owner: str, lease_seconds: int = 60
|
||||
) -> Execution | None:
|
||||
"""Atomically lease a queued execution; a stale lease may be recovered."""
|
||||
now = datetime.now(UTC)
|
||||
expires = now + timedelta(seconds=lease_seconds)
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(
|
||||
Execution.id == execution_id,
|
||||
Execution.state.in_({"queued", "preparing"}),
|
||||
(Execution.lease_expires_at.is_(None)) | (Execution.lease_expires_at < now),
|
||||
)
|
||||
.values(state="preparing", lease_owner=owner, lease_expires_at=expires, heartbeat_at=now)
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return None
|
||||
execution = await db.get(Execution, execution_id)
|
||||
if execution is not None:
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
return execution
|
||||
|
||||
|
||||
async def heartbeat(
|
||||
db: AsyncSession, execution_id: str, owner: str, lease_seconds: int = 60
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(
|
||||
Execution.id == execution_id,
|
||||
Execution.lease_owner == owner,
|
||||
Execution.state.in_(ACTIVE_STATES),
|
||||
Execution.lease_expires_at >= now,
|
||||
)
|
||||
.values(heartbeat_at=now, lease_expires_at=now + timedelta(seconds=lease_seconds))
|
||||
)
|
||||
await db.commit()
|
||||
return getattr(result, "rowcount", 0) == 1
|
||||
|
||||
|
||||
async def request_cancellation(db: AsyncSession, execution_id: str) -> Execution | None:
|
||||
# Queued work has no worker and may terminate immediately; leased work asks its owner.
|
||||
now = datetime.now(UTC)
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(Execution.id == execution_id, Execution.state == "queued")
|
||||
.values(state="cancelled", reason_code="cancellation_requested", completed_at=now)
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(Execution.id == execution_id, Execution.state.in_({"preparing", "running"}))
|
||||
.values(state="cancelling", reason_code="cancellation_requested")
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return None
|
||||
execution = await db.get(Execution, execution_id)
|
||||
if execution is not None:
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
return execution
|
||||
|
||||
|
||||
async def complete_cancellation(db: AsyncSession, execution_id: str, owner: str) -> bool:
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(
|
||||
Execution.id == execution_id,
|
||||
Execution.lease_owner == owner,
|
||||
Execution.lease_expires_at >= datetime.now(UTC),
|
||||
Execution.state == "cancelling",
|
||||
)
|
||||
.values(
|
||||
state="cancelled",
|
||||
completed_at=datetime.now(UTC),
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
)
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return False
|
||||
execution = await db.get(Execution, execution_id)
|
||||
if execution is not None:
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def retry(db: AsyncSession, execution_id: str) -> Execution | None:
|
||||
execution = await db.get(Execution, execution_id)
|
||||
if execution is None:
|
||||
return None
|
||||
if execution.state != "failed" or execution.reason_code not in TRANSIENT_REASONS:
|
||||
raise EnqueueError("retry_not_allowed", "Execution failure is not retryable.")
|
||||
execution.state = "queued"
|
||||
execution.attempt += 1
|
||||
execution.lease_owner = None
|
||||
execution.lease_expires_at = None
|
||||
execution.heartbeat_at = None
|
||||
execution.reason_code = None
|
||||
execution.operator_message = None
|
||||
await record_event(db, execution)
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError as error:
|
||||
await db.rollback()
|
||||
active_id = await db.scalar(
|
||||
select(Execution.id).where(
|
||||
Execution.job_id == execution.job_id, Execution.state.in_(ACTIVE_STATES)
|
||||
)
|
||||
)
|
||||
raise EnqueueError(
|
||||
"execution_active", "Job already has an active execution.", active_id
|
||||
) from error
|
||||
await db.refresh(execution)
|
||||
return execution
|
||||
|
||||
|
||||
def _redact_progress(value: object) -> object:
|
||||
"""Apply common redaction plus execution-specific path redaction recursively."""
|
||||
if isinstance(value, Mapping):
|
||||
safe: dict[str, object] = {}
|
||||
for key, item in value.items():
|
||||
normalized = str(key).lower()
|
||||
if any(marker in normalized for marker in ("path", "secret", "token", "password")):
|
||||
safe[str(key)] = "[REDACTED]"
|
||||
else:
|
||||
safe[str(key)] = _redact_progress(item)
|
||||
return safe
|
||||
if isinstance(value, list):
|
||||
return [_redact_progress(item) for item in value]
|
||||
return redact(value)
|
||||
|
||||
|
||||
async def record_event(db: AsyncSession, execution: Execution) -> ExecutionEvent:
|
||||
"""Append the public representation as the next durable event revision."""
|
||||
execution.revision += 1
|
||||
event = ExecutionEvent(
|
||||
execution_id=execution.id,
|
||||
revision=execution.revision,
|
||||
payload=public_event(execution),
|
||||
)
|
||||
db.add(event)
|
||||
await db.flush()
|
||||
notification_type = {
|
||||
"preparing": "execution.started",
|
||||
"committed": "execution.committed",
|
||||
"failed": "execution.failed",
|
||||
"cancelled": "execution.cancelled",
|
||||
}.get(execution.state)
|
||||
if execution.state == "queued":
|
||||
if execution.reason_code == "worker_lost":
|
||||
notification_type = "execution.worker_recovered"
|
||||
elif execution.attempt > 1:
|
||||
notification_type = "execution.retry_queued"
|
||||
else:
|
||||
notification_type = "execution.queued"
|
||||
if notification_type is not None:
|
||||
await emit_event(
|
||||
db,
|
||||
notification_type,
|
||||
correlation_id=execution.id,
|
||||
resource={"execution_id": execution.id, "job_id": execution.job_id},
|
||||
payload={
|
||||
"attempt": execution.attempt,
|
||||
"state": execution.state,
|
||||
"reason_code": execution.reason_code,
|
||||
},
|
||||
deduplication_key=f"execution:{execution.id}:revision:{execution.revision}",
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
def public_event(execution: Execution) -> dict[str, object]:
|
||||
"""Return redacted progress suitable for polling or SSE."""
|
||||
progress = _redact_progress(execution.progress)
|
||||
return {
|
||||
"id": execution.id,
|
||||
"state": execution.state,
|
||||
"attempt": execution.attempt,
|
||||
"revision": execution.revision,
|
||||
"reason_code": execution.reason_code,
|
||||
"progress": progress,
|
||||
}
|
||||
|
||||
|
||||
async def recover_stale(db: AsyncSession) -> int:
|
||||
"""Atomically recover expired executions and append one event per mutation."""
|
||||
now = datetime.now(UTC)
|
||||
executions = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(Execution).where(
|
||||
Execution.state.in_({"preparing", "running", "verifying", "cancelling"}),
|
||||
Execution.lease_expires_at < now,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
for execution in executions:
|
||||
if execution.state == "cancelling":
|
||||
execution.state = "cancelled"
|
||||
execution.completed_at = now
|
||||
execution.reason_code = "cancellation_requested"
|
||||
else:
|
||||
execution.state = "queued"
|
||||
execution.reason_code = "worker_lost"
|
||||
execution.lease_owner = None
|
||||
execution.lease_expires_at = None
|
||||
execution.heartbeat_at = None
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
return len(executions)
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class FaultInjector(Protocol):
|
||||
def hit(self, point: str) -> None: ...
|
||||
|
||||
|
||||
class NoFault:
|
||||
def hit(self, point: str) -> None:
|
||||
del point
|
||||
|
||||
|
||||
class InjectedCrash(BaseException):
|
||||
"""Test-only abrupt worker termination at a named durable fault point."""
|
||||
|
||||
|
||||
class CrashAt:
|
||||
def __init__(self, point: str) -> None:
|
||||
self.point = point
|
||||
|
||||
def hit(self, point: str) -> None:
|
||||
if point == self.point:
|
||||
raise InjectedCrash(point)
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.db.models import Backup, Execution, Job, Repository
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.retention import BackupLike, RetentionPolicy, retained_ids
|
||||
from backup_tool.security.repository_crypto import RepositoryKeyError, decrypt_object, object_aad
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GcReport:
|
||||
tombstoned: int
|
||||
purged_manifests: int
|
||||
purged_blobs: int
|
||||
quarantined: int
|
||||
|
||||
|
||||
async def tombstone_expired(db: AsyncSession, now: datetime | None = None) -> int:
|
||||
reference = now or datetime.now(UTC)
|
||||
backups = list((await db.scalars(select(Backup))).all())
|
||||
executions = {item.id: item for item in (await db.scalars(select(Execution))).all()}
|
||||
jobs = {item.id: item for item in (await db.scalars(select(Job))).all()}
|
||||
grouped: dict[str, list[Backup]] = {}
|
||||
for backup in backups:
|
||||
execution = executions.get(backup.execution_id)
|
||||
if execution is None or execution.job_id not in jobs:
|
||||
continue
|
||||
grouped.setdefault(execution.job_id, []).append(backup)
|
||||
tombstoned = 0
|
||||
for job_id, items in grouped.items():
|
||||
if not jobs[job_id].retention:
|
||||
continue
|
||||
policy = RetentionPolicy.from_dict(jobs[job_id].retention)
|
||||
keep = retained_ids(cast(list[BackupLike], items), policy, reference)
|
||||
for backup in items:
|
||||
if backup.id not in keep and backup.tombstoned_at is None:
|
||||
backup.tombstoned_at = reference
|
||||
await emit_event(
|
||||
db,
|
||||
"retention.tombstoned",
|
||||
correlation_id=backup.id,
|
||||
resource={"backup_id": backup.id, "job_id": job_id},
|
||||
payload={"outcome": "tombstoned"},
|
||||
deduplication_key=f"backup:{backup.id}:tombstoned",
|
||||
)
|
||||
tombstoned += 1
|
||||
await db.commit()
|
||||
return tombstoned
|
||||
|
||||
|
||||
async def process_retention_gc(db: AsyncSession, now: datetime | None = None) -> GcReport:
|
||||
"""Run durable retention tombstoning and repository GC from the worker role."""
|
||||
reference = now or datetime.now(UTC)
|
||||
tombstoned = await tombstone_expired(db, reference)
|
||||
reports: list[GcReport] = []
|
||||
repositories = list((await db.scalars(select(Repository))).all())
|
||||
for repository in repositories:
|
||||
manifest_ids = set(
|
||||
(
|
||||
await db.scalars(
|
||||
select(Backup.manifest_id)
|
||||
.join(Execution, Backup.execution_id == Execution.id)
|
||||
.join(Job, Execution.job_id == Job.id)
|
||||
.where(
|
||||
Job.repository_id == repository.id,
|
||||
Backup.tombstoned_at.is_not(None),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if manifest_ids:
|
||||
reports.append(purge_repository(Path(repository.root), manifest_ids, now=reference))
|
||||
return GcReport(
|
||||
tombstoned=tombstoned,
|
||||
purged_manifests=sum(report.purged_manifests for report in reports),
|
||||
purged_blobs=sum(report.purged_blobs for report in reports),
|
||||
quarantined=sum(report.quarantined for report in reports),
|
||||
)
|
||||
|
||||
|
||||
def _manifest_digests(
|
||||
path: Path,
|
||||
*,
|
||||
repository_id: str | None = None,
|
||||
manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
|
||||
) -> set[str] | None:
|
||||
try:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return None
|
||||
raw = path.read_bytes()
|
||||
if raw.startswith(b"BTENC\x01"):
|
||||
key_record = manifest_keys.get(path.stem) if manifest_keys is not None else None
|
||||
if key_record is None or repository_id is None:
|
||||
return None
|
||||
key_id, key = key_record
|
||||
raw = decrypt_object(key, object_aad(repository_id, key_id, "manifest", path.stem), raw)
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
except (OSError, RepositoryKeyError, UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(payload, dict) or not isinstance(entries := payload.get("entries"), list):
|
||||
return None
|
||||
digests: set[str] = set()
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
digest = entry.get("blob_digest")
|
||||
if digest is None:
|
||||
continue
|
||||
if (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
return None
|
||||
digests.add(digest)
|
||||
return digests
|
||||
|
||||
|
||||
def purge_repository(
|
||||
root: Path,
|
||||
tombstoned_manifest_ids: set[str],
|
||||
*,
|
||||
repository_id: str | None = None,
|
||||
manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
|
||||
grace: timedelta = timedelta(days=7),
|
||||
now: datetime | None = None,
|
||||
) -> GcReport:
|
||||
reference = now or datetime.now(UTC)
|
||||
manifests = root / "manifests"
|
||||
blobs = root / "blobs" / "sha256"
|
||||
quarantine = root / "quarantine"
|
||||
quarantine.mkdir(exist_ok=True)
|
||||
manifest_digests: dict[Path, set[str]] = {}
|
||||
for manifest in manifests.glob("*.json"):
|
||||
digests = _manifest_digests(
|
||||
manifest,
|
||||
repository_id=repository_id,
|
||||
manifest_keys=manifest_keys,
|
||||
)
|
||||
if digests is None:
|
||||
return GcReport(0, 0, 0, 0)
|
||||
manifest_digests[manifest] = digests
|
||||
|
||||
purged_manifests = 0
|
||||
for manifest_id in tombstoned_manifest_ids:
|
||||
candidate = manifests / f"{manifest_id}.json"
|
||||
if not candidate.is_file() or candidate.is_symlink():
|
||||
continue
|
||||
age = reference - datetime.fromtimestamp(candidate.stat().st_mtime, UTC)
|
||||
if age >= grace:
|
||||
candidate.unlink()
|
||||
purged_manifests += 1
|
||||
referenced: set[str] = set()
|
||||
for manifest, digests in manifest_digests.items():
|
||||
if manifest.exists():
|
||||
referenced.update(digests)
|
||||
purged_blobs = 0
|
||||
quarantined = 0
|
||||
if blobs.exists():
|
||||
for blob in blobs.iterdir():
|
||||
if not blob.is_file():
|
||||
continue
|
||||
if len(blob.name) != 64 or any(char not in "0123456789abcdef" for char in blob.name):
|
||||
try:
|
||||
shutil.move(str(blob), quarantine / blob.name)
|
||||
except OSError:
|
||||
continue
|
||||
quarantined += 1
|
||||
elif blob.name not in referenced and (
|
||||
reference - datetime.fromtimestamp(blob.stat().st_mtime, UTC) >= grace
|
||||
):
|
||||
blob.unlink()
|
||||
purged_blobs += 1
|
||||
return GcReport(0, purged_manifests, purged_blobs, quarantined)
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
_lock = threading.Lock()
|
||||
_last_millisecond = -1
|
||||
_last_random = 0
|
||||
_RANDOM_MASK = (1 << 74) - 1
|
||||
|
||||
|
||||
def new_uuid7() -> UUID:
|
||||
"""Return a process-monotonic RFC 9562 UUIDv7."""
|
||||
global _last_millisecond, _last_random
|
||||
with _lock:
|
||||
millisecond = time.time_ns() // 1_000_000
|
||||
if millisecond > _last_millisecond:
|
||||
_last_millisecond = millisecond
|
||||
_last_random = secrets.randbits(74)
|
||||
else:
|
||||
millisecond = _last_millisecond
|
||||
_last_random = (_last_random + 1) & _RANDOM_MASK
|
||||
if _last_random == 0:
|
||||
_last_millisecond += 1
|
||||
millisecond = _last_millisecond
|
||||
random_a = (_last_random >> 62) & 0xFFF
|
||||
random_b = _last_random & ((1 << 62) - 1)
|
||||
integer = (millisecond & ((1 << 48) - 1)) << 80
|
||||
integer |= 0x7 << 76
|
||||
integer |= random_a << 64
|
||||
integer |= 0b10 << 62
|
||||
integer |= random_b
|
||||
return UUID(int=integer)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Durable, worker-dispatched operational notifications."""
|
||||
|
||||
from .events import EVENT_CATALOG, emit_event, validate_filters
|
||||
|
||||
__all__ = ["EVENT_CATALOG", "emit_event", "validate_filters"]
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Worker-owned leased outbox dispatcher; sends happen only after a committed lease."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.models import (
|
||||
NotificationDelivery,
|
||||
NotificationDeliveryAttempt,
|
||||
NotificationEmailSettings,
|
||||
NotificationEvent,
|
||||
NotificationSigningKey,
|
||||
NotificationSubscription,
|
||||
Secret,
|
||||
)
|
||||
from backup_tool.notifications.email import EmailTransportError, deliver_email
|
||||
from backup_tool.notifications.retry import (
|
||||
RetryDecision,
|
||||
retry_delay,
|
||||
transport_decision,
|
||||
webhook_decision,
|
||||
)
|
||||
from backup_tool.notifications.webhook import (
|
||||
SigningMaterial,
|
||||
WebhookTransportError,
|
||||
deliver_webhook,
|
||||
)
|
||||
from backup_tool.security.secrets import EnvelopeCipher
|
||||
from backup_tool.security.ssrf import Resolver, system_resolver
|
||||
|
||||
|
||||
async def recover_notification_leases(db: AsyncSession) -> int:
|
||||
"""An interrupted post-send lease becomes eligible again (at-least-once by design)."""
|
||||
now = datetime.now(UTC)
|
||||
deliveries = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(NotificationDelivery).where(
|
||||
NotificationDelivery.state == "leased",
|
||||
NotificationDelivery.lease_expires_at < now,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
for delivery in deliveries:
|
||||
attempt = await db.scalar(
|
||||
select(NotificationDeliveryAttempt).where(
|
||||
NotificationDeliveryAttempt.delivery_id == delivery.id,
|
||||
NotificationDeliveryAttempt.number == delivery.attempt_count,
|
||||
NotificationDeliveryAttempt.outcome == "started",
|
||||
)
|
||||
)
|
||||
if attempt is not None:
|
||||
attempt.completed_at = now
|
||||
attempt.outcome = "retry"
|
||||
attempt.response_class = "lease_expired"
|
||||
attempt.diagnostic = "abandoned_lease"
|
||||
delivery.state = "retry"
|
||||
delivery.lease_owner = None
|
||||
delivery.lease_expires_at = None
|
||||
delivery.due_at = now
|
||||
if deliveries:
|
||||
await db.commit()
|
||||
return len(deliveries)
|
||||
|
||||
|
||||
async def _claim_due(
|
||||
db: AsyncSession, owner: str, lease_seconds: int
|
||||
) -> tuple[NotificationDelivery, NotificationSubscription, NotificationEvent] | None:
|
||||
now = datetime.now(UTC)
|
||||
delivery_id = await db.scalar(
|
||||
select(NotificationDelivery.id)
|
||||
.where(
|
||||
NotificationDelivery.state.in_(("pending", "retry")),
|
||||
NotificationDelivery.due_at <= now,
|
||||
)
|
||||
.order_by(NotificationDelivery.due_at, NotificationDelivery.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if delivery_id is None:
|
||||
return None
|
||||
result = await db.execute(
|
||||
update(NotificationDelivery)
|
||||
.where(
|
||||
NotificationDelivery.id == delivery_id,
|
||||
NotificationDelivery.state.in_(("pending", "retry")),
|
||||
NotificationDelivery.due_at <= now,
|
||||
)
|
||||
.values(
|
||||
state="leased",
|
||||
lease_owner=owner,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
attempt_count=NotificationDelivery.attempt_count + 1,
|
||||
)
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return None
|
||||
delivery = await db.get(NotificationDelivery, delivery_id)
|
||||
if delivery is None: # pragma: no cover - guarded by update
|
||||
await db.rollback()
|
||||
return None
|
||||
subscription = await db.get(NotificationSubscription, delivery.subscription_id)
|
||||
event = await db.get(NotificationEvent, delivery.event_id)
|
||||
if subscription is None or event is None or subscription.state != "active":
|
||||
delivery.state = "failed"
|
||||
delivery.terminal_reason = "subscription_unavailable"
|
||||
delivery.lease_owner = None
|
||||
delivery.lease_expires_at = None
|
||||
await db.commit()
|
||||
return None
|
||||
# Persist a token bucket before starting an attempt, so restarts cannot bypass
|
||||
# the subscription rate limit. Global process rate is intentionally a config
|
||||
# ceiling; the durable subscription bucket protects cross-restart behavior.
|
||||
last = subscription.rate_updated_at or now
|
||||
elapsed = max(0.0, (now - last).total_seconds())
|
||||
capacity = subscription.rate_limit_per_minute
|
||||
try:
|
||||
token_capacity = float(capacity)
|
||||
tokens = min(token_capacity, subscription.rate_tokens + elapsed * capacity / 60)
|
||||
except (TypeError, ValueError, ZeroDivisionError) as error:
|
||||
raise RuntimeError("notification rate limit is invalid") from error
|
||||
if tokens < 1:
|
||||
try:
|
||||
delay = max(1, int((1 - tokens) * 60 / capacity) + 1)
|
||||
except (TypeError, ValueError, ZeroDivisionError) as error:
|
||||
raise RuntimeError("notification rate limit is invalid") from error
|
||||
delivery.state = "retry"
|
||||
delivery.due_at = now + timedelta(seconds=delay)
|
||||
delivery.lease_owner = None
|
||||
delivery.lease_expires_at = None
|
||||
subscription.rate_tokens = tokens
|
||||
subscription.rate_updated_at = now
|
||||
await db.commit()
|
||||
return None
|
||||
subscription.rate_tokens = tokens - 1
|
||||
subscription.rate_updated_at = now
|
||||
db.add(
|
||||
NotificationDeliveryAttempt(
|
||||
delivery_id=delivery.id,
|
||||
number=delivery.attempt_count,
|
||||
started_at=now,
|
||||
outcome="started",
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return delivery, subscription, event
|
||||
|
||||
|
||||
async def _finish(
|
||||
db: AsyncSession,
|
||||
delivery_id: str,
|
||||
owner: str,
|
||||
*,
|
||||
delivered: bool,
|
||||
retryable: bool,
|
||||
response_class: str,
|
||||
reason: str,
|
||||
retry_cap: int,
|
||||
max_attempts: int,
|
||||
) -> None:
|
||||
delivery = await db.get(NotificationDelivery, delivery_id)
|
||||
if delivery is None or delivery.lease_owner != owner or delivery.state != "leased":
|
||||
await db.rollback()
|
||||
return
|
||||
attempt = await db.scalar(
|
||||
select(NotificationDeliveryAttempt).where(
|
||||
NotificationDeliveryAttempt.delivery_id == delivery.id,
|
||||
NotificationDeliveryAttempt.number == delivery.attempt_count,
|
||||
)
|
||||
)
|
||||
if attempt is None: # pragma: no cover - an invariant of _claim_due
|
||||
await db.rollback()
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
attempt.completed_at = now
|
||||
attempt.response_class = response_class
|
||||
attempt.diagnostic = reason[:512]
|
||||
delivery.response_class = response_class
|
||||
delivery.response_summary = reason[:512]
|
||||
delivery.lease_owner = None
|
||||
delivery.lease_expires_at = None
|
||||
if delivered:
|
||||
delivery.state = "delivered"
|
||||
attempt.outcome = "delivered"
|
||||
elif retryable and delivery.attempt_count < max_attempts:
|
||||
delivery.state = "retry"
|
||||
delivery.due_at = now + timedelta(seconds=retry_delay(delivery.attempt_count, retry_cap))
|
||||
attempt.outcome = "retry"
|
||||
else:
|
||||
delivery.state = "failed"
|
||||
delivery.terminal_reason = reason
|
||||
attempt.outcome = "failed"
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def dispatch_one(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
cipher: EnvelopeCipher,
|
||||
owner: str,
|
||||
*,
|
||||
resolver: Resolver = system_resolver,
|
||||
) -> bool:
|
||||
claimed = await _claim_due(db, owner, settings.notification_delivery_lease_seconds)
|
||||
if claimed is None:
|
||||
return False
|
||||
delivery, subscription, event = claimed
|
||||
max_attempts = settings.notification_max_attempts
|
||||
try:
|
||||
if subscription.channel == "webhook":
|
||||
key_rows = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(NotificationSigningKey).where(
|
||||
NotificationSigningKey.subscription_id == subscription.id,
|
||||
NotificationSigningKey.state.in_(("active", "overlap")),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
keys: list[SigningMaterial] = []
|
||||
now = datetime.now(UTC)
|
||||
for key in key_rows:
|
||||
if (
|
||||
key.state == "overlap"
|
||||
and key.overlap_expires_at is not None
|
||||
and key.overlap_expires_at <= now
|
||||
):
|
||||
key.state = "retired"
|
||||
continue
|
||||
secret = await db.get(Secret, key.secret_id)
|
||||
if secret is None:
|
||||
raise WebhookTransportError("webhook signing secret is unavailable")
|
||||
keys.append(
|
||||
SigningMaterial(
|
||||
key_id=key.id,
|
||||
version=key.version,
|
||||
secret=cipher.decrypt(
|
||||
secret.ciphertext,
|
||||
purpose=secret.purpose,
|
||||
version=secret.version,
|
||||
),
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
webhook_result = await deliver_webhook(
|
||||
str(subscription.destination_config["url"]),
|
||||
event.canonical_envelope.encode(),
|
||||
event_id=event.id,
|
||||
event_type=event.type,
|
||||
timestamp=event.occurred_at.isoformat(),
|
||||
keys=keys,
|
||||
resolver=resolver,
|
||||
connect_timeout=settings.notification_connect_timeout_seconds,
|
||||
read_timeout=settings.notification_read_timeout_seconds,
|
||||
max_response_bytes=settings.notification_max_response_bytes,
|
||||
)
|
||||
decision = webhook_decision(webhook_result.status_code)
|
||||
elif subscription.channel == "email":
|
||||
email_settings = await db.get(NotificationEmailSettings, 1)
|
||||
if email_settings is None:
|
||||
raise EmailTransportError("SMTP settings are unavailable")
|
||||
max_attempts = email_settings.max_attempts
|
||||
password_secret = await db.get(Secret, email_settings.password_secret_id)
|
||||
if password_secret is None:
|
||||
raise EmailTransportError("SMTP password is unavailable")
|
||||
email_result = await deliver_email(
|
||||
email_settings,
|
||||
cipher.decrypt(
|
||||
password_secret.ciphertext,
|
||||
purpose=password_secret.purpose,
|
||||
version=password_secret.version,
|
||||
),
|
||||
event,
|
||||
subscription.destination_config["recipients"],
|
||||
)
|
||||
decision = RetryDecision(False, email_result.response_class, "delivered")
|
||||
else: # guarded by DB constraint
|
||||
raise WebhookTransportError("notification channel is unavailable")
|
||||
await _finish(
|
||||
db,
|
||||
delivery.id,
|
||||
owner,
|
||||
delivered=decision.reason == "delivered",
|
||||
retryable=decision.retry,
|
||||
response_class=decision.response_class,
|
||||
reason=decision.reason,
|
||||
retry_cap=settings.notification_retry_cap_seconds,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
except EmailTransportError as error:
|
||||
decision = transport_decision(error.transient, "smtp_transport")
|
||||
await _finish(
|
||||
db,
|
||||
delivery.id,
|
||||
owner,
|
||||
delivered=False,
|
||||
retryable=decision.retry,
|
||||
response_class=decision.response_class,
|
||||
reason=str(error),
|
||||
retry_cap=settings.notification_retry_cap_seconds,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
except WebhookTransportError as error:
|
||||
decision = transport_decision(error.transient, "webhook_transport")
|
||||
await _finish(
|
||||
db,
|
||||
delivery.id,
|
||||
owner,
|
||||
delivered=False,
|
||||
retryable=decision.retry,
|
||||
response_class=decision.response_class,
|
||||
reason=str(error),
|
||||
retry_cap=settings.notification_retry_cap_seconds,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
except (KeyError, ValueError):
|
||||
decision = transport_decision(False, "webhook_validation")
|
||||
await _finish(
|
||||
db,
|
||||
delivery.id,
|
||||
owner,
|
||||
delivered=False,
|
||||
retryable=False,
|
||||
response_class=decision.response_class,
|
||||
reason=decision.reason,
|
||||
retry_cap=settings.notification_retry_cap_seconds,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Authenticated, certificate-verified STARTTLS email notification transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import smtplib
|
||||
import ssl
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from email.message import EmailMessage
|
||||
from email.utils import formataddr
|
||||
from typing import Protocol, Self, cast
|
||||
|
||||
from backup_tool.db.models import NotificationEmailSettings, NotificationEvent
|
||||
|
||||
|
||||
class SMTPClient(Protocol):
|
||||
def __enter__(self) -> Self: ...
|
||||
|
||||
def __exit__(self, *args: object) -> None: ...
|
||||
|
||||
def ehlo(self) -> object: ...
|
||||
|
||||
def starttls(self, *, context: ssl.SSLContext) -> object: ...
|
||||
|
||||
def login(self, user: str, password: str) -> object: ...
|
||||
|
||||
def send_message(self, msg: EmailMessage) -> object: ...
|
||||
|
||||
|
||||
class EmailTransportError(RuntimeError):
|
||||
def __init__(self, message: str, *, transient: bool = False) -> None:
|
||||
super().__init__(message)
|
||||
self.transient = transient
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EmailResult:
|
||||
response_class: str
|
||||
|
||||
|
||||
def validate_address(value: str) -> str:
|
||||
if not value or len(value) > 320 or any(character in value for character in "\r\n"):
|
||||
raise EmailTransportError("email address is invalid")
|
||||
local, separator, domain = value.rpartition("@")
|
||||
if not separator or not local or not domain or any(character.isspace() for character in value):
|
||||
raise EmailTransportError("email address is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def validate_recipients(values: Sequence[str]) -> list[str]:
|
||||
if not values or len(values) > 20:
|
||||
raise EmailTransportError("one to 20 email recipients are required")
|
||||
recipients: list[str] = []
|
||||
for value in values:
|
||||
address = validate_address(value)
|
||||
if address not in recipients:
|
||||
recipients.append(address)
|
||||
return recipients
|
||||
|
||||
|
||||
def _message(
|
||||
settings: NotificationEmailSettings,
|
||||
event: NotificationEvent,
|
||||
recipients: Sequence[str],
|
||||
) -> EmailMessage:
|
||||
sender = validate_address(settings.sender)
|
||||
safe_recipients = validate_recipients(recipients)
|
||||
message = EmailMessage()
|
||||
message["From"] = formataddr(("Backup Tool", sender))
|
||||
message["To"] = ", ".join(safe_recipients)
|
||||
message["Subject"] = f"Backup Tool: {event.type} ({event.severity})"
|
||||
message["X-Backup-Event-ID"] = event.id
|
||||
# Do not put the full envelope, paths, raw errors, or credentials into mail.
|
||||
message.set_content(
|
||||
"Backup Tool operational event\n"
|
||||
f"Event ID: {event.id}\n"
|
||||
f"Type: {event.type}\n"
|
||||
f"Severity: {event.severity}\n"
|
||||
f"Occurred: {event.occurred_at.isoformat()}\n"
|
||||
)
|
||||
return message
|
||||
|
||||
|
||||
def _deliver_sync(
|
||||
settings: NotificationEmailSettings,
|
||||
password: str,
|
||||
event: NotificationEvent,
|
||||
recipients: Sequence[str],
|
||||
smtp_factory: Callable[..., SMTPClient],
|
||||
) -> EmailResult:
|
||||
message = _message(settings, event, recipients)
|
||||
try:
|
||||
with smtp_factory(settings.host, settings.port, timeout=10) as client:
|
||||
client.ehlo()
|
||||
context = ssl.create_default_context()
|
||||
client.starttls(context=context)
|
||||
client.ehlo()
|
||||
client.login(settings.username, password)
|
||||
client.send_message(message)
|
||||
except smtplib.SMTPResponseException as error:
|
||||
raise EmailTransportError(
|
||||
f"smtp_{error.smtp_code}", transient=400 <= error.smtp_code < 500
|
||||
) from error
|
||||
except (smtplib.SMTPException, OSError) as error:
|
||||
raise EmailTransportError("smtp_transport_failed", transient=True) from error
|
||||
return EmailResult(response_class="smtp_2xx")
|
||||
|
||||
|
||||
async def deliver_email(
|
||||
settings: NotificationEmailSettings,
|
||||
password: str,
|
||||
event: NotificationEvent,
|
||||
recipients: Sequence[str],
|
||||
*,
|
||||
smtp_factory: Callable[..., SMTPClient] | None = None,
|
||||
) -> EmailResult:
|
||||
"""Run blocking SMTP only in the worker thread, never in the web process."""
|
||||
factory = smtp_factory or cast(Callable[..., SMTPClient], smtplib.SMTP)
|
||||
return await asyncio.to_thread(_deliver_sync, settings, password, event, recipients, factory)
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Versioned notification event catalog and transactional outbox fan-out."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.db.models import NotificationDelivery, NotificationEvent, NotificationSubscription
|
||||
from backup_tool.ids import new_uuid7
|
||||
from backup_tool.security.redaction import redact
|
||||
from backup_tool.security.ssrf import SSRFError, validate_webhook_url
|
||||
|
||||
EVENT_SCHEMA_VERSION = 1
|
||||
# Live-events-only policy: every public type below has a current production emitter.
|
||||
_EVENT_TYPES = (
|
||||
"execution.queued",
|
||||
"execution.started",
|
||||
"execution.committed",
|
||||
"execution.failed",
|
||||
"execution.cancelled",
|
||||
"execution.retry_queued",
|
||||
"execution.worker_recovered",
|
||||
"schedule.created",
|
||||
"schedule.updated",
|
||||
"schedule.deleted",
|
||||
"schedule.enabled",
|
||||
"schedule.disabled",
|
||||
"schedule.occurrence_enqueued",
|
||||
"schedule.occurrence_misfired",
|
||||
"schedule.occurrence_blocked",
|
||||
"backup.committed",
|
||||
"backup.verification_succeeded",
|
||||
"restore.queued",
|
||||
"restore.committed",
|
||||
"restore.failed",
|
||||
"retention.tombstoned",
|
||||
"notification.test_requested",
|
||||
)
|
||||
EVENT_CATALOG: dict[str, dict[str, Any]] = {
|
||||
event_type: {
|
||||
"event_schema_version": EVENT_SCHEMA_VERSION,
|
||||
"severity": "error" if event_type.endswith(("failed", "blocked", "rejected")) else "info",
|
||||
"payload_keys": (
|
||||
"attempt",
|
||||
"count",
|
||||
"dry_run",
|
||||
"integrity",
|
||||
"message",
|
||||
"outcome",
|
||||
"reason_code",
|
||||
"requested_mode",
|
||||
"effective_mode",
|
||||
"state",
|
||||
),
|
||||
"reserved": False,
|
||||
}
|
||||
for event_type in _EVENT_TYPES
|
||||
}
|
||||
_SAFE_RESOURCE_KEYS = frozenset(
|
||||
{
|
||||
"execution_id",
|
||||
"job_id",
|
||||
"schedule_id",
|
||||
"backup_id",
|
||||
"restore_id",
|
||||
"repository_id",
|
||||
"subscription_id",
|
||||
}
|
||||
)
|
||||
_SAFE_PAYLOAD_KEYS = frozenset().union(
|
||||
*(set(spec["payload_keys"]) for spec in EVENT_CATALOG.values())
|
||||
)
|
||||
|
||||
|
||||
class NotificationEventError(ValueError):
|
||||
"""A caller attempted to produce data outside the stable public catalog."""
|
||||
|
||||
|
||||
def _as_uuid(value: str, field: str) -> str:
|
||||
try:
|
||||
parsed = UUID(value)
|
||||
except (TypeError, ValueError, AttributeError) as error:
|
||||
raise NotificationEventError(f"{field} must be a UUID") from error
|
||||
return str(parsed)
|
||||
|
||||
|
||||
def _safe_value(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return redact(value)[:512]
|
||||
if isinstance(value, list):
|
||||
if len(value) > 32:
|
||||
raise NotificationEventError("payload arrays are limited to 32 values")
|
||||
return [_safe_value(item) for item in value]
|
||||
if isinstance(value, Mapping):
|
||||
if len(value) > 32:
|
||||
raise NotificationEventError("payload objects are limited to 32 fields")
|
||||
return {str(key)[:64]: _safe_value(item) for key, item in value.items()}
|
||||
raise NotificationEventError("payload contains an unsupported value")
|
||||
|
||||
|
||||
def _is_filter_match(filter_value: str, event_type: str) -> bool:
|
||||
if filter_value == event_type:
|
||||
return True
|
||||
family, wildcard = filter_value.rsplit(".", 1) if "." in filter_value else ("", "")
|
||||
return wildcard == "*" and event_type.startswith(f"{family}.")
|
||||
|
||||
|
||||
def validate_filters(filters: Sequence[str]) -> list[str]:
|
||||
if not filters:
|
||||
raise NotificationEventError("at least one event filter is required")
|
||||
if len(filters) > len(EVENT_CATALOG):
|
||||
raise NotificationEventError("too many event filters")
|
||||
output: list[str] = []
|
||||
for filter_value in filters:
|
||||
if not isinstance(filter_value, str) or len(filter_value) > 96:
|
||||
raise NotificationEventError("invalid event filter")
|
||||
if filter_value.endswith(".*"):
|
||||
family = filter_value[:-2]
|
||||
if not family or not any(item.startswith(f"{family}.") for item in EVENT_CATALOG):
|
||||
raise NotificationEventError("unknown event filter")
|
||||
elif filter_value not in EVENT_CATALOG:
|
||||
raise NotificationEventError("unknown event filter")
|
||||
if filter_value not in output:
|
||||
output.append(filter_value)
|
||||
return output
|
||||
|
||||
|
||||
def validate_destination(channel: str, destination: Mapping[str, Any]) -> dict[str, Any]:
|
||||
resource_filters = destination.get("resource_filters", {})
|
||||
if not isinstance(resource_filters, Mapping):
|
||||
raise NotificationEventError("resource filters are invalid")
|
||||
unknown = set(resource_filters) - {"job_ids", "repository_ids", "severities"}
|
||||
if unknown:
|
||||
raise NotificationEventError("resource filter is unknown")
|
||||
normalized_filters: dict[str, list[str]] = {}
|
||||
for key in ("job_ids", "repository_ids"):
|
||||
values = resource_filters.get(key)
|
||||
if values is None:
|
||||
continue
|
||||
if not isinstance(values, list) or not values:
|
||||
raise NotificationEventError("resource filter is invalid")
|
||||
normalized_filters[key] = [_as_uuid(value, key) for value in values]
|
||||
severities = resource_filters.get("severities")
|
||||
if severities is not None:
|
||||
if not isinstance(severities, list) or not severities:
|
||||
raise NotificationEventError("severity filter is invalid")
|
||||
if any(value not in {"info", "warning", "error", "security"} for value in severities):
|
||||
raise NotificationEventError("severity filter is invalid")
|
||||
normalized_filters["severities"] = list(severities)
|
||||
if channel == "webhook":
|
||||
url = destination.get("url")
|
||||
if not isinstance(url, str) or len(url) > 2048:
|
||||
raise NotificationEventError("webhook URL is required")
|
||||
try:
|
||||
validate_webhook_url(url)
|
||||
except SSRFError as error:
|
||||
raise NotificationEventError("webhook URL is invalid") from error
|
||||
return {"url": url, "resource_filters": normalized_filters}
|
||||
if channel == "email":
|
||||
recipients = destination.get("recipients")
|
||||
if not isinstance(recipients, list) or not recipients or len(recipients) > 20:
|
||||
raise NotificationEventError("one to 20 email recipients are required")
|
||||
safe_recipients: list[str] = []
|
||||
for recipient in recipients:
|
||||
if not isinstance(recipient, str) or any(char in recipient for char in "\r\n"):
|
||||
raise NotificationEventError("invalid email recipient")
|
||||
if "@" not in recipient or len(recipient) > 320:
|
||||
raise NotificationEventError("invalid email recipient")
|
||||
if recipient not in safe_recipients:
|
||||
safe_recipients.append(recipient)
|
||||
return {"recipients": safe_recipients, "resource_filters": normalized_filters}
|
||||
raise NotificationEventError("unsupported notification channel")
|
||||
|
||||
|
||||
def _matches(subscription: NotificationSubscription, event: dict[str, Any]) -> bool:
|
||||
if subscription.state != "active":
|
||||
return False
|
||||
if not any(_is_filter_match(item, str(event["type"])) for item in subscription.event_filters):
|
||||
return False
|
||||
filters = subscription.destination_config.get("resource_filters", {})
|
||||
if not isinstance(filters, Mapping):
|
||||
return False
|
||||
resources = event["resource"]
|
||||
for key in ("job_ids", "repository_ids"):
|
||||
selected = filters.get(key)
|
||||
resource_key = key[:-1]
|
||||
if selected is not None and resources.get(resource_key) not in selected:
|
||||
return False
|
||||
severities = filters.get("severities")
|
||||
return severities is None or event["severity"] in severities
|
||||
|
||||
|
||||
async def emit_event(
|
||||
db: AsyncSession,
|
||||
event_type: str,
|
||||
*,
|
||||
correlation_id: str,
|
||||
resource: Mapping[str, str] | None = None,
|
||||
payload: Mapping[str, Any] | None = None,
|
||||
severity: str | None = None,
|
||||
deduplication_key: str | None = None,
|
||||
occurred_at: datetime | None = None,
|
||||
only_subscription_id: str | None = None,
|
||||
) -> NotificationEvent:
|
||||
"""Append an immutable event and matching deliveries; intentionally never commits."""
|
||||
if event_type not in EVENT_CATALOG:
|
||||
raise NotificationEventError("unknown operational event type")
|
||||
correlation_id = _as_uuid(correlation_id, "correlation_id")
|
||||
resource = resource or {}
|
||||
if set(resource) - _SAFE_RESOURCE_KEYS:
|
||||
raise NotificationEventError("unknown resource reference")
|
||||
safe_resource = {key: _as_uuid(value, key) for key, value in resource.items()}
|
||||
payload = payload or {}
|
||||
if set(payload) - _SAFE_PAYLOAD_KEYS:
|
||||
raise NotificationEventError("payload key is not allowlisted")
|
||||
safe_payload = {key: _safe_value(value) for key, value in payload.items()}
|
||||
event_severity = severity or str(EVENT_CATALOG[event_type]["severity"])
|
||||
if event_severity not in {"info", "warning", "error", "security"}:
|
||||
raise NotificationEventError("invalid severity")
|
||||
if deduplication_key is not None and (not deduplication_key or len(deduplication_key) > 255):
|
||||
raise NotificationEventError("invalid deduplication key")
|
||||
if deduplication_key is not None:
|
||||
existing = await db.scalar(
|
||||
select(NotificationEvent).where(
|
||||
NotificationEvent.deduplication_key == deduplication_key
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return existing
|
||||
event_id = str(new_uuid7())
|
||||
timestamp = (occurred_at or datetime.now(UTC)).astimezone(UTC)
|
||||
envelope = {
|
||||
"event_schema_version": EVENT_SCHEMA_VERSION,
|
||||
"id": event_id,
|
||||
"type": event_type,
|
||||
"occurred_at": timestamp.isoformat(),
|
||||
"correlation_id": correlation_id,
|
||||
"severity": event_severity,
|
||||
"resource": safe_resource,
|
||||
"payload": safe_payload,
|
||||
}
|
||||
event = NotificationEvent(
|
||||
id=event_id,
|
||||
type=event_type,
|
||||
schema_version=EVENT_SCHEMA_VERSION,
|
||||
occurred_at=timestamp,
|
||||
correlation_id=correlation_id,
|
||||
severity=event_severity,
|
||||
resource_refs=safe_resource,
|
||||
payload=safe_payload,
|
||||
canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
|
||||
deduplication_key=deduplication_key,
|
||||
)
|
||||
db.add(event)
|
||||
await db.flush()
|
||||
active_subscriptions = select(NotificationSubscription).where(
|
||||
NotificationSubscription.state == "active"
|
||||
)
|
||||
subscriptions = list((await db.scalars(active_subscriptions)).all())
|
||||
for subscription in subscriptions:
|
||||
selected_for_test = only_subscription_id == subscription.id
|
||||
if selected_for_test or (only_subscription_id is None and _matches(subscription, envelope)):
|
||||
db.add(
|
||||
NotificationDelivery(
|
||||
event_id=event.id,
|
||||
subscription_id=subscription.id,
|
||||
due_at=timestamp,
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
return event
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Deterministic bounded retry classification for notification delivery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetryDecision:
|
||||
retry: bool
|
||||
response_class: str
|
||||
reason: str
|
||||
|
||||
|
||||
def retry_delay(attempt: int, cap_seconds: int) -> int:
|
||||
"""Bounded exponential delay; no jitter keeps durable tests/restarts deterministic."""
|
||||
exponent = max(0, attempt - 1)
|
||||
return min(cap_seconds, 1 << exponent)
|
||||
|
||||
|
||||
def webhook_decision(status_code: int) -> RetryDecision:
|
||||
if 200 <= status_code < 300:
|
||||
return RetryDecision(False, "http_2xx", "delivered")
|
||||
if status_code in {408, 425, 429} or status_code >= 500:
|
||||
return RetryDecision(True, f"http_{status_code}", "http_transient")
|
||||
if 300 <= status_code < 400:
|
||||
return RetryDecision(False, f"http_{status_code}", "redirect_rejected")
|
||||
return RetryDecision(False, f"http_{status_code}", "http_permanent")
|
||||
|
||||
|
||||
def transport_decision(transient: bool, response_class: str) -> RetryDecision:
|
||||
reason = "transport_transient" if transient else "transport_failed"
|
||||
return RetryDecision(transient, response_class, reason)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Canonical, dual-key HMAC webhook requests on a DNS-pinned HTTPX transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import ssl
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
|
||||
import httpx
|
||||
|
||||
from backup_tool.security.ssrf import (
|
||||
ResolvedWebhookTarget,
|
||||
Resolver,
|
||||
SSRFError,
|
||||
resolve_webhook_target,
|
||||
verify_connected_peer,
|
||||
)
|
||||
|
||||
SIGNATURE_VERSION = "v1"
|
||||
|
||||
|
||||
class WebhookTransportError(RuntimeError):
|
||||
def __init__(self, message: str, *, transient: bool = False) -> None:
|
||||
super().__init__(message)
|
||||
self.transient = transient
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SigningMaterial:
|
||||
key_id: str
|
||||
version: int
|
||||
secret: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebhookResult:
|
||||
status_code: int
|
||||
response_bytes: int
|
||||
|
||||
|
||||
def canonical_signing_input(timestamp: str, body: bytes) -> bytes:
|
||||
return SIGNATURE_VERSION.encode() + b"." + timestamp.encode("ascii") + b"." + body
|
||||
|
||||
|
||||
def signatures(timestamp: str, body: bytes, keys: Sequence[SigningMaterial]) -> list[str]:
|
||||
signing_input = canonical_signing_input(timestamp, body)
|
||||
return [
|
||||
f"{SIGNATURE_VERSION};key_id={key.key_id};key_version={key.version};sha256="
|
||||
f"{hmac.new(key.secret.encode(), signing_input, sha256).hexdigest()}"
|
||||
for key in keys
|
||||
]
|
||||
|
||||
|
||||
class PinnedWebhookTransport(httpx.AsyncBaseTransport):
|
||||
"""HTTPX transport that never lets a post-validation DNS lookup choose a peer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
target: ResolvedWebhookTarget,
|
||||
connect_timeout: float,
|
||||
read_timeout: float,
|
||||
max_response_bytes: int,
|
||||
) -> None:
|
||||
self._target = target
|
||||
self._connect_timeout = connect_timeout
|
||||
self._read_timeout = read_timeout
|
||||
self._max_response_bytes = max_response_bytes
|
||||
|
||||
async def _connect(
|
||||
self, target: ResolvedWebhookTarget
|
||||
) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]:
|
||||
hostname = target.url.hostname
|
||||
if hostname is None: # guarded by resolve_webhook_target
|
||||
raise WebhookTransportError("webhook hostname is unavailable")
|
||||
context = ssl.create_default_context() if target.url.scheme == "https" else None
|
||||
last_error: OSError | None = None
|
||||
for address in target.addresses:
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(
|
||||
address,
|
||||
target.port,
|
||||
ssl=context,
|
||||
server_hostname=hostname if context is not None else None,
|
||||
),
|
||||
timeout=self._connect_timeout,
|
||||
)
|
||||
verify_connected_peer(writer.get_extra_info("peername"), target.addresses)
|
||||
return reader, writer
|
||||
except (TimeoutError, OSError, ssl.SSLError, SSRFError) as error:
|
||||
last_error = error if isinstance(error, OSError) else None
|
||||
raise WebhookTransportError("webhook connection failed", transient=True) from last_error
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
target = self._target
|
||||
if str(request.url) != target.url.geturl():
|
||||
raise WebhookTransportError("webhook target changed")
|
||||
body = await request.aread()
|
||||
if len(body) > 65_536:
|
||||
raise WebhookTransportError("webhook body is too large")
|
||||
reader, writer = await self._connect(target)
|
||||
try:
|
||||
raw_path = target.url.path or "/"
|
||||
if target.url.query:
|
||||
raw_path += "?" + target.url.query
|
||||
headers = [(key, value) for key, value in request.headers.multi_items()]
|
||||
header_names = {key.lower() for key, _ in headers}
|
||||
if "host" not in header_names:
|
||||
host = target.url.hostname or ""
|
||||
headers.append(("Host", host))
|
||||
if "content-length" not in header_names:
|
||||
headers.append(("Content-Length", str(len(body))))
|
||||
headers.append(("Connection", "close"))
|
||||
serialized = [f"{request.method} {raw_path} HTTP/1.1\r\n".encode()]
|
||||
serialized.extend(f"{key}: {value}\r\n".encode("ascii") for key, value in headers)
|
||||
writer.write(b"".join(serialized) + b"\r\n" + body)
|
||||
await asyncio.wait_for(writer.drain(), timeout=self._read_timeout)
|
||||
head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=self._read_timeout)
|
||||
if len(head) > 16_384:
|
||||
raise WebhookTransportError("webhook response headers are too large")
|
||||
lines = head.decode("iso-8859-1").split("\r\n")
|
||||
try:
|
||||
_protocol, code, _reason = lines[0].split(" ", 2)
|
||||
status_code = int(code)
|
||||
except (IndexError, ValueError) as error:
|
||||
raise WebhookTransportError("webhook response is malformed") from error
|
||||
response_headers: list[tuple[str, str]] = []
|
||||
for line in lines[1:]:
|
||||
if not line:
|
||||
continue
|
||||
key, separator, value = line.partition(":")
|
||||
if not separator:
|
||||
raise WebhookTransportError("webhook response headers are malformed")
|
||||
response_headers.append((key.strip(), value.strip()))
|
||||
content = await asyncio.wait_for(
|
||||
reader.read(self._max_response_bytes + 1), timeout=self._read_timeout
|
||||
)
|
||||
if len(content) > self._max_response_bytes:
|
||||
raise WebhookTransportError("webhook response is too large")
|
||||
return httpx.Response(
|
||||
status_code,
|
||||
headers=response_headers,
|
||||
content=content,
|
||||
request=request,
|
||||
)
|
||||
except (TimeoutError, OSError, asyncio.IncompleteReadError) as error:
|
||||
raise WebhookTransportError("webhook request failed", transient=True) from error
|
||||
finally:
|
||||
writer.close()
|
||||
with __import__("contextlib").suppress(OSError):
|
||||
await writer.wait_closed()
|
||||
|
||||
|
||||
def webhook_headers(
|
||||
event_id: str,
|
||||
event_type: str,
|
||||
timestamp: str,
|
||||
body: bytes,
|
||||
keys: Sequence[SigningMaterial],
|
||||
) -> list[tuple[str, str]]:
|
||||
headers: list[tuple[str, str]] = [
|
||||
("Content-Type", "application/json"),
|
||||
("X-Backup-Event-ID", event_id),
|
||||
("X-Backup-Event-Type", event_type),
|
||||
("X-Backup-Signature-Version", SIGNATURE_VERSION),
|
||||
("X-Backup-Timestamp", timestamp),
|
||||
]
|
||||
headers.extend(("X-Backup-Signature", value) for value in signatures(timestamp, body, keys))
|
||||
return headers
|
||||
|
||||
|
||||
async def deliver_webhook(
|
||||
url: str,
|
||||
body: bytes,
|
||||
*,
|
||||
event_id: str,
|
||||
event_type: str,
|
||||
timestamp: str,
|
||||
keys: Sequence[SigningMaterial],
|
||||
resolver: Resolver,
|
||||
connect_timeout: float,
|
||||
read_timeout: float,
|
||||
max_response_bytes: int,
|
||||
) -> WebhookResult:
|
||||
if not keys:
|
||||
raise WebhookTransportError("webhook subscription has no active signing key")
|
||||
try:
|
||||
# Resolve immediately before this individual attempt. The resulting
|
||||
# addresses are passed to the transport, so it cannot rebind at connect.
|
||||
target = await resolve_webhook_target(url, resolver)
|
||||
except SSRFError as error:
|
||||
raise WebhookTransportError("webhook_target_rejected") from error
|
||||
transport = PinnedWebhookTransport(
|
||||
target=target,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
max_response_bytes=max_response_bytes,
|
||||
)
|
||||
headers = webhook_headers(event_id, event_type, timestamp, body, keys)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport, follow_redirects=False, trust_env=False
|
||||
) as client:
|
||||
response = await client.post(url, content=body, headers=headers)
|
||||
if 300 <= response.status_code < 400:
|
||||
raise WebhookTransportError("redirect_rejected")
|
||||
return WebhookResult(status_code=response.status_code, response_bytes=len(response.content))
|
||||
@@ -0,0 +1 @@
|
||||
"""Operational logging, metrics, and readiness primitives."""
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Readiness checks shared by HTTP and background process roles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from backup_tool.cli import build_alembic_config
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import assert_schema_current, create_engine
|
||||
|
||||
|
||||
class ReadinessError(RuntimeError):
|
||||
"""A dependency needed by the selected role is unavailable."""
|
||||
|
||||
|
||||
def _require_access(path: object, mode: int) -> None:
|
||||
try:
|
||||
candidate = path if isinstance(path, str) else str(path)
|
||||
if not os.path.isdir(candidate) or not os.access(candidate, mode):
|
||||
raise ReadinessError("required storage is unavailable")
|
||||
except OSError as error:
|
||||
raise ReadinessError("required storage is unavailable") from error
|
||||
|
||||
|
||||
async def check_role_readiness(settings: Settings, role: str) -> None:
|
||||
engine = create_engine(settings)
|
||||
try:
|
||||
await assert_schema_current(engine, build_alembic_config(settings))
|
||||
except Exception as error:
|
||||
raise ReadinessError("metadata is unavailable") from error
|
||||
finally:
|
||||
await engine.dispose()
|
||||
if role == "scheduler":
|
||||
return
|
||||
_require_access(settings.data_dir, os.R_OK | os.W_OK | os.X_OK)
|
||||
for root in settings.repository_roots + settings.restore_roots:
|
||||
_require_access(root, os.R_OK | os.W_OK | os.X_OK)
|
||||
for root in settings.local_source_roots:
|
||||
_require_access(root, os.R_OK | os.X_OK)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""JSON logging that keeps operational context machine-readable and secret-free."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
_STANDARD_RECORD_KEYS = frozenset(logging.makeLogRecord({}).__dict__)
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"event": record.getMessage(),
|
||||
"level": record.levelname.lower(),
|
||||
"logger": record.name,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_RECORD_KEYS and key not in {"message", "asctime"}:
|
||||
payload[key] = value
|
||||
return json.dumps(payload, default=str, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def configure_logging(role: str, level: str) -> None:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
root.addHandler(handler)
|
||||
root.setLevel(level)
|
||||
logging.getLogger("backup_tool").info("role_started", extra={"role": role})
|
||||
|
||||
|
||||
def log_event(name: str, **fields: object) -> None:
|
||||
logging.getLogger("backup_tool").info(name, extra=fields)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Small dependency-free Prometheus exposition for the single-node appliance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.models import Backup, Execution, Repository, Schedule
|
||||
from backup_tool.execution import ACTIVE_STATES
|
||||
|
||||
|
||||
class Metrics:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
|
||||
self._durations: dict[tuple[str, str], tuple[int, float]] = {}
|
||||
|
||||
def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None:
|
||||
route = path if path in {"/livez", "/readyz", "/metrics"} else "/api"
|
||||
with self._lock:
|
||||
self._requests[(method, route, status)] += 1
|
||||
count, total = self._durations.get((method, route), (0, 0.0))
|
||||
self._durations[(method, route)] = (count + 1, total + duration_seconds)
|
||||
|
||||
def render(self, operational: Iterable[tuple[str, float]]) -> str:
|
||||
lines = [
|
||||
"# HELP backup_tool_http_requests_total HTTP requests handled by the web role.",
|
||||
"# TYPE backup_tool_http_requests_total counter",
|
||||
]
|
||||
with self._lock:
|
||||
for (method, path, status), value in sorted(self._requests.items()):
|
||||
labels = f'method="{method}",path="{path}",status="{status}"'
|
||||
lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}")
|
||||
lines.extend(
|
||||
[
|
||||
"# HELP backup_tool_http_request_duration_seconds HTTP request duration.",
|
||||
"# TYPE backup_tool_http_request_duration_seconds summary",
|
||||
]
|
||||
)
|
||||
for (method, path), (count, total) in sorted(self._durations.items()):
|
||||
labels = f'method="{method}",path="{path}"'
|
||||
lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}")
|
||||
lines.append(
|
||||
f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}"
|
||||
)
|
||||
lines.extend(f"{name} {value}" for name, value in operational)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
async def collect_operational_metrics(
|
||||
settings: Settings, db: AsyncSession
|
||||
) -> list[tuple[str, float]]:
|
||||
now = datetime.now(UTC)
|
||||
active = await db.scalar(
|
||||
select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES))
|
||||
)
|
||||
stale = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Execution)
|
||||
.where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now)
|
||||
)
|
||||
failed = await db.scalar(
|
||||
select(func.count()).select_from(Execution).where(Execution.state == "failed")
|
||||
)
|
||||
corrupt = await db.scalar(
|
||||
select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt")
|
||||
)
|
||||
schedule_lag = await db.scalar(
|
||||
select(func.min(Schedule.next_nominal_at)).where(
|
||||
Schedule.enabled, Schedule.next_nominal_at.is_not(None)
|
||||
)
|
||||
)
|
||||
values = [
|
||||
("backup_tool_active_executions", active or 0),
|
||||
("backup_tool_stale_execution_leases", stale or 0),
|
||||
("backup_tool_failed_executions", failed or 0),
|
||||
("backup_tool_corrupt_backups", corrupt or 0),
|
||||
(
|
||||
"backup_tool_schedule_lag_seconds",
|
||||
max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0,
|
||||
),
|
||||
]
|
||||
roots = list(settings.repository_roots) + list(settings.restore_roots)
|
||||
for index, root in enumerate(roots):
|
||||
try:
|
||||
stats = os.statvfs(root)
|
||||
except OSError:
|
||||
continue
|
||||
name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}'
|
||||
values.append((name, stats.f_bavail * stats.f_frsize))
|
||||
unavailable = await db.scalar(
|
||||
select(func.count()).select_from(Repository).where(Repository.state == "unavailable")
|
||||
)
|
||||
values.append(("backup_tool_unavailable_repositories", unavailable or 0))
|
||||
return values
|
||||
@@ -0,0 +1,512 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.ids import new_uuid7
|
||||
from backup_tool.security.repository_crypto import RepositoryKeyError, create_data_key
|
||||
|
||||
|
||||
class RepositoryError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InitializedRepository:
|
||||
root: Path
|
||||
repository_id: str
|
||||
format_version: int = 1
|
||||
compression: str = "none"
|
||||
encryption: str = "none"
|
||||
signing_key_id: str = ""
|
||||
signing_public_key: str = ""
|
||||
signing_key_path: Path | None = None
|
||||
data_key_id: str | None = None
|
||||
data_key_path: Path | None = None
|
||||
|
||||
|
||||
def blob_digest(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def _contained(root: Path, relative_path: str) -> Path:
|
||||
relative = Path(relative_path)
|
||||
if not relative_path or relative.is_absolute():
|
||||
raise RepositoryError("repository path must be a non-empty relative child")
|
||||
lexical = root
|
||||
for part in relative.parts:
|
||||
lexical /= part
|
||||
if lexical.is_symlink():
|
||||
raise RepositoryError("repository path contains a symlink")
|
||||
candidate = lexical.resolve()
|
||||
resolved_root = root.resolve()
|
||||
if candidate == resolved_root:
|
||||
raise RepositoryError("repository path must be a non-empty relative child")
|
||||
if resolved_root not in candidate.parents:
|
||||
raise RepositoryError("repository path escapes configured roots")
|
||||
return candidate
|
||||
|
||||
|
||||
def _canonical_payload(compression: str, encryption: str) -> dict[str, object]:
|
||||
if compression != "none" or encryption not in {"none", "aes-256-gcm"}:
|
||||
raise RepositoryError("requested repository policy is unavailable")
|
||||
return {
|
||||
"repository_id": str(new_uuid7()),
|
||||
"format_version": 1,
|
||||
"digest_algorithm": "sha256",
|
||||
"compression": compression,
|
||||
"encryption": {"mode": encryption, "key_id": None},
|
||||
"created_at": datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(payload: dict[str, object]) -> str:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n"
|
||||
|
||||
|
||||
def replace_active_data_key(root: Path, expected_key_id: str, new_key_id: str) -> None:
|
||||
"""Durably replace the active, non-secret epoch reference in repository metadata."""
|
||||
metadata = root / "repository.json"
|
||||
try:
|
||||
payload = json.loads(metadata.read_text(encoding="utf-8"))
|
||||
encryption = payload.get("encryption") if isinstance(payload, dict) else None
|
||||
if (
|
||||
not isinstance(encryption, dict)
|
||||
or encryption.get("mode") != "aes-256-gcm"
|
||||
or encryption.get("key_id") != expected_key_id
|
||||
):
|
||||
raise ValueError
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
raise RepositoryError("repository metadata is invalid") from error
|
||||
encryption["key_id"] = new_key_id
|
||||
staging = root / f".repository.json.{os.urandom(8).hex()}.tmp"
|
||||
try:
|
||||
descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(_canonical_json(payload))
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(staging, metadata)
|
||||
descriptor = os.open(root, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
except OSError as error:
|
||||
raise RepositoryError("repository metadata cannot be updated") from error
|
||||
finally:
|
||||
staging.unlink(missing_ok=True)
|
||||
|
||||
|
||||
_ROTATION_JOURNAL = ".key-rotation.json"
|
||||
|
||||
|
||||
def begin_key_rotation(
|
||||
root: Path,
|
||||
repository_database_id: str,
|
||||
repository_id: str,
|
||||
old_key_id: str,
|
||||
new_key_id: str,
|
||||
) -> None:
|
||||
"""Persist an intent record before changing durable key-epoch state."""
|
||||
if not all((repository_database_id, repository_id, old_key_id, new_key_id)):
|
||||
raise RepositoryError("repository rotation journal is invalid")
|
||||
journal = root / _ROTATION_JOURNAL
|
||||
payload: dict[str, object] = {
|
||||
"new_key_id": new_key_id,
|
||||
"old_key_id": old_key_id,
|
||||
"repository_database_id": repository_database_id,
|
||||
"repository_id": repository_id,
|
||||
"version": 1,
|
||||
}
|
||||
try:
|
||||
descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(_canonical_json(payload))
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
descriptor = os.open(root, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
except OSError as error:
|
||||
raise RepositoryError("repository rotation is already in progress") from error
|
||||
|
||||
|
||||
def read_key_rotation(root: Path) -> dict[str, str] | None:
|
||||
journal = root / _ROTATION_JOURNAL
|
||||
if not journal.exists():
|
||||
return None
|
||||
try:
|
||||
if journal.is_symlink() or stat.S_IMODE(journal.stat().st_mode) != 0o600:
|
||||
raise ValueError
|
||||
payload = json.loads(journal.read_text(encoding="utf-8"))
|
||||
expected = {
|
||||
"new_key_id",
|
||||
"old_key_id",
|
||||
"repository_database_id",
|
||||
"repository_id",
|
||||
"version",
|
||||
}
|
||||
if (
|
||||
not isinstance(payload, dict)
|
||||
or set(payload) != expected
|
||||
or payload.get("version") != 1
|
||||
or not all(
|
||||
isinstance(payload[key], str) and payload[key] for key in expected - {"version"}
|
||||
)
|
||||
):
|
||||
raise ValueError
|
||||
return {key: payload[key] for key in expected - {"version"}}
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
raise RepositoryError("repository rotation journal is invalid") from error
|
||||
|
||||
|
||||
def finish_key_rotation(root: Path) -> None:
|
||||
journal = root / _ROTATION_JOURNAL
|
||||
try:
|
||||
journal.unlink()
|
||||
descriptor = os.open(root, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
except OSError as error:
|
||||
raise RepositoryError("repository rotation journal cannot be cleared") from error
|
||||
|
||||
|
||||
async def reconcile_key_rotations(settings: Settings, db: AsyncSession) -> int:
|
||||
"""Converge a journaled key rotation after a process crash.
|
||||
|
||||
A rotation whose DB transaction did not commit remains on the old active
|
||||
epoch. A committed transaction deterministically advances repository.json.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from backup_tool.db.models import Repository, RepositoryDataKeyEpoch
|
||||
from backup_tool.security.repository_crypto import load_data_key
|
||||
|
||||
repositories = list((await db.scalars(select(Repository))).all())
|
||||
reconciled = 0
|
||||
for repository in repositories:
|
||||
if repository.encryption != "aes-256-gcm":
|
||||
continue
|
||||
root = Path(repository.root)
|
||||
journal = read_key_rotation(root)
|
||||
if journal is None:
|
||||
continue
|
||||
inspected = inspect_repository(settings, root)
|
||||
if (
|
||||
journal["repository_database_id"] != repository.id
|
||||
or journal["repository_id"] != inspected.repository_id
|
||||
or repository.active_data_key_id is None
|
||||
):
|
||||
raise RepositoryError("repository rotation journal does not match metadata")
|
||||
epochs = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(RepositoryDataKeyEpoch).where(
|
||||
RepositoryDataKeyEpoch.repository_id == repository.id
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
active = [epoch for epoch in epochs if epoch.state == "active"]
|
||||
old_key_id = journal["old_key_id"]
|
||||
new_key_id = journal["new_key_id"]
|
||||
if len(active) != 1:
|
||||
raise RepositoryError("repository rotation epochs are invalid")
|
||||
if repository.active_data_key_id == old_key_id and active[0].key_id == old_key_id:
|
||||
if inspected.data_key_id != old_key_id:
|
||||
raise RepositoryError("repository rotation metadata is invalid")
|
||||
new_path = (
|
||||
settings.data_dir
|
||||
/ "repository-data-keys"
|
||||
/ (f"{inspected.repository_id}.{new_key_id}.key")
|
||||
)
|
||||
# Clear the durable intent before deleting an unreferenced key. A
|
||||
# crash afterward leaves only an orphaned key, not a journal whose
|
||||
# retry depends on a key that no longer exists.
|
||||
if new_path.exists() or new_path.is_symlink():
|
||||
try:
|
||||
load_data_key(settings, inspected.repository_id, new_key_id)
|
||||
except RepositoryKeyError as error:
|
||||
raise RepositoryError("repository rotation key is unavailable") from error
|
||||
finish_key_rotation(inspected.root)
|
||||
if new_path.exists() or new_path.is_symlink():
|
||||
# The journal is already gone, so a cleanup failure only leaves
|
||||
# an unreferenced key and must not disable the old-active epoch.
|
||||
with suppress(OSError):
|
||||
new_path.unlink()
|
||||
descriptor = os.open(new_path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
reconciled += 1
|
||||
continue
|
||||
if repository.active_data_key_id == new_key_id and active[0].key_id == new_key_id:
|
||||
try:
|
||||
load_data_key(settings, inspected.repository_id, new_key_id)
|
||||
except RepositoryKeyError as error:
|
||||
raise RepositoryError("repository rotation key is unavailable") from error
|
||||
if inspected.data_key_id == old_key_id:
|
||||
replace_active_data_key(inspected.root, old_key_id, new_key_id)
|
||||
elif inspected.data_key_id != new_key_id:
|
||||
raise RepositoryError("repository rotation metadata is invalid")
|
||||
finish_key_rotation(inspected.root)
|
||||
reconciled += 1
|
||||
continue
|
||||
raise RepositoryError("repository rotation state is invalid")
|
||||
return reconciled
|
||||
|
||||
|
||||
def assert_capacity(settings: Settings, root: Path) -> None:
|
||||
capacity_root = root
|
||||
while not capacity_root.exists():
|
||||
parent = capacity_root.parent
|
||||
if parent == capacity_root:
|
||||
raise RepositoryError("repository root is unreadable")
|
||||
capacity_root = parent
|
||||
usage = shutil.disk_usage(capacity_root)
|
||||
free_percent = usage.free * 100 / usage.total
|
||||
if usage.free < settings.min_free_bytes or free_percent < settings.min_free_percent:
|
||||
raise RepositoryError("repository root does not meet minimum free capacity")
|
||||
|
||||
|
||||
def _signing_key_directory(settings: Settings) -> Path:
|
||||
directory = settings.data_dir / "repository-keys"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
if directory.is_symlink() or not directory.is_dir():
|
||||
raise RepositoryError("repository signing key directory is unsafe")
|
||||
if stat.S_IMODE(directory.stat().st_mode) != 0o700:
|
||||
raise RepositoryError("repository signing key directory permissions must be 0700")
|
||||
return directory
|
||||
|
||||
|
||||
def _write_private_key_staging(directory: Path, repository_id: str, private_key: bytes) -> Path:
|
||||
staging = directory / f".{repository_id}.{os.urandom(8).hex()}.tmp"
|
||||
descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(private_key)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
return staging
|
||||
|
||||
|
||||
def initialize(
|
||||
settings: Settings, relative_path: str, compression: str, encryption: str
|
||||
) -> InitializedRepository:
|
||||
payload = _canonical_payload(compression, encryption)
|
||||
repository_id = str(payload["repository_id"])
|
||||
data_key_id: str | None = None
|
||||
data_key_path: Path | None = None
|
||||
if encryption == "aes-256-gcm":
|
||||
try:
|
||||
data_key_id, data_key_path = create_data_key(settings, repository_id)
|
||||
except RepositoryKeyError as error:
|
||||
raise RepositoryError("repository data key is unavailable") from error
|
||||
payload["encryption"] = {"mode": encryption, "key_id": data_key_id}
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
private_bytes = private_key.private_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PrivateFormat.Raw,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PublicFormat.Raw,
|
||||
)
|
||||
signing_key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
|
||||
key_directory = _signing_key_directory(settings)
|
||||
key_path = key_directory / f"{repository_id}.ed25519"
|
||||
key_staging = _write_private_key_staging(key_directory, repository_id, private_bytes)
|
||||
root = _contained(settings.repository_roots[0], relative_path)
|
||||
assert_capacity(settings, root.parent)
|
||||
if root.exists():
|
||||
key_staging.unlink(missing_ok=True)
|
||||
raise RepositoryError("repository path already exists")
|
||||
staging = root.with_name(f".{root.name}.staging-{os.urandom(8).hex()}")
|
||||
try:
|
||||
staging.mkdir(parents=True)
|
||||
(staging / "blobs" / "sha256").mkdir(parents=True)
|
||||
(staging / "manifests").mkdir()
|
||||
metadata = staging / "repository.json"
|
||||
metadata.write_text(_canonical_json(payload), encoding="utf-8")
|
||||
with metadata.open("rb") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(staging, root)
|
||||
os.replace(key_staging, key_path)
|
||||
except Exception:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
shutil.rmtree(root, ignore_errors=True)
|
||||
key_staging.unlink(missing_ok=True)
|
||||
key_path.unlink(missing_ok=True)
|
||||
if data_key_path is not None:
|
||||
data_key_path.unlink(missing_ok=True)
|
||||
raise
|
||||
return InitializedRepository(
|
||||
root=root,
|
||||
repository_id=repository_id,
|
||||
compression=compression,
|
||||
encryption=encryption,
|
||||
signing_key_id=signing_key_id,
|
||||
signing_public_key=public_bytes.hex(),
|
||||
signing_key_path=key_path,
|
||||
data_key_id=data_key_id,
|
||||
data_key_path=data_key_path,
|
||||
)
|
||||
|
||||
|
||||
def remove_repository(
|
||||
root: Path,
|
||||
signing_key_path: Path | None = None,
|
||||
data_key_path: Path | None = None,
|
||||
) -> None:
|
||||
try: # noqa: SIM105 - cleanup must not race a concurrent remover
|
||||
shutil.rmtree(root)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
for key_path in (signing_key_path, data_key_path):
|
||||
if key_path is None:
|
||||
continue
|
||||
try: # noqa: SIM105 - cleanup must not race a concurrent remover
|
||||
key_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def install_signing_key(
|
||||
settings: Settings,
|
||||
repository_id: str,
|
||||
expected_key_id: str,
|
||||
expected_public_key: str,
|
||||
private_bytes: bytes,
|
||||
) -> Path:
|
||||
"""Install a recovered signing key once after checking its public trust anchor."""
|
||||
if len(private_bytes) != 32 or "/" in repository_id:
|
||||
raise RepositoryError("repository signing key is invalid")
|
||||
try:
|
||||
private_key = Ed25519PrivateKey.from_private_bytes(private_bytes)
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PublicFormat.Raw,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise RepositoryError("repository signing key is invalid") from error
|
||||
key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
|
||||
if key_id != expected_key_id or public_bytes.hex() != expected_public_key:
|
||||
raise RepositoryError("repository signing key is invalid")
|
||||
path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
try:
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(private_bytes)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
|
||||
path.unlink(missing_ok=True)
|
||||
raise RepositoryError("repository signing key is unsafe")
|
||||
except OSError as error:
|
||||
raise RepositoryError("repository signing key cannot be installed") from error
|
||||
return path
|
||||
|
||||
|
||||
def load_signing_key(
|
||||
settings: Settings,
|
||||
repository_id: str,
|
||||
expected_key_id: str,
|
||||
expected_public_key: str,
|
||||
) -> Ed25519PrivateKey:
|
||||
key_path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
|
||||
try:
|
||||
if key_path.is_symlink() or stat.S_IMODE(key_path.stat().st_mode) != 0o600:
|
||||
raise RepositoryError("repository signing key is unsafe")
|
||||
private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes())
|
||||
except (OSError, ValueError) as error:
|
||||
raise RepositoryError("repository signing key is unreadable") from error
|
||||
public_key = private_key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw,
|
||||
serialization.PublicFormat.Raw,
|
||||
)
|
||||
key_id = f"ed25519-{hashlib.sha256(public_key).hexdigest()[:32]}"
|
||||
if key_id != expected_key_id or public_key.hex() != expected_public_key:
|
||||
raise RepositoryError("repository signing key does not match its trust anchor")
|
||||
return private_key
|
||||
|
||||
|
||||
def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
|
||||
try:
|
||||
resolved_root = root.resolve(strict=True)
|
||||
except OSError as error:
|
||||
raise RepositoryError("repository root is unreadable") from error
|
||||
if root.is_symlink() or not any(
|
||||
resolved_root != allowed.resolve() and allowed.resolve() in resolved_root.parents
|
||||
for allowed in settings.repository_roots
|
||||
):
|
||||
raise RepositoryError("repository root escapes configured roots")
|
||||
metadata = resolved_root / "repository.json"
|
||||
try:
|
||||
raw = metadata.read_text(encoding="utf-8")
|
||||
payload = json.loads(raw)
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise RepositoryError("repository metadata is unreadable") from error
|
||||
if not isinstance(payload, dict):
|
||||
raise RepositoryError("repository metadata is invalid")
|
||||
required = {
|
||||
"repository_id",
|
||||
"format_version",
|
||||
"digest_algorithm",
|
||||
"compression",
|
||||
"encryption",
|
||||
"created_at",
|
||||
}
|
||||
if (
|
||||
set(payload) != required
|
||||
or payload.get("format_version") != 1
|
||||
or payload.get("digest_algorithm") != "sha256"
|
||||
):
|
||||
raise RepositoryError("repository metadata is not canonical")
|
||||
encryption = payload.get("encryption")
|
||||
if payload.get("compression") != "none" or not isinstance(encryption, dict):
|
||||
raise RepositoryError("repository metadata is invalid")
|
||||
mode = encryption.get("mode")
|
||||
key_id = encryption.get("key_id")
|
||||
valid_policy = (mode == "none" and key_id is None) or (
|
||||
mode == "aes-256-gcm" and isinstance(key_id, str) and bool(key_id)
|
||||
)
|
||||
if not valid_policy:
|
||||
raise RepositoryError("repository metadata is invalid")
|
||||
try:
|
||||
from uuid import UUID
|
||||
|
||||
UUID(str(payload["repository_id"]))
|
||||
created_at = str(payload["created_at"])
|
||||
if not created_at.endswith("Z"):
|
||||
raise ValueError
|
||||
datetime.fromisoformat(created_at[:-1] + "+00:00")
|
||||
except (ValueError, TypeError) as error:
|
||||
raise RepositoryError("repository metadata is invalid") from error
|
||||
return InitializedRepository(
|
||||
root=resolved_root,
|
||||
repository_id=str(payload["repository_id"]),
|
||||
compression="none",
|
||||
encryption=str(encryption["mode"]),
|
||||
data_key_id=key_id if isinstance(key_id, str) else None,
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class RetentionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class BackupLike(Protocol):
|
||||
id: str
|
||||
created_at: datetime
|
||||
pinned: bool
|
||||
tombstoned_at: datetime | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetentionPolicy:
|
||||
keep_last: int = 1
|
||||
keep_days: int = 0
|
||||
keep_daily: int = 0
|
||||
keep_weekly: int = 0
|
||||
keep_monthly: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, object]) -> RetentionPolicy:
|
||||
allowed = {"keep_last", "keep_days", "keep_daily", "keep_weekly", "keep_monthly"}
|
||||
if set(value) - allowed:
|
||||
raise RetentionError("retention policy contains unknown keys")
|
||||
kwargs: dict[str, int] = {}
|
||||
for key in allowed:
|
||||
raw = value.get(key, 0 if key != "keep_last" else 1)
|
||||
if not isinstance(raw, int) or isinstance(raw, bool) or raw < 0:
|
||||
raise RetentionError("retention values must be non-negative integers")
|
||||
kwargs[key] = raw
|
||||
return cls(**kwargs)
|
||||
|
||||
|
||||
def retained_ids(
|
||||
backups: Iterable[BackupLike], policy: RetentionPolicy, now: datetime | None = None
|
||||
) -> set[str]:
|
||||
"""Return union retention set; the newest non-tombstoned backup is always protected."""
|
||||
reference = (now or datetime.now(UTC)).astimezone(UTC)
|
||||
items = sorted(
|
||||
(item for item in backups if item.tombstoned_at is None),
|
||||
key=lambda item: item.created_at.astimezone(UTC),
|
||||
reverse=True,
|
||||
)
|
||||
if not items:
|
||||
return set()
|
||||
kept = {items[0].id}
|
||||
kept.update(item.id for item in items[: policy.keep_last])
|
||||
kept.update(item.id for item in items if item.pinned)
|
||||
if policy.keep_days:
|
||||
cutoff = reference - timedelta(days=policy.keep_days)
|
||||
kept.update(item.id for item in items if item.created_at.astimezone(UTC) >= cutoff)
|
||||
for count, key in (
|
||||
(policy.keep_daily, lambda stamp: stamp.date()),
|
||||
(policy.keep_weekly, lambda stamp: stamp.isocalendar()[:2]),
|
||||
(policy.keep_monthly, lambda stamp: (stamp.year, stamp.month)),
|
||||
):
|
||||
buckets: set[object] = set()
|
||||
for item in items:
|
||||
stamp = item.created_at.astimezone(UTC)
|
||||
bucket = key(stamp)
|
||||
if len(buckets) >= count and bucket not in buckets:
|
||||
continue
|
||||
buckets.add(bucket)
|
||||
if bucket in buckets:
|
||||
kept.add(item.id)
|
||||
return kept
|
||||
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import signal
|
||||
from datetime import UTC, datetime
|
||||
from typing import cast
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger # type: ignore[import-untyped]
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import create_engine
|
||||
from backup_tool.db.models import Schedule
|
||||
from backup_tool.execution import EnqueueError, enqueue
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.observability.logging import configure_logging, log_event
|
||||
|
||||
|
||||
class ScheduleError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def next_nominal(cron: str, timezone: str, after: datetime | None = None) -> datetime:
|
||||
if len(cron.split()) != 5:
|
||||
raise ScheduleError("cron must contain exactly five fields")
|
||||
try:
|
||||
zone = ZoneInfo(timezone)
|
||||
trigger = CronTrigger.from_crontab(cron, timezone=zone)
|
||||
except (ValueError, ZoneInfoNotFoundError) as error:
|
||||
raise ScheduleError("cron or timezone is invalid") from error
|
||||
reference = (after or datetime.now(UTC)).astimezone(zone)
|
||||
next_run = trigger.get_next_fire_time(None, reference)
|
||||
if next_run is None:
|
||||
raise ScheduleError("cron has no future occurrence")
|
||||
return cast(datetime, next_run.astimezone(UTC))
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
"""Dedicated scheduler role using the same transactional enqueue service."""
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.engine = create_engine(settings)
|
||||
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
|
||||
self._stopping = asyncio.Event()
|
||||
|
||||
async def run_once(self) -> int:
|
||||
async with self.sessions() as db:
|
||||
return await deliver_due(db)
|
||||
|
||||
async def run(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
await self.run_once()
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(self._stopping.wait(), timeout=0.25)
|
||||
await self.engine.dispose()
|
||||
|
||||
def stop(self) -> None:
|
||||
log_event("role_stopping", role="scheduler")
|
||||
self._stopping.set()
|
||||
|
||||
|
||||
def run_scheduler(settings: Settings) -> int:
|
||||
configure_logging("scheduler", settings.log_level)
|
||||
service = SchedulerService(settings)
|
||||
loop = asyncio.new_event_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
with contextlib.suppress(NotImplementedError):
|
||||
loop.add_signal_handler(sig, service.stop)
|
||||
try:
|
||||
loop.run_until_complete(service.run())
|
||||
finally:
|
||||
loop.close()
|
||||
log_event("role_stopped", role="scheduler")
|
||||
return 0
|
||||
|
||||
|
||||
async def deliver_due(db: AsyncSession, now: datetime | None = None) -> int:
|
||||
current = now or datetime.now(UTC)
|
||||
schedules = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(Schedule).where(
|
||||
Schedule.enabled,
|
||||
Schedule.next_nominal_at.is_not(None),
|
||||
Schedule.next_nominal_at <= current,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
delivered = 0
|
||||
for schedule in schedules:
|
||||
nominal = schedule.next_nominal_at
|
||||
if nominal is None:
|
||||
continue
|
||||
schedule.next_nominal_at = next_nominal(schedule.cron, schedule.timezone, nominal)
|
||||
if (current - nominal).total_seconds() > schedule.misfire_grace_seconds:
|
||||
schedule.last_enqueue_outcome = "misfire"
|
||||
await emit_event(
|
||||
db,
|
||||
"schedule.occurrence_misfired",
|
||||
correlation_id=schedule.id,
|
||||
resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
|
||||
payload={"outcome": "misfire"},
|
||||
deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:misfire",
|
||||
)
|
||||
continue
|
||||
try:
|
||||
await enqueue(
|
||||
db,
|
||||
schedule.job_id,
|
||||
"schedule",
|
||||
schedule_id=schedule.id,
|
||||
nominal_run_at=nominal,
|
||||
)
|
||||
except EnqueueError as error:
|
||||
schedule.last_enqueue_outcome = error.code
|
||||
await emit_event(
|
||||
db,
|
||||
"schedule.occurrence_blocked",
|
||||
correlation_id=schedule.id,
|
||||
resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
|
||||
payload={"reason_code": error.code},
|
||||
deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:blocked",
|
||||
)
|
||||
else:
|
||||
schedule.last_enqueue_outcome = "enqueued"
|
||||
await emit_event(
|
||||
db,
|
||||
"schedule.occurrence_enqueued",
|
||||
correlation_id=schedule.id,
|
||||
resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
|
||||
payload={"outcome": "enqueued"},
|
||||
deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:enqueued",
|
||||
)
|
||||
delivered += 1
|
||||
await db.commit()
|
||||
return delivered
|
||||
@@ -0,0 +1 @@
|
||||
"""Security primitives and authentication services."""
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
|
||||
_PASSWORDS = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _PASSWORDS.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _PASSWORDS.verify(password_hash, password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def new_csrf_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def _session_key(master_key_file: Path) -> bytes:
|
||||
return hashlib.sha256(b"backup-tool-session\0" + master_key_file.read_bytes()).digest()
|
||||
|
||||
|
||||
def sign_session(
|
||||
user_id: str,
|
||||
csrf: str,
|
||||
master_key_file: Path,
|
||||
*,
|
||||
expires_at: datetime | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> str:
|
||||
issued_at = datetime.now(UTC)
|
||||
expires_at = expires_at or issued_at.replace(microsecond=0) + timedelta(hours=8)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"sub": user_id,
|
||||
"csrf": csrf,
|
||||
"iat": issued_at.isoformat(),
|
||||
"exp": expires_at.astimezone(UTC).isoformat(),
|
||||
"sid": session_id or secrets.token_urlsafe(24),
|
||||
},
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
signature = hmac.new(
|
||||
_session_key(master_key_file), encoded.encode(), hashlib.sha256
|
||||
).hexdigest()
|
||||
return f"{encoded}.{signature}"
|
||||
|
||||
|
||||
def verify_session(value: str, master_key_file: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
encoded, supplied = value.rsplit(".", 1)
|
||||
expected = hmac.new(
|
||||
_session_key(master_key_file), encoded.encode(), hashlib.sha256
|
||||
).hexdigest()
|
||||
if not hmac.compare_digest(supplied, expected):
|
||||
return None
|
||||
padded = encoded + "=" * (-len(encoded) % 4)
|
||||
decoded = json.loads(base64.urlsafe_b64decode(padded))
|
||||
if (
|
||||
not isinstance(decoded, dict)
|
||||
or not isinstance(decoded.get("sub"), str)
|
||||
or not isinstance(decoded.get("sid"), str)
|
||||
or not isinstance(decoded.get("exp"), str)
|
||||
or datetime.fromisoformat(decoded["exp"]).astimezone(UTC) <= datetime.now(UTC)
|
||||
):
|
||||
return None
|
||||
return decoded
|
||||
except (ValueError, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return None
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Offline, passphrase-protected recovery bundle codec.
|
||||
|
||||
The binary format is deliberately small and versioned so validation can reject
|
||||
unsupported inputs before attempting expensive password derivation. Every
|
||||
failure while parsing or authenticating a bundle is reported as the same error
|
||||
so callers cannot distinguish a malformed bundle from a wrong passphrase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from argon2.low_level import Type, hash_secret_raw
|
||||
from cryptography.exceptions import InvalidTag
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
|
||||
class RecoveryBundleError(ValueError):
|
||||
"""A non-disclosing recovery bundle validation failure."""
|
||||
|
||||
|
||||
class RecoveryBundlePathError(ValueError):
|
||||
"""A requested recovery bundle path cannot be used safely."""
|
||||
|
||||
|
||||
_MAGIC = b"BTREC"
|
||||
_VERSION = 1
|
||||
_KDF_ARGON2ID = 1
|
||||
_SALT_BYTES = 16
|
||||
_NONCE_BYTES = 12
|
||||
_KEY_BYTES = 32
|
||||
_TAG_BYTES = 16
|
||||
_TIME_COST = 3
|
||||
_MEMORY_COST_KIB = 65_536
|
||||
_PARALLELISM = 1
|
||||
_MAX_PASSPHRASE_BYTES = 4_096
|
||||
_MAX_PLAINTEXT_BYTES = 8 * 1024 * 1024
|
||||
# magic, version, KDF id, Argon2 time/memory/parallelism, salt/nonce lengths,
|
||||
# and the AES-GCM ciphertext (including tag) length.
|
||||
_HEADER = struct.Struct(">5sBBIIHBBQ")
|
||||
_MAX_BUNDLE_BYTES = _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _MAX_PLAINTEXT_BYTES + _TAG_BYTES
|
||||
_ERROR = "recovery bundle is invalid"
|
||||
|
||||
|
||||
def _invalid() -> RecoveryBundleError:
|
||||
return RecoveryBundleError(_ERROR)
|
||||
|
||||
|
||||
def _canonical_json(payload: Mapping[str, Any]) -> bytes:
|
||||
try:
|
||||
encoded = json.dumps(
|
||||
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||
).encode("utf-8")
|
||||
except (TypeError, ValueError) as error:
|
||||
raise _invalid() from error
|
||||
if not encoded or len(encoded) > _MAX_PLAINTEXT_BYTES:
|
||||
raise _invalid()
|
||||
return encoded
|
||||
|
||||
|
||||
def _passphrase(value: bytes) -> bytes:
|
||||
if not isinstance(value, bytes) or not value or len(value) > _MAX_PASSPHRASE_BYTES:
|
||||
raise _invalid()
|
||||
return value
|
||||
|
||||
|
||||
def _derive_key(passphrase: bytes, salt: bytes) -> bytes:
|
||||
return hash_secret_raw(
|
||||
secret=passphrase,
|
||||
salt=salt,
|
||||
time_cost=_TIME_COST,
|
||||
memory_cost=_MEMORY_COST_KIB,
|
||||
parallelism=_PARALLELISM,
|
||||
hash_len=_KEY_BYTES,
|
||||
type=Type.ID,
|
||||
)
|
||||
|
||||
|
||||
def encrypt_bundle(payload: Mapping[str, Any], passphrase: bytes) -> bytes:
|
||||
"""Serialize and encrypt a canonical recovery payload as a BTREC v1 bundle."""
|
||||
plaintext = _canonical_json(payload)
|
||||
secret = _passphrase(passphrase)
|
||||
salt = os.urandom(_SALT_BYTES)
|
||||
nonce = os.urandom(_NONCE_BYTES)
|
||||
ciphertext_length = len(plaintext) + _TAG_BYTES
|
||||
header = _HEADER.pack(
|
||||
_MAGIC,
|
||||
_VERSION,
|
||||
_KDF_ARGON2ID,
|
||||
_TIME_COST,
|
||||
_MEMORY_COST_KIB,
|
||||
_PARALLELISM,
|
||||
_SALT_BYTES,
|
||||
_NONCE_BYTES,
|
||||
ciphertext_length,
|
||||
)
|
||||
ciphertext = AESGCM(_derive_key(secret, salt)).encrypt(nonce, plaintext, header)
|
||||
return header + salt + nonce + ciphertext
|
||||
|
||||
|
||||
def decrypt_bundle(encoded: bytes, passphrase: bytes) -> dict[str, Any]:
|
||||
"""Authenticate and decode a BTREC v1 bundle without disclosing failure cause."""
|
||||
try:
|
||||
if not isinstance(encoded, bytes) or len(encoded) > _MAX_BUNDLE_BYTES:
|
||||
raise _invalid()
|
||||
if len(encoded) < _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _TAG_BYTES:
|
||||
raise _invalid()
|
||||
(
|
||||
magic,
|
||||
version,
|
||||
kdf_id,
|
||||
time_cost,
|
||||
memory_cost,
|
||||
parallelism,
|
||||
salt_length,
|
||||
nonce_length,
|
||||
ciphertext_length,
|
||||
) = _HEADER.unpack(encoded[: _HEADER.size])
|
||||
if (
|
||||
magic != _MAGIC
|
||||
or version != _VERSION
|
||||
or kdf_id != _KDF_ARGON2ID
|
||||
or time_cost != _TIME_COST
|
||||
or memory_cost != _MEMORY_COST_KIB
|
||||
or parallelism != _PARALLELISM
|
||||
or salt_length != _SALT_BYTES
|
||||
or nonce_length != _NONCE_BYTES
|
||||
or ciphertext_length < _TAG_BYTES
|
||||
or ciphertext_length > _MAX_PLAINTEXT_BYTES + _TAG_BYTES
|
||||
or len(encoded) != _HEADER.size + salt_length + nonce_length + ciphertext_length
|
||||
):
|
||||
raise _invalid()
|
||||
secret = _passphrase(passphrase)
|
||||
salt_start = _HEADER.size
|
||||
nonce_start = salt_start + salt_length
|
||||
ciphertext_start = nonce_start + nonce_length
|
||||
plaintext = AESGCM(_derive_key(secret, encoded[salt_start:nonce_start])).decrypt(
|
||||
encoded[nonce_start:ciphertext_start],
|
||||
encoded[ciphertext_start:],
|
||||
encoded[: _HEADER.size],
|
||||
)
|
||||
if not plaintext or len(plaintext) > _MAX_PLAINTEXT_BYTES:
|
||||
raise _invalid()
|
||||
payload = json.loads(plaintext.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise _invalid()
|
||||
# Reject non-canonical encodings to make catalog serialization deterministic.
|
||||
if _canonical_json(payload) != plaintext:
|
||||
raise _invalid()
|
||||
return payload
|
||||
except (
|
||||
InvalidTag,
|
||||
UnicodeDecodeError,
|
||||
json.JSONDecodeError,
|
||||
struct.error,
|
||||
ValueError,
|
||||
) as error:
|
||||
if isinstance(error, RecoveryBundleError):
|
||||
raise error
|
||||
raise _invalid() from error
|
||||
|
||||
|
||||
def _check_path_components(path: Path) -> None:
|
||||
if not path.is_absolute() or path.name in {"", ".", ".."}:
|
||||
raise RecoveryBundlePathError("recovery bundle path is unsafe")
|
||||
current = Path(path.anchor)
|
||||
for component in path.parts[1:-1]:
|
||||
current /= component
|
||||
try:
|
||||
info = current.lstat()
|
||||
except OSError as error:
|
||||
raise RecoveryBundlePathError("recovery bundle path is unsafe") from error
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
||||
raise RecoveryBundlePathError("recovery bundle path is unsafe")
|
||||
|
||||
|
||||
def write_bundle_exclusive(path: Path, encoded: bytes) -> None:
|
||||
"""Write a bundle once with restrictive permissions and no symlink following."""
|
||||
if not isinstance(encoded, bytes) or not encoded or len(encoded) > _MAX_BUNDLE_BYTES:
|
||||
raise RecoveryBundlePathError("recovery bundle output is unsafe")
|
||||
_check_path_components(path)
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
try:
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(encoded)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
except OSError as error:
|
||||
raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
|
||||
try:
|
||||
info = path.lstat()
|
||||
if (
|
||||
stat.S_ISLNK(info.st_mode)
|
||||
or not stat.S_ISREG(info.st_mode)
|
||||
or stat.S_IMODE(info.st_mode) != 0o600
|
||||
):
|
||||
path.unlink(missing_ok=True)
|
||||
raise RecoveryBundlePathError("recovery bundle output is unsafe")
|
||||
except OSError as error:
|
||||
raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
|
||||
|
||||
|
||||
def read_bundle_file(path: Path) -> bytes:
|
||||
"""Read a regular, non-symlink bundle with a bounded size."""
|
||||
_check_path_components(path)
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
try:
|
||||
descriptor = os.open(path, flags)
|
||||
with os.fdopen(descriptor, "rb") as handle:
|
||||
info = os.fstat(handle.fileno())
|
||||
if (
|
||||
not stat.S_ISREG(info.st_mode)
|
||||
or info.st_size <= 0
|
||||
or info.st_size > _MAX_BUNDLE_BYTES
|
||||
):
|
||||
raise RecoveryBundlePathError("recovery bundle input is unsafe")
|
||||
return handle.read()
|
||||
except RecoveryBundlePathError:
|
||||
raise
|
||||
except OSError as error:
|
||||
raise RecoveryBundlePathError("recovery bundle input is unsafe") from error
|
||||
|
||||
|
||||
def read_passphrase_fd(fd: int) -> bytes:
|
||||
"""Read one newline-terminated passphrase from an inherited file descriptor."""
|
||||
if not isinstance(fd, int) or fd < 0:
|
||||
raise RecoveryBundleError("recovery passphrase is unavailable")
|
||||
try:
|
||||
value = os.read(fd, _MAX_PASSPHRASE_BYTES + 2)
|
||||
except OSError as error:
|
||||
raise RecoveryBundleError("recovery passphrase is unavailable") from error
|
||||
if value.endswith(b"\r\n"):
|
||||
value = value[:-2]
|
||||
elif value.endswith(b"\n"):
|
||||
value = value[:-1]
|
||||
if not value or len(value) > _MAX_PASSPHRASE_BYTES:
|
||||
raise RecoveryBundleError("recovery passphrase is unavailable")
|
||||
return value
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
SECRET_KEYS = frozenset(
|
||||
{
|
||||
"password",
|
||||
"password_hash",
|
||||
"secret",
|
||||
"token",
|
||||
"token_hash",
|
||||
"authorization",
|
||||
"cookie",
|
||||
"ciphertext",
|
||||
"master_key",
|
||||
"private_key",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def redact(value: Any, *, canaries: Sequence[str] = ()) -> Any:
|
||||
"""Return a recursively redacted copy safe for operator-visible sinks."""
|
||||
if isinstance(value, Mapping):
|
||||
return {
|
||||
str(key): (
|
||||
"[REDACTED]" if str(key).lower() in SECRET_KEYS else redact(item, canaries=canaries)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact(item, canaries=canaries) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact(item, canaries=canaries) for item in value)
|
||||
if isinstance(value, str):
|
||||
result = value
|
||||
for canary in canaries:
|
||||
if canary:
|
||||
result = result.replace(canary, "[REDACTED]")
|
||||
return result
|
||||
return value
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.exceptions import InvalidTag
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.ids import new_uuid7
|
||||
|
||||
|
||||
class RepositoryKeyError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _directory(settings: Settings) -> Path:
|
||||
directory = settings.data_dir / "repository-data-keys"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
if (
|
||||
directory.is_symlink()
|
||||
or not directory.is_dir()
|
||||
or stat.S_IMODE(directory.stat().st_mode) != 0o700
|
||||
):
|
||||
raise RepositoryKeyError("repository data key directory is unsafe")
|
||||
return directory
|
||||
|
||||
|
||||
def create_data_key(settings: Settings, repository_id: str) -> tuple[str, Path]:
|
||||
key_id = str(new_uuid7())
|
||||
directory = _directory(settings)
|
||||
path = directory / f"{repository_id}.{key_id}.key"
|
||||
try:
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(os.urandom(32))
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
except OSError as error:
|
||||
raise RepositoryKeyError("repository data key cannot be created") from error
|
||||
if stat.S_IMODE(path.stat().st_mode) != 0o600 or path.is_symlink():
|
||||
path.unlink(missing_ok=True)
|
||||
raise RepositoryKeyError("repository data key is unsafe")
|
||||
return key_id, path
|
||||
|
||||
|
||||
def install_data_key(settings: Settings, repository_id: str, key_id: str, key: bytes) -> Path:
|
||||
"""Install recovered key material once; never replace an existing key file."""
|
||||
if len(key) != 32 or "/" in repository_id or "/" in key_id:
|
||||
raise RepositoryKeyError("repository data key is invalid")
|
||||
path = _directory(settings) / f"{repository_id}.{key_id}.key"
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
try:
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(key)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
|
||||
path.unlink(missing_ok=True)
|
||||
raise RepositoryKeyError("repository data key is unsafe")
|
||||
except OSError as error:
|
||||
raise RepositoryKeyError("repository data key cannot be installed") from error
|
||||
return path
|
||||
|
||||
|
||||
def object_aad(repository_id: str, key_id: str, kind: str, identity: str) -> bytes:
|
||||
if kind not in {"blob", "manifest"} or not all((repository_id, key_id, identity)):
|
||||
raise RepositoryKeyError("encrypted object metadata is invalid")
|
||||
return f"BTENC:1:{repository_id}:{key_id}:{kind}:{identity}".encode()
|
||||
|
||||
|
||||
def encrypt_object(key: bytes, aad: bytes, plaintext: bytes) -> bytes:
|
||||
if len(key) != 32:
|
||||
raise RepositoryKeyError("repository data key is unavailable")
|
||||
nonce = os.urandom(12)
|
||||
return b"BTENC\x01" + nonce + AESGCM(key).encrypt(nonce, plaintext, aad)
|
||||
|
||||
|
||||
def decrypt_object(key: bytes, aad: bytes, stored: bytes) -> bytes:
|
||||
if len(key) != 32 or not stored.startswith(b"BTENC\x01") or len(stored) < 35:
|
||||
raise RepositoryKeyError("encrypted object is invalid")
|
||||
try:
|
||||
return AESGCM(key).decrypt(stored[6:18], stored[18:], aad)
|
||||
except InvalidTag as error:
|
||||
raise RepositoryKeyError("encrypted object is invalid") from error
|
||||
|
||||
|
||||
def load_data_key(settings: Settings, repository_id: str, key_id: str) -> bytes:
|
||||
path = _directory(settings) / f"{repository_id}.{key_id}.key"
|
||||
try:
|
||||
if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
|
||||
raise RepositoryKeyError("repository data key is unsafe")
|
||||
key = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise RepositoryKeyError("repository data key is unavailable") from error
|
||||
if len(key) != 32:
|
||||
raise RepositoryKeyError("repository data key is unavailable")
|
||||
return key
|
||||
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
|
||||
class EnvelopeCipher:
|
||||
"""Purpose-bound authenticated encryption for persisted secret values."""
|
||||
|
||||
def __init__(self, master_key: bytes):
|
||||
self._key = hashlib.sha256(master_key).digest()
|
||||
self.key_id = hashlib.sha256(b"backup-tool-key-id\0" + self._key).hexdigest()[:24]
|
||||
self._cipher = AESGCM(self._key)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> EnvelopeCipher:
|
||||
return cls(path.read_bytes())
|
||||
|
||||
@staticmethod
|
||||
def _associated_data(purpose: str, version: int) -> bytes:
|
||||
return f"backup-tool-secret:{purpose}:v{version}".encode()
|
||||
|
||||
def encrypt(self, value: str, *, purpose: str, version: int) -> tuple[bytes, str]:
|
||||
nonce = os.urandom(12)
|
||||
encrypted = self._cipher.encrypt(
|
||||
nonce, value.encode(), self._associated_data(purpose, version)
|
||||
)
|
||||
return nonce + encrypted, self.key_id
|
||||
|
||||
def decrypt(self, ciphertext: bytes, *, purpose: str, version: int) -> str:
|
||||
nonce, encrypted = ciphertext[:12], ciphertext[12:]
|
||||
plaintext = self._cipher.decrypt(nonce, encrypted, self._associated_data(purpose, version))
|
||||
return plaintext.decode()
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Fail-closed, DNS-rebinding-resistant webhook egress validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import SplitResult, urlsplit
|
||||
|
||||
|
||||
class SSRFError(ValueError):
|
||||
"""The callback target is not safe for the notification egress boundary."""
|
||||
|
||||
|
||||
Resolver = Callable[[str, int], Awaitable[Sequence[str]]]
|
||||
_ALLOWED_PORTS = frozenset({80, 443, 8080, 8443})
|
||||
|
||||
|
||||
def _is_global(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
"""ipaddress.is_global misses some policy-important mapped/special ranges."""
|
||||
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
|
||||
return _is_global(address.ipv4_mapped)
|
||||
return bool(address.is_global) and not any(
|
||||
(
|
||||
address.is_loopback,
|
||||
address.is_private,
|
||||
address.is_link_local,
|
||||
address.is_multicast,
|
||||
address.is_unspecified,
|
||||
address.is_reserved,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def validate_webhook_url(value: str) -> SplitResult:
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise SSRFError("webhook URL is malformed") from error
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise SSRFError("webhook URL must be absolute HTTP(S)")
|
||||
if parsed.username is not None or parsed.password is not None or parsed.fragment:
|
||||
raise SSRFError("webhook URL credentials and fragments are forbidden")
|
||||
if len(value) > 2048 or any(character.isspace() for character in value):
|
||||
raise SSRFError("webhook URL is malformed")
|
||||
if port is not None and port not in _ALLOWED_PORTS:
|
||||
raise SSRFError("webhook URL port is not permitted")
|
||||
try:
|
||||
ipaddress.ip_address(parsed.hostname)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
# Callback literals are never accepted: names are resolved immediately
|
||||
# before every request and connected addresses are pinned/rechecked.
|
||||
raise SSRFError("literal IP webhook targets are forbidden")
|
||||
return parsed
|
||||
|
||||
|
||||
async def system_resolver(hostname: str, port: int) -> Sequence[str]:
|
||||
records = await asyncio.get_running_loop().getaddrinfo(
|
||||
hostname, port, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP
|
||||
)
|
||||
addresses: set[str] = set()
|
||||
for record in records:
|
||||
socket_address = record[4]
|
||||
if socket_address and isinstance(socket_address[0], str):
|
||||
addresses.add(socket_address[0])
|
||||
return tuple(sorted(addresses))
|
||||
|
||||
|
||||
async def resolve_public_addresses(
|
||||
hostname: str, port: int, resolver: Resolver = system_resolver
|
||||
) -> tuple[str, ...]:
|
||||
try:
|
||||
candidates = tuple(await resolver(hostname, port))
|
||||
except (TimeoutError, OSError) as error:
|
||||
raise SSRFError("webhook DNS resolution failed") from error
|
||||
if not candidates:
|
||||
raise SSRFError("webhook hostname has no addresses")
|
||||
approved: list[str] = []
|
||||
for candidate in candidates:
|
||||
try:
|
||||
address = ipaddress.ip_address(candidate)
|
||||
except ValueError as error:
|
||||
raise SSRFError("webhook resolver returned an invalid address") from error
|
||||
if not _is_global(address):
|
||||
# One unsafe answer poisons the hostname, including mixed public/private.
|
||||
raise SSRFError("webhook hostname resolves to a non-public address")
|
||||
approved.append(str(address))
|
||||
return tuple(approved)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedWebhookTarget:
|
||||
url: SplitResult
|
||||
port: int
|
||||
addresses: tuple[str, ...]
|
||||
|
||||
|
||||
async def resolve_webhook_target(
|
||||
value: str, resolver: Resolver = system_resolver
|
||||
) -> ResolvedWebhookTarget:
|
||||
parsed = validate_webhook_url(value)
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
addresses = await resolve_public_addresses(parsed.hostname or "", port, resolver)
|
||||
return ResolvedWebhookTarget(url=parsed, port=port, addresses=addresses)
|
||||
|
||||
|
||||
def verify_connected_peer(peername: object, approved: Sequence[str]) -> str:
|
||||
if not isinstance(peername, tuple) or not peername or not isinstance(peername[0], str):
|
||||
raise SSRFError("webhook peer address is unavailable")
|
||||
try:
|
||||
peer = str(ipaddress.ip_address(peername[0]))
|
||||
except ValueError as error:
|
||||
raise SSRFError("webhook peer address is invalid") from error
|
||||
if peer not in approved:
|
||||
raise SSRFError("webhook peer changed after DNS resolution")
|
||||
return peer
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,288 @@
|
||||
"""Pinned-host-key, forced-SFTP-only source reader.
|
||||
|
||||
The adapter has no command-channel API. It authenticates with the caller's
|
||||
already decrypted private key only after verifying the configured host key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import io
|
||||
import socket
|
||||
import stat
|
||||
from collections.abc import AsyncIterator, Callable, Iterator
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
import paramiko # type: ignore[import-untyped]
|
||||
|
||||
from backup_tool.adapters import Entry, SourceError
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.ssh_source import SSHSourcePublicConfig
|
||||
|
||||
TransportFactory = Callable[[str, int, float], Any]
|
||||
SFTPFactory = Callable[[Any], Any]
|
||||
|
||||
|
||||
def _transport_for(hostname: str, port: int, timeout: float) -> Any:
|
||||
connection = socket.create_connection((hostname, port), timeout=timeout)
|
||||
connection.settimeout(timeout)
|
||||
return paramiko.Transport(connection)
|
||||
|
||||
|
||||
def _sftp_for(transport: Any) -> Any:
|
||||
return paramiko.SFTPClient.from_transport(transport)
|
||||
|
||||
|
||||
def _next_or_none(iterator: Iterator[Any]) -> Any | None:
|
||||
try:
|
||||
return next(iterator)
|
||||
except StopIteration:
|
||||
return None
|
||||
|
||||
|
||||
def load_private_key(value: str) -> Any:
|
||||
"""Load only unencrypted Ed25519, ECDSA, or sufficiently strong RSA keys."""
|
||||
for key_type in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey):
|
||||
try:
|
||||
key = key_type.from_private_key(io.StringIO(value), password=None)
|
||||
except (paramiko.SSHException, ValueError):
|
||||
continue
|
||||
if isinstance(key, paramiko.RSAKey) and key.get_bits() < 3072:
|
||||
raise SourceError(
|
||||
"SSH private key algorithm is not permitted", reason_code="source_auth"
|
||||
)
|
||||
return key
|
||||
raise SourceError("SSH private key algorithm is not permitted", reason_code="source_auth")
|
||||
|
||||
|
||||
class SSHAdapter:
|
||||
"""A stateful SFTP reader rooted at the forced-SFTP account chroot only."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: SSHSourcePublicConfig,
|
||||
private_key: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
transport_factory: TransportFactory = _transport_for,
|
||||
sftp_factory: SFTPFactory = _sftp_for,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self._private_key = load_private_key(private_key)
|
||||
self.settings = settings
|
||||
self._transport_factory = transport_factory
|
||||
self._sftp_factory = sftp_factory
|
||||
self._transport: Any | None = None
|
||||
self._sftp: Any | None = None
|
||||
self._files: dict[str, tuple[int, int]] = {}
|
||||
|
||||
def validate_config(self) -> None:
|
||||
if self.config.root != "/": # defensive: persisted JSON can bypass API validation
|
||||
raise SourceError(
|
||||
"SSH source root must be the forced-SFTP chroot", reason_code="source_invalid"
|
||||
)
|
||||
|
||||
def _connect(self) -> None:
|
||||
if self._sftp is not None:
|
||||
return
|
||||
self.validate_config()
|
||||
transport = self._transport_factory(
|
||||
self.config.hostname, self.config.port, self.settings.ssh_connect_timeout_seconds
|
||||
)
|
||||
self._transport = transport
|
||||
try:
|
||||
transport.start_client(timeout=self.settings.ssh_connect_timeout_seconds)
|
||||
algorithm, encoded_key = self.config.host_key.split(" ", 1)
|
||||
server_key = transport.get_remote_server_key()
|
||||
if server_key.get_name() != algorithm or not hmac.compare_digest(
|
||||
server_key.get_base64(), encoded_key
|
||||
):
|
||||
raise SourceError(
|
||||
"SSH host key does not match configured pin", reason_code="source_trust"
|
||||
)
|
||||
# Authentication deliberately occurs only after the exact pin comparison.
|
||||
transport.auth_publickey(self.config.username, self._private_key)
|
||||
sftp = self._sftp_factory(transport)
|
||||
channel = sftp.get_channel()
|
||||
channel.settimeout(self.settings.ssh_operation_timeout_seconds)
|
||||
self._sftp = sftp
|
||||
except SourceError:
|
||||
self._close_sync()
|
||||
raise
|
||||
except paramiko.AuthenticationException as error:
|
||||
self._close_sync()
|
||||
raise SourceError("SSH authentication failed", reason_code="source_auth") from error
|
||||
except (TimeoutError, OSError) as error:
|
||||
self._close_sync()
|
||||
raise SourceError(
|
||||
"SSH source is unavailable", reason_code="source_unavailable"
|
||||
) from error
|
||||
except (paramiko.SSHException, ValueError) as error:
|
||||
self._close_sync()
|
||||
raise SourceError(
|
||||
"SSH source connection failed", reason_code="source_unavailable"
|
||||
) from error
|
||||
|
||||
def _sftp_client(self) -> Any:
|
||||
if self._sftp is None:
|
||||
raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
|
||||
return self._sftp
|
||||
|
||||
def _read_client(self) -> Any:
|
||||
if self._transport is None:
|
||||
raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
|
||||
client = self._sftp_factory(self._transport)
|
||||
client.get_channel().settimeout(self.settings.ssh_operation_timeout_seconds)
|
||||
return client
|
||||
|
||||
@staticmethod
|
||||
def _relative(parent: str, name: object) -> str:
|
||||
if not isinstance(name, str) or not name or name in {".", ".."}:
|
||||
raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
|
||||
if "/" in name or "\\" in name or "\x00" in name:
|
||||
raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
|
||||
return name if not parent else f"{parent}/{name}"
|
||||
|
||||
@staticmethod
|
||||
def _remote_path(relative: str) -> str:
|
||||
if (
|
||||
not relative
|
||||
or relative.startswith("/")
|
||||
or "\\" in relative
|
||||
or ".." in relative.split("/")
|
||||
):
|
||||
raise SourceError("SSH source entry is invalid", reason_code="source_invalid")
|
||||
return f"/{relative}"
|
||||
|
||||
@staticmethod
|
||||
def _entry_from_attributes(path: str, attributes: Any) -> Entry:
|
||||
mode = getattr(attributes, "st_mode", None)
|
||||
if not isinstance(mode, int):
|
||||
raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
|
||||
if stat.S_ISLNK(mode):
|
||||
raise SourceError("SSH source symlinks are not supported", reason_code="source_invalid")
|
||||
size = getattr(attributes, "st_size", 0)
|
||||
mtime = getattr(attributes, "st_mtime", 0)
|
||||
if not isinstance(size, int) or size < 0 or not isinstance(mtime, int):
|
||||
raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
|
||||
if stat.S_ISDIR(mode):
|
||||
return Entry(path, "directory", 0, stat.S_IMODE(mode), mtime * 1_000_000_000)
|
||||
if stat.S_ISREG(mode):
|
||||
return Entry(path, "file", size, stat.S_IMODE(mode), mtime * 1_000_000_000)
|
||||
raise SourceError("SSH source contains an unsupported entry", reason_code="source_invalid")
|
||||
|
||||
async def probe(self) -> dict[str, int]:
|
||||
count = 0
|
||||
try:
|
||||
async for entry in self.enumerate_entries():
|
||||
if entry.kind == "file":
|
||||
count += 1
|
||||
return {"entry_count": count}
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
async def enumerate_entries(self) -> AsyncIterator[Entry]:
|
||||
await asyncio.to_thread(self._connect)
|
||||
pending = [""]
|
||||
entry_count = 0
|
||||
try:
|
||||
while pending:
|
||||
parent = pending.pop()
|
||||
remote_parent = "/" if not parent else self._remote_path(parent)
|
||||
try:
|
||||
iterator = await asyncio.to_thread(
|
||||
self._sftp_client().listdir_iter,
|
||||
remote_parent,
|
||||
read_aheads=self.settings.ssh_list_read_aheads,
|
||||
)
|
||||
while (
|
||||
attributes := await asyncio.to_thread(_next_or_none, iterator)
|
||||
) is not None:
|
||||
path = self._relative(parent, getattr(attributes, "filename", None))
|
||||
entry_count += 1
|
||||
if entry_count > self.settings.ssh_max_entries:
|
||||
raise SourceError(
|
||||
"SSH source entry limit exceeded", reason_code="source_limit"
|
||||
)
|
||||
entry = self._entry_from_attributes(path, attributes)
|
||||
if entry.kind == "directory":
|
||||
if path.count("/") + 1 > self.settings.ssh_max_traversal_depth:
|
||||
raise SourceError(
|
||||
"SSH source traversal depth exceeded",
|
||||
reason_code="source_limit",
|
||||
)
|
||||
pending.append(path)
|
||||
else:
|
||||
self._files[path] = (entry.size, entry.mtime_ns)
|
||||
yield entry
|
||||
except SourceError:
|
||||
raise
|
||||
except (TimeoutError, OSError, paramiko.SSHException) as error:
|
||||
raise SourceError(
|
||||
"SSH source enumeration failed", reason_code="source_unavailable"
|
||||
) from error
|
||||
except BaseException:
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def open_content(self, path: str) -> AsyncIterator[bytes]:
|
||||
expected = self._files.get(path)
|
||||
if expected is None:
|
||||
raise SourceError("SSH source entry was not enumerated", reason_code="source_invalid")
|
||||
remote_path = self._remote_path(path)
|
||||
handle: Any | None = None
|
||||
reader: Any | None = None
|
||||
try:
|
||||
await asyncio.to_thread(self._connect)
|
||||
reader = await asyncio.to_thread(self._read_client)
|
||||
assert reader is not None
|
||||
attributes = await asyncio.to_thread(reader.lstat, remote_path)
|
||||
entry = self._entry_from_attributes(path, attributes)
|
||||
if entry.kind != "file" or (entry.size, entry.mtime_ns) != expected:
|
||||
raise SourceError(
|
||||
"SSH source file changed during backup", reason_code="source_changed"
|
||||
)
|
||||
handle = await asyncio.to_thread(
|
||||
reader.open,
|
||||
remote_path,
|
||||
"rb",
|
||||
self.settings.ssh_read_chunk_bytes,
|
||||
)
|
||||
assert handle is not None
|
||||
while chunk := await asyncio.to_thread(handle.read, self.settings.ssh_read_chunk_bytes):
|
||||
if not isinstance(chunk, bytes) or len(chunk) > self.settings.ssh_read_chunk_bytes:
|
||||
raise SourceError(
|
||||
"SSH source returned an invalid read", reason_code="source_invalid"
|
||||
)
|
||||
yield chunk
|
||||
after = await asyncio.to_thread(reader.lstat, remote_path)
|
||||
verified = self._entry_from_attributes(path, after)
|
||||
if verified.kind != "file" or (verified.size, verified.mtime_ns) != expected:
|
||||
raise SourceError(
|
||||
"SSH source file changed during backup", reason_code="source_changed"
|
||||
)
|
||||
except SourceError:
|
||||
raise
|
||||
except (TimeoutError, OSError, paramiko.SSHException) as error:
|
||||
raise SourceError("SSH source read failed", reason_code="source_unavailable") from error
|
||||
finally:
|
||||
if handle is not None:
|
||||
await asyncio.to_thread(handle.close)
|
||||
if reader is not None and reader is not self._sftp:
|
||||
await asyncio.to_thread(reader.close)
|
||||
|
||||
def _close_sync(self) -> None:
|
||||
sftp, transport = self._sftp, self._transport
|
||||
self._sftp = None
|
||||
self._transport = None
|
||||
if sftp is not None:
|
||||
with suppress(OSError, paramiko.SSHException):
|
||||
sftp.close()
|
||||
if transport is not None:
|
||||
with suppress(OSError, paramiko.SSHException):
|
||||
transport.close()
|
||||
|
||||
async def close(self) -> None:
|
||||
await asyncio.to_thread(self._close_sync)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Closed public configuration for the staged SSH source capability.
|
||||
|
||||
This module deliberately describes configuration only. It must not create a
|
||||
network transport or load a private key; those capabilities are outside this
|
||||
slice.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
SSH_PRIVATE_KEY_PURPOSE = "ssh_private_key"
|
||||
_ALLOWED_HOST_KEY_ALGORITHMS = frozenset(
|
||||
{
|
||||
"ssh-ed25519",
|
||||
"ecdsa-sha2-nistp256",
|
||||
"ecdsa-sha2-nistp384",
|
||||
"ecdsa-sha2-nistp521",
|
||||
"rsa-sha2-256",
|
||||
"rsa-sha2-512",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_control_or_space(value: str) -> bool:
|
||||
return any(character.isspace() or ord(character) < 32 for character in value)
|
||||
|
||||
|
||||
class SSHSourcePublicConfig(BaseModel):
|
||||
"""The public, SFTP-chroot-only portion of an SSH source definition."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
hostname: str = Field(min_length=1, max_length=253)
|
||||
port: int = Field(ge=1, le=65535)
|
||||
username: str = Field(min_length=1, max_length=255)
|
||||
host_key: str = Field(min_length=1, max_length=16384)
|
||||
# The server-side forced-SFTP account's chroot is the only permitted root.
|
||||
root: Literal["/"]
|
||||
|
||||
@field_validator("hostname")
|
||||
@classmethod
|
||||
def validate_hostname(cls, value: str) -> str:
|
||||
if _has_control_or_space(value) or any(character in value for character in "/\\@?#"):
|
||||
raise ValueError("SSH hostname is invalid")
|
||||
return value
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def validate_username(cls, value: str) -> str:
|
||||
if _has_control_or_space(value) or any(character in value for character in "/\\:@"):
|
||||
raise ValueError("SSH username is invalid")
|
||||
return value
|
||||
|
||||
@field_validator("host_key")
|
||||
@classmethod
|
||||
def validate_host_key(cls, value: str) -> str:
|
||||
algorithm, separator, encoded_key = value.partition(" ")
|
||||
if not separator or not algorithm or not encoded_key or " " in encoded_key:
|
||||
raise ValueError("SSH host key must be an algorithm and base64 key")
|
||||
if algorithm not in _ALLOWED_HOST_KEY_ALGORITHMS:
|
||||
raise ValueError("SSH host key algorithm is not supported")
|
||||
try:
|
||||
decoded = base64.b64decode(encoded_key, validate=True)
|
||||
except (ValueError, UnicodeEncodeError) as error:
|
||||
raise ValueError("SSH host key is not valid base64") from error
|
||||
if not decoded:
|
||||
raise ValueError("SSH host key is empty")
|
||||
return value
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Dedicated HTTP runtime role."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
|
||||
import uvicorn
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.observability.logging import configure_logging, log_event
|
||||
|
||||
|
||||
def _prepare_socket(settings: Settings) -> str:
|
||||
path = settings.web_socket_path
|
||||
path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||
try:
|
||||
existing = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return str(path)
|
||||
if not stat.S_ISSOCK(existing.st_mode):
|
||||
raise RuntimeError("web socket path is not a socket")
|
||||
path.unlink()
|
||||
return str(path)
|
||||
|
||||
|
||||
def run_web(settings: Settings) -> int:
|
||||
"""Run exactly one ASGI server without reload or embedded background roles."""
|
||||
from backup_tool.api.app import create_app
|
||||
|
||||
configure_logging("web", settings.log_level)
|
||||
server = uvicorn.Server(
|
||||
uvicorn.Config(
|
||||
create_app(settings),
|
||||
uds=_prepare_socket(settings),
|
||||
log_level=settings.log_level.lower(),
|
||||
reload=False,
|
||||
workers=1,
|
||||
log_config=None,
|
||||
)
|
||||
)
|
||||
server.run()
|
||||
log_event("role_stopped", role="web")
|
||||
return 0
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Single-node durable worker role.
|
||||
|
||||
The worker owns leases and performs repository I/O outside the API process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import importlib
|
||||
import signal
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import create_engine
|
||||
from backup_tool.db.models import Backup, Execution, Job, Repository, Restore, Source
|
||||
from backup_tool.execution import (
|
||||
claim,
|
||||
complete_cancellation,
|
||||
heartbeat,
|
||||
record_event,
|
||||
recover_stale,
|
||||
transition,
|
||||
)
|
||||
from backup_tool.faults import FaultInjector, NoFault
|
||||
from backup_tool.gc import process_retention_gc
|
||||
from backup_tool.notifications.dispatcher import dispatch_one, recover_notification_leases
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.observability.logging import configure_logging, log_event
|
||||
from backup_tool.repository import reconcile_key_rotations
|
||||
from backup_tool.security.secrets import EnvelopeCipher
|
||||
|
||||
snapshot = importlib.import_module("backup_tool.snapshot")
|
||||
SnapshotError = snapshot.SnapshotError
|
||||
SnapshotIntegrityError = snapshot.SnapshotIntegrityError
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
*,
|
||||
owner: str | None = None,
|
||||
fault_injector: FaultInjector | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.owner = owner or f"worker-{uuid4()}"
|
||||
self.fault_injector = fault_injector or NoFault()
|
||||
self._stopping = asyncio.Event()
|
||||
self.engine = create_engine(settings)
|
||||
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
|
||||
self.cipher = EnvelopeCipher.from_file(settings.master_key_file)
|
||||
self._execution_turns = 0
|
||||
self._next_maintenance_at: datetime | None = None
|
||||
|
||||
async def startup(self) -> int:
|
||||
async with self.sessions() as db:
|
||||
rotations = await reconcile_key_rotations(self.settings, db)
|
||||
publications = cast(int, await snapshot.reconcile_publications(self.settings, db))
|
||||
restored = cast(int, await snapshot.reconcile_restores(self.settings, db))
|
||||
recovered_executions = await recover_stale(db)
|
||||
recovered_deliveries = await recover_notification_leases(db)
|
||||
maintenance = await process_retention_gc(db)
|
||||
self._next_maintenance_at = datetime.now(UTC) + timedelta(seconds=60)
|
||||
return (
|
||||
rotations
|
||||
+ publications
|
||||
+ restored
|
||||
+ recovered_executions
|
||||
+ recovered_deliveries
|
||||
+ maintenance.tombstoned
|
||||
)
|
||||
|
||||
async def _run_restore(self, db: AsyncSession) -> bool:
|
||||
restore_id = await db.scalar(
|
||||
select(Restore.id)
|
||||
.where(Restore.state == "queued")
|
||||
.order_by(Restore.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if restore_id is None:
|
||||
return False
|
||||
result = await db.execute(
|
||||
update(Restore)
|
||||
.where(Restore.id == restore_id, Restore.state == "queued")
|
||||
.values(state="running")
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return False
|
||||
await db.commit()
|
||||
restore = await db.get(Restore, restore_id)
|
||||
if restore is None:
|
||||
return False
|
||||
try:
|
||||
backup = await db.get(Backup, restore.backup_id)
|
||||
if backup is None or backup.integrity != "verified" or backup.tombstoned_at is not None:
|
||||
raise SnapshotError("backup is unavailable")
|
||||
execution = await db.get(Execution, backup.execution_id)
|
||||
if execution is None:
|
||||
raise SnapshotError("backup execution is unavailable")
|
||||
job = await db.get(Job, execution.job_id)
|
||||
if job is None:
|
||||
raise SnapshotError("backup job is unavailable")
|
||||
repository = await db.get(Repository, job.repository_id)
|
||||
if repository is None:
|
||||
raise SnapshotError("backup repository is unavailable")
|
||||
restore.result = await snapshot.restore_full_snapshot(
|
||||
self.settings, db, restore, backup, repository
|
||||
)
|
||||
restore.state = "committed"
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.committed",
|
||||
correlation_id=restore.id,
|
||||
resource={"restore_id": restore.id, "backup_id": restore.backup_id},
|
||||
payload={"dry_run": restore.dry_run, "outcome": "committed"},
|
||||
deduplication_key=f"restore:{restore.id}:committed",
|
||||
)
|
||||
await db.commit()
|
||||
except SnapshotIntegrityError:
|
||||
await db.rollback()
|
||||
failed = await db.get(Restore, restore_id)
|
||||
if failed is not None:
|
||||
corrupted_backup = await db.get(Backup, failed.backup_id)
|
||||
if corrupted_backup is not None:
|
||||
corrupted_backup.integrity = "corrupt"
|
||||
failed.state = "failed"
|
||||
failed.result = {"reason": "restore_failed"}
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.failed",
|
||||
correlation_id=failed.id,
|
||||
resource={"restore_id": failed.id, "backup_id": failed.backup_id},
|
||||
payload={"reason_code": "restore_failed"},
|
||||
deduplication_key=f"restore:{failed.id}:failed",
|
||||
)
|
||||
await db.commit()
|
||||
except SnapshotError:
|
||||
await db.rollback()
|
||||
failed = await db.get(Restore, restore_id)
|
||||
if failed is not None:
|
||||
failed.state = "failed"
|
||||
failed.result = {"reason": "restore_failed"}
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.failed",
|
||||
correlation_id=failed.id,
|
||||
resource={"restore_id": failed.id, "backup_id": failed.backup_id},
|
||||
payload={"reason_code": "restore_failed"},
|
||||
deduplication_key=f"restore:{failed.id}:failed",
|
||||
)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def _run_maintenance(self, db: AsyncSession) -> None:
|
||||
now = datetime.now(UTC)
|
||||
if self._next_maintenance_at is None or now >= self._next_maintenance_at:
|
||||
await process_retention_gc(db, now)
|
||||
self._next_maintenance_at = now + timedelta(seconds=60)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
if self._stopping.is_set():
|
||||
return False
|
||||
async with self.sessions() as db:
|
||||
# Retention/GC is a worker responsibility, but is rate-limited so it
|
||||
# cannot turn a sustained backup queue into a metadata polling loop.
|
||||
await self._run_maintenance(db)
|
||||
# Never let an always-nonempty execution queue starve due notifications.
|
||||
if self._execution_turns >= 1 and await dispatch_one(
|
||||
db, self.settings, self.cipher, self.owner
|
||||
):
|
||||
self._execution_turns = 0
|
||||
return True
|
||||
if self._stopping.is_set():
|
||||
return False
|
||||
execution_id = await db.scalar(
|
||||
select(Execution.id)
|
||||
.where(Execution.state == "queued")
|
||||
.order_by(Execution.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if execution_id is None:
|
||||
if await self._run_restore(db):
|
||||
return True
|
||||
return await dispatch_one(db, self.settings, self.cipher, self.owner)
|
||||
execution = await claim(db, execution_id, self.owner)
|
||||
if execution is None:
|
||||
return False
|
||||
self._execution_turns += 1
|
||||
# Reload after the claim: a control request can race the lease acquisition.
|
||||
await db.refresh(execution)
|
||||
if execution.state == "cancelling":
|
||||
await complete_cancellation(db, execution.id, self.owner)
|
||||
return True
|
||||
now = datetime.now(UTC)
|
||||
result = await db.execute(
|
||||
update(Execution)
|
||||
.where(
|
||||
Execution.id == execution.id,
|
||||
Execution.lease_owner == self.owner,
|
||||
Execution.lease_expires_at >= now,
|
||||
Execution.state == "preparing",
|
||||
)
|
||||
.values(state=transition("preparing", "running"), started_at=now)
|
||||
)
|
||||
if getattr(result, "rowcount", 0) == 1:
|
||||
await db.refresh(execution)
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
await heartbeat(db, execution.id, self.owner)
|
||||
failure_detail: str | None = None
|
||||
failure_reason = "transient_io"
|
||||
try:
|
||||
job = await db.get(Job, execution.job_id)
|
||||
if job is None:
|
||||
raise SnapshotError("execution job is unavailable")
|
||||
source = await db.get(Source, job.source_id)
|
||||
repository = await db.get(Repository, job.repository_id)
|
||||
if source is None or repository is None:
|
||||
raise SnapshotError("execution source or repository is unavailable")
|
||||
backup = await snapshot.publish_full_snapshot(
|
||||
self.settings,
|
||||
db,
|
||||
execution,
|
||||
job,
|
||||
source,
|
||||
repository,
|
||||
self.fault_injector,
|
||||
cipher=self.cipher,
|
||||
)
|
||||
await db.flush()
|
||||
await emit_event(
|
||||
db,
|
||||
"backup.committed",
|
||||
correlation_id=execution.id,
|
||||
resource={
|
||||
"execution_id": execution.id,
|
||||
"job_id": job.id,
|
||||
"repository_id": repository.id,
|
||||
"backup_id": backup.id,
|
||||
},
|
||||
payload={"integrity": backup.integrity, "effective_mode": job.requested_mode},
|
||||
deduplication_key=f"backup:{backup.id}:committed",
|
||||
)
|
||||
await emit_event(
|
||||
db,
|
||||
"backup.verification_succeeded",
|
||||
correlation_id=execution.id,
|
||||
resource={"execution_id": execution.id, "backup_id": backup.id},
|
||||
payload={"integrity": backup.integrity},
|
||||
deduplication_key=f"backup:{backup.id}:verified",
|
||||
)
|
||||
execution.state = transition("running", "verifying")
|
||||
await record_event(db, execution)
|
||||
self.fault_injector.hit("metadata.before_commit")
|
||||
await db.commit()
|
||||
self.fault_injector.hit("metadata.after_commit")
|
||||
execution.state = transition("verifying", "committed")
|
||||
execution.completed_at = datetime.now(UTC)
|
||||
execution.lease_owner = None
|
||||
execution.lease_expires_at = None
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
snapshot.finalize_publication(Path(repository.root), execution.id)
|
||||
except SnapshotError as error:
|
||||
await db.rollback()
|
||||
failure_detail = str(error)
|
||||
failure_reason = error.reason_code
|
||||
if failure_detail is not None:
|
||||
failed = await db.get(Execution, execution_id)
|
||||
if failed is None or failed.state not in {"preparing", "running", "verifying"}:
|
||||
return True
|
||||
failed.state = "failed"
|
||||
failed.reason_code = failure_reason
|
||||
failed.operator_message = failure_detail
|
||||
failed.completed_at = datetime.now(UTC)
|
||||
failed.lease_owner = None
|
||||
failed.lease_expires_at = None
|
||||
await record_event(db, failed)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def run(self) -> None:
|
||||
await self.startup()
|
||||
while not self._stopping.is_set():
|
||||
if not await self.run_once():
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(self._stopping.wait(), timeout=0.25)
|
||||
await self.engine.dispose()
|
||||
|
||||
def stop(self) -> None:
|
||||
log_event("role_stopping", role="worker")
|
||||
self._stopping.set()
|
||||
|
||||
|
||||
def run_worker(settings: Settings) -> int:
|
||||
configure_logging("worker", settings.log_level)
|
||||
worker = Worker(settings)
|
||||
loop = asyncio.new_event_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
with contextlib.suppress(NotImplementedError):
|
||||
loop.add_signal_handler(sig, worker.stop)
|
||||
try:
|
||||
loop.run_until_complete(worker.run())
|
||||
finally:
|
||||
loop.close()
|
||||
log_event("role_stopped", role="worker")
|
||||
return 0
|
||||
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user