diff --git a/.dockerignore b/.dockerignore index 84399c1..fbb39d6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ wheels/ # Virtual environments venv/ +.venv/ env/ ENV/ @@ -64,6 +65,9 @@ logs/ # Tests tests/ +!tests/ssh-fixture/ +!tests/ssh-fixture/Dockerfile +!tests/ssh-fixture/sshd_config .pytest_cache/ .coverage diff --git a/.gitignore b/.gitignore index 77846ab..c3ce7dc 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ Thumbs.db # Frontend frontend/node_modules/ frontend/dist/ +frontend/test-results/ frontend/yarn.lock frontend/pnpm-lock.yaml diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..557d005 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,25 @@ +# Agent Operating Instructions + +## Continuous milestone execution + +When the user asks to continue, work autonomously through the active milestone +and its verification. Do **not** send progress-only, acknowledgement, empty, or +status final responses. Reply only when: + +1. the user asks for status; +2. a real product, security, credentials, or destructive-action decision needs + the user's input; or +3. the requested milestone is fully implemented and verified. + +Use one foreground milestone batch where possible. If work must run in a +background subagent, avoid notifying the user manually; inspect and verify the +result before replying at the same completion boundary. + +A chat turn still necessarily ends after a model response. This file prevents +unnecessary model-generated completion messages; it cannot suppress +harness-generated tool or subagent notifications. + +## Verification + +Before claiming a milestone boundary, run `make check` and record focused +acceptance evidence under `docs/release/`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d620bea --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# syntax=docker/dockerfile:1 +# Pinned by manifest digest; update tag and digest together. +FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS builder + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /build +COPY backend/pyproject.toml ./pyproject.toml +COPY backend/src ./src +RUN python -m pip install --upgrade pip==25.3 && \ + python -m pip wheel --wheel-dir /wheels . + +FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS runtime + +ENV PATH="/opt/venv/bin:${PATH}" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + BACKUP_TOOL_ALEMBIC_ROOT=/app + +RUN groupadd --gid 10001 backup-tool && \ + useradd --uid 10001 --gid backup-tool --create-home --home-dir /home/backup-tool \ + --shell /usr/sbin/nologin backup-tool && \ + python -m venv /opt/venv && \ + install --directory --owner=backup-tool --group=backup-tool --mode=0750 \ + /var/lib/backup-tool \ + /var/lib/backup-tool/repositories \ + /var/lib/backup-tool/restores \ + /run/backup-tool + +COPY --from=builder /wheels /wheels +RUN python -m pip install --no-index --find-links=/wheels backup-tool==2.0.0.dev0 && \ + rm -rf /wheels +COPY --chown=backup-tool:backup-tool backend/alembic.ini /app/alembic.ini +COPY --chown=backup-tool:backup-tool backend/alembic /app/alembic + +WORKDIR /app +USER backup-tool:backup-tool + +ENTRYPOINT ["backup-tool"] +CMD ["web"] diff --git a/Makefile b/Makefile index 6f43bb4..ac33448 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= .venv/bin/python BOOTSTRAP_PYTHON ?= python3 NPM ?= npm -.PHONY: setup install test-fast test-integration test-security lint typecheck frontend-build check-v1-absent check +.PHONY: setup install test-fast test-integration test-fault test-security test-e2e test-ssh-integration lint typecheck frontend-build check-v1-absent check-openapi check setup: test -x $(PYTHON) || $(BOOTSTRAP_PYTHON) -m venv .venv @@ -18,9 +18,18 @@ test-fast: test-integration: $(PYTHON) -m pytest tests/integration -q +test-fault: + $(PYTHON) -m pytest tests/fault -q + test-security: $(PYTHON) -m pytest tests/security -q +test-e2e: + BACKUP_TOOL_COMPOSE_E2E=1 $(PYTHON) -m pytest tests/e2e/test_compose_v2.py -q + +test-ssh-integration: + BACKUP_TOOL_SSH_INTEGRATION=1 $(PYTHON) -m pytest tests/integration/test_ssh_backup_restore.py -q + lint: $(PYTHON) -m ruff check --config backend/pyproject.toml backend/src backend/alembic tests tools $(PYTHON) -m ruff format --check --config backend/pyproject.toml backend/src backend/alembic tests tools @@ -32,7 +41,12 @@ typecheck: frontend-build: $(NPM) --prefix frontend run build +check-openapi: + $(PYTHON) tools/export_openapi.py + $(NPM) --prefix frontend run api:generate + git diff --exit-code -- openapi/v2.json frontend/src/api/generated + check-v1-absent: $(PYTHON) tools/forbidden_v1_scan.py . -check: check-v1-absent test-fast test-integration test-security lint typecheck frontend-build +check: check-v1-absent check-openapi test-fast test-integration test-fault test-security lint typecheck frontend-build diff --git a/README.md b/README.md index bb2e30d..a46b737 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A modern backup management application with a FastAPI backend and React frontend ## Features -- **Multiple Source Types**: Local filesystem, SSH/SFTP, and database (PostgreSQL, MySQL) +- **Source Type**: Local filesystem - **Backup Strategies**: Full and incremental backups - **Scheduled Backups**: Cron-based scheduling with APScheduler - **Retention Policies**: Count-based and days-based backup retention @@ -22,7 +22,7 @@ backup-tool/ │ │ ├── schemas.py # Pydantic schemas │ │ └── main.py # Application entry point │ ├── backup/ # Backup engine -│ │ ├── adapters/ # Source adapters (local, SSH, database) +│ │ ├── adapters/ # Local source adapter │ │ ├── engine.py # Backup execution engine │ │ ├── scheduler.py # Job scheduler │ │ └── retention.py # Retention policies @@ -51,9 +51,10 @@ docker compose --profile dev up -d ``` Access the application: -- Backend API: http://localhost:8000 -- Frontend: http://localhost:3000 -- API Docs: http://localhost:8000/docs + +- Backend API: +- Frontend: +- API Docs: ### Option 2: Manual Setup @@ -96,15 +97,16 @@ uvicorn app.main:app --host 0.0.0.0 --port 8000 ## API Documentation Once the backend is running, visit: -- Swagger UI: http://localhost:8000/docs -- ReDoc: http://localhost:8000/redoc + +- Swagger UI: +- ReDoc: ## Configuration ### Environment Variables | Variable | Description | Default | -|----------|-------------|---------| +| ---------- | ------------- | --------- | | `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///./backup_tool.db` | | `CORS_ORIGINS` | Comma-separated allowed CORS origins | `http://localhost:3000` | | `SQL_ECHO` | Enable SQL query logging | `false` | @@ -113,17 +115,20 @@ Once the backend is running, visit: ### Docker-Specific Configuration When running with Docker Compose, the following volumes are mounted: + - `backup-data`: Persisted SQLite database at `/app/data` - `backup-storage`: Backup files at `/app/backups` ### Development vs Production **Development Mode** (`docker compose --profile dev up -d`): + - Backend hot reload enabled - Frontend Vite dev server with HMR - Source code mounted as volumes **Production Mode** (`docker compose --profile prod up -d`): + - Optimized frontend build served via nginx - Backend without reload - Static assets compiled @@ -133,6 +138,7 @@ When running with Docker Compose, the following volumes are mounted: ### Docker Issues **Port already in use** + ```bash # Check what's using port 8000 lsof -i :8000 @@ -141,6 +147,7 @@ lsof -i :8000 ``` **Container fails to start** + ```bash # Check logs docker logs backup-tool-backend @@ -150,6 +157,7 @@ docker compose build --no-cache ``` **Permission denied on data directory** + ```bash # Fix permissions docker compose exec backend chown -R backup-tool:backup-tool /app/data @@ -157,6 +165,7 @@ docker compose exec backend chown -R backup-tool:backup-tool /app/data **Tests fail in Docker** Tests require development dependencies. Install with: + ```bash docker compose exec backend pip install -e ".[dev]" ``` @@ -165,11 +174,13 @@ docker compose exec backend pip install -e ".[dev]" **Python version incompatibility** Ensure Python 3.11+ is installed: + ```bash python --version ``` **Node modules conflicts** + ```bash cd frontend rm -rf node_modules package-lock.json @@ -182,7 +193,7 @@ npm install 1. Clone the repository 2. Run `docker compose --profile prod up -d` -3. Access at http://localhost:3000 +3. Access at ### Manual Deployment diff --git a/backend/alembic/versions/0004_repository_signing_keys.py b/backend/alembic/versions/0004_repository_signing_keys.py new file mode 100644 index 0000000..7964485 --- /dev/null +++ b/backend/alembic/versions/0004_repository_signing_keys.py @@ -0,0 +1,29 @@ +"""bind repositories to manifest signing public keys + +Revision ID: 0004_repository_signing_keys +Revises: 0003_execution_events +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0004_repository_signing_keys" +down_revision = "0003_execution_events" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("repositories") as batch: + batch.add_column( + sa.Column("signing_key_id", sa.String(length=64), nullable=False, server_default="") + ) + batch.add_column( + sa.Column("signing_public_key", sa.String(length=64), nullable=False, server_default="") + ) + + +def downgrade() -> None: + with op.batch_alter_table("repositories") as batch: + batch.drop_column("signing_public_key") + batch.drop_column("signing_key_id") diff --git a/backend/alembic/versions/0005_restore_dry_run.py b/backend/alembic/versions/0005_restore_dry_run.py new file mode 100644 index 0000000..044e2dd --- /dev/null +++ b/backend/alembic/versions/0005_restore_dry_run.py @@ -0,0 +1,23 @@ +"""persist restore dry-run intent + +Revision ID: 0005_restore_dry_run +Revises: 0004_repository_signing_keys +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0005_restore_dry_run" +down_revision = "0004_repository_signing_keys" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("restores") as batch: + batch.add_column(sa.Column("dry_run", sa.Boolean(), nullable=False, server_default="0")) + + +def downgrade() -> None: + with op.batch_alter_table("restores") as batch: + batch.drop_column("dry_run") diff --git a/backend/alembic/versions/0006_local_sources_only.py b/backend/alembic/versions/0006_local_sources_only.py new file mode 100644 index 0000000..02f0ebd --- /dev/null +++ b/backend/alembic/versions/0006_local_sources_only.py @@ -0,0 +1,43 @@ +"""restrict persisted sources to local + +Revision ID: 0006_local_sources_only +Revises: 0005_restore_dry_run +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0006_local_sources_only" +down_revision = "0005_restore_dry_run" +branch_labels = None +depends_on = None + + +def _reject_nonlocal_sources() -> None: + connection = op.get_bind() + sources = sa.table("sources", sa.column("kind")) + count = connection.scalar( + sa.select(sa.func.count()).select_from(sources).where(sources.c.kind != "local") + ) + if count is None: + raise RuntimeError("Cannot inspect persisted source kinds before migration.") + if count: + raise RuntimeError( + "Cannot restrict sources to local: " + f"found {count} non-local source row(s). Remove or migrate them before upgrading." + ) + + +def upgrade() -> None: + _reject_nonlocal_sources() + with op.batch_alter_table("sources") as batch: + batch.drop_constraint(op.f("ck_sources_kind"), type_="check") + batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'") + + +def downgrade() -> None: + with op.batch_alter_table("sources") as batch: + batch.drop_constraint(op.f("ck_sources_kind"), type_="check") + batch.create_check_constraint( + op.f("ck_sources_kind"), "kind IN ('local','sftp','postgresql','mysql')" + ) diff --git a/backend/alembic/versions/0007_repository_data_key_epochs.py b/backend/alembic/versions/0007_repository_data_key_epochs.py new file mode 100644 index 0000000..de91d87 --- /dev/null +++ b/backend/alembic/versions/0007_repository_data_key_epochs.py @@ -0,0 +1,87 @@ +"""add repository data key epochs + +Revision ID: 0007_repository_data_key_epochs +Revises: 0006_local_sources_only +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0007_repository_data_key_epochs" +down_revision = "0006_local_sources_only" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("repositories", sa.Column("active_data_key_id", sa.String(36), nullable=True)) + op.add_column("backups", sa.Column("data_key_id", sa.String(36), nullable=True)) + op.create_table( + "repository_data_key_epochs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "repository_id", + sa.String(36), + sa.ForeignKey("repositories.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("key_id", sa.String(36), nullable=False), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("retired_at", sa.DateTime(timezone=True)), + sa.UniqueConstraint( + "repository_id", "key_id", name="uq_repository_data_key_epochs_repository_key_epoch" + ), + sa.CheckConstraint( + "state IN ('active','retired')", + name="ck_repository_data_key_epochs_repository_data_key_epoch_state", + ), + ) + op.create_index( + "ix_repository_data_key_epochs_repository_id", + "repository_data_key_epochs", + ["repository_id"], + ) + op.create_index( + "uq_repository_data_key_epochs_active", + "repository_data_key_epochs", + ["repository_id"], + unique=True, + sqlite_where=sa.text("state = 'active'"), + ) + + +def downgrade() -> None: + connection = op.get_bind() + epochs = sa.table("repository_data_key_epochs") + repositories = sa.table("repositories", sa.column("active_data_key_id")) + backups = sa.table("backups", sa.column("data_key_id")) + epoch_count = connection.scalar(sa.select(sa.func.count()).select_from(epochs)) + active_key_count = connection.scalar( + sa.select(sa.func.count()) + .select_from(repositories) + .where(repositories.c.active_data_key_id.is_not(None)) + ) + backup_key_count = connection.scalar( + sa.select(sa.func.count()).select_from(backups).where(backups.c.data_key_id.is_not(None)) + ) + if epoch_count or active_key_count or backup_key_count: + raise RuntimeError("cannot downgrade while repository data key metadata exists") + op.drop_index("uq_repository_data_key_epochs_active", table_name="repository_data_key_epochs") + op.drop_index( + "ix_repository_data_key_epochs_repository_id", table_name="repository_data_key_epochs" + ) + op.drop_table("repository_data_key_epochs") + op.drop_column("backups", "data_key_id") + op.drop_column("repositories", "active_data_key_id") diff --git a/backend/alembic/versions/0008_notification_outbox.py b/backend/alembic/versions/0008_notification_outbox.py new file mode 100644 index 0000000..45ced6d --- /dev/null +++ b/backend/alembic/versions/0008_notification_outbox.py @@ -0,0 +1,445 @@ +"""replace notification attempt stub with a durable M12 outbox + +Revision ID: 0008_notification_outbox +Revises: 0007_repository_data_key_epochs +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +import sqlalchemy as sa +from alembic import op +from backup_tool.ids import new_uuid7 + +revision = "0008_notification_outbox" +down_revision = "0007_repository_data_key_epochs" +branch_labels = None +depends_on = None + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _parameterized_execute(connection: sa.Connection, statement: object) -> Any: + """Execute only SQLAlchemy Core statements, never dynamic SQL strings.""" + database = connection.execution_options() + return database.execute(statement) # type: ignore[arg-type] + + +def upgrade() -> None: + connection = op.get_bind() + # SQLite batch-rebuilds notification_subscriptions. Its old delivery table + # references this table, so enforcement must be suspended for this migration + # only while the legacy rows are copied into the replacement outbox shape. + if connection.dialect.name == "sqlite": + connection.exec_driver_sql("PRAGMA foreign_keys=OFF") + # Extend the existing subscription rows first: pre-M12 rows stay disabled until + # an operator explicitly configures a new credential/key. + with op.batch_alter_table("notification_subscriptions") as batch: + batch.add_column( + sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60") + ) + batch.add_column(sa.Column("rate_tokens", sa.Float(), nullable=False, server_default="60")) + batch.add_column(sa.Column("rate_updated_at", sa.DateTime(timezone=True), nullable=True)) + batch.add_column(sa.Column("revision", sa.Integer(), nullable=False, server_default="1")) + batch.create_check_constraint("rate_positive", "rate_limit_per_minute > 0") + batch.create_check_constraint("rate_tokens_nonnegative", "rate_tokens >= 0") + batch.create_check_constraint("revision_positive", "revision > 0") + + op.create_table( + "notification_events", + sa.Column("type", sa.String(96), nullable=False), + sa.Column("schema_version", sa.Integer(), nullable=False, server_default="1"), + sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("correlation_id", sa.String(36), nullable=False), + sa.Column("severity", sa.String(16), nullable=False), + sa.Column("resource_refs", sa.JSON(), nullable=False), + sa.Column("payload", sa.JSON(), nullable=False), + sa.Column("canonical_envelope", sa.Text(), nullable=False), + sa.Column("deduplication_key", sa.String(255), nullable=True), + sa.Column("id", sa.String(36), primary_key=True), + sa.CheckConstraint("schema_version = 1", name="ck_notification_events_schema_version"), + sa.CheckConstraint( + "severity IN ('info','warning','error','security')", + name="ck_notification_events_severity", + ), + sa.UniqueConstraint("deduplication_key", name="uq_notification_events_deduplication_key"), + ) + op.create_index( + "ix_notification_events_type_occurred", "notification_events", ["type", "occurred_at"] + ) + + # Preserve unexpected rows from the unused baseline shape. Renaming first + # keeps the original data intact if an upgrade is interrupted before copy. + op.rename_table("notification_deliveries", "notification_deliveries_legacy") + op.create_table( + "notification_deliveries", + sa.Column( + "event_id", + sa.String(36), + sa.ForeignKey("notification_events.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column( + "subscription_id", + sa.String(36), + sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("state", sa.String(32), nullable=False, server_default="pending"), + sa.Column("due_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("lease_owner", sa.String(255), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("terminal_reason", sa.String(96), nullable=True), + sa.Column("response_class", sa.String(64), nullable=True), + sa.Column("response_summary", sa.String(512), nullable=True), + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.CheckConstraint( + "attempt_count >= 0", name="ck_notification_deliveries_attempt_count_nonnegative" + ), + sa.CheckConstraint( + "state IN ('pending','leased','delivered','retry','failed')", + name="ck_notification_deliveries_state", + ), + sa.UniqueConstraint( + "event_id", "subscription_id", name="uq_notification_deliveries_event_subscription" + ), + ) + op.create_index( + "ix_notification_deliveries_due", "notification_deliveries", ["state", "due_at"] + ) + op.create_index( + "ix_notification_deliveries_lease", "notification_deliveries", ["state", "lease_expires_at"] + ) + op.create_table( + "notification_delivery_attempts", + sa.Column( + "delivery_id", + sa.String(36), + sa.ForeignKey("notification_deliveries.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("number", sa.Integer(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("outcome", sa.String(32), nullable=False, server_default="started"), + sa.Column("response_class", sa.String(64), nullable=True), + sa.Column("diagnostic", sa.String(512), nullable=True), + sa.Column("id", sa.String(36), primary_key=True), + sa.CheckConstraint("number > 0", name="ck_notification_delivery_attempts_number_positive"), + sa.CheckConstraint( + "outcome IN ('started','delivered','retry','failed')", + name="ck_notification_delivery_attempts_outcome", + ), + sa.UniqueConstraint( + "delivery_id", "number", name="uq_notification_delivery_attempts_delivery_number" + ), + ) + op.create_index( + "ix_notification_attempts_delivery", + "notification_delivery_attempts", + ["delivery_id", "number"], + ) + + legacy = sa.table( + "notification_deliveries_legacy", + sa.column("id"), + sa.column("event_id"), + sa.column("subscription_id"), + sa.column("attempt"), + sa.column("state"), + sa.column("response_class"), + sa.column("next_attempt_at"), + sa.column("created_at"), + sa.column("updated_at"), + ) + rows = _parameterized_execute(connection, sa.select(legacy)).mappings().all() + events = sa.table( + "notification_events", + sa.column("id"), + sa.column("type"), + sa.column("schema_version"), + sa.column("occurred_at"), + sa.column("correlation_id"), + sa.column("severity"), + sa.column("resource_refs", sa.JSON()), + sa.column("payload", sa.JSON()), + sa.column("canonical_envelope"), + sa.column("deduplication_key"), + ) + deliveries = sa.table( + "notification_deliveries", + *[ + sa.column(name) + for name in ( + "id", + "event_id", + "subscription_id", + "state", + "due_at", + "attempt_count", + "terminal_reason", + "response_class", + "response_summary", + "created_at", + "updated_at", + ) + ], + ) + attempts = sa.table( + "notification_delivery_attempts", + *[ + sa.column(name) + for name in ( + "id", + "delivery_id", + "number", + "started_at", + "completed_at", + "outcome", + "response_class", + "diagnostic", + ) + ], + ) + for row in rows: + occurred = row["created_at"] or _now() + event_id, delivery_id, attempt_id = (str(new_uuid7()), str(new_uuid7()), str(new_uuid7())) + payload = {"legacy_event_id": str(row["event_id"]), "legacy_delivery_id": str(row["id"])} + envelope = { + "event_schema_version": 1, + "id": event_id, + "type": "notification.legacy", + "occurred_at": occurred.isoformat() + if hasattr(occurred, "isoformat") + else str(occurred), + "correlation_id": event_id, + "severity": "warning", + "resource": {"subscription_id": str(row["subscription_id"])}, + "payload": payload, + } + _parameterized_execute( + connection, + events.insert().values( + id=event_id, + type="notification.legacy", + schema_version=1, + occurred_at=occurred, + correlation_id=event_id, + severity="warning", + resource_refs=envelope["resource"], + payload=payload, + canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")), + deduplication_key=f"legacy:{row['id']}", + ), + ) + old_state = str(row["state"]) + new_state = "retry" if old_state == "pending" else old_state + try: + legacy_attempt = max(1, int(row["attempt"])) + except (TypeError, ValueError) as error: + raise RuntimeError("legacy notification delivery has an invalid attempt") from error + _parameterized_execute( + connection, + deliveries.insert().values( + id=delivery_id, + event_id=event_id, + subscription_id=row["subscription_id"], + state=new_state, + due_at=row["next_attempt_at"] or occurred, + attempt_count=legacy_attempt, + terminal_reason="legacy_migrated" if new_state == "failed" else None, + response_class=row["response_class"], + response_summary="legacy delivery migrated", + created_at=occurred, + updated_at=row["updated_at"] or occurred, + ), + ) + _parameterized_execute( + connection, + attempts.insert().values( + id=attempt_id, + delivery_id=delivery_id, + number=legacy_attempt, + started_at=occurred, + completed_at=row["updated_at"] if new_state in {"delivered", "failed"} else None, + outcome="delivered" + if new_state == "delivered" + else ("failed" if new_state == "failed" else "retry"), + response_class=row["response_class"], + diagnostic="legacy delivery migrated", + ), + ) + op.drop_table("notification_deliveries_legacy") + + op.create_table( + "notification_signing_keys", + sa.Column( + "subscription_id", + sa.String(36), + sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("version", sa.Integer(), nullable=False), + sa.Column( + "secret_id", + sa.String(36), + sa.ForeignKey("secrets.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("state", sa.String(16), nullable=False, server_default="active"), + sa.Column("overlap_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.CheckConstraint("version > 0", name="ck_notification_signing_keys_version_positive"), + sa.CheckConstraint( + "state IN ('active','overlap','retired')", name="ck_notification_signing_keys_state" + ), + sa.UniqueConstraint( + "subscription_id", "version", name="uq_notification_signing_keys_subscription_version" + ), + ) + op.create_index( + "ix_notification_signing_keys_subscription", + "notification_signing_keys", + ["subscription_id", "state"], + ) + op.create_index( + "uq_notification_signing_keys_active", + "notification_signing_keys", + ["subscription_id"], + unique=True, + sqlite_where=sa.text("state = 'active'"), + ) + op.create_index( + "uq_notification_signing_keys_overlap", + "notification_signing_keys", + ["subscription_id"], + unique=True, + sqlite_where=sa.text("state = 'overlap'"), + ) + op.create_table( + "notification_email_settings", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("host", sa.String(255), nullable=False), + sa.Column("port", sa.Integer(), nullable=False, server_default="587"), + sa.Column("username", sa.String(255), nullable=False), + sa.Column( + "password_secret_id", + sa.String(36), + sa.ForeignKey("secrets.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("sender", sa.String(320), nullable=False), + sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="5"), + sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60"), + sa.CheckConstraint("id = 1", name="ck_notification_email_settings_singleton"), + sa.CheckConstraint("port > 0 AND port < 65536", name="ck_notification_email_settings_port"), + sa.CheckConstraint("max_attempts > 0", name="ck_notification_email_settings_max_attempts"), + sa.CheckConstraint( + "rate_limit_per_minute > 0", name="ck_notification_email_settings_rate_positive" + ), + ) + if connection.dialect.name == "sqlite": + connection.exec_driver_sql("PRAGMA foreign_keys=ON") + + +def downgrade() -> None: + bind = op.get_bind() + for table in ( + "notification_events", + "notification_delivery_attempts", + "notification_signing_keys", + "notification_email_settings", + ): + if bind.scalar(sa.text(f"SELECT count(*) FROM {table}")): + raise RuntimeError( + "cannot downgrade while M12 notification history or configuration exists" + ) + # A fresh M12 schema can safely return to the historical stub shape. + op.drop_table("notification_email_settings") + op.drop_index("uq_notification_signing_keys_overlap", table_name="notification_signing_keys") + op.drop_index("uq_notification_signing_keys_active", table_name="notification_signing_keys") + op.drop_index( + "ix_notification_signing_keys_subscription", table_name="notification_signing_keys" + ) + op.drop_table("notification_signing_keys") + op.drop_index("ix_notification_attempts_delivery", table_name="notification_delivery_attempts") + op.drop_table("notification_delivery_attempts") + op.drop_index("ix_notification_deliveries_lease", table_name="notification_deliveries") + op.drop_index("ix_notification_deliveries_due", table_name="notification_deliveries") + op.drop_table("notification_deliveries") + op.drop_index("ix_notification_events_type_occurred", table_name="notification_events") + op.drop_table("notification_events") + op.create_table( + "notification_deliveries", + sa.Column("event_id", sa.String(36), nullable=False), + sa.Column("subscription_id", sa.String(36), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("response_class", sa.String(64)), + sa.Column("next_attempt_at", sa.DateTime(timezone=True)), + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.CheckConstraint("attempt > 0", name="ck_notification_deliveries_attempt_positive"), + sa.CheckConstraint( + "state IN ('pending','delivered','retry','failed')", + name="ck_notification_deliveries_state", + ), + sa.ForeignKeyConstraint( + ["subscription_id"], ["notification_subscriptions.id"], ondelete="RESTRICT" + ), + sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"), + ) + op.create_index( + "ix_notification_deliveries_state_next", + "notification_deliveries", + ["state", "next_attempt_at"], + ) + with op.batch_alter_table("notification_subscriptions") as batch: + batch.drop_constraint("revision_positive", type_="check") + batch.drop_constraint("rate_tokens_nonnegative", type_="check") + batch.drop_constraint("rate_positive", type_="check") + batch.drop_column("revision") + batch.drop_column("rate_updated_at") + batch.drop_column("rate_tokens") + batch.drop_column("rate_limit_per_minute") diff --git a/backend/alembic/versions/0009_allow_ssh_sources.py b/backend/alembic/versions/0009_allow_ssh_sources.py new file mode 100644 index 0000000..89af8d6 --- /dev/null +++ b/backend/alembic/versions/0009_allow_ssh_sources.py @@ -0,0 +1,43 @@ +"""allow staged SSH source definitions + +Revision ID: 0009_allow_ssh_sources +Revises: 0008_notification_outbox +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "0009_allow_ssh_sources" +down_revision = "0008_notification_outbox" +branch_labels = None +depends_on = None + + +def _reject_ssh_sources() -> None: + connection = op.get_bind() + sources = sa.table("sources", sa.column("kind")) + count = connection.scalar( + sa.select(sa.func.count()).select_from(sources).where(sources.c.kind == "ssh") + ) + if count is None: + raise RuntimeError("Cannot inspect persisted SSH sources before migration.") + if count: + raise RuntimeError( + "Cannot restrict sources to local: " + f"found {count} SSH source row(s). Remove them before downgrading." + ) + + +def upgrade() -> None: + with op.batch_alter_table("sources") as batch: + batch.drop_constraint(op.f("ck_sources_kind"), type_="check") + batch.create_check_constraint(op.f("ck_sources_kind"), "kind IN ('local','ssh')") + + +def downgrade() -> None: + _reject_ssh_sources() + with op.batch_alter_table("sources") as batch: + batch.drop_constraint(op.f("ck_sources_kind"), type_="check") + batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 33fec84..3272ad7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -18,9 +18,9 @@ dependencies = [ "cryptography==49.0.0", "fastapi==0.136.1", "httpx==0.28.1", - "paramiko==5.0.0", "pydantic==2.13.4", "pydantic-settings==2.14.2", + "paramiko==5.0.0", "sqlalchemy[asyncio]==2.0.49", "uvicorn[standard]==0.51.0", ] diff --git a/backend/src/backup_tool/adapters.py b/backend/src/backup_tool/adapters.py index 9777740..8116955 100644 --- a/backend/src/backup_tool/adapters.py +++ b/backend/src/backup_tool/adapters.py @@ -1,21 +1,43 @@ from __future__ import annotations +import os +import stat from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path +from typing import Protocol from backup_tool.config import Settings class SourceError(ValueError): - pass + """A redacted source failure with a stable execution reason.""" + + def __init__(self, message: str, *, reason_code: str = "source_invalid") -> None: + super().__init__(message) + self.reason_code = reason_code @dataclass(frozen=True) class Entry: path: str kind: str - size: int | None = None + size: int + mode: int + mtime_ns: int + link_target: str | None = None + + +class SourceReader(Protocol): + def validate_config(self) -> None: ... + + async def probe(self) -> dict[str, int]: ... + + def enumerate_entries(self) -> AsyncIterator[Entry]: ... + + def open_content(self, path: str) -> AsyncIterator[bytes]: ... + + async def close(self) -> None: ... class LocalAdapter: @@ -36,22 +58,59 @@ class LocalAdapter: async def enumerate_entries(self) -> AsyncIterator[Entry]: self.validate_config() - for item in self.root.rglob("*"): + for item in sorted(self.root.rglob("*"), key=lambda candidate: candidate.as_posix()): relative = item.relative_to(self.root).as_posix() - if item.is_symlink(): - yield Entry(relative, "symlink") - elif item.is_file(): - yield Entry(relative, "file", item.stat().st_size) - elif item.is_dir(): - yield Entry(relative, "directory") + metadata = item.lstat() + if stat.S_ISLNK(metadata.st_mode): + target = os.readlink(item) + target_path = Path(target) + if ( + target_path.is_absolute() + or "\\" in target + or ".." in target_path.parts + or not target + ): + raise SourceError(f"symlink target for {relative!r} is unsafe") + yield Entry( + relative, + "symlink", + 0, + stat.S_IMODE(metadata.st_mode), + metadata.st_mtime_ns, + target, + ) + elif stat.S_ISREG(metadata.st_mode): + yield Entry( + relative, + "file", + metadata.st_size, + stat.S_IMODE(metadata.st_mode), + metadata.st_mtime_ns, + ) + elif stat.S_ISDIR(metadata.st_mode): + yield Entry( + relative, + "directory", + 0, + stat.S_IMODE(metadata.st_mode), + metadata.st_mtime_ns, + ) + else: + raise SourceError(f"local source entry {relative!r} has an unsupported type") + + async def close(self) -> None: + return None async def open_content(self, path: str) -> AsyncIterator[bytes]: - candidate = (self.root / path).resolve() - if ( - not candidate.is_relative_to(self.root) - or not candidate.is_file() - or candidate.is_symlink() - ): + requested = Path(path) + if not path or requested.is_absolute() or "\\" in path or ".." in requested.parts: + raise SourceError("invalid local source entry") + candidate = self.root / requested + metadata = candidate.lstat() + if not stat.S_ISREG(metadata.st_mode) or candidate.is_symlink(): + raise SourceError("invalid local source entry") + resolved = candidate.resolve() + if not resolved.is_relative_to(self.root): raise SourceError("invalid local source entry") with candidate.open("rb") as handle: while chunk := handle.read(1024 * 1024): diff --git a/backend/src/backup_tool/api/app.py b/backend/src/backup_tool/api/app.py index 0271e4a..8154f17 100644 --- a/backend/src/backup_tool/api/app.py +++ b/backend/src/backup_tool/api/app.py @@ -5,28 +5,37 @@ import json from collections.abc import AsyncGenerator, AsyncIterator from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Annotated, Any, cast +from time import perf_counter +from typing import Annotated, Any, Literal, cast from fastapi import Depends, FastAPI, Header, Request, Response from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, StreamingResponse -from pydantic import BaseModel, Field -from sqlalchemy import desc, select +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import desc, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from backup_tool.adapters import LocalAdapter, SourceError -from backup_tool.cli import build_alembic_config from backup_tool.config import Settings -from backup_tool.db.engine import SchemaNotCurrentError, assert_schema_current, create_engine +from backup_tool.db.engine import create_engine from backup_tool.db.models import ( ApiToken, AuditEvent, + Backup, Execution, ExecutionEvent, IdempotencyRecord, Job, + NotificationDelivery, + NotificationDeliveryAttempt, + NotificationEmailSettings, + NotificationSigningKey, + NotificationSubscription, Repository, + RepositoryDataKeyEpoch, + Restore, + Schedule, Secret, Session, Source, @@ -40,12 +49,23 @@ from backup_tool.execution import ( request_cancellation, retry, ) +from backup_tool.notifications.events import ( + EVENT_CATALOG, + NotificationEventError, + emit_event, + validate_destination, + validate_filters, +) +from backup_tool.observability.health import ReadinessError, check_role_readiness +from backup_tool.observability.logging import log_event +from backup_tool.observability.metrics import Metrics, collect_operational_metrics from backup_tool.repository import ( RepositoryError, initialize, inspect_repository, remove_repository, ) +from backup_tool.scheduler import ScheduleError, next_nominal from backup_tool.security.auth import ( hash_password, hash_token, @@ -57,6 +77,8 @@ from backup_tool.security.auth import ( ) from backup_tool.security.redaction import redact from backup_tool.security.secrets import EnvelopeCipher +from backup_tool.snapshot import SnapshotError, source_adapter, verify_backup_snapshot +from backup_tool.ssh_source import SSH_PRIVATE_KEY_PURPOSE, SSHSourcePublicConfig async def execution_events_stream( @@ -123,12 +145,187 @@ class LoginInput(BaseModel): password: str = Field(min_length=1, max_length=1024) -class SourceInput(BaseModel): - name: str = Field(min_length=1, max_length=255) +class AuthenticatedUser(BaseModel): + id: str + username: str + + +class SessionUser(AuthenticatedUser): + state: str + + +class RepositorySummary(BaseModel): + id: str + name: str + format_version: int + compression: str + encryption: str + state: str + + +class RepositoryList(BaseModel): + items: list[RepositorySummary] + + +class SourceSummary(BaseModel): + id: str + name: str kind: str + state: str public_config: dict[str, Any] +class SourceList(BaseModel): + items: list[SourceSummary] + + +class ScheduleSummary(BaseModel): + id: str + cron: str + timezone: str + enabled: bool + next_nominal_at: datetime | None + last_enqueue_outcome: str | None + + +class JobSummary(BaseModel): + id: str + name: str + source_id: str + repository_id: str + requested_mode: str + enabled: bool + state: str + schedule: ScheduleSummary | None + + +class JobList(BaseModel): + items: list[JobSummary] + + +class ExecutionSummary(BaseModel): + id: str + state: str + attempt: int + revision: int + reason_code: str | None + progress: dict[str, Any] + + +class ExecutionList(BaseModel): + items: list[ExecutionSummary] + + +class BackupSummary(BaseModel): + id: str + execution_id: str + manifest_id: str + logical_bytes: int + stored_bytes: int + integrity: str + pinned: bool + tombstoned_at: datetime | None + created_at: datetime + + +class BackupList(BaseModel): + items: list[BackupSummary] + + +class BackupDeletePreview(BaseModel): + backup_id: str + eligible: bool + reason: str | None + destructive_action: str + + +class RecoveryStatus(BaseModel): + recovery_mode: Literal["cli_only"] + runbook: str + encrypted_repository_count: int + + +class NotificationSubscriptionSummary(BaseModel): + id: str + channel: str + event_filters: list[str] + destination: dict[str, Any] + state: str + rate_limit_per_minute: int + revision: int + created_at: str + updated_at: str + + +class NotificationSubscriptionList(BaseModel): + items: list[NotificationSubscriptionSummary] + + +class NotificationDeliverySummary(BaseModel): + id: str + event_id: str + subscription_id: str + state: str + attempt_count: int + response_class: str | None + response_summary: str | None + terminal_reason: str | None + due_at: str + + +class NotificationDeliveryList(BaseModel): + items: list[NotificationDeliverySummary] + + +class NotificationAttemptSummary(BaseModel): + number: int + outcome: str + response_class: str | None + diagnostic: str | None + started_at: str + completed_at: str | None + + +class NotificationAttemptList(BaseModel): + items: list[NotificationAttemptSummary] + + +class AuditSummary(BaseModel): + id: str + action: str + resource_type: str + resource_id: str | None + outcome: str + request_id: str + created_at: str + details: dict[str, Any] + + +class AuditList(BaseModel): + items: list[AuditSummary] + next_cursor: str | None + + +class LocalSourceInput(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, max_length=255) + kind: Literal["local"] + public_config: dict[str, Any] + + +class SSHSourceInput(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, max_length=255) + kind: Literal["ssh"] + public_config: SSHSourcePublicConfig + private_key_secret_id: str = Field(min_length=1, max_length=36) + + +SourceInput = Annotated[LocalSourceInput | SSHSourceInput, Field(discriminator="kind")] + + class JobInput(BaseModel): name: str = Field(min_length=1, max_length=255) source_id: str @@ -166,8 +363,59 @@ class RepositoryPatch(BaseModel): encryption: str | None = None -def _etag(user: User) -> str: - return f'"{user.id}:{user.updated_at.isoformat()}"' +class ScheduleInput(BaseModel): + cron: str = Field(min_length=1, max_length=255) + timezone: str = Field(min_length=1, max_length=255) + misfire_grace_seconds: int = Field(default=900, ge=0) + enabled: bool = True + + +class RestoreInput(BaseModel): + destination: str = Field(min_length=1, max_length=4096) + selection: list[str] = Field(default_factory=list) + dry_run: bool = False + overwrite_policy: str = "fail" + + +class NotificationSubscriptionInput(BaseModel): + channel: Literal["webhook", "email"] + event_filters: list[str] = Field(min_length=1, max_length=48) + destination: dict[str, Any] + signing_secret: str | None = Field(default=None, min_length=16, max_length=65_536) + rate_limit_per_minute: int = Field(default=60, ge=1, le=10_000) + + +class NotificationSubscriptionPatch(BaseModel): + event_filters: list[str] | None = Field(default=None, min_length=1, max_length=48) + destination: dict[str, Any] | None = None + state: Literal["active", "disabled", "archived"] | None = None + rate_limit_per_minute: int | None = Field(default=None, ge=1, le=10_000) + + +class SigningKeyRotateInput(BaseModel): + secret: str = Field(min_length=16, max_length=65_536) + overlap_seconds: int = Field(default=3600, ge=60, le=86_400) + + +class EmailSettingsInput(BaseModel): + host: str = Field(min_length=1, max_length=255) + port: int = Field(default=587, ge=1, le=65535) + username: str = Field(min_length=1, max_length=255) + password: str = Field(min_length=1, max_length=65_536) + sender: str = Field(min_length=3, max_length=320) + max_attempts: int = Field(default=5, ge=1, le=20) + rate_limit_per_minute: int = Field(default=60, ge=1, le=10_000) + + +def _etag(resource: Any) -> str: + return f'"{resource.id}:{resource.updated_at.isoformat()}"' + + +def _notification_rate_tokens(value: int) -> float: + try: + return float(value) + except (TypeError, ValueError) as error: + raise Problem(422, "validation_failed", "Notification rate limit is invalid.") from error def _cursor(item_id: str) -> str: @@ -203,14 +451,28 @@ def create_app(settings: Settings) -> FastAPI: app.state.sessions = async_sessionmaker(app.state.engine, expire_on_commit=False) app.state.cipher = EnvelopeCipher.from_file(settings.master_key_file) app.state.setup_lock = asyncio.Lock() + app.state.metrics = Metrics() @app.middleware("http") async def request_id_middleware(request: Request, call_next: Any) -> Response: from backup_tool.ids import new_uuid7 request.state.request_id = str(new_uuid7()) + started = perf_counter() response = cast(Response, await call_next(request)) response.headers["X-Request-ID"] = request.state.request_id + duration = perf_counter() - started + app.state.metrics.observe_request( + request.method, request.url.path, response.status_code, duration + ) + log_event( + "http_request", + request_id=request.state.request_id, + method=request.method, + path=request.url.path, + status=response.status_code, + duration_seconds=round(duration, 6), + ) return response @app.exception_handler(Problem) @@ -336,16 +598,24 @@ def create_app(settings: Settings) -> FastAPI: return {"status": "alive"} @app.get("/readyz") - async def readyz(db: Annotated[AsyncSession, Depends(session)]) -> dict[str, str]: + async def readyz() -> dict[str, str]: try: - await assert_schema_current(app.state.engine, build_alembic_config(settings)) - except SchemaNotCurrentError as error: - raise Problem(503, "schema_not_current", "Metadata schema is not current.") from error - if await db.scalar(select(User.id).limit(1)) is None: - raise Problem(503, "setup_required", "Initial administrator setup is required.") + await check_role_readiness(settings, "web") + except ReadinessError as error: + raise Problem( + 503, "dependency_unavailable", "Required runtime dependency is unavailable." + ) from error return {"status": "ready"} - @app.post("/api/v2/setup", status_code=201) + @app.get("/metrics", include_in_schema=False) + async def metrics(db: Annotated[AsyncSession, Depends(session)]) -> Response: + values = await collect_operational_metrics(settings, db) + return Response( + app.state.metrics.render(values), + media_type="text/plain; version=0.0.4; charset=utf-8", + ) + + @app.post("/api/v2/setup", status_code=201, response_model=AuthenticatedUser) async def setup( input_: SetupInput, request: Request, @@ -374,7 +644,7 @@ def create_app(settings: Settings) -> FastAPI: await set_session(db, response, user.id) return {"id": user.id, "username": user.username} - @app.post("/api/v2/auth/login") + @app.post("/api/v2/auth/login", response_model=AuthenticatedUser) async def login( input_: LoginInput, request: Request, @@ -412,7 +682,7 @@ def create_app(settings: Settings) -> FastAPI: response.delete_cookie("backup_tool_session", path="/") response.delete_cookie("backup_tool_csrf", path="/") - @app.get("/api/v2/auth/session") + @app.get("/api/v2/auth/session", response_model=SessionUser) async def get_session( identity: Annotated[tuple[User, set[str], bool], Depends(actor)], ) -> dict[str, str]: @@ -599,15 +869,30 @@ def create_app(settings: Settings) -> FastAPI: format_version=initialized.format_version, compression=initialized.compression, encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, ) try: db.add(repository) await db.flush() + if initialized.data_key_id is not None: + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) await audit(db, request, "create", "repository", repository.id, "success", user.id) await db.commit() except Exception as error: await db.rollback() - remove_repository(initialized.root) + remove_repository( + initialized.root, + initialized.signing_key_path, + initialized.data_key_path, + ) raise Problem( 409, "repository_create_failed", "Repository metadata could not be stored." ) from error @@ -619,7 +904,7 @@ def create_app(settings: Settings) -> FastAPI: "encryption": repository.encryption, } - @app.get("/api/v2/repositories") + @app.get("/api/v2/repositories", response_model=RepositoryList) async def list_repositories( db: Annotated[AsyncSession, Depends(session)], _: Annotated[tuple[User, set[str], bool], Depends(actor)], @@ -687,24 +972,56 @@ def create_app(settings: Settings) -> FastAPI: raise Problem(409, "repository_policy_immutable", "Repository policy is immutable.") raise Problem(422, "validation_failed", "No mutable fields supplied.") + @app.get("/api/v2/sources", response_model=SourceList) + async def list_sources( + db: Annotated[AsyncSession, Depends(session)], + _: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + items = list((await db.scalars(select(Source).order_by(Source.name))).all()) + return { + "items": [ + { + "id": item.id, + "name": item.name, + "kind": item.kind, + "state": item.state, + "public_config": item.public_config, + } + for item in items + ] + } + @app.post("/api/v2/sources", status_code=201) async def create_source( input_: SourceInput, db: Annotated[AsyncSession, Depends(session)], _: Annotated[tuple[User, set[str], bool], Depends(require)], ) -> dict[str, Any]: - if input_.kind != "local": - raise Problem(422, "validation_failed", "Only local sources are available.") - root = input_.public_config.get("root") - if not isinstance(root, str): - raise Problem(422, "validation_failed", "Local source root is required.") - try: - LocalAdapter(Path(root), settings).validate_config() - except SourceError as error: - raise Problem(422, "validation_failed", str(error)) from error - source = Source( - name=input_.name, kind="local", public_config={"root": root}, secret_refs=[] - ) + if isinstance(input_, LocalSourceInput): + root = input_.public_config.get("root") + if not isinstance(root, str): + raise Problem(422, "validation_failed", "Local source root is required.") + try: + LocalAdapter(Path(root), settings).validate_config() + except SourceError as error: + raise Problem(422, "validation_failed", str(error)) from error + source = Source( + name=input_.name, kind="local", public_config={"root": root}, secret_refs=[] + ) + else: + secret = await db.get(Secret, input_.private_key_secret_id) + if secret is None or secret.purpose != SSH_PRIVATE_KEY_PURPOSE: + raise Problem( + 422, + "validation_failed", + "SSH source requires an existing SSH private-key secret.", + ) + source = Source( + name=input_.name, + kind="ssh", + public_config=input_.public_config.model_dump(), + secret_refs=[secret.id], + ) db.add(source) try: await db.commit() @@ -731,10 +1048,15 @@ def create_app(settings: Settings) -> FastAPI: raise Problem(404, "resource_not_found", "Source was not found.") if source.state != "active": raise Problem(409, "source_archived", "Source is archived.") + adapter = None try: - result = await LocalAdapter(Path(source.public_config["root"]), settings).probe() + adapter = await source_adapter(settings, db, source, app.state.cipher) + result = await adapter.probe() except SourceError as error: raise Problem(409, "source_probe_failed", str(error)) from error + finally: + if adapter is not None: + await adapter.close() source.last_probe = result await db.commit() return result @@ -752,6 +1074,40 @@ def create_app(settings: Settings) -> FastAPI: await db.commit() return Response(status_code=204) + @app.get("/api/v2/jobs", response_model=JobList) + async def list_jobs( + db: Annotated[AsyncSession, Depends(session)], + _: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + jobs = list((await db.scalars(select(Job).order_by(Job.name))).all()) + schedules = {item.job_id: item for item in (await db.scalars(select(Schedule))).all()} + return { + "items": [ + { + "id": item.id, + "name": item.name, + "source_id": item.source_id, + "repository_id": item.repository_id, + "requested_mode": item.requested_mode, + "enabled": item.enabled, + "state": item.state, + "schedule": ( + { + "id": schedules[item.id].id, + "cron": schedules[item.id].cron, + "timezone": schedules[item.id].timezone, + "enabled": schedules[item.id].enabled, + "next_nominal_at": schedules[item.id].next_nominal_at, + "last_enqueue_outcome": schedules[item.id].last_enqueue_outcome, + } + if item.id in schedules + else None + ), + } + for item in jobs + ] + } + @app.post("/api/v2/jobs", status_code=201) async def create_job( input_: JobInput, @@ -796,6 +1152,134 @@ def create_app(settings: Settings) -> FastAPI: "state": job.state, } + @app.post("/api/v2/jobs/{job_id}/schedule", status_code=201) + async def create_schedule( + job_id: str, + input_: ScheduleInput, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:write") + job = await db.get(Job, job_id) + if job is None or job.state != "active" or not job.enabled: + raise Problem(409, "job_disabled", "Job is unavailable for scheduling.") + try: + next_run = next_nominal(input_.cron, input_.timezone) if input_.enabled else None + except ScheduleError as error: + raise Problem(422, "validation_failed", str(error)) from error + schedule = Schedule( + job_id=job.id, + cron=input_.cron, + timezone=input_.timezone, + misfire_grace_seconds=input_.misfire_grace_seconds, + enabled=input_.enabled, + next_nominal_at=next_run, + ) + db.add(schedule) + try: + await db.flush() + await emit_event( + db, + "schedule.created", + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"state": "enabled" if schedule.enabled else "disabled"}, + deduplication_key=f"schedule:{schedule.id}:created", + ) + await db.commit() + except IntegrityError as error: + await db.rollback() + raise Problem(409, "resource_conflict", "Job already has a schedule.") from error + return { + "id": schedule.id, + "job_id": schedule.job_id, + "next_nominal_at": schedule.next_nominal_at, + } + + @app.get("/api/v2/jobs/{job_id}/schedule") + async def get_schedule( + job_id: str, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:write") + schedule = await db.scalar(select(Schedule).where(Schedule.job_id == job_id)) + if schedule is None: + raise Problem(404, "resource_not_found", "Schedule was not found.") + return { + "id": schedule.id, + "job_id": schedule.job_id, + "cron": schedule.cron, + "timezone": schedule.timezone, + "misfire_grace_seconds": schedule.misfire_grace_seconds, + "enabled": schedule.enabled, + "next_nominal_at": schedule.next_nominal_at, + "last_enqueue_outcome": schedule.last_enqueue_outcome, + } + + @app.patch("/api/v2/jobs/{job_id}/schedule") + async def patch_schedule( + job_id: str, + input_: ScheduleInput, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:write") + schedule = await db.scalar(select(Schedule).where(Schedule.job_id == job_id)) + if schedule is None: + raise Problem(404, "resource_not_found", "Schedule was not found.") + try: + next_run = next_nominal(input_.cron, input_.timezone) if input_.enabled else None + except ScheduleError as error: + raise Problem(422, "validation_failed", str(error)) from error + prior_enabled = schedule.enabled + schedule.cron = input_.cron + schedule.timezone = input_.timezone + schedule.misfire_grace_seconds = input_.misfire_grace_seconds + schedule.enabled = input_.enabled + schedule.next_nominal_at = next_run + event_type = "schedule.updated" + if prior_enabled != schedule.enabled: + event_type = "schedule.enabled" if schedule.enabled else "schedule.disabled" + await emit_event( + db, + event_type, + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"state": "enabled" if schedule.enabled else "disabled"}, + deduplication_key=( + f"schedule:{schedule.id}:{event_type}:{input_.cron}:{input_.timezone}:" + f"{input_.enabled}:{input_.misfire_grace_seconds}" + ), + ) + await db.commit() + return await get_schedule(job_id, db, identity) + + @app.delete("/api/v2/jobs/{job_id}/schedule", status_code=204) + async def delete_schedule( + job_id: str, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> Response: + enforce_scope(identity[1], "admin:write") + schedule = await db.scalar(select(Schedule).where(Schedule.job_id == job_id)) + if schedule is None: + raise Problem(404, "resource_not_found", "Schedule was not found.") + await emit_event( + db, + "schedule.deleted", + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"state": "deleted"}, + deduplication_key=f"schedule:{schedule.id}:deleted", + ) + await db.delete(schedule) + await db.commit() + return Response(status_code=204) + @app.post("/api/v2/jobs/{job_id}/executions", status_code=202) async def enqueue_execution( job_id: str, @@ -823,7 +1307,197 @@ def create_app(settings: Settings) -> FastAPI: "attempt": execution.attempt, } - @app.get("/api/v2/executions/{execution_id}") + @app.get("/api/v2/backups", response_model=BackupList) + async def list_backups( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + items = list((await db.scalars(select(Backup).order_by(desc(Backup.created_at)))).all()) + return { + "items": [ + { + "id": item.id, + "execution_id": item.execution_id, + "manifest_id": item.manifest_id, + "logical_bytes": item.logical_bytes, + "stored_bytes": item.stored_bytes, + "integrity": item.integrity, + "pinned": item.pinned, + "tombstoned_at": item.tombstoned_at, + "created_at": item.created_at, + } + for item in items + ] + } + + @app.get("/api/v2/backups/{backup_id}", response_model=BackupSummary) + async def get_backup( + backup_id: str, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + item = await db.get(Backup, backup_id) + if item is None: + raise Problem(404, "resource_not_found", "Backup was not found.") + return { + "id": item.id, + "execution_id": item.execution_id, + "manifest_id": item.manifest_id, + "logical_bytes": item.logical_bytes, + "stored_bytes": item.stored_bytes, + "integrity": item.integrity, + "pinned": item.pinned, + "tombstoned_at": item.tombstoned_at, + "created_at": item.created_at, + } + + @app.post("/api/v2/backups/{backup_id}/verify", response_model=BackupSummary) + async def verify_backup( + backup_id: str, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + item = await db.get(Backup, backup_id) + if item is None: + raise Problem(404, "resource_not_found", "Backup was not found.") + execution = await db.get(Execution, item.execution_id) + repository = await db.get(Job, execution.job_id) if execution is not None else None + target = await db.get(Repository, repository.repository_id) if repository else None + if target is None: + raise Problem(409, "backup_unavailable", "Backup repository is unavailable.") + try: + await verify_backup_snapshot(settings, db, item, target) + except SnapshotError as error: + item.integrity = "corrupt" + await audit(db, request, "verify", "backup", item.id, "failure", user.id) + await db.commit() + raise Problem( + 409, "backup_verification_failed", "Backup verification failed." + ) from error + item.integrity = "verified" + await emit_event( + db, + "backup.verification_succeeded", + correlation_id=item.id, + resource={"backup_id": item.id, "repository_id": target.id}, + payload={"integrity": item.integrity, "outcome": "verified"}, + deduplication_key=f"backup:{item.id}:verification_succeeded", + ) + await audit(db, request, "verify", "backup", item.id, "success", user.id) + await db.commit() + return await get_backup(backup_id, db, (user, scopes, False)) + + @app.get("/api/v2/backups/{backup_id}/delete-preview", response_model=BackupDeletePreview) + async def backup_delete_preview( + backup_id: str, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:write") + item = await db.get(Backup, backup_id) + if item is None: + raise Problem(404, "resource_not_found", "Backup was not found.") + if item.pinned: + reason = "Pinned backups cannot be deleted." + elif item.tombstoned_at is not None: + reason = "Backup is already tombstoned." + else: + reason = None + return { + "backup_id": item.id, + "eligible": reason is None, + "reason": reason, + "destructive_action": ( + "Deletion is performed by retention and garbage collection after its grace period." + ), + } + + @app.post("/api/v2/backups/{backup_id}/restores", status_code=202) + async def create_restore( + backup_id: str, + input_: RestoreInput, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + backup = await db.get(Backup, backup_id) + if backup is None: + raise Problem(404, "resource_not_found", "Backup was not found.") + if backup.integrity != "verified" or backup.tombstoned_at is not None: + raise Problem(409, "backup_unavailable", "Backup is not available for restore.") + if input_.overwrite_policy not in {"fail", "skip", "replace"}: + raise Problem(422, "validation_failed", "Restore overwrite policy is invalid.") + restore = Restore( + backup_id=backup.id, + destination=input_.destination, + selection=input_.selection, + dry_run=input_.dry_run, + overwrite_policy=input_.overwrite_policy, + ) + db.add(restore) + await db.flush() + await audit(db, request, "create", "restore", restore.id, "success", user.id) + await emit_event( + db, + "restore.queued", + correlation_id=restore.id, + resource={"restore_id": restore.id, "backup_id": restore.backup_id}, + payload={"dry_run": restore.dry_run, "state": restore.state}, + deduplication_key=f"restore:{restore.id}:queued", + ) + await db.commit() + await db.refresh(restore) + return { + "id": restore.id, + "backup_id": restore.backup_id, + "state": restore.state, + "destination": restore.destination, + "selection": restore.selection, + "dry_run": restore.dry_run, + "overwrite_policy": restore.overwrite_policy, + "result": restore.result, + } + + @app.get("/api/v2/restores/{restore_id}") + async def get_restore( + restore_id: str, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:write") + restore = await db.get(Restore, restore_id) + if restore is None: + raise Problem(404, "resource_not_found", "Restore was not found.") + return { + "id": restore.id, + "backup_id": restore.backup_id, + "state": restore.state, + "destination": restore.destination, + "selection": restore.selection, + "dry_run": restore.dry_run, + "overwrite_policy": restore.overwrite_policy, + "result": restore.result, + } + + @app.get("/api/v2/executions", response_model=ExecutionList) + async def list_executions( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "execution:read") + items = list( + (await db.scalars(select(Execution).order_by(desc(Execution.created_at)))).all() + ) + return {"items": [public_event(item) for item in items]} + + @app.get("/api/v2/executions/{execution_id}", response_model=ExecutionSummary) async def get_execution( execution_id: str, db: Annotated[AsyncSession, Depends(session)], @@ -862,7 +1536,10 @@ def create_app(settings: Settings) -> FastAPI: raise Problem(404, "resource_not_found", "Execution was not found.") return public_event(execution) - @app.get("/api/v2/executions/{execution_id}/events") + @app.get( + "/api/v2/executions/{execution_id}/events", + responses={200: {"content": {"text/event-stream": {}}}}, + ) async def execution_events( execution_id: str, db: Annotated[AsyncSession, Depends(session)], @@ -878,7 +1555,433 @@ def create_app(settings: Settings) -> FastAPI: headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) - @app.get("/api/v2/audit") + def public_subscription(item: NotificationSubscription) -> dict[str, Any]: + if item.channel == "webhook": + destination: dict[str, Any] = {"configured": True} + else: + recipients = item.destination_config.get("recipients", []) + destination = {"recipient_count": len(recipients)} + return { + "id": item.id, + "channel": item.channel, + "event_filters": item.event_filters, + "destination": destination, + "state": item.state, + "rate_limit_per_minute": item.rate_limit_per_minute, + "revision": item.revision, + "created_at": item.created_at.isoformat(), + "updated_at": item.updated_at.isoformat(), + } + + @app.get("/api/v2/security/recovery/status", response_model=RecoveryStatus) + async def recovery_status( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + encrypted_count = await db.scalar( + select(func.count()) + .select_from(Repository) + .where(Repository.encryption == "aes-256-gcm") + ) + return { + "recovery_mode": "cli_only", + "runbook": "docs/runbooks/recovery-bundle.md", + "encrypted_repository_count": encrypted_count or 0, + } + + @app.get("/api/v2/notifications/event-catalog") + async def notification_catalog( + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + return {"event_schema_version": 1, "events": EVENT_CATALOG} + + @app.get("/api/v2/notifications/subscriptions", response_model=NotificationSubscriptionList) + async def list_notification_subscriptions( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + items = list((await db.scalars(select(NotificationSubscription))).all()) + return {"items": [public_subscription(item) for item in items]} + + @app.post("/api/v2/notifications/subscriptions", status_code=201) + async def create_notification_subscription( + input_: NotificationSubscriptionInput, + request: Request, + response: Response, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + try: + filters = validate_filters(input_.event_filters) + destination = validate_destination(input_.channel, input_.destination) + except NotificationEventError as error: + raise Problem(422, "validation_failed", str(error)) from error + if input_.channel == "webhook" and input_.signing_secret is None: + raise Problem(422, "validation_failed", "Webhook signing secret is required.") + item = NotificationSubscription( + channel=input_.channel, + event_filters=filters, + destination_config=destination, + rate_limit_per_minute=input_.rate_limit_per_minute, + rate_tokens=_notification_rate_tokens(input_.rate_limit_per_minute), + rate_updated_at=datetime.now(UTC), + ) + db.add(item) + await db.flush() + if input_.channel == "webhook": + ciphertext, key_id = app.state.cipher.encrypt( + input_.signing_secret or "", purpose="notification_webhook", version=1 + ) + secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_webhook") + db.add(secret) + await db.flush() + db.add(NotificationSigningKey(subscription_id=item.id, version=1, secret_id=secret.id)) + await audit(db, request, "create", "notification_subscription", item.id, "success", user.id) + await db.commit() + await db.refresh(item) + response.headers["ETag"] = _etag(item) + return public_subscription(item) + + @app.get("/api/v2/notifications/subscriptions/{subscription_id}") + async def get_notification_subscription( + subscription_id: str, + response: Response, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + item = await db.get(NotificationSubscription, subscription_id) + if item is None: + raise Problem(404, "resource_not_found", "Subscription was not found.") + response.headers["ETag"] = _etag(item) + return public_subscription(item) + + @app.patch("/api/v2/notifications/subscriptions/{subscription_id}") + async def patch_notification_subscription( + subscription_id: str, + input_: NotificationSubscriptionPatch, + request: Request, + response: Response, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + item = await db.get(NotificationSubscription, subscription_id) + if item is None: + raise Problem(404, "resource_not_found", "Subscription was not found.") + if if_match != _etag(item): + raise Problem(412, "etag_mismatch", "Resource was modified by another request.") + try: + if input_.event_filters is not None: + item.event_filters = validate_filters(input_.event_filters) + if input_.destination is not None: + item.destination_config = validate_destination(item.channel, input_.destination) + except NotificationEventError as error: + raise Problem(422, "validation_failed", str(error)) from error + if input_.state is not None: + item.state = input_.state + if input_.rate_limit_per_minute is not None: + item.rate_limit_per_minute = input_.rate_limit_per_minute + item.rate_tokens = min( + item.rate_tokens, + _notification_rate_tokens(input_.rate_limit_per_minute), + ) + item.revision += 1 + item.updated_at = datetime.now(UTC) + await audit(db, request, "update", "notification_subscription", item.id, "success", user.id) + await db.commit() + response.headers["ETag"] = _etag(item) + return public_subscription(item) + + @app.get("/api/v2/notifications/email-settings") + async def get_notification_email_settings( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + item = await db.get(NotificationEmailSettings, 1) + if item is None: + return {"configured": False} + return { + "configured": True, + "host": item.host, + "port": item.port, + "username": item.username, + "sender": item.sender, + "max_attempts": item.max_attempts, + "rate_limit_per_minute": item.rate_limit_per_minute, + "password_configured": True, + } + + @app.put("/api/v2/notifications/email-settings") + async def put_notification_email_settings( + input_: EmailSettingsInput, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + if any(character in input_.sender for character in "\r\n") or "@" not in input_.sender: + raise Problem(422, "validation_failed", "Sender address is invalid.") + ciphertext, key_id = app.state.cipher.encrypt( + input_.password, purpose="notification_smtp", version=1 + ) + secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp") + db.add(secret) + await db.flush() + item = await db.get(NotificationEmailSettings, 1) + if item is None: + item = NotificationEmailSettings( + id=1, + host=input_.host, + port=input_.port, + username=input_.username, + password_secret_id=secret.id, + sender=input_.sender, + max_attempts=input_.max_attempts, + rate_limit_per_minute=input_.rate_limit_per_minute, + ) + db.add(item) + else: + item.host, item.port, item.username = input_.host, input_.port, input_.username + item.password_secret_id, item.sender = secret.id, input_.sender + item.max_attempts = input_.max_attempts + item.rate_limit_per_minute = input_.rate_limit_per_minute + await audit(db, request, "update", "notification_email_settings", "1", "success", user.id) + await db.commit() + return {"configured": True, "password_configured": True} + + @app.post("/api/v2/notifications/subscriptions/{subscription_id}/signing-keys/rotate") + async def rotate_notification_signing_key( + subscription_id: str, + input_: SigningKeyRotateInput, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, + ) -> dict[str, Any]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + if not idempotency_key: + raise Problem(400, "idempotency_key_required", "Idempotency-Key is required.") + # The request secret is deliberately excluded: idempotency must not + # create an offline plaintext-secret verifier in durable metadata. + digest = hashlib.sha256(f"{subscription_id}:{input_.overlap_seconds}".encode()).hexdigest() + existing = await db.scalar( + select(IdempotencyRecord).where( + IdempotencyRecord.actor_id == user.id, + IdempotencyRecord.key == idempotency_key, + IdempotencyRecord.operation == "rotate_notification_signing_key", + ) + ) + if existing is not None: + if existing.request_digest != digest: + raise Problem( + 409, + "idempotency_mismatch", + "Idempotency-Key was used for another request.", + ) + key = await db.get(NotificationSigningKey, existing.response_resource_id) + if key is None: # pragma: no cover - protected by foreign-key lifetime + raise Problem(409, "idempotency_conflict", "Signing key is unavailable.") + return {"id": key.id, "version": key.version, "state": key.state} + item = await db.get(NotificationSubscription, subscription_id) + if item is None or item.channel != "webhook": + raise Problem(404, "resource_not_found", "Webhook subscription was not found.") + active = await db.scalar( + select(NotificationSigningKey).where( + NotificationSigningKey.subscription_id == item.id, + NotificationSigningKey.state == "active", + ) + ) + if active is None: + raise Problem(409, "signing_key_unavailable", "Active signing key is unavailable.") + overlap = await db.scalar( + select(NotificationSigningKey).where( + NotificationSigningKey.subscription_id == item.id, + NotificationSigningKey.state == "overlap", + ) + ) + if overlap is not None: + overlap.state = "retired" + ciphertext, key_id = app.state.cipher.encrypt( + input_.secret, purpose="notification_webhook", version=1 + ) + secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_webhook") + db.add(secret) + await db.flush() + active.state = "overlap" + active.overlap_expires_at = datetime.now(UTC) + timedelta(seconds=input_.overlap_seconds) + key = NotificationSigningKey( + subscription_id=item.id, + version=active.version + 1, + secret_id=secret.id, + state="active", + ) + db.add(key) + await db.flush() + db.add( + IdempotencyRecord( + actor_id=user.id, + key=idempotency_key, + operation="rotate_notification_signing_key", + request_digest=digest, + response_resource_type="notification_signing_key", + response_resource_id=key.id, + ) + ) + await audit(db, request, "rotate", "notification_signing_key", key.id, "success", user.id) + await db.commit() + return {"id": key.id, "version": key.version, "state": key.state} + + @app.get("/api/v2/notifications/deliveries", response_model=NotificationDeliveryList) + async def list_notification_deliveries( + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + limit: int = 50, + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + if not 1 <= limit <= 100: + raise Problem(422, "validation_failed", "Limit must be between 1 and 100.") + statement = ( + select(NotificationDelivery) + .order_by(desc(NotificationDelivery.created_at)) + .limit(limit) + ) + items = list((await db.scalars(statement)).all()) + return { + "items": [ + { + "id": item.id, + "event_id": item.event_id, + "subscription_id": item.subscription_id, + "state": item.state, + "attempt_count": item.attempt_count, + "response_class": item.response_class, + "response_summary": item.response_summary, + "terminal_reason": item.terminal_reason, + "due_at": item.due_at.isoformat(), + } + for item in items + ] + } + + @app.get( + "/api/v2/notifications/deliveries/{delivery_id}/attempts", + response_model=NotificationAttemptList, + ) + async def list_notification_attempts( + delivery_id: str, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(actor)], + ) -> dict[str, Any]: + enforce_scope(identity[1], "admin:read") + statement = ( + select(NotificationDeliveryAttempt) + .where(NotificationDeliveryAttempt.delivery_id == delivery_id) + .order_by(NotificationDeliveryAttempt.number) + ) + attempts = list((await db.scalars(statement)).all()) + return { + "items": [ + { + "number": item.number, + "outcome": item.outcome, + "response_class": item.response_class, + "diagnostic": item.diagnostic, + "started_at": item.started_at.isoformat(), + "completed_at": item.completed_at.isoformat() if item.completed_at else None, + } + for item in attempts + ] + } + + @app.post("/api/v2/notifications/subscriptions/{subscription_id}/test", status_code=202) + async def test_notification_subscription( + subscription_id: str, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, + ) -> dict[str, str]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + if not idempotency_key: + raise Problem(400, "idempotency_key_required", "Idempotency-Key is required.") + item = await db.get(NotificationSubscription, subscription_id) + if item is None or item.state != "active": + raise Problem(409, "subscription_unavailable", "Subscription is unavailable.") + event = await emit_event( + db, + "notification.test_requested", + correlation_id=request.state.request_id, + resource={"subscription_id": item.id}, + payload={"outcome": "requested"}, + deduplication_key=f"notification-test:{item.id}:{idempotency_key}", + only_subscription_id=item.id, + ) + await audit(db, request, "test", "notification_subscription", item.id, "success", user.id) + await db.commit() + return {"event_id": event.id} + + @app.post("/api/v2/notifications/deliveries/{delivery_id}/retry", status_code=202) + async def retry_notification_delivery( + delivery_id: str, + request: Request, + db: Annotated[AsyncSession, Depends(session)], + identity: Annotated[tuple[User, set[str], bool], Depends(require)], + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, + ) -> dict[str, str]: + user, scopes, _ = identity + enforce_scope(scopes, "admin:write") + if not idempotency_key: + raise Problem(400, "idempotency_key_required", "Idempotency-Key is required.") + digest = hashlib.sha256(delivery_id.encode()).hexdigest() + existing = await db.scalar( + select(IdempotencyRecord).where( + IdempotencyRecord.actor_id == user.id, + IdempotencyRecord.key == idempotency_key, + IdempotencyRecord.operation == "retry_notification_delivery", + ) + ) + if existing is not None: + if existing.request_digest != digest: + raise Problem( + 409, + "idempotency_mismatch", + "Idempotency-Key was used for another request.", + ) + return {"delivery_id": existing.response_resource_id} + delivery = await db.get(NotificationDelivery, delivery_id) + if delivery is None: + raise Problem(404, "resource_not_found", "Delivery was not found.") + if delivery.state != "failed": + raise Problem(409, "retry_not_allowed", "Delivery is not terminally failed.") + delivery.state, delivery.due_at = "retry", datetime.now(UTC) + delivery.terminal_reason = None + db.add( + IdempotencyRecord( + actor_id=user.id, + key=idempotency_key, + operation="retry_notification_delivery", + request_digest=digest, + response_resource_type="notification_delivery", + response_resource_id=delivery.id, + ) + ) + await audit(db, request, "retry", "notification_delivery", delivery.id, "success", user.id) + await db.commit() + return {"delivery_id": delivery.id} + + @app.get("/api/v2/audit", response_model=AuditList) async def list_audit( db: Annotated[AsyncSession, Depends(session)], identity: Annotated[tuple[User, set[str], bool], Depends(actor)], diff --git a/backend/src/backup_tool/cli.py b/backend/src/backup_tool/cli.py index c1faf9c..c4a4e2b 100644 --- a/backend/src/backup_tool/cli.py +++ b/backend/src/backup_tool/cli.py @@ -4,16 +4,62 @@ from __future__ import annotations import argparse import asyncio +import base64 +import binascii +import json +import os +import socket from collections.abc import Callable, Sequence +from contextlib import suppress +from datetime import UTC, datetime from pathlib import Path +from typing import Any from alembic import command from alembic.config import Config +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker from backup_tool import __version__ from backup_tool.config import Settings from .db.engine import assert_schema_current, create_engine +from .db.models import ( + Backup, + Base, + Execution, + Job, + Repository, + RepositoryDataKeyEpoch, + Source, +) +from .repository import ( + begin_key_rotation, + finish_key_rotation, + inspect_repository, + install_signing_key, + load_signing_key, + reconcile_key_rotations, + replace_active_data_key, +) +from .scheduler import run_scheduler +from .security.recovery_bundle import ( + RecoveryBundleError, + decrypt_bundle, + encrypt_bundle, + read_bundle_file, + read_passphrase_fd, + write_bundle_exclusive, +) +from .security.repository_crypto import ( + RepositoryKeyError, + create_data_key, + install_data_key, + load_data_key, +) +from .web import run_web from .worker import run_worker RoleHandler = Callable[[Settings], int] @@ -25,8 +71,8 @@ def _placeholder_role(_settings: Settings) -> int: ROLE_HANDLERS: dict[str, RoleHandler] = { - "web": _placeholder_role, - "scheduler": _placeholder_role, + "web": run_web, + "scheduler": run_scheduler, "worker": run_worker, "admin": _placeholder_role, } @@ -37,14 +83,34 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--version", action="version", version=__version__) subparsers = parser.add_subparsers(dest="role", required=True) for role in DATABASE_ROLES: - subparsers.add_parser(role, help=f"run the {role} role") + role_parser = subparsers.add_parser(role, help=f"run the {role} role") + if role == "admin": + admin_parsers = role_parser.add_subparsers(dest="admin_command") + repository_key = admin_parsers.add_parser("repository-key") + repository_key_parsers = repository_key.add_subparsers(dest="repository_key_command") + rotate = repository_key_parsers.add_parser("rotate") + rotate.add_argument("--repository-id", required=True) + recovery = admin_parsers.add_parser("recovery") + recovery_parsers = recovery.add_subparsers(dest="recovery_command", required=True) + export = recovery_parsers.add_parser("export") + export.add_argument("--output", required=True, type=Path) + export.add_argument("--passphrase-fd", required=True, type=int) + validate = recovery_parsers.add_parser("validate") + validate.add_argument("--input", required=True, type=Path) + validate.add_argument("--passphrase-fd", required=True, type=int) + import_ = recovery_parsers.add_parser("import") + import_.add_argument("--input", required=True, type=Path) + import_.add_argument("--passphrase-fd", required=True, type=int) migrate = subparsers.add_parser("migrate", help="manage the metadata schema") migrate.add_argument("action", choices=("upgrade", "downgrade", "current")) + health = subparsers.add_parser("health", help="check runtime role readiness") + health.add_argument("health_role", choices=("web", "scheduler", "worker")) return parser def build_alembic_config(settings: Settings) -> Config: - backend_root = Path(__file__).resolve().parents[2] + default_root = Path(__file__).resolve().parents[2] + backend_root = Path(os.environ.get("BACKUP_TOOL_ALEMBIC_ROOT", default_root)) migration = Config(backend_root / "alembic.ini") migration.set_main_option("sqlalchemy.url", settings.database_url) return migration @@ -61,6 +127,24 @@ def require_current_schema(settings: Settings) -> None: asyncio.run(check()) +def run_role_healthcheck(settings: Settings, role: str) -> int: + """Verify the selected isolated runtime role is ready to serve work.""" + if role != "web": + from backup_tool.observability.health import check_role_readiness + + asyncio.run(check_role_readiness(settings, role)) + return 0 + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(2) + client.connect(str(settings.web_socket_path)) + request = b"GET /readyz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + client.sendall(request) + status_line = client.recv(64).split(b"\r\n", 1)[0] + if status_line != b"HTTP/1.1 200 OK": + raise RuntimeError("web role is not ready") + return 0 + + def run_migration(action: str, settings: Settings) -> int: migration = build_alembic_config(settings) if action == "upgrade": @@ -76,12 +160,792 @@ def load_settings() -> Settings: return Settings() # type: ignore[call-arg] +async def rotate_repository_key( + settings: Settings, + repository_id: str, + *, + after_database_commit: Callable[[], None] | None = None, +) -> dict[str, str]: + """Journal a key rotation so a crash always has a deterministic recovery path.""" + engine = create_engine(settings) + sessions = async_sessionmaker(engine, expire_on_commit=False) + new_key_path: Path | None = None + database_committed = False + journal_started = False + root: Path | None = None + old_key_id = "" + new_key_id = "" + try: + async with sessions() as db: + await reconcile_key_rotations(settings, db) + await db.rollback() + async with db.begin(): + repository = await db.scalar( + select(Repository).where(Repository.id == repository_id).with_for_update() + ) + if repository is None: + raise ValueError("repository was not found") + if repository.encryption != "aes-256-gcm" or repository.active_data_key_id is None: + raise ValueError("repository encryption is not enabled") + inspected = inspect_repository(settings, Path(repository.root)) + root = inspected.root + if ( + inspected.encryption != "aes-256-gcm" + or inspected.data_key_id != repository.active_data_key_id + ): + raise ValueError("repository encryption metadata is invalid") + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch).where( + RepositoryDataKeyEpoch.repository_id == repository.id + ) + ) + ).all() + ) + active = [epoch for epoch in epochs if epoch.state == "active"] + if len(active) != 1 or active[0].key_id != repository.active_data_key_id: + raise ValueError("repository key epochs are invalid") + old_key_id = active[0].key_id + new_key_id, new_key_path = create_data_key(settings, inspected.repository_id) + begin_key_rotation( + inspected.root, + repository.id, + inspected.repository_id, + old_key_id, + new_key_id, + ) + journal_started = True + active[0].state = "retired" + active[0].retired_at = datetime.now(UTC) + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=new_key_id, + state="active", + ) + ) + repository.active_data_key_id = new_key_id + database_committed = True + if after_database_commit is not None: + after_database_commit() + if root is None: + raise ValueError("repository rotation metadata is invalid") + replace_active_data_key(root, old_key_id, new_key_id) + finish_key_rotation(root) + journal_started = False + return { + "repository_id": repository_id, + "retired_key_id": old_key_id, + "active_key_id": new_key_id, + } + finally: + if not database_committed: + if journal_started and root is not None: + with suppress(ValueError): + finish_key_rotation(root) + if new_key_path is not None: + new_key_path.unlink(missing_ok=True) + await engine.dispose() + + +async def recovery_export_payload(settings: Settings) -> dict[str, Any]: + """Collect the current repository trust catalog and offline key material.""" + engine = create_engine(settings) + sessions = async_sessionmaker(engine, expire_on_commit=False) + try: + async with sessions() as db: + await reconcile_key_rotations(settings, db) + await db.rollback() + repositories = list( + (await db.scalars(select(Repository).order_by(Repository.id))).all() + ) + catalog_repositories: list[dict[str, Any]] = [] + key_records: list[dict[str, Any]] = [] + for repository in repositories: + inspected = inspect_repository(settings, Path(repository.root)) + if inspected.encryption != repository.encryption or ( + repository.encryption == "aes-256-gcm" + and repository.active_data_key_id != inspected.data_key_id + ): + raise RecoveryBundleError("recovery bundle export is unavailable") + private_key = load_signing_key( + settings, + inspected.repository_id, + repository.signing_key_id, + repository.signing_public_key, + ) + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch) + .where(RepositoryDataKeyEpoch.repository_id == repository.id) + .order_by(RepositoryDataKeyEpoch.key_id) + ) + ).all() + ) + active = [epoch for epoch in epochs if epoch.state == "active"] + if repository.encryption == "aes-256-gcm": + if ( + repository.active_data_key_id is None + or len(active) != 1 + or active[0].key_id != repository.active_data_key_id + ): + raise RecoveryBundleError("recovery bundle export is unavailable") + elif epochs: + raise RecoveryBundleError("recovery bundle export is unavailable") + data_keys: list[dict[str, str]] = [] + for epoch in epochs: + data_keys.append( + { + "key_id": epoch.key_id, + "key": base64.b64encode( + load_data_key(settings, inspected.repository_id, epoch.key_id) + ).decode("ascii"), + } + ) + catalog_repositories.append( + { + "active_data_key_id": repository.active_data_key_id, + "compression": repository.compression, + "data_key_epochs": [ + { + "key_id": epoch.key_id, + "retired_at": ( + epoch.retired_at.isoformat() + if epoch.retired_at is not None + else None + ), + "state": epoch.state, + } + for epoch in epochs + ], + "database_id": repository.id, + "encryption": repository.encryption, + "format_version": repository.format_version, + "name": repository.name, + "repository_id": inspected.repository_id, + "root": repository.root, + "signing_key_id": repository.signing_key_id, + "signing_public_key": repository.signing_public_key, + "state": repository.state, + } + ) + key_records.append( + { + "data_keys": data_keys, + "repository_id": inspected.repository_id, + "signing_private_key": base64.b64encode( + private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + ).decode("ascii"), + } + ) + sources = list((await db.scalars(select(Source).order_by(Source.id))).all()) + jobs = list((await db.scalars(select(Job).order_by(Job.id))).all()) + executions = list((await db.scalars(select(Execution).order_by(Execution.id))).all()) + backups = list((await db.scalars(select(Backup).order_by(Backup.id))).all()) + if any(source.secret_refs for source in sources): + raise RecoveryBundleError("recovery bundle export is unavailable") + return { + "catalog": { + "backups": [ + { + "created_at": backup.created_at.isoformat(), + "data_key_id": backup.data_key_id, + "execution_id": backup.execution_id, + "id": backup.id, + "integrity": backup.integrity, + "logical_bytes": backup.logical_bytes, + "manifest_digest": backup.manifest_digest, + "manifest_id": backup.manifest_id, + "parent_backup_id": backup.parent_backup_id, + "pinned": backup.pinned, + "stored_bytes": backup.stored_bytes, + "tombstoned_at": ( + backup.tombstoned_at.isoformat() + if backup.tombstoned_at is not None + else None + ), + } + for backup in backups + ], + "executions": [ + { + "attempt": execution.attempt, + "id": execution.id, + "job_id": execution.job_id, + "progress": execution.progress, + "revision": execution.revision, + "trigger": execution.trigger, + } + for execution in executions + ], + "format": "backup-tool-recovery-catalog", + "jobs": [ + { + "allow_empty": job.allow_empty, + "exclusions": job.exclusions, + "id": job.id, + "name": job.name, + "repository_id": job.repository_id, + "requested_mode": job.requested_mode, + "retention": job.retention, + "source_id": job.source_id, + } + for job in jobs + ], + "repositories": catalog_repositories, + "sources": [ + { + "id": source.id, + "kind": source.kind, + "name": source.name, + "public_config": source.public_config, + } + for source in sources + ], + "version": 2, + }, + "keys": key_records, + } + except (OSError, RepositoryKeyError, ValueError) as error: + if isinstance(error, RecoveryBundleError): + raise + raise RecoveryBundleError("recovery bundle export is unavailable") from error + finally: + await engine.dispose() + + +def _validate_legacy_recovery_payload(payload: dict[str, Any]) -> int: + """Validate the authenticated export shape without exposing key material.""" + try: + if set(payload) != {"catalog", "keys"}: + raise ValueError + catalog = payload["catalog"] + keys = payload["keys"] + if ( + not isinstance(catalog, dict) + or set(catalog) != {"format", "repositories", "version"} + or catalog["format"] != "backup-tool-recovery-catalog" + or catalog["version"] != 1 + or not isinstance(catalog["repositories"], list) + or not isinstance(keys, list) + or len(catalog["repositories"]) != len(keys) + ): + raise ValueError + seen: set[str] = set() + for repository, key_record in zip(catalog["repositories"], keys, strict=True): + if not isinstance(repository, dict) or not isinstance(key_record, dict): + raise ValueError + required_repository = { + "active_data_key_id", + "compression", + "data_key_epochs", + "database_id", + "encryption", + "format_version", + "name", + "repository_id", + "root", + "signing_key_id", + "signing_public_key", + "state", + } + if set(repository) != required_repository or set(key_record) != { + "data_keys", + "repository_id", + "signing_private_key", + }: + raise ValueError + repository_id = repository["repository_id"] + if ( + not isinstance(repository_id, str) + or not repository_id + or repository_id in seen + or key_record["repository_id"] != repository_id + or not isinstance(repository["data_key_epochs"], list) + or not isinstance(key_record["data_keys"], list) + ): + raise ValueError + seen.add(repository_id) + private_key = base64.b64decode(key_record["signing_private_key"], validate=True) + public_key = bytes.fromhex(repository["signing_public_key"]) + if len(private_key) != 32 or len(public_key) != 32: + raise ValueError + signing_key = Ed25519PrivateKey.from_private_bytes(private_key) + derived_public = signing_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + if derived_public != public_key: + raise ValueError + epochs = repository["data_key_epochs"] + data_keys = key_record["data_keys"] + if len(epochs) != len(data_keys): + raise ValueError + epoch_ids: set[str] = set() + for epoch, data_key in zip(epochs, data_keys, strict=True): + if ( + not isinstance(epoch, dict) + or set(epoch) != {"key_id", "retired_at", "state"} + or not isinstance(data_key, dict) + or set(data_key) != {"key", "key_id"} + or not isinstance(epoch["key_id"], str) + or epoch["key_id"] in epoch_ids + or data_key["key_id"] != epoch["key_id"] + or len(base64.b64decode(data_key["key"], validate=True)) != 32 + ): + raise ValueError + epoch_ids.add(epoch["key_id"]) + active = [epoch for epoch in epochs if epoch["state"] == "active"] + if repository["encryption"] == "aes-256-gcm": + if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]: + raise ValueError + elif ( + repository["encryption"] != "none" + or epochs + or repository["active_data_key_id"] is not None + ): + raise ValueError + return len(seen) + except (binascii.Error, KeyError, TypeError, ValueError) as error: + raise RecoveryBundleError("recovery bundle is invalid") from error + + +def _validated_recovery_catalog( + payload: dict[str, Any], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Validate a v2 catalog before any filesystem or metadata mutation.""" + try: + if set(payload) != {"catalog", "keys"}: + raise ValueError + catalog = payload["catalog"] + keys = payload["keys"] + required_catalog = { + "backups", + "executions", + "format", + "jobs", + "repositories", + "sources", + "version", + } + if ( + not isinstance(catalog, dict) + or set(catalog) != required_catalog + or catalog["format"] != "backup-tool-recovery-catalog" + or catalog["version"] != 2 + or not isinstance(keys, list) + or any( + not isinstance(catalog[name], list) + for name in required_catalog - {"format", "version"} + ) + or len(catalog["repositories"]) != len(keys) + ): + raise ValueError + repositories = catalog["repositories"] + key_by_repository: dict[str, dict[str, Any]] = {} + repository_by_id: dict[str, dict[str, Any]] = {} + canonical_ids: set[str] = set() + for repository, key_record in zip(repositories, keys, strict=True): + if not isinstance(repository, dict) or not isinstance(key_record, dict): + raise ValueError + required_repository = { + "active_data_key_id", + "compression", + "data_key_epochs", + "database_id", + "encryption", + "format_version", + "name", + "repository_id", + "root", + "signing_key_id", + "signing_public_key", + "state", + } + if set(repository) != required_repository or set(key_record) != { + "data_keys", + "repository_id", + "signing_private_key", + }: + raise ValueError + database_id = repository["database_id"] + canonical_id = repository["repository_id"] + if ( + not isinstance(database_id, str) + or not isinstance(canonical_id, str) + or not database_id + or not canonical_id + or database_id in repository_by_id + or canonical_id in canonical_ids + or key_record["repository_id"] != canonical_id + or not isinstance(repository["data_key_epochs"], list) + or not isinstance(key_record["data_keys"], list) + ): + raise ValueError + private_key = base64.b64decode(key_record["signing_private_key"], validate=True) + public_key = bytes.fromhex(repository["signing_public_key"]) + signing_key = Ed25519PrivateKey.from_private_bytes(private_key) + if ( + len(public_key) != 32 + or signing_key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + != public_key + ): + raise ValueError + epochs = repository["data_key_epochs"] + data_keys = key_record["data_keys"] + if len(epochs) != len(data_keys): + raise ValueError + epoch_ids: set[str] = set() + for epoch, data_key in zip(epochs, data_keys, strict=True): + if ( + not isinstance(epoch, dict) + or set(epoch) != {"key_id", "retired_at", "state"} + or epoch.get("state") not in {"active", "retired"} + or not isinstance(epoch.get("key_id"), str) + or epoch["key_id"] in epoch_ids + or not isinstance(data_key, dict) + or set(data_key) != {"key", "key_id"} + or data_key["key_id"] != epoch["key_id"] + or len(base64.b64decode(data_key["key"], validate=True)) != 32 + ): + raise ValueError + epoch_ids.add(epoch["key_id"]) + active = [epoch for epoch in epochs if epoch["state"] == "active"] + if repository["encryption"] == "aes-256-gcm": + if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]: + raise ValueError + elif ( + repository["encryption"] != "none" + or epochs + or repository["active_data_key_id"] is not None + ): + raise ValueError + repository_by_id[database_id] = repository + canonical_ids.add(canonical_id) + key_by_repository[canonical_id] = key_record + source_ids = _catalog_ids(catalog["sources"], {"id", "kind", "name", "public_config"}) + job_ids = _catalog_ids( + catalog["jobs"], + { + "allow_empty", + "exclusions", + "id", + "name", + "repository_id", + "requested_mode", + "retention", + "source_id", + }, + ) + for job in catalog["jobs"]: + if job["source_id"] not in source_ids or job["repository_id"] not in repository_by_id: + raise ValueError + execution_ids = _catalog_ids( + catalog["executions"], {"attempt", "id", "job_id", "progress", "revision", "trigger"} + ) + for execution in catalog["executions"]: + if execution["job_id"] not in job_ids: + raise ValueError + backup_ids = _catalog_ids( + catalog["backups"], + { + "created_at", + "data_key_id", + "execution_id", + "id", + "integrity", + "logical_bytes", + "manifest_digest", + "manifest_id", + "parent_backup_id", + "pinned", + "stored_bytes", + "tombstoned_at", + }, + ) + for backup in catalog["backups"]: + if backup["execution_id"] not in execution_ids: + raise ValueError + _catalog_datetime(backup["created_at"]) + _catalog_datetime(backup["tombstoned_at"], allow_none=True) + if ( + backup["parent_backup_id"] is not None + and backup["parent_backup_id"] not in backup_ids + ): + raise ValueError + return ( + catalog, + [key_by_repository[repository["repository_id"]] for repository in repositories], + ) + except (binascii.Error, KeyError, TypeError, ValueError) as error: + raise RecoveryBundleError("recovery bundle is invalid") from error + + +def _catalog_ids(rows: list[Any], required: set[str]) -> set[str]: + identifiers: set[str] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != required or not isinstance(row.get("id"), str): + raise ValueError + if not row["id"] or row["id"] in identifiers: + raise ValueError + identifiers.add(row["id"]) + return identifiers + + +def _catalog_datetime(value: object, *, allow_none: bool = False) -> datetime | None: + if value is None and allow_none: + return None + if not isinstance(value, str): + raise ValueError + parsed = datetime.fromisoformat(value.replace("Z", "+00")) + if parsed.tzinfo is None: + raise ValueError + return parsed.astimezone(UTC) + + +def _validate_recovery_payload(payload: dict[str, Any]) -> int: + catalog = payload.get("catalog") + if isinstance(catalog, dict) and catalog.get("version") == 1: + return _validate_legacy_recovery_payload(payload) + validated_catalog, _keys = _validated_recovery_catalog(payload) + return len(validated_catalog["repositories"]) + + +def _install_or_verify_recovery_keys( + settings: Settings, + repository: dict[str, Any], + key_record: dict[str, Any], + installed_paths: list[Path], +) -> None: + inspected = inspect_repository(settings, Path(repository["root"])) + if ( + inspected.repository_id != repository["repository_id"] + or inspected.encryption != repository["encryption"] + or inspected.data_key_id != repository["active_data_key_id"] + ): + raise RecoveryBundleError("recovery bundle is invalid") + private_key = base64.b64decode(key_record["signing_private_key"], validate=True) + signing_path = settings.data_dir / "repository-keys" / f"{inspected.repository_id}.ed25519" + if signing_path.exists() or signing_path.is_symlink(): + loaded = load_signing_key( + settings, + inspected.repository_id, + repository["signing_key_id"], + repository["signing_public_key"], + ) + if ( + loaded.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + != private_key + ): + raise RecoveryBundleError("recovery destination has conflicting keys") + else: + installed_paths.append( + install_signing_key( + settings, + inspected.repository_id, + repository["signing_key_id"], + repository["signing_public_key"], + private_key, + ) + ) + for data_key in key_record["data_keys"]: + key_value = base64.b64decode(data_key["key"], validate=True) + key_path = ( + settings.data_dir + / "repository-data-keys" + / f"{inspected.repository_id}.{data_key['key_id']}.key" + ) + if key_path.exists() or key_path.is_symlink(): + if load_data_key(settings, inspected.repository_id, data_key["key_id"]) != key_value: + raise RecoveryBundleError("recovery destination has conflicting keys") + else: + installed_paths.append( + install_data_key(settings, inspected.repository_id, data_key["key_id"], key_value) + ) + + +async def import_recovery_payload( + settings: Settings, + payload: dict[str, Any], + *, + after_key_install: Callable[[], None] | None = None, +) -> int: + """Install a complete authenticated catalog into an empty, migrated database.""" + catalog, key_records = _validated_recovery_catalog(payload) + engine = create_engine(settings) + sessions = async_sessionmaker(engine, expire_on_commit=False) + installed_paths: list[Path] = [] + completed = False + try: + async with sessions() as db: + for table in Base.metadata.sorted_tables: + if await db.scalar(select(func.count()).select_from(table)): + raise RecoveryBundleError("recovery destination is not empty") + await db.rollback() + for repository, key_record in zip(catalog["repositories"], key_records, strict=True): + _install_or_verify_recovery_keys(settings, repository, key_record, installed_paths) + if after_key_install is not None: + after_key_install() + async with db.begin(): + for repository in catalog["repositories"]: + db.add( + Repository( + id=repository["database_id"], + name=repository["name"], + root=repository["root"], + format_version=repository["format_version"], + compression=repository["compression"], + encryption=repository["encryption"], + signing_key_id=repository["signing_key_id"], + signing_public_key=repository["signing_public_key"], + active_data_key_id=repository["active_data_key_id"], + state=repository["state"], + ) + ) + for repository in catalog["repositories"]: + for epoch in repository["data_key_epochs"]: + db.add( + RepositoryDataKeyEpoch( + repository_id=repository["database_id"], + key_id=epoch["key_id"], + state=epoch["state"], + ) + ) + await db.flush() + for source in catalog["sources"]: + db.add( + Source( + id=source["id"], + name=source["name"], + kind=source["kind"], + public_config=source["public_config"], + secret_refs=[], + state="unavailable", + ) + ) + await db.flush() + for job in catalog["jobs"]: + db.add( + Job( + id=job["id"], + name=job["name"], + source_id=job["source_id"], + repository_id=job["repository_id"], + requested_mode=job["requested_mode"], + exclusions=job["exclusions"], + retention=job["retention"], + enabled=False, + allow_empty=job["allow_empty"], + state="archived", + ) + ) + await db.flush() + for execution in catalog["executions"]: + db.add( + Execution( + id=execution["id"], + job_id=execution["job_id"], + schedule_id=None, + trigger=execution["trigger"], + state="committed", + attempt=execution["attempt"], + progress=execution["progress"], + revision=execution["revision"], + ) + ) + await db.flush() + for backup in catalog["backups"]: + db.add( + Backup( + id=backup["id"], + execution_id=backup["execution_id"], + parent_backup_id=None, + manifest_id=backup["manifest_id"], + manifest_digest=backup["manifest_digest"], + logical_bytes=backup["logical_bytes"], + stored_bytes=backup["stored_bytes"], + integrity=backup["integrity"], + data_key_id=backup["data_key_id"], + pinned=backup["pinned"], + created_at=_catalog_datetime(backup["created_at"]), + tombstoned_at=_catalog_datetime( + backup["tombstoned_at"], allow_none=True + ), + ) + ) + await db.flush() + for backup in catalog["backups"]: + if backup["parent_backup_id"] is not None: + imported = await db.get(Backup, backup["id"]) + if imported is None: + raise RecoveryBundleError("recovery bundle is invalid") + imported.parent_backup_id = backup["parent_backup_id"] + completed = True + return len(catalog["repositories"]) + except (OSError, RepositoryKeyError, ValueError) as error: + if isinstance(error, RecoveryBundleError): + raise + raise RecoveryBundleError("recovery import failed") from error + finally: + await engine.dispose() + if not completed: + for path in installed_paths: + path.unlink(missing_ok=True) + + +def run_admin(settings: Settings, args: argparse.Namespace) -> int: + if ( + getattr(args, "admin_command", None) == "repository-key" + and getattr(args, "repository_key_command", None) == "rotate" + ): + result = asyncio.run(rotate_repository_key(settings, args.repository_id)) + print(json.dumps(result, sort_keys=True)) + return 0 + if getattr(args, "admin_command", None) == "recovery": + passphrase = read_passphrase_fd(args.passphrase_fd) + if args.recovery_command == "export": + payload = asyncio.run(recovery_export_payload(settings)) + write_bundle_exclusive(args.output, encrypt_bundle(payload, passphrase)) + print(json.dumps({"repositories": len(payload["keys"]), "status": "exported"})) + return 0 + if args.recovery_command == "validate": + payload = decrypt_bundle(read_bundle_file(args.input), passphrase) + validation_result: dict[str, object] = { + "repositories": _validate_recovery_payload(payload), + "status": "valid", + } + print(json.dumps(validation_result)) + return 0 + if args.recovery_command == "import": + payload = decrypt_bundle(read_bundle_file(args.input), passphrase) + imported = asyncio.run(import_recovery_payload(settings, payload)) + print(json.dumps({"repositories": imported, "status": "imported"})) + return 0 + raise ValueError("admin command is invalid") + + def main(argv: Sequence[str] | None = None, *, settings: Settings | None = None) -> int: args = build_parser().parse_args(argv) runtime_settings = settings or load_settings() if args.role == "migrate": return run_migration(args.action, runtime_settings) + if args.role == "health": + return run_role_healthcheck(runtime_settings, args.health_role) require_current_schema(runtime_settings) + if args.role == "admin": + return run_admin(runtime_settings, args) return ROLE_HANDLERS[args.role](runtime_settings) diff --git a/backend/src/backup_tool/config.py b/backend/src/backup_tool/config.py index 4a8b1c9..c119a6d 100644 --- a/backend/src/backup_tool/config.py +++ b/backend/src/backup_tool/config.py @@ -19,6 +19,7 @@ class Settings(BaseSettings): ) data_dir: Path = Path("/var/lib/backup-tool") + web_socket_path: Path = Path("/tmp/backup-tool-web.sock") database_url: str = "sqlite+aiosqlite:////var/lib/backup-tool/metadata.db" repository_roots: tuple[Path, ...] local_source_roots: tuple[Path, ...] @@ -29,6 +30,21 @@ class Settings(BaseSettings): session_ttl_seconds: int = Field(default=28_800, ge=60, le=2_592_000) cors_origins: tuple[str, ...] = () worker_concurrency: Literal[1] = 1 + notification_delivery_lease_seconds: int = Field(default=60, ge=5, le=3600) + notification_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60) + notification_read_timeout_seconds: float = Field(default=10.0, gt=0, le=120) + notification_max_attempts: int = Field(default=5, ge=1, le=20) + notification_retry_cap_seconds: int = Field(default=3600, ge=1, le=86_400) + notification_global_rate_per_minute: int = Field(default=120, ge=1, le=10_000) + notification_default_rate_per_minute: int = Field(default=60, ge=1, le=10_000) + notification_max_webhook_body_bytes: int = Field(default=65_536, ge=256, le=1_048_576) + notification_max_response_bytes: int = Field(default=16_384, ge=256, le=1_048_576) + ssh_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60) + ssh_operation_timeout_seconds: float = Field(default=10.0, gt=0, le=120) + ssh_read_chunk_bytes: int = Field(default=1_048_576, ge=4096, le=8_388_608) + ssh_list_read_aheads: int = Field(default=32, ge=1, le=256) + ssh_max_entries: int = Field(default=1_000_000, ge=1, le=10_000_000) + ssh_max_traversal_depth: int = Field(default=128, ge=1, le=1024) min_free_bytes: int = Field(default=1_073_741_824, ge=0) min_free_percent: int = Field(default=5, gt=0, lt=100) sqlite_busy_timeout_ms: int = Field(default=5_000, ge=1_000, le=120_000) @@ -37,6 +53,7 @@ class Settings(BaseSettings): @field_validator( "data_dir", "master_key_file", + "web_socket_path", mode="before", ) @classmethod diff --git a/backend/src/backup_tool/db/models.py b/backend/src/backup_tool/db/models.py index efcbe07..bba9905 100644 --- a/backend/src/backup_tool/db/models.py +++ b/backend/src/backup_tool/db/models.py @@ -95,6 +95,9 @@ class Repository(IdentityMixin, TimestampMixin, Base): format_version: Mapped[int] = mapped_column(Integer, nullable=False) compression: Mapped[str] = mapped_column(String(32), nullable=False) encryption: Mapped[str] = mapped_column(String(32), nullable=False) + signing_key_id: Mapped[str] = mapped_column(String(64), nullable=False, server_default="") + signing_public_key: Mapped[str] = mapped_column(String(64), nullable=False, server_default="") + active_data_key_id: Mapped[str | None] = mapped_column(String(36)) state: Mapped[str] = mapped_column(String(32), nullable=False, default="active") __table_args__ = ( CheckConstraint("format_version > 0", name="format_version_positive"), @@ -102,6 +105,25 @@ class Repository(IdentityMixin, TimestampMixin, Base): ) +class RepositoryDataKeyEpoch(IdentityMixin, TimestampMixin, Base): + __tablename__ = "repository_data_key_epochs" + repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT")) + key_id: Mapped[str] = mapped_column(String(36), nullable=False) + state: Mapped[str] = mapped_column(String(16), nullable=False) + retired_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + __table_args__ = ( + UniqueConstraint("repository_id", "key_id", name="repository_key_epoch"), + CheckConstraint("state IN ('active','retired')", name="repository_data_key_epoch_state"), + Index("ix_repository_data_key_epochs_repository_id", "repository_id"), + Index( + "uq_repository_data_key_epochs_active", + "repository_id", + unique=True, + sqlite_where=text("state = 'active'"), + ), + ) + + class Source(IdentityMixin, TimestampMixin, Base): __tablename__ = "sources" name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) @@ -111,7 +133,7 @@ class Source(IdentityMixin, TimestampMixin, Base): state: Mapped[str] = mapped_column(String(32), nullable=False, default="active") last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON) __table_args__ = ( - CheckConstraint("kind IN ('local','sftp','postgresql','mysql')", name="kind"), + CheckConstraint("kind IN ('local','ssh')", name="kind"), CheckConstraint("state IN ('active','archived','unavailable')", name="state"), ) @@ -216,6 +238,7 @@ class Backup(IdentityMixin, Base): logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False) stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False) integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified") + data_key_id: Mapped[str | None] = mapped_column(String(36)) pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) tombstoned_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) created_at: Mapped[datetime] = mapped_column( @@ -236,6 +259,9 @@ class Restore(IdentityMixin, TimestampMixin, Base): backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT")) destination: Mapped[str] = mapped_column(Text, nullable=False) selection: Mapped[list[str]] = mapped_column(JSON, nullable=False) + dry_run: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0" + ) overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False) state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued") result: Mapped[dict[str, Any] | None] = mapped_column(JSON) @@ -268,33 +294,143 @@ class AuditEvent(IdentityMixin, Base): class NotificationSubscription(IdentityMixin, TimestampMixin, Base): + """A typed, mutable outbound channel configuration; credential material is never here.""" + __tablename__ = "notification_subscriptions" channel: Mapped[str] = mapped_column(String(32), nullable=False) event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False) destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + # Retained only to make the migration from the unused v1-shaped table lossless. + # New webhook secrets live in NotificationSigningKey and SMTP secrets in settings. secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT")) state: Mapped[str] = mapped_column(String(32), nullable=False, default="active") + rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60) + rate_tokens: Mapped[float] = mapped_column(nullable=False, default=60.0) + rate_updated_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1) __table_args__ = ( CheckConstraint("channel IN ('webhook','email')", name="channel"), CheckConstraint("state IN ('active','disabled','archived')", name="state"), + CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"), + CheckConstraint("rate_tokens >= 0", name="rate_tokens_nonnegative"), + CheckConstraint("revision > 0", name="revision_positive"), + ) + + +class NotificationEvent(IdentityMixin, Base): + """Immutable canonical outbox envelope, created in the source mutation transaction.""" + + __tablename__ = "notification_events" + type: Mapped[str] = mapped_column(String(96), nullable=False) + schema_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + occurred_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False) + correlation_id: Mapped[str] = mapped_column(String(36), nullable=False) + severity: Mapped[str] = mapped_column(String(16), nullable=False) + resource_refs: Mapped[dict[str, str]] = mapped_column(JSON, nullable=False) + payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + canonical_envelope: Mapped[str] = mapped_column(Text, nullable=False) + deduplication_key: Mapped[str | None] = mapped_column(String(255), unique=True) + __table_args__ = ( + CheckConstraint("schema_version = 1", name="schema_version"), + CheckConstraint("severity IN ('info','warning','error','security')", name="severity"), + Index("ix_notification_events_type_occurred", "type", "occurred_at"), ) class NotificationDelivery(IdentityMixin, TimestampMixin, Base): + """One durable delivery per matching subscription, with a lease-owned lifecycle.""" + __tablename__ = "notification_deliveries" - event_id: Mapped[str] = mapped_column(String(36), nullable=False) - subscription_id: Mapped[str] = mapped_column( - ForeignKey("notification_subscriptions.id", ondelete="RESTRICT") + event_id: Mapped[str] = mapped_column( + ForeignKey("notification_events.id", ondelete="RESTRICT"), nullable=False ) - attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1) - state: Mapped[str] = mapped_column(String(32), nullable=False) + subscription_id: Mapped[str] = mapped_column( + ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False + ) + state: Mapped[str] = mapped_column(String(32), nullable=False, default="pending") + due_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False) + lease_owner: Mapped[str | None] = mapped_column(String(255)) + lease_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + terminal_reason: Mapped[str | None] = mapped_column(String(96)) response_class: Mapped[str | None] = mapped_column(String(64)) - next_attempt_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + response_summary: Mapped[str | None] = mapped_column(String(512)) __table_args__ = ( - CheckConstraint("attempt > 0", name="attempt_positive"), - CheckConstraint("state IN ('pending','delivered','retry','failed')", name="state"), - UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"), - Index("ix_notification_deliveries_state_next", "state", "next_attempt_at"), + CheckConstraint("attempt_count >= 0", name="attempt_count_nonnegative"), + CheckConstraint("state IN ('pending','leased','delivered','retry','failed')", name="state"), + UniqueConstraint("event_id", "subscription_id", name="event_subscription"), + Index("ix_notification_deliveries_due", "state", "due_at"), + Index("ix_notification_deliveries_lease", "state", "lease_expires_at"), + ) + + +class NotificationDeliveryAttempt(IdentityMixin, Base): + """Append-only history. Diagnostics are bounded/redacted before persistence.""" + + __tablename__ = "notification_delivery_attempts" + delivery_id: Mapped[str] = mapped_column( + ForeignKey("notification_deliveries.id", ondelete="RESTRICT"), nullable=False + ) + number: Mapped[int] = mapped_column(Integer, nullable=False) + started_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False) + completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + outcome: Mapped[str] = mapped_column(String(32), nullable=False, default="started") + response_class: Mapped[str | None] = mapped_column(String(64)) + diagnostic: Mapped[str | None] = mapped_column(String(512)) + __table_args__ = ( + CheckConstraint("number > 0", name="number_positive"), + CheckConstraint("outcome IN ('started','delivered','retry','failed')", name="outcome"), + UniqueConstraint("delivery_id", "number", name="delivery_number"), + Index("ix_notification_attempts_delivery", "delivery_id", "number"), + ) + + +class NotificationSigningKey(IdentityMixin, TimestampMixin, Base): + __tablename__ = "notification_signing_keys" + subscription_id: Mapped[str] = mapped_column( + ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False + ) + version: Mapped[int] = mapped_column(Integer, nullable=False) + secret_id: Mapped[str] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT")) + state: Mapped[str] = mapped_column(String(16), nullable=False, default="active") + overlap_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + __table_args__ = ( + CheckConstraint("version > 0", name="version_positive"), + CheckConstraint("state IN ('active','overlap','retired')", name="state"), + UniqueConstraint("subscription_id", "version", name="subscription_version"), + Index("ix_notification_signing_keys_subscription", "subscription_id", "state"), + Index( + "uq_notification_signing_keys_active", + "subscription_id", + unique=True, + sqlite_where=text("state = 'active'"), + ), + Index( + "uq_notification_signing_keys_overlap", + "subscription_id", + unique=True, + sqlite_where=text("state = 'overlap'"), + ), + ) + + +class NotificationEmailSettings(Base): + __tablename__ = "notification_email_settings" + id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1) + host: Mapped[str] = mapped_column(String(255), nullable=False) + port: Mapped[int] = mapped_column(Integer, nullable=False, default=587) + username: Mapped[str] = mapped_column(String(255), nullable=False) + password_secret_id: Mapped[str] = mapped_column( + ForeignKey("secrets.id", ondelete="RESTRICT"), nullable=False + ) + sender: Mapped[str] = mapped_column(String(320), nullable=False) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=5) + rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60) + __table_args__ = ( + CheckConstraint("id = 1", name="singleton"), + CheckConstraint("port > 0 AND port < 65536", name="port"), + CheckConstraint("max_attempts > 0", name="max_attempts"), + CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"), ) diff --git a/backend/src/backup_tool/exclusions.py b/backend/src/backup_tool/exclusions.py new file mode 100644 index 0000000..3b3a6e0 --- /dev/null +++ b/backend/src/backup_tool/exclusions.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from pathlib import PurePosixPath + + +class ExclusionError(ValueError): + pass + + +def _validate(value: str) -> None: + path = PurePosixPath(value) + if not value or "\\" in value or path.is_absolute() or ".." in path.parts: + raise ExclusionError("exclusion paths must be normalized relative POSIX paths") + + +def matches(path: str, patterns: list[str]) -> bool: + """Return whether normalized relative `path` is excluded by ordered gitignore-like rules.""" + _validate(path) + excluded = False + candidate = PurePosixPath(path) + for pattern in patterns: + negated = pattern.startswith("!") + raw = pattern[1:] if negated else pattern + if not raw or raw.startswith("/") or "\\" in raw or ".." in PurePosixPath(raw).parts: + raise ExclusionError("exclusion pattern is invalid") + directory = raw.endswith("/") + raw = raw.rstrip("/") + if not raw: + raise ExclusionError("exclusion pattern is invalid") + matched = candidate.match(raw) or candidate.match(f"**/{raw}") + if directory: + matched = matched or any( + parent.match(raw) or parent.match(f"**/{raw}") for parent in candidate.parents + ) + if matched: + excluded = not negated + return excluded diff --git a/backend/src/backup_tool/execution.py b/backend/src/backup_tool/execution.py index 49bb2c9..2a002af 100644 --- a/backend/src/backup_tool/execution.py +++ b/backend/src/backup_tool/execution.py @@ -8,6 +8,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from backup_tool.db.models import Execution, ExecutionEvent, Job +from backup_tool.notifications.events import emit_event from backup_tool.security.redaction import redact ACTIVE_STATES = frozenset({"queued", "preparing", "running", "verifying", "cancelling"}) @@ -44,7 +45,14 @@ def transition(current: str, target: str) -> str: return target -async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Execution: +async def enqueue( + db: AsyncSession, + job_id: str, + trigger: str = "manual", + *, + schedule_id: str | None = None, + nominal_run_at: datetime | None = None, +) -> Execution: """Create exactly one active execution for an enabled active job.""" job = await db.get(Job, job_id) if job is None: @@ -52,7 +60,13 @@ async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Exe if job.state != "active" or not job.enabled: raise EnqueueError("job_disabled", "Job is disabled or archived.") job_identifier = job.id - execution = Execution(job_id=job_identifier, trigger=trigger, progress={}) + execution = Execution( + job_id=job_identifier, + trigger=trigger, + schedule_id=schedule_id, + nominal_run_at=nominal_run_at, + progress={}, + ) db.add(execution) try: await db.flush() @@ -221,6 +235,32 @@ async def record_event(db: AsyncSession, execution: Execution) -> ExecutionEvent ) db.add(event) await db.flush() + notification_type = { + "preparing": "execution.started", + "committed": "execution.committed", + "failed": "execution.failed", + "cancelled": "execution.cancelled", + }.get(execution.state) + if execution.state == "queued": + if execution.reason_code == "worker_lost": + notification_type = "execution.worker_recovered" + elif execution.attempt > 1: + notification_type = "execution.retry_queued" + else: + notification_type = "execution.queued" + if notification_type is not None: + await emit_event( + db, + notification_type, + correlation_id=execution.id, + resource={"execution_id": execution.id, "job_id": execution.job_id}, + payload={ + "attempt": execution.attempt, + "state": execution.state, + "reason_code": execution.reason_code, + }, + deduplication_key=f"execution:{execution.id}:revision:{execution.revision}", + ) return event diff --git a/backend/src/backup_tool/faults.py b/backend/src/backup_tool/faults.py new file mode 100644 index 0000000..c1d54f4 --- /dev/null +++ b/backend/src/backup_tool/faults.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import Protocol + + +class FaultInjector(Protocol): + def hit(self, point: str) -> None: ... + + +class NoFault: + def hit(self, point: str) -> None: + del point + + +class InjectedCrash(BaseException): + """Test-only abrupt worker termination at a named durable fault point.""" + + +class CrashAt: + def __init__(self, point: str) -> None: + self.point = point + + def hit(self, point: str) -> None: + if point == self.point: + raise InjectedCrash(point) diff --git a/backend/src/backup_tool/gc.py b/backend/src/backup_tool/gc.py new file mode 100644 index 0000000..969ade2 --- /dev/null +++ b/backend/src/backup_tool/gc.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import shutil +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import cast + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backup_tool.db.models import Backup, Execution, Job, Repository +from backup_tool.notifications.events import emit_event +from backup_tool.retention import BackupLike, RetentionPolicy, retained_ids +from backup_tool.security.repository_crypto import RepositoryKeyError, decrypt_object, object_aad + + +@dataclass(frozen=True) +class GcReport: + tombstoned: int + purged_manifests: int + purged_blobs: int + quarantined: int + + +async def tombstone_expired(db: AsyncSession, now: datetime | None = None) -> int: + reference = now or datetime.now(UTC) + backups = list((await db.scalars(select(Backup))).all()) + executions = {item.id: item for item in (await db.scalars(select(Execution))).all()} + jobs = {item.id: item for item in (await db.scalars(select(Job))).all()} + grouped: dict[str, list[Backup]] = {} + for backup in backups: + execution = executions.get(backup.execution_id) + if execution is None or execution.job_id not in jobs: + continue + grouped.setdefault(execution.job_id, []).append(backup) + tombstoned = 0 + for job_id, items in grouped.items(): + if not jobs[job_id].retention: + continue + policy = RetentionPolicy.from_dict(jobs[job_id].retention) + keep = retained_ids(cast(list[BackupLike], items), policy, reference) + for backup in items: + if backup.id not in keep and backup.tombstoned_at is None: + backup.tombstoned_at = reference + await emit_event( + db, + "retention.tombstoned", + correlation_id=backup.id, + resource={"backup_id": backup.id, "job_id": job_id}, + payload={"outcome": "tombstoned"}, + deduplication_key=f"backup:{backup.id}:tombstoned", + ) + tombstoned += 1 + await db.commit() + return tombstoned + + +async def process_retention_gc(db: AsyncSession, now: datetime | None = None) -> GcReport: + """Run durable retention tombstoning and repository GC from the worker role.""" + reference = now or datetime.now(UTC) + tombstoned = await tombstone_expired(db, reference) + reports: list[GcReport] = [] + repositories = list((await db.scalars(select(Repository))).all()) + for repository in repositories: + manifest_ids = set( + ( + await db.scalars( + select(Backup.manifest_id) + .join(Execution, Backup.execution_id == Execution.id) + .join(Job, Execution.job_id == Job.id) + .where( + Job.repository_id == repository.id, + Backup.tombstoned_at.is_not(None), + ) + ) + ).all() + ) + if manifest_ids: + reports.append(purge_repository(Path(repository.root), manifest_ids, now=reference)) + return GcReport( + tombstoned=tombstoned, + purged_manifests=sum(report.purged_manifests for report in reports), + purged_blobs=sum(report.purged_blobs for report in reports), + quarantined=sum(report.quarantined for report in reports), + ) + + +def _manifest_digests( + path: Path, + *, + repository_id: str | None = None, + manifest_keys: Mapping[str, tuple[str, bytes]] | None = None, +) -> set[str] | None: + try: + if path.is_symlink() or not path.is_file(): + return None + raw = path.read_bytes() + if raw.startswith(b"BTENC\x01"): + key_record = manifest_keys.get(path.stem) if manifest_keys is not None else None + if key_record is None or repository_id is None: + return None + key_id, key = key_record + raw = decrypt_object(key, object_aad(repository_id, key_id, "manifest", path.stem), raw) + payload = json.loads(raw.decode("utf-8")) + except (OSError, RepositoryKeyError, UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict) or not isinstance(entries := payload.get("entries"), list): + return None + digests: set[str] = set() + for entry in entries: + if not isinstance(entry, dict): + return None + digest = entry.get("blob_digest") + if digest is None: + continue + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + return None + digests.add(digest) + return digests + + +def purge_repository( + root: Path, + tombstoned_manifest_ids: set[str], + *, + repository_id: str | None = None, + manifest_keys: Mapping[str, tuple[str, bytes]] | None = None, + grace: timedelta = timedelta(days=7), + now: datetime | None = None, +) -> GcReport: + reference = now or datetime.now(UTC) + manifests = root / "manifests" + blobs = root / "blobs" / "sha256" + quarantine = root / "quarantine" + quarantine.mkdir(exist_ok=True) + manifest_digests: dict[Path, set[str]] = {} + for manifest in manifests.glob("*.json"): + digests = _manifest_digests( + manifest, + repository_id=repository_id, + manifest_keys=manifest_keys, + ) + if digests is None: + return GcReport(0, 0, 0, 0) + manifest_digests[manifest] = digests + + purged_manifests = 0 + for manifest_id in tombstoned_manifest_ids: + candidate = manifests / f"{manifest_id}.json" + if not candidate.is_file() or candidate.is_symlink(): + continue + age = reference - datetime.fromtimestamp(candidate.stat().st_mtime, UTC) + if age >= grace: + candidate.unlink() + purged_manifests += 1 + referenced: set[str] = set() + for manifest, digests in manifest_digests.items(): + if manifest.exists(): + referenced.update(digests) + purged_blobs = 0 + quarantined = 0 + if blobs.exists(): + for blob in blobs.iterdir(): + if not blob.is_file(): + continue + if len(blob.name) != 64 or any(char not in "0123456789abcdef" for char in blob.name): + try: + shutil.move(str(blob), quarantine / blob.name) + except OSError: + continue + quarantined += 1 + elif blob.name not in referenced and ( + reference - datetime.fromtimestamp(blob.stat().st_mtime, UTC) >= grace + ): + blob.unlink() + purged_blobs += 1 + return GcReport(0, purged_manifests, purged_blobs, quarantined) diff --git a/backend/src/backup_tool/notifications/__init__.py b/backend/src/backup_tool/notifications/__init__.py new file mode 100644 index 0000000..535e8c3 --- /dev/null +++ b/backend/src/backup_tool/notifications/__init__.py @@ -0,0 +1,5 @@ +"""Durable, worker-dispatched operational notifications.""" + +from .events import EVENT_CATALOG, emit_event, validate_filters + +__all__ = ["EVENT_CATALOG", "emit_event", "validate_filters"] diff --git a/backend/src/backup_tool/notifications/dispatcher.py b/backend/src/backup_tool/notifications/dispatcher.py new file mode 100644 index 0000000..e88d191 --- /dev/null +++ b/backend/src/backup_tool/notifications/dispatcher.py @@ -0,0 +1,335 @@ +"""Worker-owned leased outbox dispatcher; sends happen only after a committed lease.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from backup_tool.config import Settings +from backup_tool.db.models import ( + NotificationDelivery, + NotificationDeliveryAttempt, + NotificationEmailSettings, + NotificationEvent, + NotificationSigningKey, + NotificationSubscription, + Secret, +) +from backup_tool.notifications.email import EmailTransportError, deliver_email +from backup_tool.notifications.retry import ( + RetryDecision, + retry_delay, + transport_decision, + webhook_decision, +) +from backup_tool.notifications.webhook import ( + SigningMaterial, + WebhookTransportError, + deliver_webhook, +) +from backup_tool.security.secrets import EnvelopeCipher +from backup_tool.security.ssrf import Resolver, system_resolver + + +async def recover_notification_leases(db: AsyncSession) -> int: + """An interrupted post-send lease becomes eligible again (at-least-once by design).""" + now = datetime.now(UTC) + deliveries = list( + ( + await db.scalars( + select(NotificationDelivery).where( + NotificationDelivery.state == "leased", + NotificationDelivery.lease_expires_at < now, + ) + ) + ).all() + ) + for delivery in deliveries: + attempt = await db.scalar( + select(NotificationDeliveryAttempt).where( + NotificationDeliveryAttempt.delivery_id == delivery.id, + NotificationDeliveryAttempt.number == delivery.attempt_count, + NotificationDeliveryAttempt.outcome == "started", + ) + ) + if attempt is not None: + attempt.completed_at = now + attempt.outcome = "retry" + attempt.response_class = "lease_expired" + attempt.diagnostic = "abandoned_lease" + delivery.state = "retry" + delivery.lease_owner = None + delivery.lease_expires_at = None + delivery.due_at = now + if deliveries: + await db.commit() + return len(deliveries) + + +async def _claim_due( + db: AsyncSession, owner: str, lease_seconds: int +) -> tuple[NotificationDelivery, NotificationSubscription, NotificationEvent] | None: + now = datetime.now(UTC) + delivery_id = await db.scalar( + select(NotificationDelivery.id) + .where( + NotificationDelivery.state.in_(("pending", "retry")), + NotificationDelivery.due_at <= now, + ) + .order_by(NotificationDelivery.due_at, NotificationDelivery.created_at) + .limit(1) + ) + if delivery_id is None: + return None + result = await db.execute( + update(NotificationDelivery) + .where( + NotificationDelivery.id == delivery_id, + NotificationDelivery.state.in_(("pending", "retry")), + NotificationDelivery.due_at <= now, + ) + .values( + state="leased", + lease_owner=owner, + lease_expires_at=now + timedelta(seconds=lease_seconds), + attempt_count=NotificationDelivery.attempt_count + 1, + ) + ) + if getattr(result, "rowcount", 0) != 1: + await db.rollback() + return None + delivery = await db.get(NotificationDelivery, delivery_id) + if delivery is None: # pragma: no cover - guarded by update + await db.rollback() + return None + subscription = await db.get(NotificationSubscription, delivery.subscription_id) + event = await db.get(NotificationEvent, delivery.event_id) + if subscription is None or event is None or subscription.state != "active": + delivery.state = "failed" + delivery.terminal_reason = "subscription_unavailable" + delivery.lease_owner = None + delivery.lease_expires_at = None + await db.commit() + return None + # Persist a token bucket before starting an attempt, so restarts cannot bypass + # the subscription rate limit. Global process rate is intentionally a config + # ceiling; the durable subscription bucket protects cross-restart behavior. + last = subscription.rate_updated_at or now + elapsed = max(0.0, (now - last).total_seconds()) + capacity = subscription.rate_limit_per_minute + try: + token_capacity = float(capacity) + tokens = min(token_capacity, subscription.rate_tokens + elapsed * capacity / 60) + except (TypeError, ValueError, ZeroDivisionError) as error: + raise RuntimeError("notification rate limit is invalid") from error + if tokens < 1: + try: + delay = max(1, int((1 - tokens) * 60 / capacity) + 1) + except (TypeError, ValueError, ZeroDivisionError) as error: + raise RuntimeError("notification rate limit is invalid") from error + delivery.state = "retry" + delivery.due_at = now + timedelta(seconds=delay) + delivery.lease_owner = None + delivery.lease_expires_at = None + subscription.rate_tokens = tokens + subscription.rate_updated_at = now + await db.commit() + return None + subscription.rate_tokens = tokens - 1 + subscription.rate_updated_at = now + db.add( + NotificationDeliveryAttempt( + delivery_id=delivery.id, + number=delivery.attempt_count, + started_at=now, + outcome="started", + ) + ) + await db.commit() + return delivery, subscription, event + + +async def _finish( + db: AsyncSession, + delivery_id: str, + owner: str, + *, + delivered: bool, + retryable: bool, + response_class: str, + reason: str, + retry_cap: int, + max_attempts: int, +) -> None: + delivery = await db.get(NotificationDelivery, delivery_id) + if delivery is None or delivery.lease_owner != owner or delivery.state != "leased": + await db.rollback() + return + attempt = await db.scalar( + select(NotificationDeliveryAttempt).where( + NotificationDeliveryAttempt.delivery_id == delivery.id, + NotificationDeliveryAttempt.number == delivery.attempt_count, + ) + ) + if attempt is None: # pragma: no cover - an invariant of _claim_due + await db.rollback() + return + now = datetime.now(UTC) + attempt.completed_at = now + attempt.response_class = response_class + attempt.diagnostic = reason[:512] + delivery.response_class = response_class + delivery.response_summary = reason[:512] + delivery.lease_owner = None + delivery.lease_expires_at = None + if delivered: + delivery.state = "delivered" + attempt.outcome = "delivered" + elif retryable and delivery.attempt_count < max_attempts: + delivery.state = "retry" + delivery.due_at = now + timedelta(seconds=retry_delay(delivery.attempt_count, retry_cap)) + attempt.outcome = "retry" + else: + delivery.state = "failed" + delivery.terminal_reason = reason + attempt.outcome = "failed" + await db.commit() + + +async def dispatch_one( + db: AsyncSession, + settings: Settings, + cipher: EnvelopeCipher, + owner: str, + *, + resolver: Resolver = system_resolver, +) -> bool: + claimed = await _claim_due(db, owner, settings.notification_delivery_lease_seconds) + if claimed is None: + return False + delivery, subscription, event = claimed + max_attempts = settings.notification_max_attempts + try: + if subscription.channel == "webhook": + key_rows = list( + ( + await db.scalars( + select(NotificationSigningKey).where( + NotificationSigningKey.subscription_id == subscription.id, + NotificationSigningKey.state.in_(("active", "overlap")), + ) + ) + ).all() + ) + keys: list[SigningMaterial] = [] + now = datetime.now(UTC) + for key in key_rows: + if ( + key.state == "overlap" + and key.overlap_expires_at is not None + and key.overlap_expires_at <= now + ): + key.state = "retired" + continue + secret = await db.get(Secret, key.secret_id) + if secret is None: + raise WebhookTransportError("webhook signing secret is unavailable") + keys.append( + SigningMaterial( + key_id=key.id, + version=key.version, + secret=cipher.decrypt( + secret.ciphertext, + purpose=secret.purpose, + version=secret.version, + ), + ) + ) + await db.commit() + webhook_result = await deliver_webhook( + str(subscription.destination_config["url"]), + event.canonical_envelope.encode(), + event_id=event.id, + event_type=event.type, + timestamp=event.occurred_at.isoformat(), + keys=keys, + resolver=resolver, + connect_timeout=settings.notification_connect_timeout_seconds, + read_timeout=settings.notification_read_timeout_seconds, + max_response_bytes=settings.notification_max_response_bytes, + ) + decision = webhook_decision(webhook_result.status_code) + elif subscription.channel == "email": + email_settings = await db.get(NotificationEmailSettings, 1) + if email_settings is None: + raise EmailTransportError("SMTP settings are unavailable") + max_attempts = email_settings.max_attempts + password_secret = await db.get(Secret, email_settings.password_secret_id) + if password_secret is None: + raise EmailTransportError("SMTP password is unavailable") + email_result = await deliver_email( + email_settings, + cipher.decrypt( + password_secret.ciphertext, + purpose=password_secret.purpose, + version=password_secret.version, + ), + event, + subscription.destination_config["recipients"], + ) + decision = RetryDecision(False, email_result.response_class, "delivered") + else: # guarded by DB constraint + raise WebhookTransportError("notification channel is unavailable") + await _finish( + db, + delivery.id, + owner, + delivered=decision.reason == "delivered", + retryable=decision.retry, + response_class=decision.response_class, + reason=decision.reason, + retry_cap=settings.notification_retry_cap_seconds, + max_attempts=max_attempts, + ) + except EmailTransportError as error: + decision = transport_decision(error.transient, "smtp_transport") + await _finish( + db, + delivery.id, + owner, + delivered=False, + retryable=decision.retry, + response_class=decision.response_class, + reason=str(error), + retry_cap=settings.notification_retry_cap_seconds, + max_attempts=max_attempts, + ) + except WebhookTransportError as error: + decision = transport_decision(error.transient, "webhook_transport") + await _finish( + db, + delivery.id, + owner, + delivered=False, + retryable=decision.retry, + response_class=decision.response_class, + reason=str(error), + retry_cap=settings.notification_retry_cap_seconds, + max_attempts=max_attempts, + ) + except (KeyError, ValueError): + decision = transport_decision(False, "webhook_validation") + await _finish( + db, + delivery.id, + owner, + delivered=False, + retryable=False, + response_class=decision.response_class, + reason=decision.reason, + retry_cap=settings.notification_retry_cap_seconds, + max_attempts=max_attempts, + ) + return True diff --git a/backend/src/backup_tool/notifications/email.py b/backend/src/backup_tool/notifications/email.py new file mode 100644 index 0000000..f7d7501 --- /dev/null +++ b/backend/src/backup_tool/notifications/email.py @@ -0,0 +1,120 @@ +"""Authenticated, certificate-verified STARTTLS email notification transport.""" + +from __future__ import annotations + +import asyncio +import smtplib +import ssl +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from email.message import EmailMessage +from email.utils import formataddr +from typing import Protocol, Self, cast + +from backup_tool.db.models import NotificationEmailSettings, NotificationEvent + + +class SMTPClient(Protocol): + def __enter__(self) -> Self: ... + + def __exit__(self, *args: object) -> None: ... + + def ehlo(self) -> object: ... + + def starttls(self, *, context: ssl.SSLContext) -> object: ... + + def login(self, user: str, password: str) -> object: ... + + def send_message(self, msg: EmailMessage) -> object: ... + + +class EmailTransportError(RuntimeError): + def __init__(self, message: str, *, transient: bool = False) -> None: + super().__init__(message) + self.transient = transient + + +@dataclass(frozen=True) +class EmailResult: + response_class: str + + +def validate_address(value: str) -> str: + if not value or len(value) > 320 or any(character in value for character in "\r\n"): + raise EmailTransportError("email address is invalid") + local, separator, domain = value.rpartition("@") + if not separator or not local or not domain or any(character.isspace() for character in value): + raise EmailTransportError("email address is invalid") + return value + + +def validate_recipients(values: Sequence[str]) -> list[str]: + if not values or len(values) > 20: + raise EmailTransportError("one to 20 email recipients are required") + recipients: list[str] = [] + for value in values: + address = validate_address(value) + if address not in recipients: + recipients.append(address) + return recipients + + +def _message( + settings: NotificationEmailSettings, + event: NotificationEvent, + recipients: Sequence[str], +) -> EmailMessage: + sender = validate_address(settings.sender) + safe_recipients = validate_recipients(recipients) + message = EmailMessage() + message["From"] = formataddr(("Backup Tool", sender)) + message["To"] = ", ".join(safe_recipients) + message["Subject"] = f"Backup Tool: {event.type} ({event.severity})" + message["X-Backup-Event-ID"] = event.id + # Do not put the full envelope, paths, raw errors, or credentials into mail. + message.set_content( + "Backup Tool operational event\n" + f"Event ID: {event.id}\n" + f"Type: {event.type}\n" + f"Severity: {event.severity}\n" + f"Occurred: {event.occurred_at.isoformat()}\n" + ) + return message + + +def _deliver_sync( + settings: NotificationEmailSettings, + password: str, + event: NotificationEvent, + recipients: Sequence[str], + smtp_factory: Callable[..., SMTPClient], +) -> EmailResult: + message = _message(settings, event, recipients) + try: + with smtp_factory(settings.host, settings.port, timeout=10) as client: + client.ehlo() + context = ssl.create_default_context() + client.starttls(context=context) + client.ehlo() + client.login(settings.username, password) + client.send_message(message) + except smtplib.SMTPResponseException as error: + raise EmailTransportError( + f"smtp_{error.smtp_code}", transient=400 <= error.smtp_code < 500 + ) from error + except (smtplib.SMTPException, OSError) as error: + raise EmailTransportError("smtp_transport_failed", transient=True) from error + return EmailResult(response_class="smtp_2xx") + + +async def deliver_email( + settings: NotificationEmailSettings, + password: str, + event: NotificationEvent, + recipients: Sequence[str], + *, + smtp_factory: Callable[..., SMTPClient] | None = None, +) -> EmailResult: + """Run blocking SMTP only in the worker thread, never in the web process.""" + factory = smtp_factory or cast(Callable[..., SMTPClient], smtplib.SMTP) + return await asyncio.to_thread(_deliver_sync, settings, password, event, recipients, factory) diff --git a/backend/src/backup_tool/notifications/events.py b/backend/src/backup_tool/notifications/events.py new file mode 100644 index 0000000..b30b7f4 --- /dev/null +++ b/backend/src/backup_tool/notifications/events.py @@ -0,0 +1,279 @@ +"""Versioned notification event catalog and transactional outbox fan-out.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backup_tool.db.models import NotificationDelivery, NotificationEvent, NotificationSubscription +from backup_tool.ids import new_uuid7 +from backup_tool.security.redaction import redact +from backup_tool.security.ssrf import SSRFError, validate_webhook_url + +EVENT_SCHEMA_VERSION = 1 +# Live-events-only policy: every public type below has a current production emitter. +_EVENT_TYPES = ( + "execution.queued", + "execution.started", + "execution.committed", + "execution.failed", + "execution.cancelled", + "execution.retry_queued", + "execution.worker_recovered", + "schedule.created", + "schedule.updated", + "schedule.deleted", + "schedule.enabled", + "schedule.disabled", + "schedule.occurrence_enqueued", + "schedule.occurrence_misfired", + "schedule.occurrence_blocked", + "backup.committed", + "backup.verification_succeeded", + "restore.queued", + "restore.committed", + "restore.failed", + "retention.tombstoned", + "notification.test_requested", +) +EVENT_CATALOG: dict[str, dict[str, Any]] = { + event_type: { + "event_schema_version": EVENT_SCHEMA_VERSION, + "severity": "error" if event_type.endswith(("failed", "blocked", "rejected")) else "info", + "payload_keys": ( + "attempt", + "count", + "dry_run", + "integrity", + "message", + "outcome", + "reason_code", + "requested_mode", + "effective_mode", + "state", + ), + "reserved": False, + } + for event_type in _EVENT_TYPES +} +_SAFE_RESOURCE_KEYS = frozenset( + { + "execution_id", + "job_id", + "schedule_id", + "backup_id", + "restore_id", + "repository_id", + "subscription_id", + } +) +_SAFE_PAYLOAD_KEYS = frozenset().union( + *(set(spec["payload_keys"]) for spec in EVENT_CATALOG.values()) +) + + +class NotificationEventError(ValueError): + """A caller attempted to produce data outside the stable public catalog.""" + + +def _as_uuid(value: str, field: str) -> str: + try: + parsed = UUID(value) + except (TypeError, ValueError, AttributeError) as error: + raise NotificationEventError(f"{field} must be a UUID") from error + return str(parsed) + + +def _safe_value(value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + return redact(value)[:512] + if isinstance(value, list): + if len(value) > 32: + raise NotificationEventError("payload arrays are limited to 32 values") + return [_safe_value(item) for item in value] + if isinstance(value, Mapping): + if len(value) > 32: + raise NotificationEventError("payload objects are limited to 32 fields") + return {str(key)[:64]: _safe_value(item) for key, item in value.items()} + raise NotificationEventError("payload contains an unsupported value") + + +def _is_filter_match(filter_value: str, event_type: str) -> bool: + if filter_value == event_type: + return True + family, wildcard = filter_value.rsplit(".", 1) if "." in filter_value else ("", "") + return wildcard == "*" and event_type.startswith(f"{family}.") + + +def validate_filters(filters: Sequence[str]) -> list[str]: + if not filters: + raise NotificationEventError("at least one event filter is required") + if len(filters) > len(EVENT_CATALOG): + raise NotificationEventError("too many event filters") + output: list[str] = [] + for filter_value in filters: + if not isinstance(filter_value, str) or len(filter_value) > 96: + raise NotificationEventError("invalid event filter") + if filter_value.endswith(".*"): + family = filter_value[:-2] + if not family or not any(item.startswith(f"{family}.") for item in EVENT_CATALOG): + raise NotificationEventError("unknown event filter") + elif filter_value not in EVENT_CATALOG: + raise NotificationEventError("unknown event filter") + if filter_value not in output: + output.append(filter_value) + return output + + +def validate_destination(channel: str, destination: Mapping[str, Any]) -> dict[str, Any]: + resource_filters = destination.get("resource_filters", {}) + if not isinstance(resource_filters, Mapping): + raise NotificationEventError("resource filters are invalid") + unknown = set(resource_filters) - {"job_ids", "repository_ids", "severities"} + if unknown: + raise NotificationEventError("resource filter is unknown") + normalized_filters: dict[str, list[str]] = {} + for key in ("job_ids", "repository_ids"): + values = resource_filters.get(key) + if values is None: + continue + if not isinstance(values, list) or not values: + raise NotificationEventError("resource filter is invalid") + normalized_filters[key] = [_as_uuid(value, key) for value in values] + severities = resource_filters.get("severities") + if severities is not None: + if not isinstance(severities, list) or not severities: + raise NotificationEventError("severity filter is invalid") + if any(value not in {"info", "warning", "error", "security"} for value in severities): + raise NotificationEventError("severity filter is invalid") + normalized_filters["severities"] = list(severities) + if channel == "webhook": + url = destination.get("url") + if not isinstance(url, str) or len(url) > 2048: + raise NotificationEventError("webhook URL is required") + try: + validate_webhook_url(url) + except SSRFError as error: + raise NotificationEventError("webhook URL is invalid") from error + return {"url": url, "resource_filters": normalized_filters} + if channel == "email": + recipients = destination.get("recipients") + if not isinstance(recipients, list) or not recipients or len(recipients) > 20: + raise NotificationEventError("one to 20 email recipients are required") + safe_recipients: list[str] = [] + for recipient in recipients: + if not isinstance(recipient, str) or any(char in recipient for char in "\r\n"): + raise NotificationEventError("invalid email recipient") + if "@" not in recipient or len(recipient) > 320: + raise NotificationEventError("invalid email recipient") + if recipient not in safe_recipients: + safe_recipients.append(recipient) + return {"recipients": safe_recipients, "resource_filters": normalized_filters} + raise NotificationEventError("unsupported notification channel") + + +def _matches(subscription: NotificationSubscription, event: dict[str, Any]) -> bool: + if subscription.state != "active": + return False + if not any(_is_filter_match(item, str(event["type"])) for item in subscription.event_filters): + return False + filters = subscription.destination_config.get("resource_filters", {}) + if not isinstance(filters, Mapping): + return False + resources = event["resource"] + for key in ("job_ids", "repository_ids"): + selected = filters.get(key) + resource_key = key[:-1] + if selected is not None and resources.get(resource_key) not in selected: + return False + severities = filters.get("severities") + return severities is None or event["severity"] in severities + + +async def emit_event( + db: AsyncSession, + event_type: str, + *, + correlation_id: str, + resource: Mapping[str, str] | None = None, + payload: Mapping[str, Any] | None = None, + severity: str | None = None, + deduplication_key: str | None = None, + occurred_at: datetime | None = None, + only_subscription_id: str | None = None, +) -> NotificationEvent: + """Append an immutable event and matching deliveries; intentionally never commits.""" + if event_type not in EVENT_CATALOG: + raise NotificationEventError("unknown operational event type") + correlation_id = _as_uuid(correlation_id, "correlation_id") + resource = resource or {} + if set(resource) - _SAFE_RESOURCE_KEYS: + raise NotificationEventError("unknown resource reference") + safe_resource = {key: _as_uuid(value, key) for key, value in resource.items()} + payload = payload or {} + if set(payload) - _SAFE_PAYLOAD_KEYS: + raise NotificationEventError("payload key is not allowlisted") + safe_payload = {key: _safe_value(value) for key, value in payload.items()} + event_severity = severity or str(EVENT_CATALOG[event_type]["severity"]) + if event_severity not in {"info", "warning", "error", "security"}: + raise NotificationEventError("invalid severity") + if deduplication_key is not None and (not deduplication_key or len(deduplication_key) > 255): + raise NotificationEventError("invalid deduplication key") + if deduplication_key is not None: + existing = await db.scalar( + select(NotificationEvent).where( + NotificationEvent.deduplication_key == deduplication_key + ) + ) + if existing is not None: + return existing + event_id = str(new_uuid7()) + timestamp = (occurred_at or datetime.now(UTC)).astimezone(UTC) + envelope = { + "event_schema_version": EVENT_SCHEMA_VERSION, + "id": event_id, + "type": event_type, + "occurred_at": timestamp.isoformat(), + "correlation_id": correlation_id, + "severity": event_severity, + "resource": safe_resource, + "payload": safe_payload, + } + event = NotificationEvent( + id=event_id, + type=event_type, + schema_version=EVENT_SCHEMA_VERSION, + occurred_at=timestamp, + correlation_id=correlation_id, + severity=event_severity, + resource_refs=safe_resource, + payload=safe_payload, + canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")), + deduplication_key=deduplication_key, + ) + db.add(event) + await db.flush() + active_subscriptions = select(NotificationSubscription).where( + NotificationSubscription.state == "active" + ) + subscriptions = list((await db.scalars(active_subscriptions)).all()) + for subscription in subscriptions: + selected_for_test = only_subscription_id == subscription.id + if selected_for_test or (only_subscription_id is None and _matches(subscription, envelope)): + db.add( + NotificationDelivery( + event_id=event.id, + subscription_id=subscription.id, + due_at=timestamp, + ) + ) + await db.flush() + return event diff --git a/backend/src/backup_tool/notifications/retry.py b/backend/src/backup_tool/notifications/retry.py new file mode 100644 index 0000000..1257131 --- /dev/null +++ b/backend/src/backup_tool/notifications/retry.py @@ -0,0 +1,33 @@ +"""Deterministic bounded retry classification for notification delivery.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RetryDecision: + retry: bool + response_class: str + reason: str + + +def retry_delay(attempt: int, cap_seconds: int) -> int: + """Bounded exponential delay; no jitter keeps durable tests/restarts deterministic.""" + exponent = max(0, attempt - 1) + return min(cap_seconds, 1 << exponent) + + +def webhook_decision(status_code: int) -> RetryDecision: + if 200 <= status_code < 300: + return RetryDecision(False, "http_2xx", "delivered") + if status_code in {408, 425, 429} or status_code >= 500: + return RetryDecision(True, f"http_{status_code}", "http_transient") + if 300 <= status_code < 400: + return RetryDecision(False, f"http_{status_code}", "redirect_rejected") + return RetryDecision(False, f"http_{status_code}", "http_permanent") + + +def transport_decision(transient: bool, response_class: str) -> RetryDecision: + reason = "transport_transient" if transient else "transport_failed" + return RetryDecision(transient, response_class, reason) diff --git a/backend/src/backup_tool/notifications/webhook.py b/backend/src/backup_tool/notifications/webhook.py new file mode 100644 index 0000000..38c47a7 --- /dev/null +++ b/backend/src/backup_tool/notifications/webhook.py @@ -0,0 +1,210 @@ +"""Canonical, dual-key HMAC webhook requests on a DNS-pinned HTTPX transport.""" + +from __future__ import annotations + +import asyncio +import hmac +import ssl +from collections.abc import Sequence +from dataclasses import dataclass +from hashlib import sha256 + +import httpx + +from backup_tool.security.ssrf import ( + ResolvedWebhookTarget, + Resolver, + SSRFError, + resolve_webhook_target, + verify_connected_peer, +) + +SIGNATURE_VERSION = "v1" + + +class WebhookTransportError(RuntimeError): + def __init__(self, message: str, *, transient: bool = False) -> None: + super().__init__(message) + self.transient = transient + + +@dataclass(frozen=True) +class SigningMaterial: + key_id: str + version: int + secret: str + + +@dataclass(frozen=True) +class WebhookResult: + status_code: int + response_bytes: int + + +def canonical_signing_input(timestamp: str, body: bytes) -> bytes: + return SIGNATURE_VERSION.encode() + b"." + timestamp.encode("ascii") + b"." + body + + +def signatures(timestamp: str, body: bytes, keys: Sequence[SigningMaterial]) -> list[str]: + signing_input = canonical_signing_input(timestamp, body) + return [ + f"{SIGNATURE_VERSION};key_id={key.key_id};key_version={key.version};sha256=" + f"{hmac.new(key.secret.encode(), signing_input, sha256).hexdigest()}" + for key in keys + ] + + +class PinnedWebhookTransport(httpx.AsyncBaseTransport): + """HTTPX transport that never lets a post-validation DNS lookup choose a peer.""" + + def __init__( + self, + *, + target: ResolvedWebhookTarget, + connect_timeout: float, + read_timeout: float, + max_response_bytes: int, + ) -> None: + self._target = target + self._connect_timeout = connect_timeout + self._read_timeout = read_timeout + self._max_response_bytes = max_response_bytes + + async def _connect( + self, target: ResolvedWebhookTarget + ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + hostname = target.url.hostname + if hostname is None: # guarded by resolve_webhook_target + raise WebhookTransportError("webhook hostname is unavailable") + context = ssl.create_default_context() if target.url.scheme == "https" else None + last_error: OSError | None = None + for address in target.addresses: + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection( + address, + target.port, + ssl=context, + server_hostname=hostname if context is not None else None, + ), + timeout=self._connect_timeout, + ) + verify_connected_peer(writer.get_extra_info("peername"), target.addresses) + return reader, writer + except (TimeoutError, OSError, ssl.SSLError, SSRFError) as error: + last_error = error if isinstance(error, OSError) else None + raise WebhookTransportError("webhook connection failed", transient=True) from last_error + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + target = self._target + if str(request.url) != target.url.geturl(): + raise WebhookTransportError("webhook target changed") + body = await request.aread() + if len(body) > 65_536: + raise WebhookTransportError("webhook body is too large") + reader, writer = await self._connect(target) + try: + raw_path = target.url.path or "/" + if target.url.query: + raw_path += "?" + target.url.query + headers = [(key, value) for key, value in request.headers.multi_items()] + header_names = {key.lower() for key, _ in headers} + if "host" not in header_names: + host = target.url.hostname or "" + headers.append(("Host", host)) + if "content-length" not in header_names: + headers.append(("Content-Length", str(len(body)))) + headers.append(("Connection", "close")) + serialized = [f"{request.method} {raw_path} HTTP/1.1\r\n".encode()] + serialized.extend(f"{key}: {value}\r\n".encode("ascii") for key, value in headers) + writer.write(b"".join(serialized) + b"\r\n" + body) + await asyncio.wait_for(writer.drain(), timeout=self._read_timeout) + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=self._read_timeout) + if len(head) > 16_384: + raise WebhookTransportError("webhook response headers are too large") + lines = head.decode("iso-8859-1").split("\r\n") + try: + _protocol, code, _reason = lines[0].split(" ", 2) + status_code = int(code) + except (IndexError, ValueError) as error: + raise WebhookTransportError("webhook response is malformed") from error + response_headers: list[tuple[str, str]] = [] + for line in lines[1:]: + if not line: + continue + key, separator, value = line.partition(":") + if not separator: + raise WebhookTransportError("webhook response headers are malformed") + response_headers.append((key.strip(), value.strip())) + content = await asyncio.wait_for( + reader.read(self._max_response_bytes + 1), timeout=self._read_timeout + ) + if len(content) > self._max_response_bytes: + raise WebhookTransportError("webhook response is too large") + return httpx.Response( + status_code, + headers=response_headers, + content=content, + request=request, + ) + except (TimeoutError, OSError, asyncio.IncompleteReadError) as error: + raise WebhookTransportError("webhook request failed", transient=True) from error + finally: + writer.close() + with __import__("contextlib").suppress(OSError): + await writer.wait_closed() + + +def webhook_headers( + event_id: str, + event_type: str, + timestamp: str, + body: bytes, + keys: Sequence[SigningMaterial], +) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [ + ("Content-Type", "application/json"), + ("X-Backup-Event-ID", event_id), + ("X-Backup-Event-Type", event_type), + ("X-Backup-Signature-Version", SIGNATURE_VERSION), + ("X-Backup-Timestamp", timestamp), + ] + headers.extend(("X-Backup-Signature", value) for value in signatures(timestamp, body, keys)) + return headers + + +async def deliver_webhook( + url: str, + body: bytes, + *, + event_id: str, + event_type: str, + timestamp: str, + keys: Sequence[SigningMaterial], + resolver: Resolver, + connect_timeout: float, + read_timeout: float, + max_response_bytes: int, +) -> WebhookResult: + if not keys: + raise WebhookTransportError("webhook subscription has no active signing key") + try: + # Resolve immediately before this individual attempt. The resulting + # addresses are passed to the transport, so it cannot rebind at connect. + target = await resolve_webhook_target(url, resolver) + except SSRFError as error: + raise WebhookTransportError("webhook_target_rejected") from error + transport = PinnedWebhookTransport( + target=target, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + max_response_bytes=max_response_bytes, + ) + headers = webhook_headers(event_id, event_type, timestamp, body, keys) + async with httpx.AsyncClient( + transport=transport, follow_redirects=False, trust_env=False + ) as client: + response = await client.post(url, content=body, headers=headers) + if 300 <= response.status_code < 400: + raise WebhookTransportError("redirect_rejected") + return WebhookResult(status_code=response.status_code, response_bytes=len(response.content)) diff --git a/backend/src/backup_tool/observability/__init__.py b/backend/src/backup_tool/observability/__init__.py new file mode 100644 index 0000000..b520077 --- /dev/null +++ b/backend/src/backup_tool/observability/__init__.py @@ -0,0 +1 @@ +"""Operational logging, metrics, and readiness primitives.""" diff --git a/backend/src/backup_tool/observability/health.py b/backend/src/backup_tool/observability/health.py new file mode 100644 index 0000000..88e350e --- /dev/null +++ b/backend/src/backup_tool/observability/health.py @@ -0,0 +1,39 @@ +"""Readiness checks shared by HTTP and background process roles.""" + +from __future__ import annotations + +import os + +from backup_tool.cli import build_alembic_config +from backup_tool.config import Settings +from backup_tool.db.engine import assert_schema_current, create_engine + + +class ReadinessError(RuntimeError): + """A dependency needed by the selected role is unavailable.""" + + +def _require_access(path: object, mode: int) -> None: + try: + candidate = path if isinstance(path, str) else str(path) + if not os.path.isdir(candidate) or not os.access(candidate, mode): + raise ReadinessError("required storage is unavailable") + except OSError as error: + raise ReadinessError("required storage is unavailable") from error + + +async def check_role_readiness(settings: Settings, role: str) -> None: + engine = create_engine(settings) + try: + await assert_schema_current(engine, build_alembic_config(settings)) + except Exception as error: + raise ReadinessError("metadata is unavailable") from error + finally: + await engine.dispose() + if role == "scheduler": + return + _require_access(settings.data_dir, os.R_OK | os.W_OK | os.X_OK) + for root in settings.repository_roots + settings.restore_roots: + _require_access(root, os.R_OK | os.W_OK | os.X_OK) + for root in settings.local_source_roots: + _require_access(root, os.R_OK | os.X_OK) diff --git a/backend/src/backup_tool/observability/logging.py b/backend/src/backup_tool/observability/logging.py new file mode 100644 index 0000000..f1e21e1 --- /dev/null +++ b/backend/src/backup_tool/observability/logging.py @@ -0,0 +1,39 @@ +"""JSON logging that keeps operational context machine-readable and secret-free.""" + +from __future__ import annotations + +import json +import logging +import sys +from datetime import UTC, datetime +from typing import Any + +_STANDARD_RECORD_KEYS = frozenset(logging.makeLogRecord({}).__dict__) + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "event": record.getMessage(), + "level": record.levelname.lower(), + "logger": record.name, + "timestamp": datetime.now(UTC).isoformat(), + } + for key, value in record.__dict__.items(): + if key not in _STANDARD_RECORD_KEYS and key not in {"message", "asctime"}: + payload[key] = value + return json.dumps(payload, default=str, separators=(",", ":"), sort_keys=True) + + +def configure_logging(role: str, level: str) -> None: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level) + logging.getLogger("backup_tool").info("role_started", extra={"role": role}) + + +def log_event(name: str, **fields: object) -> None: + logging.getLogger("backup_tool").info(name, extra=fields) diff --git a/backend/src/backup_tool/observability/metrics.py b/backend/src/backup_tool/observability/metrics.py new file mode 100644 index 0000000..93b5c78 --- /dev/null +++ b/backend/src/backup_tool/observability/metrics.py @@ -0,0 +1,102 @@ +"""Small dependency-free Prometheus exposition for the single-node appliance.""" + +from __future__ import annotations + +import os +import threading +from collections import defaultdict +from collections.abc import Iterable +from datetime import UTC, datetime + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backup_tool.config import Settings +from backup_tool.db.models import Backup, Execution, Repository, Schedule +from backup_tool.execution import ACTIVE_STATES + + +class Metrics: + def __init__(self) -> None: + self._lock = threading.Lock() + self._requests: dict[tuple[str, str, int], int] = defaultdict(int) + self._durations: dict[tuple[str, str], tuple[int, float]] = {} + + def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None: + route = path if path in {"/livez", "/readyz", "/metrics"} else "/api" + with self._lock: + self._requests[(method, route, status)] += 1 + count, total = self._durations.get((method, route), (0, 0.0)) + self._durations[(method, route)] = (count + 1, total + duration_seconds) + + def render(self, operational: Iterable[tuple[str, float]]) -> str: + lines = [ + "# HELP backup_tool_http_requests_total HTTP requests handled by the web role.", + "# TYPE backup_tool_http_requests_total counter", + ] + with self._lock: + for (method, path, status), value in sorted(self._requests.items()): + labels = f'method="{method}",path="{path}",status="{status}"' + lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}") + lines.extend( + [ + "# HELP backup_tool_http_request_duration_seconds HTTP request duration.", + "# TYPE backup_tool_http_request_duration_seconds summary", + ] + ) + for (method, path), (count, total) in sorted(self._durations.items()): + labels = f'method="{method}",path="{path}"' + lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}") + lines.append( + f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}" + ) + lines.extend(f"{name} {value}" for name, value in operational) + return "\n".join(lines) + "\n" + + +async def collect_operational_metrics( + settings: Settings, db: AsyncSession +) -> list[tuple[str, float]]: + now = datetime.now(UTC) + active = await db.scalar( + select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES)) + ) + stale = await db.scalar( + select(func.count()) + .select_from(Execution) + .where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now) + ) + failed = await db.scalar( + select(func.count()).select_from(Execution).where(Execution.state == "failed") + ) + corrupt = await db.scalar( + select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt") + ) + schedule_lag = await db.scalar( + select(func.min(Schedule.next_nominal_at)).where( + Schedule.enabled, Schedule.next_nominal_at.is_not(None) + ) + ) + values = [ + ("backup_tool_active_executions", active or 0), + ("backup_tool_stale_execution_leases", stale or 0), + ("backup_tool_failed_executions", failed or 0), + ("backup_tool_corrupt_backups", corrupt or 0), + ( + "backup_tool_schedule_lag_seconds", + max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0, + ), + ] + roots = list(settings.repository_roots) + list(settings.restore_roots) + for index, root in enumerate(roots): + try: + stats = os.statvfs(root) + except OSError: + continue + name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}' + values.append((name, stats.f_bavail * stats.f_frsize)) + unavailable = await db.scalar( + select(func.count()).select_from(Repository).where(Repository.state == "unavailable") + ) + values.append(("backup_tool_unavailable_repositories", unavailable or 0)) + return values diff --git a/backend/src/backup_tool/repository.py b/backend/src/backup_tool/repository.py index 4936715..27d0f59 100644 --- a/backend/src/backup_tool/repository.py +++ b/backend/src/backup_tool/repository.py @@ -4,12 +4,19 @@ import hashlib import json import os import shutil +import stat +from contextlib import suppress from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from sqlalchemy.ext.asyncio import AsyncSession + from backup_tool.config import Settings from backup_tool.ids import new_uuid7 +from backup_tool.security.repository_crypto import RepositoryKeyError, create_data_key class RepositoryError(ValueError): @@ -19,9 +26,15 @@ class RepositoryError(ValueError): @dataclass(frozen=True) class InitializedRepository: root: Path + repository_id: str format_version: int = 1 compression: str = "none" encryption: str = "none" + signing_key_id: str = "" + signing_public_key: str = "" + signing_key_path: Path | None = None + data_key_id: str | None = None + data_key_path: Path | None = None def blob_digest(content: bytes) -> str: @@ -47,14 +60,14 @@ def _contained(root: Path, relative_path: str) -> Path: def _canonical_payload(compression: str, encryption: str) -> dict[str, object]: - if compression != "none" or encryption != "none": + if compression != "none" or encryption not in {"none", "aes-256-gcm"}: raise RepositoryError("requested repository policy is unavailable") return { "repository_id": str(new_uuid7()), "format_version": 1, "digest_algorithm": "sha256", "compression": compression, - "encryption": {"mode": "none", "key_id": None}, + "encryption": {"mode": encryption, "key_id": None}, "created_at": datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z"), } @@ -63,7 +76,205 @@ def _canonical_json(payload: dict[str, object]) -> str: return json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" -def _assert_capacity(settings: Settings, root: Path) -> None: +def replace_active_data_key(root: Path, expected_key_id: str, new_key_id: str) -> None: + """Durably replace the active, non-secret epoch reference in repository metadata.""" + metadata = root / "repository.json" + try: + payload = json.loads(metadata.read_text(encoding="utf-8")) + encryption = payload.get("encryption") if isinstance(payload, dict) else None + if ( + not isinstance(encryption, dict) + or encryption.get("mode") != "aes-256-gcm" + or encryption.get("key_id") != expected_key_id + ): + raise ValueError + except (OSError, ValueError, json.JSONDecodeError) as error: + raise RepositoryError("repository metadata is invalid") from error + encryption["key_id"] = new_key_id + staging = root / f".repository.json.{os.urandom(8).hex()}.tmp" + try: + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(_canonical_json(payload)) + handle.flush() + os.fsync(handle.fileno()) + os.replace(staging, metadata) + descriptor = os.open(root, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as error: + raise RepositoryError("repository metadata cannot be updated") from error + finally: + staging.unlink(missing_ok=True) + + +_ROTATION_JOURNAL = ".key-rotation.json" + + +def begin_key_rotation( + root: Path, + repository_database_id: str, + repository_id: str, + old_key_id: str, + new_key_id: str, +) -> None: + """Persist an intent record before changing durable key-epoch state.""" + if not all((repository_database_id, repository_id, old_key_id, new_key_id)): + raise RepositoryError("repository rotation journal is invalid") + journal = root / _ROTATION_JOURNAL + payload: dict[str, object] = { + "new_key_id": new_key_id, + "old_key_id": old_key_id, + "repository_database_id": repository_database_id, + "repository_id": repository_id, + "version": 1, + } + try: + descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(_canonical_json(payload)) + handle.flush() + os.fsync(handle.fileno()) + descriptor = os.open(root, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as error: + raise RepositoryError("repository rotation is already in progress") from error + + +def read_key_rotation(root: Path) -> dict[str, str] | None: + journal = root / _ROTATION_JOURNAL + if not journal.exists(): + return None + try: + if journal.is_symlink() or stat.S_IMODE(journal.stat().st_mode) != 0o600: + raise ValueError + payload = json.loads(journal.read_text(encoding="utf-8")) + expected = { + "new_key_id", + "old_key_id", + "repository_database_id", + "repository_id", + "version", + } + if ( + not isinstance(payload, dict) + or set(payload) != expected + or payload.get("version") != 1 + or not all( + isinstance(payload[key], str) and payload[key] for key in expected - {"version"} + ) + ): + raise ValueError + return {key: payload[key] for key in expected - {"version"}} + except (OSError, ValueError, json.JSONDecodeError) as error: + raise RepositoryError("repository rotation journal is invalid") from error + + +def finish_key_rotation(root: Path) -> None: + journal = root / _ROTATION_JOURNAL + try: + journal.unlink() + descriptor = os.open(root, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as error: + raise RepositoryError("repository rotation journal cannot be cleared") from error + + +async def reconcile_key_rotations(settings: Settings, db: AsyncSession) -> int: + """Converge a journaled key rotation after a process crash. + + A rotation whose DB transaction did not commit remains on the old active + epoch. A committed transaction deterministically advances repository.json. + """ + from sqlalchemy import select + + from backup_tool.db.models import Repository, RepositoryDataKeyEpoch + from backup_tool.security.repository_crypto import load_data_key + + repositories = list((await db.scalars(select(Repository))).all()) + reconciled = 0 + for repository in repositories: + if repository.encryption != "aes-256-gcm": + continue + root = Path(repository.root) + journal = read_key_rotation(root) + if journal is None: + continue + inspected = inspect_repository(settings, root) + if ( + journal["repository_database_id"] != repository.id + or journal["repository_id"] != inspected.repository_id + or repository.active_data_key_id is None + ): + raise RepositoryError("repository rotation journal does not match metadata") + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch).where( + RepositoryDataKeyEpoch.repository_id == repository.id + ) + ) + ).all() + ) + active = [epoch for epoch in epochs if epoch.state == "active"] + old_key_id = journal["old_key_id"] + new_key_id = journal["new_key_id"] + if len(active) != 1: + raise RepositoryError("repository rotation epochs are invalid") + if repository.active_data_key_id == old_key_id and active[0].key_id == old_key_id: + if inspected.data_key_id != old_key_id: + raise RepositoryError("repository rotation metadata is invalid") + new_path = ( + settings.data_dir + / "repository-data-keys" + / (f"{inspected.repository_id}.{new_key_id}.key") + ) + # Clear the durable intent before deleting an unreferenced key. A + # crash afterward leaves only an orphaned key, not a journal whose + # retry depends on a key that no longer exists. + if new_path.exists() or new_path.is_symlink(): + try: + load_data_key(settings, inspected.repository_id, new_key_id) + except RepositoryKeyError as error: + raise RepositoryError("repository rotation key is unavailable") from error + finish_key_rotation(inspected.root) + if new_path.exists() or new_path.is_symlink(): + # The journal is already gone, so a cleanup failure only leaves + # an unreferenced key and must not disable the old-active epoch. + with suppress(OSError): + new_path.unlink() + descriptor = os.open(new_path.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + reconciled += 1 + continue + if repository.active_data_key_id == new_key_id and active[0].key_id == new_key_id: + try: + load_data_key(settings, inspected.repository_id, new_key_id) + except RepositoryKeyError as error: + raise RepositoryError("repository rotation key is unavailable") from error + if inspected.data_key_id == old_key_id: + replace_active_data_key(inspected.root, old_key_id, new_key_id) + elif inspected.data_key_id != new_key_id: + raise RepositoryError("repository rotation metadata is invalid") + finish_key_rotation(inspected.root) + reconciled += 1 + continue + raise RepositoryError("repository rotation state is invalid") + return reconciled + + +def assert_capacity(settings: Settings, root: Path) -> None: capacity_root = root while not capacity_root.exists(): parent = capacity_root.parent @@ -76,13 +287,57 @@ def _assert_capacity(settings: Settings, root: Path) -> None: raise RepositoryError("repository root does not meet minimum free capacity") +def _signing_key_directory(settings: Settings) -> Path: + directory = settings.data_dir / "repository-keys" + directory.mkdir(mode=0o700, exist_ok=True) + if directory.is_symlink() or not directory.is_dir(): + raise RepositoryError("repository signing key directory is unsafe") + if stat.S_IMODE(directory.stat().st_mode) != 0o700: + raise RepositoryError("repository signing key directory permissions must be 0700") + return directory + + +def _write_private_key_staging(directory: Path, repository_id: str, private_key: bytes) -> Path: + staging = directory / f".{repository_id}.{os.urandom(8).hex()}.tmp" + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(private_key) + handle.flush() + os.fsync(handle.fileno()) + return staging + + def initialize( settings: Settings, relative_path: str, compression: str, encryption: str ) -> InitializedRepository: payload = _canonical_payload(compression, encryption) + repository_id = str(payload["repository_id"]) + data_key_id: str | None = None + data_key_path: Path | None = None + if encryption == "aes-256-gcm": + try: + data_key_id, data_key_path = create_data_key(settings, repository_id) + except RepositoryKeyError as error: + raise RepositoryError("repository data key is unavailable") from error + payload["encryption"] = {"mode": encryption, "key_id": data_key_id} + private_key = Ed25519PrivateKey.generate() + private_bytes = private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + public_bytes = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + signing_key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}" + key_directory = _signing_key_directory(settings) + key_path = key_directory / f"{repository_id}.ed25519" + key_staging = _write_private_key_staging(key_directory, repository_id, private_bytes) root = _contained(settings.repository_roots[0], relative_path) - _assert_capacity(settings, root.parent) + assert_capacity(settings, root.parent) if root.exists(): + key_staging.unlink(missing_ok=True) raise RepositoryError("repository path already exists") staging = root.with_name(f".{root.name}.staging-{os.urandom(8).hex()}") try: @@ -94,17 +349,106 @@ def initialize( with metadata.open("rb") as handle: os.fsync(handle.fileno()) os.replace(staging, root) + os.replace(key_staging, key_path) except Exception: shutil.rmtree(staging, ignore_errors=True) + shutil.rmtree(root, ignore_errors=True) + key_staging.unlink(missing_ok=True) + key_path.unlink(missing_ok=True) + if data_key_path is not None: + data_key_path.unlink(missing_ok=True) raise - return InitializedRepository(root=root, compression=compression, encryption=encryption) + return InitializedRepository( + root=root, + repository_id=repository_id, + compression=compression, + encryption=encryption, + signing_key_id=signing_key_id, + signing_public_key=public_bytes.hex(), + signing_key_path=key_path, + data_key_id=data_key_id, + data_key_path=data_key_path, + ) -def remove_repository(root: Path) -> None: - try: +def remove_repository( + root: Path, + signing_key_path: Path | None = None, + data_key_path: Path | None = None, +) -> None: + try: # noqa: SIM105 - cleanup must not race a concurrent remover shutil.rmtree(root) except FileNotFoundError: - return + pass + for key_path in (signing_key_path, data_key_path): + if key_path is None: + continue + try: # noqa: SIM105 - cleanup must not race a concurrent remover + key_path.unlink() + except FileNotFoundError: + pass + + +def install_signing_key( + settings: Settings, + repository_id: str, + expected_key_id: str, + expected_public_key: str, + private_bytes: bytes, +) -> Path: + """Install a recovered signing key once after checking its public trust anchor.""" + if len(private_bytes) != 32 or "/" in repository_id: + raise RepositoryError("repository signing key is invalid") + try: + private_key = Ed25519PrivateKey.from_private_bytes(private_bytes) + public_bytes = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + except ValueError as error: + raise RepositoryError("repository signing key is invalid") from error + key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}" + if key_id != expected_key_id or public_bytes.hex() != expected_public_key: + raise RepositoryError("repository signing key is invalid") + path = _signing_key_directory(settings) / f"{repository_id}.ed25519" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(private_bytes) + handle.flush() + os.fsync(handle.fileno()) + if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600: + path.unlink(missing_ok=True) + raise RepositoryError("repository signing key is unsafe") + except OSError as error: + raise RepositoryError("repository signing key cannot be installed") from error + return path + + +def load_signing_key( + settings: Settings, + repository_id: str, + expected_key_id: str, + expected_public_key: str, +) -> Ed25519PrivateKey: + key_path = _signing_key_directory(settings) / f"{repository_id}.ed25519" + try: + if key_path.is_symlink() or stat.S_IMODE(key_path.stat().st_mode) != 0o600: + raise RepositoryError("repository signing key is unsafe") + private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes()) + except (OSError, ValueError) as error: + raise RepositoryError("repository signing key is unreadable") from error + public_key = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + key_id = f"ed25519-{hashlib.sha256(public_key).hexdigest()[:32]}" + if key_id != expected_key_id or public_key.hex() != expected_public_key: + raise RepositoryError("repository signing key does not match its trust anchor") + return private_key def inspect_repository(settings: Settings, root: Path) -> InitializedRepository: @@ -140,7 +484,14 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository: ): raise RepositoryError("repository metadata is not canonical") encryption = payload.get("encryption") - if payload.get("compression") != "none" or encryption != {"mode": "none", "key_id": None}: + if payload.get("compression") != "none" or not isinstance(encryption, dict): + raise RepositoryError("repository metadata is invalid") + mode = encryption.get("mode") + key_id = encryption.get("key_id") + valid_policy = (mode == "none" and key_id is None) or ( + mode == "aes-256-gcm" and isinstance(key_id, str) and bool(key_id) + ) + if not valid_policy: raise RepositoryError("repository metadata is invalid") try: from uuid import UUID @@ -152,4 +503,10 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository: datetime.fromisoformat(created_at[:-1] + "+00:00") except (ValueError, TypeError) as error: raise RepositoryError("repository metadata is invalid") from error - return InitializedRepository(root=resolved_root, compression="none", encryption="none") + return InitializedRepository( + root=resolved_root, + repository_id=str(payload["repository_id"]), + compression="none", + encryption=str(encryption["mode"]), + data_key_id=key_id if isinstance(key_id, str) else None, + ) diff --git a/backend/src/backup_tool/retention.py b/backend/src/backup_tool/retention.py new file mode 100644 index 0000000..523a92e --- /dev/null +++ b/backend/src/backup_tool/retention.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Protocol + + +class RetentionError(ValueError): + pass + + +class BackupLike(Protocol): + id: str + created_at: datetime + pinned: bool + tombstoned_at: datetime | None + + +@dataclass(frozen=True) +class RetentionPolicy: + keep_last: int = 1 + keep_days: int = 0 + keep_daily: int = 0 + keep_weekly: int = 0 + keep_monthly: int = 0 + + @classmethod + def from_dict(cls, value: dict[str, object]) -> RetentionPolicy: + allowed = {"keep_last", "keep_days", "keep_daily", "keep_weekly", "keep_monthly"} + if set(value) - allowed: + raise RetentionError("retention policy contains unknown keys") + kwargs: dict[str, int] = {} + for key in allowed: + raw = value.get(key, 0 if key != "keep_last" else 1) + if not isinstance(raw, int) or isinstance(raw, bool) or raw < 0: + raise RetentionError("retention values must be non-negative integers") + kwargs[key] = raw + return cls(**kwargs) + + +def retained_ids( + backups: Iterable[BackupLike], policy: RetentionPolicy, now: datetime | None = None +) -> set[str]: + """Return union retention set; the newest non-tombstoned backup is always protected.""" + reference = (now or datetime.now(UTC)).astimezone(UTC) + items = sorted( + (item for item in backups if item.tombstoned_at is None), + key=lambda item: item.created_at.astimezone(UTC), + reverse=True, + ) + if not items: + return set() + kept = {items[0].id} + kept.update(item.id for item in items[: policy.keep_last]) + kept.update(item.id for item in items if item.pinned) + if policy.keep_days: + cutoff = reference - timedelta(days=policy.keep_days) + kept.update(item.id for item in items if item.created_at.astimezone(UTC) >= cutoff) + for count, key in ( + (policy.keep_daily, lambda stamp: stamp.date()), + (policy.keep_weekly, lambda stamp: stamp.isocalendar()[:2]), + (policy.keep_monthly, lambda stamp: (stamp.year, stamp.month)), + ): + buckets: set[object] = set() + for item in items: + stamp = item.created_at.astimezone(UTC) + bucket = key(stamp) + if len(buckets) >= count and bucket not in buckets: + continue + buckets.add(bucket) + if bucket in buckets: + kept.add(item.id) + return kept diff --git a/backend/src/backup_tool/scheduler.py b/backend/src/backup_tool/scheduler.py new file mode 100644 index 0000000..4fdb1a9 --- /dev/null +++ b/backend/src/backup_tool/scheduler.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import asyncio +import contextlib +import signal +from datetime import UTC, datetime +from typing import cast +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from apscheduler.triggers.cron import CronTrigger # type: ignore[import-untyped] +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Schedule +from backup_tool.execution import EnqueueError, enqueue +from backup_tool.notifications.events import emit_event +from backup_tool.observability.logging import configure_logging, log_event + + +class ScheduleError(ValueError): + pass + + +def next_nominal(cron: str, timezone: str, after: datetime | None = None) -> datetime: + if len(cron.split()) != 5: + raise ScheduleError("cron must contain exactly five fields") + try: + zone = ZoneInfo(timezone) + trigger = CronTrigger.from_crontab(cron, timezone=zone) + except (ValueError, ZoneInfoNotFoundError) as error: + raise ScheduleError("cron or timezone is invalid") from error + reference = (after or datetime.now(UTC)).astimezone(zone) + next_run = trigger.get_next_fire_time(None, reference) + if next_run is None: + raise ScheduleError("cron has no future occurrence") + return cast(datetime, next_run.astimezone(UTC)) + + +class SchedulerService: + """Dedicated scheduler role using the same transactional enqueue service.""" + + def __init__(self, settings: Settings) -> None: + self.engine = create_engine(settings) + self.sessions = async_sessionmaker(self.engine, expire_on_commit=False) + self._stopping = asyncio.Event() + + async def run_once(self) -> int: + async with self.sessions() as db: + return await deliver_due(db) + + async def run(self) -> None: + while not self._stopping.is_set(): + await self.run_once() + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self._stopping.wait(), timeout=0.25) + await self.engine.dispose() + + def stop(self) -> None: + log_event("role_stopping", role="scheduler") + self._stopping.set() + + +def run_scheduler(settings: Settings) -> int: + configure_logging("scheduler", settings.log_level) + service = SchedulerService(settings) + loop = asyncio.new_event_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + with contextlib.suppress(NotImplementedError): + loop.add_signal_handler(sig, service.stop) + try: + loop.run_until_complete(service.run()) + finally: + loop.close() + log_event("role_stopped", role="scheduler") + return 0 + + +async def deliver_due(db: AsyncSession, now: datetime | None = None) -> int: + current = now or datetime.now(UTC) + schedules = list( + ( + await db.scalars( + select(Schedule).where( + Schedule.enabled, + Schedule.next_nominal_at.is_not(None), + Schedule.next_nominal_at <= current, + ) + ) + ).all() + ) + delivered = 0 + for schedule in schedules: + nominal = schedule.next_nominal_at + if nominal is None: + continue + schedule.next_nominal_at = next_nominal(schedule.cron, schedule.timezone, nominal) + if (current - nominal).total_seconds() > schedule.misfire_grace_seconds: + schedule.last_enqueue_outcome = "misfire" + await emit_event( + db, + "schedule.occurrence_misfired", + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"outcome": "misfire"}, + deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:misfire", + ) + continue + try: + await enqueue( + db, + schedule.job_id, + "schedule", + schedule_id=schedule.id, + nominal_run_at=nominal, + ) + except EnqueueError as error: + schedule.last_enqueue_outcome = error.code + await emit_event( + db, + "schedule.occurrence_blocked", + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"reason_code": error.code}, + deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:blocked", + ) + else: + schedule.last_enqueue_outcome = "enqueued" + await emit_event( + db, + "schedule.occurrence_enqueued", + correlation_id=schedule.id, + resource={"schedule_id": schedule.id, "job_id": schedule.job_id}, + payload={"outcome": "enqueued"}, + deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:enqueued", + ) + delivered += 1 + await db.commit() + return delivered diff --git a/backend/src/backup_tool/security/recovery_bundle.py b/backend/src/backup_tool/security/recovery_bundle.py new file mode 100644 index 0000000..98883ec --- /dev/null +++ b/backend/src/backup_tool/security/recovery_bundle.py @@ -0,0 +1,249 @@ +"""Offline, passphrase-protected recovery bundle codec. + +The binary format is deliberately small and versioned so validation can reject +unsupported inputs before attempting expensive password derivation. Every +failure while parsing or authenticating a bundle is reported as the same error +so callers cannot distinguish a malformed bundle from a wrong passphrase. +""" + +from __future__ import annotations + +import json +import os +import stat +import struct +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from argon2.low_level import Type, hash_secret_raw +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class RecoveryBundleError(ValueError): + """A non-disclosing recovery bundle validation failure.""" + + +class RecoveryBundlePathError(ValueError): + """A requested recovery bundle path cannot be used safely.""" + + +_MAGIC = b"BTREC" +_VERSION = 1 +_KDF_ARGON2ID = 1 +_SALT_BYTES = 16 +_NONCE_BYTES = 12 +_KEY_BYTES = 32 +_TAG_BYTES = 16 +_TIME_COST = 3 +_MEMORY_COST_KIB = 65_536 +_PARALLELISM = 1 +_MAX_PASSPHRASE_BYTES = 4_096 +_MAX_PLAINTEXT_BYTES = 8 * 1024 * 1024 +# magic, version, KDF id, Argon2 time/memory/parallelism, salt/nonce lengths, +# and the AES-GCM ciphertext (including tag) length. +_HEADER = struct.Struct(">5sBBIIHBBQ") +_MAX_BUNDLE_BYTES = _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _MAX_PLAINTEXT_BYTES + _TAG_BYTES +_ERROR = "recovery bundle is invalid" + + +def _invalid() -> RecoveryBundleError: + return RecoveryBundleError(_ERROR) + + +def _canonical_json(payload: Mapping[str, Any]) -> bytes: + try: + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + except (TypeError, ValueError) as error: + raise _invalid() from error + if not encoded or len(encoded) > _MAX_PLAINTEXT_BYTES: + raise _invalid() + return encoded + + +def _passphrase(value: bytes) -> bytes: + if not isinstance(value, bytes) or not value or len(value) > _MAX_PASSPHRASE_BYTES: + raise _invalid() + return value + + +def _derive_key(passphrase: bytes, salt: bytes) -> bytes: + return hash_secret_raw( + secret=passphrase, + salt=salt, + time_cost=_TIME_COST, + memory_cost=_MEMORY_COST_KIB, + parallelism=_PARALLELISM, + hash_len=_KEY_BYTES, + type=Type.ID, + ) + + +def encrypt_bundle(payload: Mapping[str, Any], passphrase: bytes) -> bytes: + """Serialize and encrypt a canonical recovery payload as a BTREC v1 bundle.""" + plaintext = _canonical_json(payload) + secret = _passphrase(passphrase) + salt = os.urandom(_SALT_BYTES) + nonce = os.urandom(_NONCE_BYTES) + ciphertext_length = len(plaintext) + _TAG_BYTES + header = _HEADER.pack( + _MAGIC, + _VERSION, + _KDF_ARGON2ID, + _TIME_COST, + _MEMORY_COST_KIB, + _PARALLELISM, + _SALT_BYTES, + _NONCE_BYTES, + ciphertext_length, + ) + ciphertext = AESGCM(_derive_key(secret, salt)).encrypt(nonce, plaintext, header) + return header + salt + nonce + ciphertext + + +def decrypt_bundle(encoded: bytes, passphrase: bytes) -> dict[str, Any]: + """Authenticate and decode a BTREC v1 bundle without disclosing failure cause.""" + try: + if not isinstance(encoded, bytes) or len(encoded) > _MAX_BUNDLE_BYTES: + raise _invalid() + if len(encoded) < _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _TAG_BYTES: + raise _invalid() + ( + magic, + version, + kdf_id, + time_cost, + memory_cost, + parallelism, + salt_length, + nonce_length, + ciphertext_length, + ) = _HEADER.unpack(encoded[: _HEADER.size]) + if ( + magic != _MAGIC + or version != _VERSION + or kdf_id != _KDF_ARGON2ID + or time_cost != _TIME_COST + or memory_cost != _MEMORY_COST_KIB + or parallelism != _PARALLELISM + or salt_length != _SALT_BYTES + or nonce_length != _NONCE_BYTES + or ciphertext_length < _TAG_BYTES + or ciphertext_length > _MAX_PLAINTEXT_BYTES + _TAG_BYTES + or len(encoded) != _HEADER.size + salt_length + nonce_length + ciphertext_length + ): + raise _invalid() + secret = _passphrase(passphrase) + salt_start = _HEADER.size + nonce_start = salt_start + salt_length + ciphertext_start = nonce_start + nonce_length + plaintext = AESGCM(_derive_key(secret, encoded[salt_start:nonce_start])).decrypt( + encoded[nonce_start:ciphertext_start], + encoded[ciphertext_start:], + encoded[: _HEADER.size], + ) + if not plaintext or len(plaintext) > _MAX_PLAINTEXT_BYTES: + raise _invalid() + payload = json.loads(plaintext.decode("utf-8")) + if not isinstance(payload, dict): + raise _invalid() + # Reject non-canonical encodings to make catalog serialization deterministic. + if _canonical_json(payload) != plaintext: + raise _invalid() + return payload + except ( + InvalidTag, + UnicodeDecodeError, + json.JSONDecodeError, + struct.error, + ValueError, + ) as error: + if isinstance(error, RecoveryBundleError): + raise error + raise _invalid() from error + + +def _check_path_components(path: Path) -> None: + if not path.is_absolute() or path.name in {"", ".", ".."}: + raise RecoveryBundlePathError("recovery bundle path is unsafe") + current = Path(path.anchor) + for component in path.parts[1:-1]: + current /= component + try: + info = current.lstat() + except OSError as error: + raise RecoveryBundlePathError("recovery bundle path is unsafe") from error + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise RecoveryBundlePathError("recovery bundle path is unsafe") + + +def write_bundle_exclusive(path: Path, encoded: bytes) -> None: + """Write a bundle once with restrictive permissions and no symlink following.""" + if not isinstance(encoded, bytes) or not encoded or len(encoded) > _MAX_BUNDLE_BYTES: + raise RecoveryBundlePathError("recovery bundle output is unsafe") + _check_path_components(path) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + raise RecoveryBundlePathError("recovery bundle output is unsafe") from error + try: + info = path.lstat() + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISREG(info.st_mode) + or stat.S_IMODE(info.st_mode) != 0o600 + ): + path.unlink(missing_ok=True) + raise RecoveryBundlePathError("recovery bundle output is unsafe") + except OSError as error: + raise RecoveryBundlePathError("recovery bundle output is unsafe") from error + + +def read_bundle_file(path: Path) -> bytes: + """Read a regular, non-symlink bundle with a bounded size.""" + _check_path_components(path) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as handle: + info = os.fstat(handle.fileno()) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_size <= 0 + or info.st_size > _MAX_BUNDLE_BYTES + ): + raise RecoveryBundlePathError("recovery bundle input is unsafe") + return handle.read() + except RecoveryBundlePathError: + raise + except OSError as error: + raise RecoveryBundlePathError("recovery bundle input is unsafe") from error + + +def read_passphrase_fd(fd: int) -> bytes: + """Read one newline-terminated passphrase from an inherited file descriptor.""" + if not isinstance(fd, int) or fd < 0: + raise RecoveryBundleError("recovery passphrase is unavailable") + try: + value = os.read(fd, _MAX_PASSPHRASE_BYTES + 2) + except OSError as error: + raise RecoveryBundleError("recovery passphrase is unavailable") from error + if value.endswith(b"\r\n"): + value = value[:-2] + elif value.endswith(b"\n"): + value = value[:-1] + if not value or len(value) > _MAX_PASSPHRASE_BYTES: + raise RecoveryBundleError("recovery passphrase is unavailable") + return value diff --git a/backend/src/backup_tool/security/repository_crypto.py b/backend/src/backup_tool/security/repository_crypto.py new file mode 100644 index 0000000..0755631 --- /dev/null +++ b/backend/src/backup_tool/security/repository_crypto.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import os +import stat +from pathlib import Path + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from backup_tool.config import Settings +from backup_tool.ids import new_uuid7 + + +class RepositoryKeyError(ValueError): + pass + + +def _directory(settings: Settings) -> Path: + directory = settings.data_dir / "repository-data-keys" + directory.mkdir(mode=0o700, exist_ok=True) + if ( + directory.is_symlink() + or not directory.is_dir() + or stat.S_IMODE(directory.stat().st_mode) != 0o700 + ): + raise RepositoryKeyError("repository data key directory is unsafe") + return directory + + +def create_data_key(settings: Settings, repository_id: str) -> tuple[str, Path]: + key_id = str(new_uuid7()) + directory = _directory(settings) + path = directory / f"{repository_id}.{key_id}.key" + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(os.urandom(32)) + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + raise RepositoryKeyError("repository data key cannot be created") from error + if stat.S_IMODE(path.stat().st_mode) != 0o600 or path.is_symlink(): + path.unlink(missing_ok=True) + raise RepositoryKeyError("repository data key is unsafe") + return key_id, path + + +def install_data_key(settings: Settings, repository_id: str, key_id: str, key: bytes) -> Path: + """Install recovered key material once; never replace an existing key file.""" + if len(key) != 32 or "/" in repository_id or "/" in key_id: + raise RepositoryKeyError("repository data key is invalid") + path = _directory(settings) / f"{repository_id}.{key_id}.key" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(key) + handle.flush() + os.fsync(handle.fileno()) + if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600: + path.unlink(missing_ok=True) + raise RepositoryKeyError("repository data key is unsafe") + except OSError as error: + raise RepositoryKeyError("repository data key cannot be installed") from error + return path + + +def object_aad(repository_id: str, key_id: str, kind: str, identity: str) -> bytes: + if kind not in {"blob", "manifest"} or not all((repository_id, key_id, identity)): + raise RepositoryKeyError("encrypted object metadata is invalid") + return f"BTENC:1:{repository_id}:{key_id}:{kind}:{identity}".encode() + + +def encrypt_object(key: bytes, aad: bytes, plaintext: bytes) -> bytes: + if len(key) != 32: + raise RepositoryKeyError("repository data key is unavailable") + nonce = os.urandom(12) + return b"BTENC\x01" + nonce + AESGCM(key).encrypt(nonce, plaintext, aad) + + +def decrypt_object(key: bytes, aad: bytes, stored: bytes) -> bytes: + if len(key) != 32 or not stored.startswith(b"BTENC\x01") or len(stored) < 35: + raise RepositoryKeyError("encrypted object is invalid") + try: + return AESGCM(key).decrypt(stored[6:18], stored[18:], aad) + except InvalidTag as error: + raise RepositoryKeyError("encrypted object is invalid") from error + + +def load_data_key(settings: Settings, repository_id: str, key_id: str) -> bytes: + path = _directory(settings) / f"{repository_id}.{key_id}.key" + try: + if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600: + raise RepositoryKeyError("repository data key is unsafe") + key = path.read_bytes() + except OSError as error: + raise RepositoryKeyError("repository data key is unavailable") from error + if len(key) != 32: + raise RepositoryKeyError("repository data key is unavailable") + return key diff --git a/backend/src/backup_tool/security/ssrf.py b/backend/src/backup_tool/security/ssrf.py new file mode 100644 index 0000000..e3bf389 --- /dev/null +++ b/backend/src/backup_tool/security/ssrf.py @@ -0,0 +1,121 @@ +"""Fail-closed, DNS-rebinding-resistant webhook egress validation.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import socket +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from urllib.parse import SplitResult, urlsplit + + +class SSRFError(ValueError): + """The callback target is not safe for the notification egress boundary.""" + + +Resolver = Callable[[str, int], Awaitable[Sequence[str]]] +_ALLOWED_PORTS = frozenset({80, 443, 8080, 8443}) + + +def _is_global(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """ipaddress.is_global misses some policy-important mapped/special ranges.""" + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None: + return _is_global(address.ipv4_mapped) + return bool(address.is_global) and not any( + ( + address.is_loopback, + address.is_private, + address.is_link_local, + address.is_multicast, + address.is_unspecified, + address.is_reserved, + ) + ) + + +def validate_webhook_url(value: str) -> SplitResult: + try: + parsed = urlsplit(value) + port = parsed.port + except ValueError as error: + raise SSRFError("webhook URL is malformed") from error + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise SSRFError("webhook URL must be absolute HTTP(S)") + if parsed.username is not None or parsed.password is not None or parsed.fragment: + raise SSRFError("webhook URL credentials and fragments are forbidden") + if len(value) > 2048 or any(character.isspace() for character in value): + raise SSRFError("webhook URL is malformed") + if port is not None and port not in _ALLOWED_PORTS: + raise SSRFError("webhook URL port is not permitted") + try: + ipaddress.ip_address(parsed.hostname) + except ValueError: + pass + else: + # Callback literals are never accepted: names are resolved immediately + # before every request and connected addresses are pinned/rechecked. + raise SSRFError("literal IP webhook targets are forbidden") + return parsed + + +async def system_resolver(hostname: str, port: int) -> Sequence[str]: + records = await asyncio.get_running_loop().getaddrinfo( + hostname, port, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP + ) + addresses: set[str] = set() + for record in records: + socket_address = record[4] + if socket_address and isinstance(socket_address[0], str): + addresses.add(socket_address[0]) + return tuple(sorted(addresses)) + + +async def resolve_public_addresses( + hostname: str, port: int, resolver: Resolver = system_resolver +) -> tuple[str, ...]: + try: + candidates = tuple(await resolver(hostname, port)) + except (TimeoutError, OSError) as error: + raise SSRFError("webhook DNS resolution failed") from error + if not candidates: + raise SSRFError("webhook hostname has no addresses") + approved: list[str] = [] + for candidate in candidates: + try: + address = ipaddress.ip_address(candidate) + except ValueError as error: + raise SSRFError("webhook resolver returned an invalid address") from error + if not _is_global(address): + # One unsafe answer poisons the hostname, including mixed public/private. + raise SSRFError("webhook hostname resolves to a non-public address") + approved.append(str(address)) + return tuple(approved) + + +@dataclass(frozen=True) +class ResolvedWebhookTarget: + url: SplitResult + port: int + addresses: tuple[str, ...] + + +async def resolve_webhook_target( + value: str, resolver: Resolver = system_resolver +) -> ResolvedWebhookTarget: + parsed = validate_webhook_url(value) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + addresses = await resolve_public_addresses(parsed.hostname or "", port, resolver) + return ResolvedWebhookTarget(url=parsed, port=port, addresses=addresses) + + +def verify_connected_peer(peername: object, approved: Sequence[str]) -> str: + if not isinstance(peername, tuple) or not peername or not isinstance(peername[0], str): + raise SSRFError("webhook peer address is unavailable") + try: + peer = str(ipaddress.ip_address(peername[0])) + except ValueError as error: + raise SSRFError("webhook peer address is invalid") from error + if peer not in approved: + raise SSRFError("webhook peer changed after DNS resolution") + return peer diff --git a/backend/src/backup_tool/snapshot.py b/backend/src/backup_tool/snapshot.py new file mode 100644 index 0000000..ac68112 --- /dev/null +++ b/backend/src/backup_tool/snapshot.py @@ -0,0 +1,1080 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +from contextlib import suppress +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from sqlalchemy import desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backup_tool.adapters import LocalAdapter, SourceError, SourceReader +from backup_tool.config import Settings +from backup_tool.db.models import ( + Backup, + Execution, + Job, + Repository, + RepositoryDataKeyEpoch, + Restore, + Source, +) +from backup_tool.exclusions import ExclusionError, matches +from backup_tool.faults import FaultInjector, NoFault +from backup_tool.ids import new_uuid7 +from backup_tool.repository import ( + InitializedRepository, + RepositoryError, + assert_capacity, + inspect_repository, + load_signing_key, +) +from backup_tool.security.repository_crypto import ( + RepositoryKeyError, + decrypt_object, + encrypt_object, + load_data_key, + object_aad, +) +from backup_tool.security.secrets import EnvelopeCipher +from backup_tool.ssh_adapter import SSHAdapter +from backup_tool.ssh_source import SSH_PRIVATE_KEY_PURPOSE, SSHSourcePublicConfig + + +class SnapshotError(ValueError): + def __init__(self, message: str, *, reason_code: str = "transient_io") -> None: + super().__init__(message) + self.reason_code = reason_code + + +class SnapshotIntegrityError(SnapshotError): + pass + + +def _canonical_json(payload: dict[str, Any]) -> bytes: + return (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def _timestamp() -> str: + return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def _hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _private_directory(path: Path, *, exist_ok: bool = False) -> None: + try: + path.mkdir(mode=0o700, exist_ok=exist_ok) + metadata = path.lstat() + except OSError as error: + raise SnapshotError("snapshot staging path is unavailable") from error + if ( + path.is_symlink() + or not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o700 + ): + raise SnapshotError("snapshot staging path is unsafe") + + +def _private_file_descriptor(path: Path) -> int: + descriptor: int | None = None + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, 0o600) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600: + raise OSError("snapshot staging file has unsafe permissions") + return descriptor + except OSError as error: + if descriptor is not None: + with suppress(OSError): + os.close(descriptor) + path.unlink(missing_ok=True) + raise SnapshotError("snapshot staging file is unavailable") from error + + +def _matches_blob_digest( + blob_path: Path, digest: str, encryption_key: bytes | None, aad: bytes | None +) -> bool: + try: + if encryption_key is None: + return _hash_file(blob_path) == digest + if aad is None: + return False + plaintext = decrypt_object(encryption_key, aad, blob_path.read_bytes()) + return hashlib.sha256(plaintext).hexdigest() == digest + except (OSError, RepositoryKeyError): + return False + + +def _install_blob( + staged_blob: Path, + blob_path: Path, + digest: str, + fault_injector: FaultInjector, + *, + encryption_key_id: str | None = None, + encryption_keys: dict[str, bytes] | None = None, + repository_id: str | None = None, +) -> str | None: + blob_path.parent.mkdir(parents=True, exist_ok=True) + if blob_path.is_symlink(): + raise SnapshotError("repository blob path is unsafe") + fault_injector.hit("blob.before_rename") + try: + os.link(staged_blob, blob_path) + except FileExistsError as error: + if encryption_keys is None: + if not _matches_blob_digest(blob_path, digest, None, None): + raise SnapshotError("existing repository blob does not match its digest") from error + used_key_id = None + else: + if repository_id is None: + raise SnapshotError("repository encryption metadata is invalid") from error + used_key_id = next( + ( + key_id + for key_id, key in encryption_keys.items() + if _matches_blob_digest( + blob_path, + digest, + key, + object_aad(repository_id, key_id, "blob", digest), + ) + ), + None, + ) + if used_key_id is None: + raise SnapshotError("existing repository blob does not match its digest") from error + else: + with blob_path.open("rb") as handle: + os.fsync(handle.fileno()) + _fsync_directory(blob_path.parent) + used_key_id = encryption_key_id + staged_blob.unlink(missing_ok=True) + return used_key_id + + +async def _copy_file( + adapter: SourceReader, + path: str, + staged_blob: Path, + fault_injector: FaultInjector, +) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + fault_injector.hit("blob.before_write") + with os.fdopen(_private_file_descriptor(staged_blob), "wb") as handle: + async for chunk in adapter.open_content(path): + digest.update(chunk) + size += len(chunk) + handle.write(chunk) + fault_injector.hit("blob.after_write") + handle.flush() + os.fsync(handle.fileno()) + fault_injector.hit("blob.after_fsync") + return digest.hexdigest(), size + + +def _encrypt_staged_blob(staged_blob: Path, key: bytes, aad: bytes) -> None: + encrypted_blob = staged_blob.with_suffix(".encrypted") + try: + with os.fdopen(_private_file_descriptor(encrypted_blob), "wb") as handle: + handle.write(encrypt_object(key, aad, staged_blob.read_bytes())) + handle.flush() + os.fsync(handle.fileno()) + os.replace(encrypted_blob, staged_blob) + except (OSError, RepositoryKeyError) as error: + raise SnapshotError("repository blob encryption failed") from error + finally: + encrypted_blob.unlink(missing_ok=True) + + +def _assert_repository_encryption_metadata( + repository: Repository, inspected: InitializedRepository +) -> None: + if repository.encryption != inspected.encryption or ( + inspected.encryption == "aes-256-gcm" + and repository.active_data_key_id != inspected.data_key_id + ): + raise SnapshotError("repository encryption metadata is invalid") + + +def _repository_data_key( + settings: Settings, repository_id: str, key_id: str | None +) -> bytes | None: + if key_id is None: + return None + try: + return load_data_key(settings, repository_id, key_id) + except RepositoryKeyError as error: + raise SnapshotError("repository data key is unavailable") from error + + +async def _repository_epoch_keys( + settings: Settings, db: AsyncSession, repository: Repository, inspected: InitializedRepository +) -> dict[str, bytes]: + if inspected.encryption == "none": + return {} + if inspected.data_key_id is None: + raise SnapshotError("repository encryption metadata is invalid") + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch).where( + RepositoryDataKeyEpoch.repository_id == repository.id + ) + ) + ).all() + ) + active = [epoch for epoch in epochs if epoch.state == "active"] + if ( + len(active) != 1 + or active[0].key_id != inspected.data_key_id + or repository.active_data_key_id != inspected.data_key_id + ): + raise SnapshotError("repository encryption metadata is invalid") + keys: dict[str, bytes] = {} + for epoch in epochs: + key = _repository_data_key(settings, inspected.repository_id, epoch.key_id) + if key is None: + raise SnapshotError("repository encryption metadata is invalid") + keys[epoch.key_id] = key + return keys + + +def _entry_key_id(entry: dict[str, Any], manifest_key_id: str | None) -> str | None: + key_id = entry.get("encryption_key_id", manifest_key_id) + if key_id is not None and not isinstance(key_id, str): + raise SnapshotIntegrityError("published manifest encryption metadata is invalid") + return key_id + + +def require_nonempty(entries: list[dict[str, Any]], allow_empty: bool) -> None: + if not entries and not allow_empty: + raise SnapshotError("source_empty") + + +def _unsigned_manifest( + backup_id: str, + repository_id: str, + source: Source, + job: Job, + execution: Execution, + entries: list[dict[str, Any]], + logical_bytes: int, + stored_bytes: int, + effective_mode: str = "full", + encryption_key_id: str | None = None, +) -> dict[str, Any]: + captured_at = _timestamp() + return { + "format_version": 1, + "backup_id": backup_id, + "repository_id": repository_id, + "source_id": source.id, + "job_id": job.id, + "execution_id": execution.id, + "requested_mode": job.requested_mode, + "effective_mode": effective_mode, + "created_at": captured_at, + "source_consistency": { + "adapter": source.kind, + "captured_at": captured_at, + "evidence": {"enumeration": "local"}, + }, + "exclusion_policy": { + "matcher": "gitignore", + "version": 1, + "patterns": job.exclusions, + }, + "entries": entries, + "aggregates": { + "entry_count": len(entries), + "logical_bytes": logical_bytes, + "stored_bytes": stored_bytes, + }, + "encryption_key_id": encryption_key_id, + } + + +def verify_published_snapshot( + root: Path, + manifest_path: Path, + expected_public_key: str, + *, + encryption_key: bytes | None = None, + encryption_key_id: str | None = None, + blob_keys: dict[str, bytes] | None = None, + expected_repository_id: str | None = None, +) -> dict[str, Any]: + try: + stored_manifest = manifest_path.read_bytes() + if encryption_key_id is not None: + if encryption_key is None or expected_repository_id is None: + raise RepositoryKeyError("repository manifest key is unavailable") + stored_manifest = decrypt_object( + encryption_key, + object_aad( + expected_repository_id, + encryption_key_id, + "manifest", + manifest_path.stem, + ), + stored_manifest, + ) + manifest = json.loads(stored_manifest) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, RepositoryKeyError) as error: + raise SnapshotIntegrityError("published manifest is unreadable") from error + if not isinstance(manifest, dict): + raise SnapshotIntegrityError("published manifest is invalid") + try: + digest = manifest.pop("manifest_digest") + signature = manifest.pop("manifest_signature") + signature_value = bytes.fromhex(signature["value"]) + public_key = Ed25519PublicKey.from_public_bytes(bytes.fromhex(expected_public_key)) + expected_digest = hashlib.sha256(_canonical_json(manifest)).hexdigest() + if digest != expected_digest: + raise SnapshotIntegrityError("published manifest digest is invalid") + public_key.verify(signature_value, bytes.fromhex(digest)) + except (InvalidSignature, KeyError, TypeError, ValueError) as error: + raise SnapshotIntegrityError("published manifest signature is invalid") from error + manifest_key_id = manifest.get("encryption_key_id") + repository_id = manifest.get("repository_id") + if ( + manifest_key_id != encryption_key_id + or not isinstance(repository_id, str) + or (expected_repository_id is not None and repository_id != expected_repository_id) + ): + raise SnapshotIntegrityError("published manifest encryption metadata is invalid") + for entry in manifest.get("entries", []): + if entry.get("type") != "file": + continue + digest = entry.get("blob_digest") + if not isinstance(digest, str): + raise SnapshotIntegrityError("published manifest file entry is invalid") + blob_path = root / "blobs" / "sha256" / digest + blob_key_id = _entry_key_id(entry, manifest_key_id) + blob_key = ( + blob_keys.get(blob_key_id) + if blob_keys is not None and blob_key_id is not None + else encryption_key + ) + aad = ( + object_aad(repository_id, blob_key_id, "blob", digest) + if blob_key_id is not None + else None + ) + if ( + (blob_key_id is not None and blob_key is None) + or blob_path.is_symlink() + or not blob_path.is_file() + or not _matches_blob_digest(blob_path, digest, blob_key, aad) + ): + raise SnapshotIntegrityError("published blob verification failed") + manifest["manifest_digest"] = expected_digest + manifest["manifest_signature"] = signature + return manifest + + +async def source_adapter( + settings: Settings, + db: AsyncSession, + source: Source, + cipher: EnvelopeCipher | None, +) -> SourceReader: + if source.kind == "local": + root = source.public_config.get("root") + if not isinstance(root, str): + raise SourceError("local source root is invalid") + return LocalAdapter(Path(root), settings) + if source.kind != "ssh": + raise SourceError("source kind is unavailable") + if cipher is None or len(source.secret_refs) != 1: + raise SourceError("SSH source key is unavailable", reason_code="source_invalid") + try: + config = SSHSourcePublicConfig.model_validate(source.public_config) + except ValueError as error: + raise SourceError( + "SSH source configuration is invalid", reason_code="source_invalid" + ) from error + from backup_tool.db.models import Secret + + secret = await db.get(Secret, source.secret_refs[0]) + if secret is None or secret.purpose != SSH_PRIVATE_KEY_PURPOSE: + raise SourceError("SSH source key is unavailable", reason_code="source_invalid") + try: + private_key = cipher.decrypt( + secret.ciphertext, purpose=secret.purpose, version=secret.version + ) + except Exception as error: + raise SourceError("SSH source key is unavailable", reason_code="source_invalid") from error + return SSHAdapter(config, private_key, settings) + + +async def publish_full_snapshot( + settings: Settings, + db: AsyncSession, + execution: Execution, + job: Job, + source: Source, + repository: Repository, + fault_injector: FaultInjector | None = None, + *, + cipher: EnvelopeCipher | None = None, +) -> Backup: + injector = fault_injector or NoFault() + if job.requested_mode not in {"full", "incremental"}: + raise SnapshotError("backup mode is invalid") + adapter: SourceReader | None = None + try: + inspected = inspect_repository(settings, Path(repository.root)) + signing_key = load_signing_key( + settings, + inspected.repository_id, + repository.signing_key_id, + repository.signing_public_key, + ) + _assert_repository_encryption_metadata(repository, inspected) + encryption_keys = await _repository_epoch_keys(settings, db, repository, inspected) + encryption_key = ( + encryption_keys[inspected.data_key_id] if inspected.data_key_id is not None else None + ) + assert_capacity(settings, inspected.root) + adapter = await source_adapter(settings, db, source, cipher) + adapter.validate_config() + except SourceError as error: + raise SnapshotError(str(error), reason_code=error.reason_code) from error + except (KeyError, RepositoryError) as error: + raise SnapshotError(str(error)) from error + + baseline = None + if job.requested_mode == "incremental": + baseline = await db.scalar( + select(Backup) + .join(Execution, Backup.execution_id == Execution.id) + .where( + Execution.job_id == job.id, + Backup.integrity == "verified", + Backup.tombstoned_at.is_(None), + ) + .order_by(desc(Backup.created_at)) + .limit(1) + ) + effective_mode = "incremental" if baseline is not None else "full" + + staging_root = inspected.root / "staging" + _private_directory(staging_root, exist_ok=True) + staging = staging_root / execution.id + _private_directory(staging) + published = False + try: + staged_blobs = staging / "blobs" + _private_directory(staged_blobs) + entries: list[dict[str, Any]] = [] + logical_bytes = 0 + stored_bytes = 0 + async for entry in adapter.enumerate_entries(): + try: + excluded = matches(entry.path, job.exclusions) + except ExclusionError as error: + raise SnapshotError(str(error)) from error + if excluded: + continue + manifest_entry: dict[str, Any] = { + "path": entry.path, + "type": entry.kind, + "size": entry.size, + "blob_digest": None, + "mode": entry.mode, + "mtime_ns": entry.mtime_ns, + "link_target": entry.link_target, + "metadata_support": ["mode", "mtime_ns"], + } + if entry.kind == "file": + staged_blob = staged_blobs / f"{len(entries)}.blob" + digest, copied_size = await _copy_file(adapter, entry.path, staged_blob, injector) + if copied_size != entry.size: + raise SnapshotError("source file changed during backup") + aad = None + if encryption_key is not None: + if inspected.data_key_id is None: + raise SnapshotError("repository encryption metadata is invalid") + aad = object_aad(inspected.repository_id, inspected.data_key_id, "blob", digest) + _encrypt_staged_blob(staged_blob, encryption_key, aad) + blob_key_id = _install_blob( + staged_blob, + inspected.root / "blobs" / "sha256" / digest, + digest, + injector, + encryption_key_id=inspected.data_key_id, + encryption_keys=encryption_keys or None, + repository_id=inspected.repository_id, + ) + manifest_entry["blob_digest"] = digest + if blob_key_id is not None: + manifest_entry["encryption_key_id"] = blob_key_id + logical_bytes += copied_size + stored_bytes += ( + staged_blob.stat().st_size + if staged_blob.exists() + else (inspected.root / "blobs" / "sha256" / digest).stat().st_size + ) + entries.append(manifest_entry) + + require_nonempty(entries, job.allow_empty) + + backup_id = str(new_uuid7()) + manifest = _unsigned_manifest( + backup_id, + inspected.repository_id, + source, + job, + execution, + entries, + logical_bytes, + stored_bytes, + effective_mode, + inspected.data_key_id, + ) + manifest_digest = hashlib.sha256(_canonical_json(manifest)).hexdigest() + manifest["manifest_digest"] = manifest_digest + manifest["manifest_signature"] = { + "algorithm": "ed25519", + "key_id": repository.signing_key_id, + "value": signing_key.sign(bytes.fromhex(manifest_digest)).hex(), + } + manifest_path = inspected.root / "manifests" / f"{backup_id}.json" + marker = { + "execution_id": execution.id, + "backup_id": backup_id, + "manifest_digest": manifest_digest, + "logical_bytes": logical_bytes, + "stored_bytes": stored_bytes, + "data_key_id": inspected.data_key_id, + } + publication_marker = staging / "publication.json" + with os.fdopen(_private_file_descriptor(publication_marker), "wb") as handle: + handle.write(_canonical_json(marker)) + handle.flush() + os.fsync(handle.fileno()) + staged_manifest = staging / "manifest.json" + manifest_payload = _canonical_json(manifest) + if encryption_key is not None: + if inspected.data_key_id is None: + raise SnapshotError("repository encryption metadata is invalid") + try: + manifest_payload = encrypt_object( + encryption_key, + object_aad( + inspected.repository_id, + inspected.data_key_id, + "manifest", + backup_id, + ), + manifest_payload, + ) + except RepositoryKeyError as error: + raise SnapshotError("repository manifest encryption failed") from error + injector.hit("manifest.before_write") + with os.fdopen(_private_file_descriptor(staged_manifest), "wb") as handle: + handle.write(manifest_payload) + handle.flush() + os.fsync(handle.fileno()) + injector.hit("manifest.after_fsync") + injector.hit("manifest.before_publish") + os.replace(staged_manifest, manifest_path) + _fsync_directory(manifest_path.parent) + published = True + verify_published_snapshot( + inspected.root, + manifest_path, + repository.signing_public_key, + encryption_key=encryption_key, + encryption_key_id=inspected.data_key_id, + blob_keys=encryption_keys, + expected_repository_id=inspected.repository_id, + ) + backup = Backup( + execution_id=execution.id, + parent_backup_id=baseline.id if baseline is not None else None, + manifest_id=backup_id, + manifest_digest=manifest_digest, + logical_bytes=logical_bytes, + stored_bytes=stored_bytes, + integrity="verified", + data_key_id=inspected.data_key_id, + ) + db.add(backup) + return backup + except SourceError as error: + raise SnapshotError(str(error), reason_code=error.reason_code) from error + finally: + if adapter is not None: + await adapter.close() + if not published: + shutil.rmtree(staging, ignore_errors=True) + + +def finalize_publication(root: Path, execution_id: str) -> None: + staging = root / "staging" / execution_id + try: # noqa: SIM105 - finalized staging can already be removed during recovery + shutil.rmtree(staging) + except FileNotFoundError: + pass + + +async def reconcile_publications(settings: Settings, db: AsyncSession) -> int: + reconciled = 0 + repositories = list((await db.scalars(select(Repository))).all()) + for repository in repositories: + try: + inspected = inspect_repository(settings, Path(repository.root)) + _assert_repository_encryption_metadata(repository, inspected) + except (RepositoryError, SnapshotError): + continue + staging_root = inspected.root / "staging" + if not staging_root.is_dir() or staging_root.is_symlink(): + continue + for staging in staging_root.iterdir(): + marker_path = staging / "publication.json" + if not staging.is_dir() or staging.is_symlink() or not marker_path.is_file(): + continue + try: + marker = json.loads(marker_path.read_text(encoding="utf-8")) + execution_id = marker["execution_id"] + backup_id = marker["backup_id"] + manifest_digest = marker["manifest_digest"] + logical_bytes = marker["logical_bytes"] + stored_bytes = marker["stored_bytes"] + data_key_id = marker.get("data_key_id") + if not ( + isinstance(execution_id, str) + and isinstance(backup_id, str) + and isinstance(manifest_digest, str) + and isinstance(logical_bytes, int) + and isinstance(stored_bytes, int) + and (data_key_id is None or isinstance(data_key_id, str)) + ): + raise ValueError + epoch_keys = await _repository_epoch_keys(settings, db, repository, inspected) + encryption_key = epoch_keys.get(data_key_id) if data_key_id is not None else None + manifest = verify_published_snapshot( + inspected.root, + inspected.root / "manifests" / f"{backup_id}.json", + repository.signing_public_key, + encryption_key=encryption_key, + encryption_key_id=data_key_id, + blob_keys=epoch_keys, + expected_repository_id=inspected.repository_id, + ) + if manifest.get("manifest_digest") != manifest_digest: + raise ValueError + execution = await db.get(Execution, execution_id) + if execution is None: + raise ValueError + except (OSError, ValueError, SnapshotError, json.JSONDecodeError): + continue + backup = await db.scalar(select(Backup).where(Backup.execution_id == execution_id)) + if backup is None: + db.add( + Backup( + execution_id=execution_id, + parent_backup_id=None, + manifest_id=backup_id, + manifest_digest=manifest_digest, + logical_bytes=logical_bytes, + stored_bytes=stored_bytes, + integrity="verified", + data_key_id=data_key_id, + ) + ) + if execution.state in {"preparing", "running", "verifying"}: + execution.state = "committed" + execution.completed_at = datetime.now(UTC) + execution.lease_owner = None + execution.lease_expires_at = None + finalize_publication(inspected.root, execution_id) + reconciled += 1 + await db.commit() + return reconciled + + +def validate_restore_destination( + settings: Settings, raw_destination: str, *, allow_existing: bool = False +) -> tuple[Path, Path]: + destination = Path(raw_destination) + if not destination.is_absolute() or destination.name in {"", ".", ".."}: + raise SnapshotError("restore destination must be an absolute directory") + for root in settings.restore_roots: + resolved_root = root.resolve() + if destination.parent.resolve() != resolved_root: + continue + if root.is_symlink() or not resolved_root.is_dir() or destination.parent.is_symlink(): + raise SnapshotError("restore root is unsafe") + if destination.is_symlink() or (destination.exists() and not allow_existing): + raise SnapshotError("restore destination already exists") + return destination, resolved_root + raise SnapshotError("restore destination is outside configured restore roots") + + +def _safe_restore_entries(manifest: dict[str, Any]) -> list[dict[str, Any]]: + entries = manifest.get("entries") + if not isinstance(entries, list): + raise SnapshotIntegrityError("published manifest entries are invalid") + paths: dict[str, str] = {} + for entry in entries: + if not isinstance(entry, dict): + raise SnapshotIntegrityError("published manifest entry is invalid") + path = entry.get("path") + entry_type = entry.get("type") + relative = Path(path) if isinstance(path, str) else None + if ( + not isinstance(path, str) + or not path + or path == ".backup-tool-restore.json" + or "\\" in path + or relative is None + or relative.is_absolute() + or ".." in relative.parts + or entry_type not in {"file", "directory", "symlink"} + or path in paths + ): + raise SnapshotIntegrityError("published manifest entry path is unsafe") + paths[path] = entry_type + if entry_type == "file": + digest = entry.get("blob_digest") + size = entry.get("size") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or digest.lower() != digest + or any(character not in "0123456789abcdef" for character in digest) + or not isinstance(size, int) + or size < 0 + ): + raise SnapshotIntegrityError("published manifest file entry is invalid") + elif entry.get("blob_digest") is not None: + raise SnapshotIntegrityError("published manifest non-file entry is invalid") + if entry_type == "symlink": + target = entry.get("link_target") + target_path = Path(target) if isinstance(target, str) else None + if ( + not isinstance(target, str) + or not target + or "\\" in target + or target_path is None + or target_path.is_absolute() + or ".." in target_path.parts + ): + raise SnapshotIntegrityError("published manifest symlink target is unsafe") + for path in paths: + parent = Path(path).parent + while parent != Path("."): + if paths.get(parent.as_posix()) in {"file", "symlink"}: + raise SnapshotIntegrityError("published manifest entry has an unsafe parent") + parent = parent.parent + return sorted(entries, key=lambda entry: (len(Path(entry["path"]).parts), entry["path"])) + + +def _select_restore_entries( + entries: list[dict[str, Any]], selection: list[str] +) -> list[dict[str, Any]]: + if not selection: + return entries + selected: set[str] = set() + for path in selection: + normalized = Path(path) if isinstance(path, str) else None + if ( + not isinstance(path, str) + or not path + or "\\" in path + or normalized is None + or normalized.is_absolute() + or ".." in normalized.parts + ): + raise SnapshotError("restore selection is invalid") + selected.add(path) + selected_entries = { + entry["path"] + for entry in entries + if any(entry["path"] == path or entry["path"].startswith(f"{path}/") for path in selected) + } + for path in tuple(selected_entries): + parent = Path(path).parent + while parent != Path("."): + selected_entries.add(parent.as_posix()) + parent = parent.parent + return [entry for entry in entries if entry["path"] in selected_entries] + + +def _copy_restore_file( + blob_path: Path, + destination: Path, + digest: str, + expected_size: int, + *, + encryption_key: bytes | None = None, + aad: bytes | None = None, +) -> None: + if blob_path.is_symlink() or not blob_path.is_file(): + raise SnapshotError("restore blob is unavailable") + if encryption_key is not None: + if aad is None: + raise SnapshotError("restore blob verification failed") + try: + plaintext = decrypt_object(encryption_key, aad, blob_path.read_bytes()) + except (OSError, RepositoryKeyError) as error: + raise SnapshotError("restore blob verification failed") from error + if len(plaintext) != expected_size or hashlib.sha256(plaintext).hexdigest() != digest: + raise SnapshotError("restore blob verification failed") + with destination.open("xb") as target: + target.write(plaintext) + target.flush() + os.fsync(target.fileno()) + return + copied = 0 + copied_digest = hashlib.sha256() + with blob_path.open("rb") as source, destination.open("xb") as target: + while chunk := source.read(1024 * 1024): + copied_digest.update(chunk) + copied += len(chunk) + target.write(chunk) + target.flush() + os.fsync(target.fileno()) + if copied != expected_size or copied_digest.hexdigest() != digest: + raise SnapshotError("restore blob verification failed") + + +def _apply_metadata(path: Path, entry: dict[str, Any], *, symlink: bool = False) -> None: + support = entry.get("metadata_support") + if not isinstance(support, list): + return + mode = entry.get("mode") + if "mode" in support and isinstance(mode, int) and not symlink: + os.chmod(path, stat.S_IMODE(mode) & 0o0777) + mtime_ns = entry.get("mtime_ns") + if "mtime_ns" in support and isinstance(mtime_ns, int): + os.utime(path, ns=(mtime_ns, mtime_ns), follow_symlinks=not symlink) + + +async def verify_backup_snapshot( + settings: Settings, + db: AsyncSession, + backup: Backup, + repository: Repository, +) -> dict[str, Any]: + """Authenticate a published backup before any API or restore operation trusts it.""" + try: + inspected = inspect_repository(settings, Path(repository.root)) + _assert_repository_encryption_metadata(repository, inspected) + except RepositoryError as error: + raise SnapshotError(str(error)) from error + epoch_keys = await _repository_epoch_keys(settings, db, repository, inspected) + encryption_key = epoch_keys.get(backup.data_key_id) if backup.data_key_id is not None else None + manifest_path = inspected.root / "manifests" / f"{backup.manifest_id}.json" + manifest = verify_published_snapshot( + inspected.root, + manifest_path, + repository.signing_public_key, + encryption_key=encryption_key, + encryption_key_id=backup.data_key_id, + blob_keys=epoch_keys, + expected_repository_id=inspected.repository_id, + ) + if ( + manifest.get("backup_id") != backup.manifest_id + or manifest.get("repository_id") != inspected.repository_id + or manifest.get("manifest_digest") != backup.manifest_digest + ): + raise SnapshotError("published manifest does not match backup metadata") + return manifest + + +async def restore_full_snapshot( + settings: Settings, + db: AsyncSession, + restore: Restore, + backup: Backup, + repository: Repository, +) -> dict[str, Any]: + if restore.overwrite_policy not in {"fail", "skip", "replace"}: + raise SnapshotError("restore overwrite policy is invalid") + manifest = await verify_backup_snapshot(settings, db, backup, repository) + try: + inspected = inspect_repository(settings, Path(repository.root)) + except RepositoryError as error: + raise SnapshotError(str(error)) from error + epoch_keys = await _repository_epoch_keys(settings, db, repository, inspected) + entries = _select_restore_entries(_safe_restore_entries(manifest), restore.selection) + if restore.dry_run: + return { + "restore_id": restore.id, + "backup_id": backup.id, + "manifest_digest": backup.manifest_digest, + "entry_count": len(entries), + "file_count": sum(entry["type"] == "file" for entry in entries), + "restored_bytes": sum(entry["size"] for entry in entries if entry["type"] == "file"), + "dry_run": True, + } + destination, restore_root = validate_restore_destination( + settings, restore.destination, allow_existing=restore.overwrite_policy != "fail" + ) + if destination.exists() and restore.overwrite_policy == "skip": + return { + "restore_id": restore.id, + "backup_id": backup.id, + "manifest_digest": backup.manifest_digest, + "entry_count": len(entries), + "file_count": 0, + "restored_bytes": 0, + "skipped": True, + } + assert_capacity(settings, restore_root) + staging = restore_root / f".{destination.name}.restore-{restore.id}" + try: + staging.mkdir(mode=0o700, exist_ok=False) + except OSError as error: + raise SnapshotError("restore staging path is unavailable") from error + try: + directories = [entry for entry in entries if entry["type"] == "directory"] + for entry in directories: + directory = staging.joinpath(*Path(entry["path"]).parts) + directory.mkdir(mode=0o700, parents=True, exist_ok=False) + restored_files = 0 + restored_bytes = 0 + for entry in entries: + if entry["type"] != "file": + continue + target = staging.joinpath(*Path(entry["path"]).parts) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + digest = entry["blob_digest"] + size = entry["size"] + if not isinstance(digest, str) or not isinstance(size, int): + raise SnapshotError("published manifest file entry is invalid") + blob_key_id = _entry_key_id(entry, backup.data_key_id) + blob_key = epoch_keys.get(blob_key_id) if blob_key_id is not None else None + if blob_key_id is not None and blob_key is None: + raise SnapshotError("repository data key is unavailable") + aad = ( + object_aad(inspected.repository_id, blob_key_id, "blob", digest) + if blob_key_id is not None + else None + ) + _copy_restore_file( + inspected.root / "blobs" / "sha256" / digest, + target, + digest, + size, + encryption_key=blob_key, + aad=aad, + ) + _apply_metadata(target, entry) + restored_files += 1 + restored_bytes += size + for entry in entries: + if entry["type"] != "symlink": + continue + target = staging.joinpath(*Path(entry["path"]).parts) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + link_target = entry.get("link_target") + if not isinstance(link_target, str): + raise SnapshotError("published manifest symlink entry is invalid") + os.symlink(link_target, target) + _apply_metadata(target, entry, symlink=True) + for entry in reversed(directories): + _apply_metadata(staging.joinpath(*Path(entry["path"]).parts), entry) + result = { + "restore_id": restore.id, + "backup_id": backup.id, + "manifest_digest": backup.manifest_digest, + "entry_count": len(entries), + "file_count": restored_files, + "restored_bytes": restored_bytes, + } + sidecar = staging / ".backup-tool-restore.json" + with sidecar.open("xb") as handle: + handle.write(_canonical_json(result)) + handle.flush() + os.fsync(handle.fileno()) + _fsync_directory(staging) + previous: Path | None = None + if destination.exists(): + previous = restore_root / f".{destination.name}.previous-{restore.id}" + if previous.exists(): + raise SnapshotError("restore replacement recovery is incomplete") + os.rename(destination, previous) + try: + os.rename(staging, destination) + except OSError as error: + if previous is not None: + os.rename(previous, destination) + raise SnapshotError("restore destination could not be published") from error + _fsync_directory(restore_root) + if previous is not None: + shutil.rmtree(previous) + return result + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + + +async def reconcile_restores(settings: Settings, db: AsyncSession) -> int: + restores = list((await db.scalars(select(Restore).where(Restore.state == "running"))).all()) + for restore in restores: + destination = Path(restore.destination) + sidecar = destination / ".backup-tool-restore.json" + if not destination.is_dir() or destination.is_symlink() or not sidecar.is_file(): + staging = destination.parent / f".{destination.name}.restore-{restore.id}" + try: # noqa: SIM105 - recovery must tolerate a previously cleaned staging directory + shutil.rmtree(staging) + except FileNotFoundError: + pass + if destination.exists(): + restore.state = "failed" + restore.result = {"reason": "restore_recovery_failed"} + else: + restore.state = "queued" + restore.result = {"reason": "worker_lost"} + continue + try: + result = json.loads(sidecar.read_text(encoding="utf-8")) + backup = await db.get(Backup, restore.backup_id) + if ( + not isinstance(result, dict) + or backup is None + or result.get("restore_id") != restore.id + or result.get("backup_id") != backup.id + or result.get("manifest_digest") != backup.manifest_digest + ): + raise ValueError + except (OSError, ValueError, json.JSONDecodeError): + restore.state = "failed" + restore.result = {"reason": "restore_recovery_failed"} + continue + restore.state = "committed" + restore.result = result + await db.commit() + return len(restores) diff --git a/backend/src/backup_tool/ssh_adapter.py b/backend/src/backup_tool/ssh_adapter.py new file mode 100644 index 0000000..eec99d3 --- /dev/null +++ b/backend/src/backup_tool/ssh_adapter.py @@ -0,0 +1,288 @@ +"""Pinned-host-key, forced-SFTP-only source reader. + +The adapter has no command-channel API. It authenticates with the caller's +already decrypted private key only after verifying the configured host key. +""" + +from __future__ import annotations + +import asyncio +import hmac +import io +import socket +import stat +from collections.abc import AsyncIterator, Callable, Iterator +from contextlib import suppress +from typing import Any + +import paramiko # type: ignore[import-untyped] + +from backup_tool.adapters import Entry, SourceError +from backup_tool.config import Settings +from backup_tool.ssh_source import SSHSourcePublicConfig + +TransportFactory = Callable[[str, int, float], Any] +SFTPFactory = Callable[[Any], Any] + + +def _transport_for(hostname: str, port: int, timeout: float) -> Any: + connection = socket.create_connection((hostname, port), timeout=timeout) + connection.settimeout(timeout) + return paramiko.Transport(connection) + + +def _sftp_for(transport: Any) -> Any: + return paramiko.SFTPClient.from_transport(transport) + + +def _next_or_none(iterator: Iterator[Any]) -> Any | None: + try: + return next(iterator) + except StopIteration: + return None + + +def load_private_key(value: str) -> Any: + """Load only unencrypted Ed25519, ECDSA, or sufficiently strong RSA keys.""" + for key_type in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey): + try: + key = key_type.from_private_key(io.StringIO(value), password=None) + except (paramiko.SSHException, ValueError): + continue + if isinstance(key, paramiko.RSAKey) and key.get_bits() < 3072: + raise SourceError( + "SSH private key algorithm is not permitted", reason_code="source_auth" + ) + return key + raise SourceError("SSH private key algorithm is not permitted", reason_code="source_auth") + + +class SSHAdapter: + """A stateful SFTP reader rooted at the forced-SFTP account chroot only.""" + + def __init__( + self, + config: SSHSourcePublicConfig, + private_key: str, + settings: Settings, + *, + transport_factory: TransportFactory = _transport_for, + sftp_factory: SFTPFactory = _sftp_for, + ) -> None: + self.config = config + self._private_key = load_private_key(private_key) + self.settings = settings + self._transport_factory = transport_factory + self._sftp_factory = sftp_factory + self._transport: Any | None = None + self._sftp: Any | None = None + self._files: dict[str, tuple[int, int]] = {} + + def validate_config(self) -> None: + if self.config.root != "/": # defensive: persisted JSON can bypass API validation + raise SourceError( + "SSH source root must be the forced-SFTP chroot", reason_code="source_invalid" + ) + + def _connect(self) -> None: + if self._sftp is not None: + return + self.validate_config() + transport = self._transport_factory( + self.config.hostname, self.config.port, self.settings.ssh_connect_timeout_seconds + ) + self._transport = transport + try: + transport.start_client(timeout=self.settings.ssh_connect_timeout_seconds) + algorithm, encoded_key = self.config.host_key.split(" ", 1) + server_key = transport.get_remote_server_key() + if server_key.get_name() != algorithm or not hmac.compare_digest( + server_key.get_base64(), encoded_key + ): + raise SourceError( + "SSH host key does not match configured pin", reason_code="source_trust" + ) + # Authentication deliberately occurs only after the exact pin comparison. + transport.auth_publickey(self.config.username, self._private_key) + sftp = self._sftp_factory(transport) + channel = sftp.get_channel() + channel.settimeout(self.settings.ssh_operation_timeout_seconds) + self._sftp = sftp + except SourceError: + self._close_sync() + raise + except paramiko.AuthenticationException as error: + self._close_sync() + raise SourceError("SSH authentication failed", reason_code="source_auth") from error + except (TimeoutError, OSError) as error: + self._close_sync() + raise SourceError( + "SSH source is unavailable", reason_code="source_unavailable" + ) from error + except (paramiko.SSHException, ValueError) as error: + self._close_sync() + raise SourceError( + "SSH source connection failed", reason_code="source_unavailable" + ) from error + + def _sftp_client(self) -> Any: + if self._sftp is None: + raise SourceError("SSH source is unavailable", reason_code="source_unavailable") + return self._sftp + + def _read_client(self) -> Any: + if self._transport is None: + raise SourceError("SSH source is unavailable", reason_code="source_unavailable") + client = self._sftp_factory(self._transport) + client.get_channel().settimeout(self.settings.ssh_operation_timeout_seconds) + return client + + @staticmethod + def _relative(parent: str, name: object) -> str: + if not isinstance(name, str) or not name or name in {".", ".."}: + raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid") + if "/" in name or "\\" in name or "\x00" in name: + raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid") + return name if not parent else f"{parent}/{name}" + + @staticmethod + def _remote_path(relative: str) -> str: + if ( + not relative + or relative.startswith("/") + or "\\" in relative + or ".." in relative.split("/") + ): + raise SourceError("SSH source entry is invalid", reason_code="source_invalid") + return f"/{relative}" + + @staticmethod + def _entry_from_attributes(path: str, attributes: Any) -> Entry: + mode = getattr(attributes, "st_mode", None) + if not isinstance(mode, int): + raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid") + if stat.S_ISLNK(mode): + raise SourceError("SSH source symlinks are not supported", reason_code="source_invalid") + size = getattr(attributes, "st_size", 0) + mtime = getattr(attributes, "st_mtime", 0) + if not isinstance(size, int) or size < 0 or not isinstance(mtime, int): + raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid") + if stat.S_ISDIR(mode): + return Entry(path, "directory", 0, stat.S_IMODE(mode), mtime * 1_000_000_000) + if stat.S_ISREG(mode): + return Entry(path, "file", size, stat.S_IMODE(mode), mtime * 1_000_000_000) + raise SourceError("SSH source contains an unsupported entry", reason_code="source_invalid") + + async def probe(self) -> dict[str, int]: + count = 0 + try: + async for entry in self.enumerate_entries(): + if entry.kind == "file": + count += 1 + return {"entry_count": count} + finally: + await self.close() + + async def enumerate_entries(self) -> AsyncIterator[Entry]: + await asyncio.to_thread(self._connect) + pending = [""] + entry_count = 0 + try: + while pending: + parent = pending.pop() + remote_parent = "/" if not parent else self._remote_path(parent) + try: + iterator = await asyncio.to_thread( + self._sftp_client().listdir_iter, + remote_parent, + read_aheads=self.settings.ssh_list_read_aheads, + ) + while ( + attributes := await asyncio.to_thread(_next_or_none, iterator) + ) is not None: + path = self._relative(parent, getattr(attributes, "filename", None)) + entry_count += 1 + if entry_count > self.settings.ssh_max_entries: + raise SourceError( + "SSH source entry limit exceeded", reason_code="source_limit" + ) + entry = self._entry_from_attributes(path, attributes) + if entry.kind == "directory": + if path.count("/") + 1 > self.settings.ssh_max_traversal_depth: + raise SourceError( + "SSH source traversal depth exceeded", + reason_code="source_limit", + ) + pending.append(path) + else: + self._files[path] = (entry.size, entry.mtime_ns) + yield entry + except SourceError: + raise + except (TimeoutError, OSError, paramiko.SSHException) as error: + raise SourceError( + "SSH source enumeration failed", reason_code="source_unavailable" + ) from error + except BaseException: + await self.close() + raise + + async def open_content(self, path: str) -> AsyncIterator[bytes]: + expected = self._files.get(path) + if expected is None: + raise SourceError("SSH source entry was not enumerated", reason_code="source_invalid") + remote_path = self._remote_path(path) + handle: Any | None = None + reader: Any | None = None + try: + await asyncio.to_thread(self._connect) + reader = await asyncio.to_thread(self._read_client) + assert reader is not None + attributes = await asyncio.to_thread(reader.lstat, remote_path) + entry = self._entry_from_attributes(path, attributes) + if entry.kind != "file" or (entry.size, entry.mtime_ns) != expected: + raise SourceError( + "SSH source file changed during backup", reason_code="source_changed" + ) + handle = await asyncio.to_thread( + reader.open, + remote_path, + "rb", + self.settings.ssh_read_chunk_bytes, + ) + assert handle is not None + while chunk := await asyncio.to_thread(handle.read, self.settings.ssh_read_chunk_bytes): + if not isinstance(chunk, bytes) or len(chunk) > self.settings.ssh_read_chunk_bytes: + raise SourceError( + "SSH source returned an invalid read", reason_code="source_invalid" + ) + yield chunk + after = await asyncio.to_thread(reader.lstat, remote_path) + verified = self._entry_from_attributes(path, after) + if verified.kind != "file" or (verified.size, verified.mtime_ns) != expected: + raise SourceError( + "SSH source file changed during backup", reason_code="source_changed" + ) + except SourceError: + raise + except (TimeoutError, OSError, paramiko.SSHException) as error: + raise SourceError("SSH source read failed", reason_code="source_unavailable") from error + finally: + if handle is not None: + await asyncio.to_thread(handle.close) + if reader is not None and reader is not self._sftp: + await asyncio.to_thread(reader.close) + + def _close_sync(self) -> None: + sftp, transport = self._sftp, self._transport + self._sftp = None + self._transport = None + if sftp is not None: + with suppress(OSError, paramiko.SSHException): + sftp.close() + if transport is not None: + with suppress(OSError, paramiko.SSHException): + transport.close() + + async def close(self) -> None: + await asyncio.to_thread(self._close_sync) diff --git a/backend/src/backup_tool/ssh_source.py b/backend/src/backup_tool/ssh_source.py new file mode 100644 index 0000000..0e3adba --- /dev/null +++ b/backend/src/backup_tool/ssh_source.py @@ -0,0 +1,72 @@ +"""Closed public configuration for the staged SSH source capability. + +This module deliberately describes configuration only. It must not create a +network transport or load a private key; those capabilities are outside this +slice. +""" + +from __future__ import annotations + +import base64 +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +SSH_PRIVATE_KEY_PURPOSE = "ssh_private_key" +_ALLOWED_HOST_KEY_ALGORITHMS = frozenset( + { + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-256", + "rsa-sha2-512", + } +) + + +def _has_control_or_space(value: str) -> bool: + return any(character.isspace() or ord(character) < 32 for character in value) + + +class SSHSourcePublicConfig(BaseModel): + """The public, SFTP-chroot-only portion of an SSH source definition.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + hostname: str = Field(min_length=1, max_length=253) + port: int = Field(ge=1, le=65535) + username: str = Field(min_length=1, max_length=255) + host_key: str = Field(min_length=1, max_length=16384) + # The server-side forced-SFTP account's chroot is the only permitted root. + root: Literal["/"] + + @field_validator("hostname") + @classmethod + def validate_hostname(cls, value: str) -> str: + if _has_control_or_space(value) or any(character in value for character in "/\\@?#"): + raise ValueError("SSH hostname is invalid") + return value + + @field_validator("username") + @classmethod + def validate_username(cls, value: str) -> str: + if _has_control_or_space(value) or any(character in value for character in "/\\:@"): + raise ValueError("SSH username is invalid") + return value + + @field_validator("host_key") + @classmethod + def validate_host_key(cls, value: str) -> str: + algorithm, separator, encoded_key = value.partition(" ") + if not separator or not algorithm or not encoded_key or " " in encoded_key: + raise ValueError("SSH host key must be an algorithm and base64 key") + if algorithm not in _ALLOWED_HOST_KEY_ALGORITHMS: + raise ValueError("SSH host key algorithm is not supported") + try: + decoded = base64.b64decode(encoded_key, validate=True) + except (ValueError, UnicodeEncodeError) as error: + raise ValueError("SSH host key is not valid base64") from error + if not decoded: + raise ValueError("SSH host key is empty") + return value diff --git a/backend/src/backup_tool/web.py b/backend/src/backup_tool/web.py new file mode 100644 index 0000000..0cedc29 --- /dev/null +++ b/backend/src/backup_tool/web.py @@ -0,0 +1,43 @@ +"""Dedicated HTTP runtime role.""" + +from __future__ import annotations + +import stat + +import uvicorn + +from backup_tool.config import Settings +from backup_tool.observability.logging import configure_logging, log_event + + +def _prepare_socket(settings: Settings) -> str: + path = settings.web_socket_path + path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + try: + existing = path.lstat() + except FileNotFoundError: + return str(path) + if not stat.S_ISSOCK(existing.st_mode): + raise RuntimeError("web socket path is not a socket") + path.unlink() + return str(path) + + +def run_web(settings: Settings) -> int: + """Run exactly one ASGI server without reload or embedded background roles.""" + from backup_tool.api.app import create_app + + configure_logging("web", settings.log_level) + server = uvicorn.Server( + uvicorn.Config( + create_app(settings), + uds=_prepare_socket(settings), + log_level=settings.log_level.lower(), + reload=False, + workers=1, + log_config=None, + ) + ) + server.run() + log_event("role_stopped", role="web") + return 0 diff --git a/backend/src/backup_tool/worker.py b/backend/src/backup_tool/worker.py index 5837443..de726b2 100644 --- a/backend/src/backup_tool/worker.py +++ b/backend/src/backup_tool/worker.py @@ -1,22 +1,25 @@ """Single-node durable worker role. -The worker owns leases; backup publishing is deliberately supplied by later M6 work. +The worker owns leases and performs repository I/O outside the API process. """ from __future__ import annotations import asyncio import contextlib +import importlib import signal -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import cast from uuid import uuid4 from sqlalchemy import select, update -from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from backup_tool.config import Settings from backup_tool.db.engine import create_engine -from backup_tool.db.models import Execution +from backup_tool.db.models import Backup, Execution, Job, Repository, Restore, Source from backup_tool.execution import ( claim, complete_cancellation, @@ -25,22 +28,158 @@ from backup_tool.execution import ( recover_stale, transition, ) +from backup_tool.faults import FaultInjector, NoFault +from backup_tool.gc import process_retention_gc +from backup_tool.notifications.dispatcher import dispatch_one, recover_notification_leases +from backup_tool.notifications.events import emit_event +from backup_tool.observability.logging import configure_logging, log_event +from backup_tool.repository import reconcile_key_rotations +from backup_tool.security.secrets import EnvelopeCipher + +snapshot = importlib.import_module("backup_tool.snapshot") +SnapshotError = snapshot.SnapshotError +SnapshotIntegrityError = snapshot.SnapshotIntegrityError class Worker: - def __init__(self, settings: Settings, *, owner: str | None = None) -> None: + def __init__( + self, + settings: Settings, + *, + owner: str | None = None, + fault_injector: FaultInjector | None = None, + ) -> None: self.settings = settings self.owner = owner or f"worker-{uuid4()}" + self.fault_injector = fault_injector or NoFault() self._stopping = asyncio.Event() self.engine = create_engine(settings) self.sessions = async_sessionmaker(self.engine, expire_on_commit=False) + self.cipher = EnvelopeCipher.from_file(settings.master_key_file) + self._execution_turns = 0 + self._next_maintenance_at: datetime | None = None async def startup(self) -> int: async with self.sessions() as db: - return await recover_stale(db) + rotations = await reconcile_key_rotations(self.settings, db) + publications = cast(int, await snapshot.reconcile_publications(self.settings, db)) + restored = cast(int, await snapshot.reconcile_restores(self.settings, db)) + recovered_executions = await recover_stale(db) + recovered_deliveries = await recover_notification_leases(db) + maintenance = await process_retention_gc(db) + self._next_maintenance_at = datetime.now(UTC) + timedelta(seconds=60) + return ( + rotations + + publications + + restored + + recovered_executions + + recovered_deliveries + + maintenance.tombstoned + ) + + async def _run_restore(self, db: AsyncSession) -> bool: + restore_id = await db.scalar( + select(Restore.id) + .where(Restore.state == "queued") + .order_by(Restore.created_at) + .limit(1) + ) + if restore_id is None: + return False + result = await db.execute( + update(Restore) + .where(Restore.id == restore_id, Restore.state == "queued") + .values(state="running") + ) + if getattr(result, "rowcount", 0) != 1: + await db.rollback() + return False + await db.commit() + restore = await db.get(Restore, restore_id) + if restore is None: + return False + try: + backup = await db.get(Backup, restore.backup_id) + if backup is None or backup.integrity != "verified" or backup.tombstoned_at is not None: + raise SnapshotError("backup is unavailable") + execution = await db.get(Execution, backup.execution_id) + if execution is None: + raise SnapshotError("backup execution is unavailable") + job = await db.get(Job, execution.job_id) + if job is None: + raise SnapshotError("backup job is unavailable") + repository = await db.get(Repository, job.repository_id) + if repository is None: + raise SnapshotError("backup repository is unavailable") + restore.result = await snapshot.restore_full_snapshot( + self.settings, db, restore, backup, repository + ) + restore.state = "committed" + await emit_event( + db, + "restore.committed", + correlation_id=restore.id, + resource={"restore_id": restore.id, "backup_id": restore.backup_id}, + payload={"dry_run": restore.dry_run, "outcome": "committed"}, + deduplication_key=f"restore:{restore.id}:committed", + ) + await db.commit() + except SnapshotIntegrityError: + await db.rollback() + failed = await db.get(Restore, restore_id) + if failed is not None: + corrupted_backup = await db.get(Backup, failed.backup_id) + if corrupted_backup is not None: + corrupted_backup.integrity = "corrupt" + failed.state = "failed" + failed.result = {"reason": "restore_failed"} + await emit_event( + db, + "restore.failed", + correlation_id=failed.id, + resource={"restore_id": failed.id, "backup_id": failed.backup_id}, + payload={"reason_code": "restore_failed"}, + deduplication_key=f"restore:{failed.id}:failed", + ) + await db.commit() + except SnapshotError: + await db.rollback() + failed = await db.get(Restore, restore_id) + if failed is not None: + failed.state = "failed" + failed.result = {"reason": "restore_failed"} + await emit_event( + db, + "restore.failed", + correlation_id=failed.id, + resource={"restore_id": failed.id, "backup_id": failed.backup_id}, + payload={"reason_code": "restore_failed"}, + deduplication_key=f"restore:{failed.id}:failed", + ) + await db.commit() + return True + + async def _run_maintenance(self, db: AsyncSession) -> None: + now = datetime.now(UTC) + if self._next_maintenance_at is None or now >= self._next_maintenance_at: + await process_retention_gc(db, now) + self._next_maintenance_at = now + timedelta(seconds=60) async def run_once(self) -> bool: + if self._stopping.is_set(): + return False async with self.sessions() as db: + # Retention/GC is a worker responsibility, but is rate-limited so it + # cannot turn a sustained backup queue into a metadata polling loop. + await self._run_maintenance(db) + # Never let an always-nonempty execution queue starve due notifications. + if self._execution_turns >= 1 and await dispatch_one( + db, self.settings, self.cipher, self.owner + ): + self._execution_turns = 0 + return True + if self._stopping.is_set(): + return False execution_id = await db.scalar( select(Execution.id) .where(Execution.state == "queued") @@ -48,10 +187,13 @@ class Worker: .limit(1) ) if execution_id is None: - return False + if await self._run_restore(db): + return True + return await dispatch_one(db, self.settings, self.cipher, self.owner) execution = await claim(db, execution_id, self.owner) if execution is None: return False + self._execution_turns += 1 # Reload after the claim: a control request can race the lease acquisition. await db.refresh(execution) if execution.state == "cancelling": @@ -73,6 +215,76 @@ class Worker: await record_event(db, execution) await db.commit() await heartbeat(db, execution.id, self.owner) + failure_detail: str | None = None + failure_reason = "transient_io" + try: + job = await db.get(Job, execution.job_id) + if job is None: + raise SnapshotError("execution job is unavailable") + source = await db.get(Source, job.source_id) + repository = await db.get(Repository, job.repository_id) + if source is None or repository is None: + raise SnapshotError("execution source or repository is unavailable") + backup = await snapshot.publish_full_snapshot( + self.settings, + db, + execution, + job, + source, + repository, + self.fault_injector, + cipher=self.cipher, + ) + await db.flush() + await emit_event( + db, + "backup.committed", + correlation_id=execution.id, + resource={ + "execution_id": execution.id, + "job_id": job.id, + "repository_id": repository.id, + "backup_id": backup.id, + }, + payload={"integrity": backup.integrity, "effective_mode": job.requested_mode}, + deduplication_key=f"backup:{backup.id}:committed", + ) + await emit_event( + db, + "backup.verification_succeeded", + correlation_id=execution.id, + resource={"execution_id": execution.id, "backup_id": backup.id}, + payload={"integrity": backup.integrity}, + deduplication_key=f"backup:{backup.id}:verified", + ) + execution.state = transition("running", "verifying") + await record_event(db, execution) + self.fault_injector.hit("metadata.before_commit") + await db.commit() + self.fault_injector.hit("metadata.after_commit") + execution.state = transition("verifying", "committed") + execution.completed_at = datetime.now(UTC) + execution.lease_owner = None + execution.lease_expires_at = None + await record_event(db, execution) + await db.commit() + snapshot.finalize_publication(Path(repository.root), execution.id) + except SnapshotError as error: + await db.rollback() + failure_detail = str(error) + failure_reason = error.reason_code + if failure_detail is not None: + failed = await db.get(Execution, execution_id) + if failed is None or failed.state not in {"preparing", "running", "verifying"}: + return True + failed.state = "failed" + failed.reason_code = failure_reason + failed.operator_message = failure_detail + failed.completed_at = datetime.now(UTC) + failed.lease_owner = None + failed.lease_expires_at = None + await record_event(db, failed) + await db.commit() return True async def run(self) -> None: @@ -84,10 +296,12 @@ class Worker: await self.engine.dispose() def stop(self) -> None: + log_event("role_stopping", role="worker") self._stopping.set() def run_worker(settings: Settings) -> int: + configure_logging("worker", settings.log_level) worker = Worker(settings) loop = asyncio.new_event_loop() for sig in (signal.SIGINT, signal.SIGTERM): @@ -97,4 +311,5 @@ def run_worker(settings: Settings) -> int: loop.run_until_complete(worker.run()) finally: loop.close() + log_event("role_stopped", role="worker") return 0 diff --git a/contracts/repository/v1/manifest.schema.json b/contracts/repository/v1/manifest.schema.json index b89ab09..8313d1d 100644 --- a/contracts/repository/v1/manifest.schema.json +++ b/contracts/repository/v1/manifest.schema.json @@ -24,7 +24,7 @@ "additionalProperties": false, "required": ["adapter", "captured_at", "evidence"], "properties": { - "adapter": {"enum": ["local", "ssh", "postgresql", "mysql"]}, + "adapter": {"enum": ["local", "postgresql", "mysql"]}, "captured_at": {"type": "string", "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\\.[0-9]+)?Z$"}, "evidence": {"type": "object"} } diff --git a/contracts/repository/v1/repository.schema.json b/contracts/repository/v1/repository.schema.json index 29af0e7..9a16166 100644 --- a/contracts/repository/v1/repository.schema.json +++ b/contracts/repository/v1/repository.schema.json @@ -5,6 +5,13 @@ "type": "object", "additionalProperties": false, "required": ["repository_id", "format_version", "digest_algorithm", "compression", "encryption", "created_at"], + "allOf": [ + { + "if": {"properties": {"encryption": {"properties": {"mode": {"const": "none"}}}}}, + "then": {"properties": {"encryption": {"properties": {"key_id": {"type": "null"}}}}}, + "else": {"properties": {"encryption": {"properties": {"key_id": {"type": "string", "format": "uuid", "minLength": 1}}}}} + } + ], "properties": { "repository_id": { "type": "string", diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..78e3e58 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,115 @@ +name: backup-tool + +x-app-base: &app-base + build: + context: . + dockerfile: Dockerfile + image: backup-tool:local + init: true + read_only: true + tmpfs: + - /tmp:mode=1777,size=64m + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + environment: + BACKUP_TOOL_DATA_DIR: /var/lib/backup-tool + BACKUP_TOOL_DATABASE_URL: sqlite+aiosqlite:////var/lib/backup-tool/metadata.db + BACKUP_TOOL_REPOSITORY_ROOTS: '["/var/lib/backup-tool/repositories"]' + BACKUP_TOOL_LOCAL_SOURCE_ROOTS: '["/mnt/sources"]' + BACKUP_TOOL_RESTORE_ROOTS: '["/var/lib/backup-tool/restores"]' + BACKUP_TOOL_MASTER_KEY_FILE: /run/backup-tool-secrets/master.key + BACKUP_TOOL_WEB_SOCKET_PATH: /run/backup-tool/web.sock + BACKUP_TOOL_PUBLIC_BASE_URL: ${BACKUP_TOOL_PUBLIC_BASE_URL:-http://localhost:8080} + volumes: + - backup-tool-data:/var/lib/backup-tool + - backup-tool-runtime:/run/backup-tool + - type: bind + source: ${BACKUP_TOOL_SOURCES_DIR:-./sources} + target: /mnt/sources + read_only: true + - type: bind + source: ${BACKUP_TOOL_MASTER_KEY_FILE:-./secrets/master.key} + target: /run/backup-tool-secrets/master.key + read_only: true + +services: + migrate: + <<: *app-base + command: ["migrate", "upgrade"] + restart: "no" + + web: + <<: *app-base + command: ["web"] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "backup-tool", "health", "web"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 0s + + scheduler: + <<: *app-base + command: ["scheduler"] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "backup-tool", "health", "scheduler"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 0s + + worker: + <<: *app-base + command: ["worker"] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "backup-tool", "health", "worker"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 0s + + admin: + <<: *app-base + profiles: ["admin"] + command: ["admin", "--help"] + depends_on: + migrate: + condition: service_completed_successfully + + proxy: + build: + context: ./frontend + dockerfile: Dockerfile + image: backup-tool-proxy:local + init: true + read_only: true + tmpfs: + - /tmp:mode=1777,size=64m + - /var/cache/nginx:uid=10001,gid=0,mode=0750,size=32m + - /var/run:uid=10001,gid=0,mode=0750,size=8m + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + ports: + - "127.0.0.1:${BACKUP_TOOL_PORT:-8080}:8080" + volumes: + - backup-tool-runtime:/run/backup-tool:ro + depends_on: + web: + condition: service_healthy + +volumes: + backup-tool-data: + backup-tool-runtime: diff --git a/docs/release/m10-evidence.md b/docs/release/m10-evidence.md new file mode 100644 index 0000000..66ccdf7 --- /dev/null +++ b/docs/release/m10-evidence.md @@ -0,0 +1,29 @@ +# M10 SSH source evidence + +The released source capability is `ssh`: private-key-only, pinned-host-key, +forced-SFTP chroot access. No password, shell, command channel, agent, default +key discovery, or arbitrary remote root is supported. + +## Fixture + +`tests/compose.ssh.yaml` builds a test-only OpenSSH server. Each run generates +host and client Ed25519 keys under pytest `tmp_path`, mounts no committed keys, +and configures a dedicated `backup` account with `ChrootDirectory /home/backup` +and `ForceCommand internal-sftp`. The ordinary operator Compose stack is not +modified. Run `make test-ssh-integration` to build, run, and tear down the +fixture. + +## Verification + +- Fake transport tests cover pin mismatch before authentication/SFTP, bounded + reads, unsafe entries, and accepted/rejected private-key algorithms. +- The opt-in live test covers a private-key probe, backup, signed verification, + and restore through the forced-SFTP fixture. +- `make test-ssh-integration` passed after fixture isolation and SFTP-channel + concurrency fixes. +- Final `make check` passed: 107 unit/contract, 73 integration (one skipped), + 15 fault, and 33 security tests; Ruff, mypy, TypeScript, and the frontend + build passed. + +See `docs/runbooks/ssh-sources.md` for deployment prerequisites, rotation, and +containment limitations. diff --git a/docs/release/m11-evidence.md b/docs/release/m11-evidence.md new file mode 100644 index 0000000..0eba981 --- /dev/null +++ b/docs/release/m11-evidence.md @@ -0,0 +1,55 @@ +# M11 recovery import evidence + +- Recovery bundles use the existing versioned `BTREC` Argon2id/AES-GCM codec + and now carry a version-2 authenticated catalog containing repositories, + key epochs, sources, jobs, executions, and backups required for encrypted + restore. +- `backup-tool admin recovery import` requires a migrated empty destination DB, + re-inspects every surviving repository through configured allowlists, writes + signing/data keys exclusively with mode `0600`, and rejects conflicts. +- Imported sources are `unavailable`; imported jobs are `archived` and disabled. + Imported metadata therefore supports existing restore records without silently + restarting backup schedules. +- The focused host-loss drill exports an encrypted backup, imports it into a + fresh metadata/key host, and restores the file byte-for-byte. It also proves + unsafe repository paths and non-empty destination metadata are rejected. +- Encrypted repository creation is enabled only after that drill passed and now + records its active data-key epoch atomically with repository metadata. +- Interrupted imports remove newly installed key files on handled failure; a + retry also safely adopts only exact, authenticated key files left by an + unclean process loss. Rotation writes an fsynced repository journal before + its DB transition; worker startup deterministically completes a committed + epoch transition or removes an uncommitted one while retaining old-active. + A stale rollback journal is cleared safely even when the unreferenced new key + was already deleted before the journal cleanup could run. +- Key-aware GC decrypts encrypted manifests using their declared epoch key and + remains fail-closed for absent, wrong, or corrupt keys/manifests. Restore + removes setuid, setgid, and sticky bits from captured modes. Migration 0007 + refuses downgrade while key metadata is populated. +- Recovery catalog import preserves backup `created_at` and `tombstoned_at`. +- Snapshot staging roots, per-execution directories, blob directories, and + plaintext temporary blobs are created owner-only (`0700`/`0600`) independent + of umask and are rejected if their permissions are unsafe. + +## Verification + +```text +pytest tests/integration/test_encrypted_repository.py -q +5 passed + +make test-fault +11 passed + +make test-security +21 passed + +make lint && make typecheck && make frontend-build +passed + +make check +86 unit/contract, 53 integration, 12 fault, and 22 security tests passed; +Ruff, mypy, TypeScript, and frontend build passed. + +git diff --check && git diff --cached --quiet +passed +``` diff --git a/docs/release/m12-evidence.md b/docs/release/m12-evidence.md new file mode 100644 index 0000000..3755ca9 --- /dev/null +++ b/docs/release/m12-evidence.md @@ -0,0 +1,43 @@ +# M12 evidence + +- Migration revision: `0008_notification_outbox` +- Event schema version: `1` +- Catalog: 22 live stable IDs only. Deferred/unimplemented operation types are not public notification contracts. +- Delivery guarantee: durable at-least-once, stable event ID, leased worker retries; receiver deduplication is required. + +## Green focused evidence + +```text +PYTHONPATH=.:backend/src .venv/bin/python -m pytest \ + tests/contract/test_notification_contract.py \ + tests/integration/test_migrations.py \ + tests/integration/test_all_operational_events_deliver.py \ + tests/integration/test_notifications.py \ + tests/fault/test_notification_retries.py \ + tests/security/test_webhook_ssrf.py -q +28 passed (live-catalog contract, fair dispatch regression; no deferred event IDs). + +# Scheduler-role service-path regression +PYTHONPATH=.:backend/src .venv/bin/python -m pytest \ + tests/integration/test_scheduler_live_sync.py -q +1 passed (scheduler role service path) +``` + +The suite uses temporary SQLite/repository roots, a fake SMTP implementation, fake resolver inputs, and dispatcher monkeypatches; it performs no real webhook DNS, HTTP, or SMTP delivery. It verifies receiver-visible versioned webhook headers/signatures, STARTTLS-before-AUTH SMTP behavior, transient/permanent SMTP classification, persisted SMTP attempt limits, lease-abandoned attempt closure, selected-only test sends, manual-retry idempotency, and the absence of a plaintext-secret idempotency verifier. Behavioral producer tests cover every live catalog family: execution, schedule, backup/verification, restore, and retention. They also prove fair dispatch under an execution backlog, scheduler-role delivery, and worker-owned retention/GC maintenance. The CLI scheduler role now runs `SchedulerService`; worker maintenance runs durable retention/GC on startup and at bounded intervals. + +## Quality evidence + +- `make test-fast`: 89 passed. +- `make test-integration`: 61 passed in 27.00s (the execution wrapper nevertheless returned exit 124 at its fixed 30s wall limit). +- `make test-fault`: focused fair-dispatch regression passed. +- `make test-security`: 28 passed. +- `make lint` and `make typecheck`: passed. +- `make frontend-build`: passed. +- `git diff --check`: passed. +- staged-file check: no staged files. + +Final verification: `make check` passed — 90 unit/contract, 61 integration, 14 fault, and 28 security tests; Ruff, mypy, TypeScript, and the frontend build passed. `git diff --check` and the staged-file check also passed. + +## Rollback + +Disable or archive subscriptions and stop worker dispatch. Do not delete notification events, deliveries, or attempts: they remain audit history. A fresh host recovery intentionally starts with no notification settings or credentials and must be reconfigured. diff --git a/docs/release/m13-foundation-evidence.md b/docs/release/m13-foundation-evidence.md new file mode 100644 index 0000000..a1a22f4 --- /dev/null +++ b/docs/release/m13-foundation-evidence.md @@ -0,0 +1,20 @@ +# M13 UI/OpenAPI evidence + +- OpenAPI is deterministically exported to `openapi/v2.json`; generated browser client drift is checked by `npm --prefix frontend run api:check`. +- `text/event-stream` is declared in OpenAPI for execution events. The generated client intentionally emits `executionEventsUrl(...) -> URL`, not a misleading JSON `Promise`; UI opens that URL using browser `EventSource`. +- Notification delivery retry requests send an `Idempotency-Key` and the browser CSRF header. +- Recovery status is CLI/runbook-only. The browser has no export, import, bundle, key, or passphrase transfer control. +- v2.1 PostgreSQL, MySQL, and TAR/download controls remain absent. + +## Green M13 checks + +```text +npm --prefix frontend test -- --run # 9 passed +npm --prefix frontend run typecheck # passed +npm --prefix frontend run build # passed +npx --prefix frontend playwright test --config frontend/playwright.config.ts # 1 passed +.venv/bin/python tools/export_openapi.py --check openapi/v2.json # current +npm --prefix frontend run api:check # current +``` + +Final verification: `make check` passed — 90 unit/contract, 61 integration, 14 fault, and 28 security tests; Ruff, mypy, TypeScript, and the frontend build passed. `git diff --check` and the staged-file check also passed. diff --git a/docs/release/m14-evidence.md b/docs/release/m14-evidence.md new file mode 100644 index 0000000..646e260 --- /dev/null +++ b/docs/release/m14-evidence.md @@ -0,0 +1,34 @@ +# M14 operations and packaging evidence + +## Delivered + +- Pinned non-root OCI application and proxy images; isolated web, scheduler, worker, + migrate, and admin roles; same-origin Unix-socket proxy; no reload or embedded roles. + The proxy port is loopback-bound by default (`127.0.0.1`), so the localhost public + URL cannot permit remote first-admin setup takeover. Operators exposing it through + an external reverse proxy must set a non-loopback public URL and bootstrap secret. +- Role-aware readiness, JSON structured logs, safe worker claim shutdown, and + dependency-free Prometheus metrics for request volume/duration, active/stale/failed + execution state, schedule lag, corrupt/unavailable repositories, and free space. +- SBOM generation at `tools/generate_sbom.py`, generated CycloneDX artifact + `m14-sbom.json`, and base-image/source provenance in `m14-provenance.md`. +- Metadata, repository, key, upgrade, disaster-recovery, and observability runbooks. + +## Green verification + +```text + docker compose config --quiet # passed (loopback port binding) + docker compose build --pull # passed + make test-e2e # 1 passed in 51.69s (loopback regression) + make check # passed after loopback regression + 91 unit/contract, 61 integration, 15 fault, 33 security + Ruff/format, mypy (40 files), TypeScript, frontend build all passed + python tools/generate_sbom.py # 256 components +``` + +`test_compose_v2.py` creates its source and host-bind key fixture in pytest temporary +directories. The actual service-owned master key is generated only in an ephemeral +Compose named volume, then `down --volumes --remove-orphans` removes it. No fixture +secret or source directory is committed. The test runs migration, starts the stack, +checks readiness and metrics, stops/restarts the worker cleanly, restarts runtime +roles, verifies setup metadata persists, and tears down the project. diff --git a/docs/release/m14-provenance.md b/docs/release/m14-provenance.md new file mode 100644 index 0000000..f70c20e --- /dev/null +++ b/docs/release/m14-provenance.md @@ -0,0 +1,13 @@ +# M14 build provenance + +- **Source revision:** `396219e776aa9a115900d2b7bfd9fb5c1cfde115` +- **Application base:** `python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7` +- **Frontend builder:** `node:22.17.1-alpine@sha256:5539840ce9d013fa13e3b9814c9353024be7ac75aca5db6d039504a56c04ea59` +- **Proxy base:** `nginx:1.29.7-alpine@sha256:e7257f1ef28ba17cf7c248cb8ccf6f0c6e0228ab9c315c152f9c203cd34cf6d1` +- **Build command:** `docker compose build --pull` +- **SBOM:** `docs/release/m14-sbom.json`, generated deterministically with + `python tools/generate_sbom.py` from the pinned backend manifest and frontend lockfile. + +The build uses digest-pinned bases and a non-root runtime user. Provenance records +inputs and generation instructions rather than embedding a mutable image tag or a +secret-bearing build environment. diff --git a/docs/release/m14-sbom.json b/docs/release/m14-sbom.json new file mode 100644 index 0000000..514d7c7 --- /dev/null +++ b/docs/release/m14-sbom.json @@ -0,0 +1,1551 @@ +{ + "bomFormat": "CycloneDX", + "components": [ + { + "name": "@adobe/css-tools", + "purl": "pkg:npm/@adobe/css-tools@4.4.4", + "type": "library", + "version": "4.4.4" + }, + { + "name": "@alloc/quick-lru", + "purl": "pkg:npm/@alloc/quick-lru@5.2.0", + "type": "library", + "version": "5.2.0" + }, + { + "name": "@asamuzakjp/css-color", + "purl": "pkg:npm/@asamuzakjp/css-color@6.0.5", + "type": "library", + "version": "6.0.5" + }, + { + "name": "@asamuzakjp/dom-selector", + "purl": "pkg:npm/@asamuzakjp/dom-selector@8.3.0", + "type": "library", + "version": "8.3.0" + }, + { + "name": "@babel/code-frame", + "purl": "pkg:npm/@babel/code-frame@7.29.0", + "type": "library", + "version": "7.29.0" + }, + { + "name": "@babel/helper-validator-identifier", + "purl": "pkg:npm/@babel/helper-validator-identifier@7.28.5", + "type": "library", + "version": "7.28.5" + }, + { + "name": "@babel/runtime", + "purl": "pkg:npm/@babel/runtime@7.29.2", + "type": "library", + "version": "7.29.2" + }, + { + "name": "@bramus/specificity", + "purl": "pkg:npm/@bramus/specificity@2.4.2", + "type": "library", + "version": "2.4.2" + }, + { + "name": "@csstools/color-helpers", + "purl": "pkg:npm/@csstools/color-helpers@6.1.0", + "type": "library", + "version": "6.1.0" + }, + { + "name": "@csstools/css-calc", + "purl": "pkg:npm/@csstools/css-calc@3.3.0", + "type": "library", + "version": "3.3.0" + }, + { + "name": "@csstools/css-color-parser", + "purl": "pkg:npm/@csstools/css-color-parser@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@csstools/css-parser-algorithms", + "purl": "pkg:npm/@csstools/css-parser-algorithms@4.0.0", + "type": "library", + "version": "4.0.0" + }, + { + "name": "@csstools/css-syntax-patches-for-csstree", + "purl": "pkg:npm/@csstools/css-syntax-patches-for-csstree@1.1.7", + "type": "library", + "version": "1.1.7" + }, + { + "name": "@csstools/css-tokenizer", + "purl": "pkg:npm/@csstools/css-tokenizer@4.0.0", + "type": "library", + "version": "4.0.0" + }, + { + "name": "@emnapi/core", + "purl": "pkg:npm/@emnapi/core@1.11.1", + "type": "library", + "version": "1.11.1" + }, + { + "name": "@emnapi/runtime", + "purl": "pkg:npm/@emnapi/runtime@1.11.1", + "type": "library", + "version": "1.11.1" + }, + { + "name": "@emnapi/wasi-threads", + "purl": "pkg:npm/@emnapi/wasi-threads@1.2.2", + "type": "library", + "version": "1.2.2" + }, + { + "name": "@exodus/bytes", + "purl": "pkg:npm/@exodus/bytes@1.15.1", + "type": "library", + "version": "1.15.1" + }, + { + "name": "@jridgewell/gen-mapping", + "purl": "pkg:npm/@jridgewell/gen-mapping@0.3.13", + "type": "library", + "version": "0.3.13" + }, + { + "name": "@jridgewell/resolve-uri", + "purl": "pkg:npm/@jridgewell/resolve-uri@3.1.2", + "type": "library", + "version": "3.1.2" + }, + { + "name": "@jridgewell/sourcemap-codec", + "purl": "pkg:npm/@jridgewell/sourcemap-codec@1.5.5", + "type": "library", + "version": "1.5.5" + }, + { + "name": "@jridgewell/trace-mapping", + "purl": "pkg:npm/@jridgewell/trace-mapping@0.3.31", + "type": "library", + "version": "0.3.31" + }, + { + "name": "@napi-rs/wasm-runtime", + "purl": "pkg:npm/@napi-rs/wasm-runtime@1.1.6", + "type": "library", + "version": "1.1.6" + }, + { + "name": "@nodelib/fs.scandir", + "purl": "pkg:npm/@nodelib/fs.scandir@2.1.5", + "type": "library", + "version": "2.1.5" + }, + { + "name": "@nodelib/fs.stat", + "purl": "pkg:npm/@nodelib/fs.stat@2.0.5", + "type": "library", + "version": "2.0.5" + }, + { + "name": "@nodelib/fs.walk", + "purl": "pkg:npm/@nodelib/fs.walk@1.2.8", + "type": "library", + "version": "1.2.8" + }, + { + "name": "@oxc-project/types", + "purl": "pkg:npm/@oxc-project/types@0.139.0", + "type": "library", + "version": "0.139.0" + }, + { + "name": "@playwright/test", + "purl": "pkg:npm/@playwright/test@1.57.0", + "type": "library", + "version": "1.57.0" + }, + { + "name": "@rolldown/binding-android-arm64", + "purl": "pkg:npm/@rolldown/binding-android-arm64@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-darwin-arm64", + "purl": "pkg:npm/@rolldown/binding-darwin-arm64@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-darwin-x64", + "purl": "pkg:npm/@rolldown/binding-darwin-x64@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-freebsd-x64", + "purl": "pkg:npm/@rolldown/binding-freebsd-x64@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-arm-gnueabihf", + "purl": "pkg:npm/@rolldown/binding-linux-arm-gnueabihf@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-arm64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-arm64-gnu@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-arm64-musl", + "purl": "pkg:npm/@rolldown/binding-linux-arm64-musl@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-ppc64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-ppc64-gnu@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-s390x-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-s390x-gnu@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-x64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-x64-gnu@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-linux-x64-musl", + "purl": "pkg:npm/@rolldown/binding-linux-x64-musl@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-openharmony-arm64", + "purl": "pkg:npm/@rolldown/binding-openharmony-arm64@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-wasm32-wasi", + "purl": "pkg:npm/@rolldown/binding-wasm32-wasi@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-win32-arm64-msvc", + "purl": "pkg:npm/@rolldown/binding-win32-arm64-msvc@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/binding-win32-x64-msvc", + "purl": "pkg:npm/@rolldown/binding-win32-x64-msvc@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "@rolldown/pluginutils", + "purl": "pkg:npm/@rolldown/pluginutils@1.0.1", + "type": "library", + "version": "1.0.1" + }, + { + "name": "@standard-schema/spec", + "purl": "pkg:npm/@standard-schema/spec@1.1.0", + "type": "library", + "version": "1.1.0" + }, + { + "name": "@testing-library/dom/node_modules/aria-query", + "purl": "pkg:npm/@testing-library/dom/node_modules/aria-query@5.3.0", + "type": "library", + "version": "5.3.0" + }, + { + "name": "@testing-library/dom/node_modules/dom-accessibility-api", + "purl": "pkg:npm/@testing-library/dom/node_modules/dom-accessibility-api@0.5.16", + "type": "library", + "version": "0.5.16" + }, + { + "name": "@testing-library/dom", + "purl": "pkg:npm/@testing-library/dom@10.4.1", + "type": "library", + "version": "10.4.1" + }, + { + "name": "@testing-library/jest-dom", + "purl": "pkg:npm/@testing-library/jest-dom@7.0.0", + "type": "library", + "version": "7.0.0" + }, + { + "name": "@testing-library/react", + "purl": "pkg:npm/@testing-library/react@16.3.2", + "type": "library", + "version": "16.3.2" + }, + { + "name": "@tybys/wasm-util", + "purl": "pkg:npm/@tybys/wasm-util@0.10.3", + "type": "library", + "version": "0.10.3" + }, + { + "name": "@types/aria-query", + "purl": "pkg:npm/@types/aria-query@5.0.4", + "type": "library", + "version": "5.0.4" + }, + { + "name": "@types/chai", + "purl": "pkg:npm/@types/chai@5.2.3", + "type": "library", + "version": "5.2.3" + }, + { + "name": "@types/deep-eql", + "purl": "pkg:npm/@types/deep-eql@4.0.2", + "type": "library", + "version": "4.0.2" + }, + { + "name": "@types/estree", + "purl": "pkg:npm/@types/estree@1.0.9", + "type": "library", + "version": "1.0.9" + }, + { + "name": "@types/react-dom", + "purl": "pkg:npm/@types/react-dom@19.2.3", + "type": "library", + "version": "19.2.3" + }, + { + "name": "@types/react", + "purl": "pkg:npm/@types/react@19.2.17", + "type": "library", + "version": "19.2.17" + }, + { + "name": "@typescript/typescript-aix-ppc64", + "purl": "pkg:npm/@typescript/typescript-aix-ppc64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-darwin-arm64", + "purl": "pkg:npm/@typescript/typescript-darwin-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-darwin-x64", + "purl": "pkg:npm/@typescript/typescript-darwin-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-freebsd-arm64", + "purl": "pkg:npm/@typescript/typescript-freebsd-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-freebsd-x64", + "purl": "pkg:npm/@typescript/typescript-freebsd-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-arm64", + "purl": "pkg:npm/@typescript/typescript-linux-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-arm", + "purl": "pkg:npm/@typescript/typescript-linux-arm@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-loong64", + "purl": "pkg:npm/@typescript/typescript-linux-loong64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-mips64el", + "purl": "pkg:npm/@typescript/typescript-linux-mips64el@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-ppc64", + "purl": "pkg:npm/@typescript/typescript-linux-ppc64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-riscv64", + "purl": "pkg:npm/@typescript/typescript-linux-riscv64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-s390x", + "purl": "pkg:npm/@typescript/typescript-linux-s390x@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-linux-x64", + "purl": "pkg:npm/@typescript/typescript-linux-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-netbsd-arm64", + "purl": "pkg:npm/@typescript/typescript-netbsd-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-netbsd-x64", + "purl": "pkg:npm/@typescript/typescript-netbsd-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-openbsd-arm64", + "purl": "pkg:npm/@typescript/typescript-openbsd-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-openbsd-x64", + "purl": "pkg:npm/@typescript/typescript-openbsd-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-sunos-x64", + "purl": "pkg:npm/@typescript/typescript-sunos-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-win32-arm64", + "purl": "pkg:npm/@typescript/typescript-win32-arm64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@typescript/typescript-win32-x64", + "purl": "pkg:npm/@typescript/typescript-win32-x64@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "@vitejs/plugin-react", + "purl": "pkg:npm/@vitejs/plugin-react@6.0.4", + "type": "library", + "version": "6.0.4" + }, + { + "name": "@vitest/expect", + "purl": "pkg:npm/@vitest/expect@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/mocker", + "purl": "pkg:npm/@vitest/mocker@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/pretty-format", + "purl": "pkg:npm/@vitest/pretty-format@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/runner", + "purl": "pkg:npm/@vitest/runner@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/snapshot", + "purl": "pkg:npm/@vitest/snapshot@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/spy", + "purl": "pkg:npm/@vitest/spy@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "@vitest/utils", + "purl": "pkg:npm/@vitest/utils@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "ansi-regex", + "purl": "pkg:npm/ansi-regex@5.0.1", + "type": "library", + "version": "5.0.1" + }, + { + "name": "any-promise", + "purl": "pkg:npm/any-promise@1.3.0", + "type": "library", + "version": "1.3.0" + }, + { + "name": "anymatch", + "purl": "pkg:npm/anymatch@3.1.3", + "type": "library", + "version": "3.1.3" + }, + { + "name": "arg", + "purl": "pkg:npm/arg@5.0.2", + "type": "library", + "version": "5.0.2" + }, + { + "name": "aria-query", + "purl": "pkg:npm/aria-query@5.3.2", + "type": "library", + "version": "5.3.2" + }, + { + "name": "assertion-error", + "purl": "pkg:npm/assertion-error@2.0.1", + "type": "library", + "version": "2.0.1" + }, + { + "name": "autoprefixer", + "purl": "pkg:npm/autoprefixer@10.5.4", + "type": "library", + "version": "10.5.4" + }, + { + "name": "baseline-browser-mapping", + "purl": "pkg:npm/baseline-browser-mapping@2.11.5", + "type": "library", + "version": "2.11.5" + }, + { + "name": "bidi-js", + "purl": "pkg:npm/bidi-js@1.0.3", + "type": "library", + "version": "1.0.3" + }, + { + "name": "binary-extensions", + "purl": "pkg:npm/binary-extensions@2.3.0", + "type": "library", + "version": "2.3.0" + }, + { + "name": "braces", + "purl": "pkg:npm/braces@3.0.3", + "type": "library", + "version": "3.0.3" + }, + { + "name": "browserslist", + "purl": "pkg:npm/browserslist@4.28.7", + "type": "library", + "version": "4.28.7" + }, + { + "name": "camelcase-css", + "purl": "pkg:npm/camelcase-css@2.0.1", + "type": "library", + "version": "2.0.1" + }, + { + "name": "caniuse-lite", + "purl": "pkg:npm/caniuse-lite@1.0.30001806", + "type": "library", + "version": "1.0.30001806" + }, + { + "name": "chai", + "purl": "pkg:npm/chai@6.2.2", + "type": "library", + "version": "6.2.2" + }, + { + "name": "chokidar/node_modules/glob-parent", + "purl": "pkg:npm/chokidar/node_modules/glob-parent@5.1.2", + "type": "library", + "version": "5.1.2" + }, + { + "name": "chokidar", + "purl": "pkg:npm/chokidar@3.6.0", + "type": "library", + "version": "3.6.0" + }, + { + "name": "commander", + "purl": "pkg:npm/commander@4.1.1", + "type": "library", + "version": "4.1.1" + }, + { + "name": "convert-source-map", + "purl": "pkg:npm/convert-source-map@2.0.0", + "type": "library", + "version": "2.0.0" + }, + { + "name": "css-tree", + "purl": "pkg:npm/css-tree@3.2.1", + "type": "library", + "version": "3.2.1" + }, + { + "name": "css.escape", + "purl": "pkg:npm/css.escape@1.5.1", + "type": "library", + "version": "1.5.1" + }, + { + "name": "cssesc", + "purl": "pkg:npm/cssesc@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "name": "csstype", + "purl": "pkg:npm/csstype@3.2.3", + "type": "library", + "version": "3.2.3" + }, + { + "name": "data-urls/node_modules/whatwg-url", + "purl": "pkg:npm/data-urls/node_modules/whatwg-url@16.0.1", + "type": "library", + "version": "16.0.1" + }, + { + "name": "data-urls", + "purl": "pkg:npm/data-urls@7.0.0", + "type": "library", + "version": "7.0.0" + }, + { + "name": "decimal.js", + "purl": "pkg:npm/decimal.js@10.6.0", + "type": "library", + "version": "10.6.0" + }, + { + "name": "dequal", + "purl": "pkg:npm/dequal@2.0.3", + "type": "library", + "version": "2.0.3" + }, + { + "name": "detect-libc", + "purl": "pkg:npm/detect-libc@2.1.2", + "type": "library", + "version": "2.1.2" + }, + { + "name": "didyoumean", + "purl": "pkg:npm/didyoumean@1.2.2", + "type": "library", + "version": "1.2.2" + }, + { + "name": "dlv", + "purl": "pkg:npm/dlv@1.1.3", + "type": "library", + "version": "1.1.3" + }, + { + "name": "dom-accessibility-api", + "purl": "pkg:npm/dom-accessibility-api@0.6.3", + "type": "library", + "version": "0.6.3" + }, + { + "name": "electron-to-chromium", + "purl": "pkg:npm/electron-to-chromium@1.5.396", + "type": "library", + "version": "1.5.396" + }, + { + "name": "entities", + "purl": "pkg:npm/entities@8.0.0", + "type": "library", + "version": "8.0.0" + }, + { + "name": "es-errors", + "purl": "pkg:npm/es-errors@1.3.0", + "type": "library", + "version": "1.3.0" + }, + { + "name": "es-module-lexer", + "purl": "pkg:npm/es-module-lexer@2.3.1", + "type": "library", + "version": "2.3.1" + }, + { + "name": "escalade", + "purl": "pkg:npm/escalade@3.2.0", + "type": "library", + "version": "3.2.0" + }, + { + "name": "estree-walker", + "purl": "pkg:npm/estree-walker@3.0.3", + "type": "library", + "version": "3.0.3" + }, + { + "name": "expect-type", + "purl": "pkg:npm/expect-type@1.4.0", + "type": "library", + "version": "1.4.0" + }, + { + "name": "fast-glob/node_modules/glob-parent", + "purl": "pkg:npm/fast-glob/node_modules/glob-parent@5.1.2", + "type": "library", + "version": "5.1.2" + }, + { + "name": "fast-glob", + "purl": "pkg:npm/fast-glob@3.3.3", + "type": "library", + "version": "3.3.3" + }, + { + "name": "fastq", + "purl": "pkg:npm/fastq@1.20.1", + "type": "library", + "version": "1.20.1" + }, + { + "name": "fill-range", + "purl": "pkg:npm/fill-range@7.1.1", + "type": "library", + "version": "7.1.1" + }, + { + "name": "fraction.js", + "purl": "pkg:npm/fraction.js@5.3.4", + "type": "library", + "version": "5.3.4" + }, + { + "name": "fsevents", + "purl": "pkg:npm/fsevents@2.3.3", + "type": "library", + "version": "2.3.3" + }, + { + "name": "function-bind", + "purl": "pkg:npm/function-bind@1.1.2", + "type": "library", + "version": "1.1.2" + }, + { + "name": "glob-parent", + "purl": "pkg:npm/glob-parent@6.0.2", + "type": "library", + "version": "6.0.2" + }, + { + "name": "hasown", + "purl": "pkg:npm/hasown@2.0.3", + "type": "library", + "version": "2.0.3" + }, + { + "name": "html-encoding-sniffer", + "purl": "pkg:npm/html-encoding-sniffer@6.0.0", + "type": "library", + "version": "6.0.0" + }, + { + "name": "indent-string", + "purl": "pkg:npm/indent-string@4.0.0", + "type": "library", + "version": "4.0.0" + }, + { + "name": "is-binary-path", + "purl": "pkg:npm/is-binary-path@2.1.0", + "type": "library", + "version": "2.1.0" + }, + { + "name": "is-core-module", + "purl": "pkg:npm/is-core-module@2.16.2", + "type": "library", + "version": "2.16.2" + }, + { + "name": "is-extglob", + "purl": "pkg:npm/is-extglob@2.1.1", + "type": "library", + "version": "2.1.1" + }, + { + "name": "is-glob", + "purl": "pkg:npm/is-glob@4.0.3", + "type": "library", + "version": "4.0.3" + }, + { + "name": "is-number", + "purl": "pkg:npm/is-number@7.0.0", + "type": "library", + "version": "7.0.0" + }, + { + "name": "is-potential-custom-element-name", + "purl": "pkg:npm/is-potential-custom-element-name@1.0.1", + "type": "library", + "version": "1.0.1" + }, + { + "name": "jiti", + "purl": "pkg:npm/jiti@1.21.7", + "type": "library", + "version": "1.21.7" + }, + { + "name": "js-tokens", + "purl": "pkg:npm/js-tokens@4.0.0", + "type": "library", + "version": "4.0.0" + }, + { + "name": "jsdom", + "purl": "pkg:npm/jsdom@30.0.0", + "type": "library", + "version": "30.0.0" + }, + { + "name": "lightningcss-android-arm64", + "purl": "pkg:npm/lightningcss-android-arm64@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-darwin-arm64", + "purl": "pkg:npm/lightningcss-darwin-arm64@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-darwin-x64", + "purl": "pkg:npm/lightningcss-darwin-x64@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-freebsd-x64", + "purl": "pkg:npm/lightningcss-freebsd-x64@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-linux-arm-gnueabihf", + "purl": "pkg:npm/lightningcss-linux-arm-gnueabihf@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-linux-arm64-gnu", + "purl": "pkg:npm/lightningcss-linux-arm64-gnu@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-linux-arm64-musl", + "purl": "pkg:npm/lightningcss-linux-arm64-musl@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-linux-x64-gnu", + "purl": "pkg:npm/lightningcss-linux-x64-gnu@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-linux-x64-musl", + "purl": "pkg:npm/lightningcss-linux-x64-musl@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-win32-arm64-msvc", + "purl": "pkg:npm/lightningcss-win32-arm64-msvc@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss-win32-x64-msvc", + "purl": "pkg:npm/lightningcss-win32-x64-msvc@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lightningcss", + "purl": "pkg:npm/lightningcss@1.33.0", + "type": "library", + "version": "1.33.0" + }, + { + "name": "lilconfig", + "purl": "pkg:npm/lilconfig@3.1.3", + "type": "library", + "version": "3.1.3" + }, + { + "name": "lines-and-columns", + "purl": "pkg:npm/lines-and-columns@1.2.4", + "type": "library", + "version": "1.2.4" + }, + { + "name": "lru-cache", + "purl": "pkg:npm/lru-cache@11.5.2", + "type": "library", + "version": "11.5.2" + }, + { + "name": "lz-string", + "purl": "pkg:npm/lz-string@1.5.0", + "type": "library", + "version": "1.5.0" + }, + { + "name": "magic-string", + "purl": "pkg:npm/magic-string@0.30.21", + "type": "library", + "version": "0.30.21" + }, + { + "name": "mdn-data", + "purl": "pkg:npm/mdn-data@2.27.1", + "type": "library", + "version": "2.27.1" + }, + { + "name": "merge2", + "purl": "pkg:npm/merge2@1.4.1", + "type": "library", + "version": "1.4.1" + }, + { + "name": "micromatch", + "purl": "pkg:npm/micromatch@4.0.8", + "type": "library", + "version": "4.0.8" + }, + { + "name": "min-indent", + "purl": "pkg:npm/min-indent@1.0.1", + "type": "library", + "version": "1.0.1" + }, + { + "name": "mz", + "purl": "pkg:npm/mz@2.7.0", + "type": "library", + "version": "2.7.0" + }, + { + "name": "nanoid", + "purl": "pkg:npm/nanoid@3.3.16", + "type": "library", + "version": "3.3.16" + }, + { + "name": "node-releases", + "purl": "pkg:npm/node-releases@2.0.51", + "type": "library", + "version": "2.0.51" + }, + { + "name": "normalize-path", + "purl": "pkg:npm/normalize-path@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "name": "object-assign", + "purl": "pkg:npm/object-assign@4.1.1", + "type": "library", + "version": "4.1.1" + }, + { + "name": "object-hash", + "purl": "pkg:npm/object-hash@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "name": "obug", + "purl": "pkg:npm/obug@2.1.4", + "type": "library", + "version": "2.1.4" + }, + { + "name": "parse5", + "purl": "pkg:npm/parse5@8.0.1", + "type": "library", + "version": "8.0.1" + }, + { + "name": "path-parse", + "purl": "pkg:npm/path-parse@1.0.7", + "type": "library", + "version": "1.0.7" + }, + { + "name": "pathe", + "purl": "pkg:npm/pathe@2.0.3", + "type": "library", + "version": "2.0.3" + }, + { + "name": "picocolors", + "purl": "pkg:npm/picocolors@1.1.1", + "type": "library", + "version": "1.1.1" + }, + { + "name": "picomatch", + "purl": "pkg:npm/picomatch@2.3.2", + "type": "library", + "version": "2.3.2" + }, + { + "name": "pify", + "purl": "pkg:npm/pify@2.3.0", + "type": "library", + "version": "2.3.0" + }, + { + "name": "pirates", + "purl": "pkg:npm/pirates@4.0.7", + "type": "library", + "version": "4.0.7" + }, + { + "name": "playwright-core", + "purl": "pkg:npm/playwright-core@1.57.0", + "type": "library", + "version": "1.57.0" + }, + { + "name": "playwright/node_modules/fsevents", + "purl": "pkg:npm/playwright/node_modules/fsevents@2.3.2", + "type": "library", + "version": "2.3.2" + }, + { + "name": "playwright", + "purl": "pkg:npm/playwright@1.57.0", + "type": "library", + "version": "1.57.0" + }, + { + "name": "postcss-import", + "purl": "pkg:npm/postcss-import@15.1.0", + "type": "library", + "version": "15.1.0" + }, + { + "name": "postcss-js", + "purl": "pkg:npm/postcss-js@4.1.0", + "type": "library", + "version": "4.1.0" + }, + { + "name": "postcss-load-config", + "purl": "pkg:npm/postcss-load-config@6.0.1", + "type": "library", + "version": "6.0.1" + }, + { + "name": "postcss-nested", + "purl": "pkg:npm/postcss-nested@6.2.0", + "type": "library", + "version": "6.2.0" + }, + { + "name": "postcss-selector-parser", + "purl": "pkg:npm/postcss-selector-parser@6.1.2", + "type": "library", + "version": "6.1.2" + }, + { + "name": "postcss-value-parser", + "purl": "pkg:npm/postcss-value-parser@4.2.0", + "type": "library", + "version": "4.2.0" + }, + { + "name": "postcss", + "purl": "pkg:npm/postcss@8.5.23", + "type": "library", + "version": "8.5.23" + }, + { + "name": "pretty-format/node_modules/ansi-styles", + "purl": "pkg:npm/pretty-format/node_modules/ansi-styles@5.2.0", + "type": "library", + "version": "5.2.0" + }, + { + "name": "pretty-format", + "purl": "pkg:npm/pretty-format@27.5.1", + "type": "library", + "version": "27.5.1" + }, + { + "name": "punycode", + "purl": "pkg:npm/punycode@2.3.1", + "type": "library", + "version": "2.3.1" + }, + { + "name": "queue-microtask", + "purl": "pkg:npm/queue-microtask@1.2.3", + "type": "library", + "version": "1.2.3" + }, + { + "name": "react-dom", + "purl": "pkg:npm/react-dom@19.2.8", + "type": "library", + "version": "19.2.8" + }, + { + "name": "react-is", + "purl": "pkg:npm/react-is@17.0.2", + "type": "library", + "version": "17.0.2" + }, + { + "name": "react", + "purl": "pkg:npm/react@19.2.8", + "type": "library", + "version": "19.2.8" + }, + { + "name": "read-cache", + "purl": "pkg:npm/read-cache@1.0.0", + "type": "library", + "version": "1.0.0" + }, + { + "name": "readdirp", + "purl": "pkg:npm/readdirp@3.6.0", + "type": "library", + "version": "3.6.0" + }, + { + "name": "redent", + "purl": "pkg:npm/redent@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "name": "require-from-string", + "purl": "pkg:npm/require-from-string@2.0.2", + "type": "library", + "version": "2.0.2" + }, + { + "name": "resolve", + "purl": "pkg:npm/resolve@1.22.12", + "type": "library", + "version": "1.22.12" + }, + { + "name": "reusify", + "purl": "pkg:npm/reusify@1.1.0", + "type": "library", + "version": "1.1.0" + }, + { + "name": "rolldown", + "purl": "pkg:npm/rolldown@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "name": "run-parallel", + "purl": "pkg:npm/run-parallel@1.2.0", + "type": "library", + "version": "1.2.0" + }, + { + "name": "saxes", + "purl": "pkg:npm/saxes@6.0.0", + "type": "library", + "version": "6.0.0" + }, + { + "name": "scheduler", + "purl": "pkg:npm/scheduler@0.27.0", + "type": "library", + "version": "0.27.0" + }, + { + "name": "siginfo", + "purl": "pkg:npm/siginfo@2.0.0", + "type": "library", + "version": "2.0.0" + }, + { + "name": "source-map-js", + "purl": "pkg:npm/source-map-js@1.2.1", + "type": "library", + "version": "1.2.1" + }, + { + "name": "stackback", + "purl": "pkg:npm/stackback@0.0.2", + "type": "library", + "version": "0.0.2" + }, + { + "name": "std-env", + "purl": "pkg:npm/std-env@4.2.0", + "type": "library", + "version": "4.2.0" + }, + { + "name": "strip-indent", + "purl": "pkg:npm/strip-indent@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "name": "sucrase", + "purl": "pkg:npm/sucrase@3.35.1", + "type": "library", + "version": "3.35.1" + }, + { + "name": "supports-preserve-symlinks-flag", + "purl": "pkg:npm/supports-preserve-symlinks-flag@1.0.0", + "type": "library", + "version": "1.0.0" + }, + { + "name": "symbol-tree", + "purl": "pkg:npm/symbol-tree@3.2.4", + "type": "library", + "version": "3.2.4" + }, + { + "name": "tailwindcss", + "purl": "pkg:npm/tailwindcss@3.4.19", + "type": "library", + "version": "3.4.19" + }, + { + "name": "thenify-all", + "purl": "pkg:npm/thenify-all@1.6.0", + "type": "library", + "version": "1.6.0" + }, + { + "name": "thenify", + "purl": "pkg:npm/thenify@3.3.1", + "type": "library", + "version": "3.3.1" + }, + { + "name": "tinybench", + "purl": "pkg:npm/tinybench@2.9.0", + "type": "library", + "version": "2.9.0" + }, + { + "name": "tinyexec", + "purl": "pkg:npm/tinyexec@1.2.4", + "type": "library", + "version": "1.2.4" + }, + { + "name": "tinyglobby/node_modules/fdir", + "purl": "pkg:npm/tinyglobby/node_modules/fdir@6.5.0", + "type": "library", + "version": "6.5.0" + }, + { + "name": "tinyglobby/node_modules/picomatch", + "purl": "pkg:npm/tinyglobby/node_modules/picomatch@4.0.4", + "type": "library", + "version": "4.0.4" + }, + { + "name": "tinyglobby", + "purl": "pkg:npm/tinyglobby@0.2.17", + "type": "library", + "version": "0.2.17" + }, + { + "name": "tinyrainbow", + "purl": "pkg:npm/tinyrainbow@3.1.0", + "type": "library", + "version": "3.1.0" + }, + { + "name": "tldts-core", + "purl": "pkg:npm/tldts-core@7.4.9", + "type": "library", + "version": "7.4.9" + }, + { + "name": "tldts", + "purl": "pkg:npm/tldts@7.4.9", + "type": "library", + "version": "7.4.9" + }, + { + "name": "to-regex-range", + "purl": "pkg:npm/to-regex-range@5.0.1", + "type": "library", + "version": "5.0.1" + }, + { + "name": "tough-cookie", + "purl": "pkg:npm/tough-cookie@6.0.2", + "type": "library", + "version": "6.0.2" + }, + { + "name": "tr46", + "purl": "pkg:npm/tr46@6.0.0", + "type": "library", + "version": "6.0.0" + }, + { + "name": "ts-interface-checker", + "purl": "pkg:npm/ts-interface-checker@0.1.13", + "type": "library", + "version": "0.1.13" + }, + { + "name": "tslib", + "purl": "pkg:npm/tslib@2.8.1", + "type": "library", + "version": "2.8.1" + }, + { + "name": "typescript", + "purl": "pkg:npm/typescript@7.0.2", + "type": "library", + "version": "7.0.2" + }, + { + "name": "undici", + "purl": "pkg:npm/undici@8.9.0", + "type": "library", + "version": "8.9.0" + }, + { + "name": "update-browserslist-db", + "purl": "pkg:npm/update-browserslist-db@1.2.3", + "type": "library", + "version": "1.2.3" + }, + { + "name": "util-deprecate", + "purl": "pkg:npm/util-deprecate@1.0.2", + "type": "library", + "version": "1.0.2" + }, + { + "name": "vite/node_modules/picomatch", + "purl": "pkg:npm/vite/node_modules/picomatch@4.0.5", + "type": "library", + "version": "4.0.5" + }, + { + "name": "vite", + "purl": "pkg:npm/vite@8.1.5", + "type": "library", + "version": "8.1.5" + }, + { + "name": "vitest/node_modules/picomatch", + "purl": "pkg:npm/vitest/node_modules/picomatch@4.0.5", + "type": "library", + "version": "4.0.5" + }, + { + "name": "vitest", + "purl": "pkg:npm/vitest@4.1.10", + "type": "library", + "version": "4.1.10" + }, + { + "name": "w3c-xmlserializer", + "purl": "pkg:npm/w3c-xmlserializer@5.0.0", + "type": "library", + "version": "5.0.0" + }, + { + "name": "webidl-conversions", + "purl": "pkg:npm/webidl-conversions@8.0.1", + "type": "library", + "version": "8.0.1" + }, + { + "name": "whatwg-mimetype", + "purl": "pkg:npm/whatwg-mimetype@5.0.0", + "type": "library", + "version": "5.0.0" + }, + { + "name": "whatwg-url", + "purl": "pkg:npm/whatwg-url@17.1.0", + "type": "library", + "version": "17.1.0" + }, + { + "name": "why-is-node-running", + "purl": "pkg:npm/why-is-node-running@2.3.0", + "type": "library", + "version": "2.3.0" + }, + { + "name": "xml-name-validator", + "purl": "pkg:npm/xml-name-validator@5.0.0", + "type": "library", + "version": "5.0.0" + }, + { + "name": "xmlchars", + "purl": "pkg:npm/xmlchars@2.2.0", + "type": "library", + "version": "2.2.0" + }, + { + "name": "aiofiles", + "purl": "pkg:pypi/aiofiles@25.1.0", + "type": "library", + "version": "25.1.0" + }, + { + "name": "aiosqlite", + "purl": "pkg:pypi/aiosqlite@0.22.1", + "type": "library", + "version": "0.22.1" + }, + { + "name": "alembic", + "purl": "pkg:pypi/alembic@1.18.5", + "type": "library", + "version": "1.18.5" + }, + { + "name": "apscheduler", + "purl": "pkg:pypi/apscheduler@3.11.3", + "type": "library", + "version": "3.11.3" + }, + { + "name": "argon2-cffi", + "purl": "pkg:pypi/argon2-cffi@25.1.0", + "type": "library", + "version": "25.1.0" + }, + { + "name": "cryptography", + "purl": "pkg:pypi/cryptography@49.0.0", + "type": "library", + "version": "49.0.0" + }, + { + "name": "fastapi", + "purl": "pkg:pypi/fastapi@0.136.1", + "type": "library", + "version": "0.136.1" + }, + { + "name": "httpx", + "purl": "pkg:pypi/httpx@0.28.1", + "type": "library", + "version": "0.28.1" + }, + { + "name": "pydantic-settings", + "purl": "pkg:pypi/pydantic-settings@2.14.2", + "type": "library", + "version": "2.14.2" + }, + { + "name": "pydantic", + "purl": "pkg:pypi/pydantic@2.13.4", + "type": "library", + "version": "2.13.4" + }, + { + "name": "sqlalchemy", + "purl": "pkg:pypi/sqlalchemy@2.0.49", + "type": "library", + "version": "2.0.49" + }, + { + "name": "uvicorn", + "purl": "pkg:pypi/uvicorn@0.51.0", + "type": "library", + "version": "0.51.0" + } + ], + "metadata": { + "component": { + "name": "backup-tool", + "purl": "pkg:generic/backup-tool@2.0.0.dev0", + "type": "library", + "version": "2.0.0.dev0" + } + }, + "specVersion": "1.5", + "version": 1 +} diff --git a/docs/release/m15-evidence.md b/docs/release/m15-evidence.md new file mode 100644 index 0000000..6e85338 --- /dev/null +++ b/docs/release/m15-evidence.md @@ -0,0 +1,23 @@ +# M15 v2.0 certification evidence + +## Reference host and method + +Current CI reference host: Linux 7.1.4, Python 3.14.6, 12 CPUs. Certification uses a deterministic synthetic metadata workload: 100 jobs, 1,000,000 declared entries, and 100,000 cataloged backups with 10 TiB **logical** bytes per backup. It does not claim a physical 10 TiB transfer. + +`m15-scale-report.json` records 100,000 backup inserts in 0.57 s, a 100-row deep pagination query in 0.004209 s, and a 5.32 MB SQLite fixture. + +## Capability certification + +```text +python tools/assert_capabilities.py --release v2.0 \ + --include local,ssh,restore,webhook,email \ + --exclude postgresql,mysql,tar_download +``` + +Passed. The released contract exposes `local`, `ssh`, `restore`, `webhook`, and `email`; PostgreSQL, MySQL, and TAR download remain disabled. + +SSH is private-key-only and requires a dedicated forced-SFTP chroot account; no password, shell, or remote-command path is released. + +## Remaining certification gates + +Run the full project, container E2E, SSH live integration, fault/security/leakage suites, and review the synthetic workload boundaries before a release commit is created. diff --git a/docs/release/m15-scale-report.json b/docs/release/m15-scale-report.json new file mode 100644 index 0000000..c796d02 --- /dev/null +++ b/docs/release/m15-scale-report.json @@ -0,0 +1,21 @@ +{ + "method": "synthetic metadata certification; logical bytes are sparse and no physical 10 TiB payload is allocated", + "reference_host": { + "cpus": 12, + "platform": "Linux-7.1.4-arch1-1-x86_64-with-glibc2.44", + "python": "3.14.6" + }, + "results": { + "backup_insert_seconds": 0.57, + "database_bytes": 5320704, + "pagination_rows": 100, + "pagination_seconds": 0.004209, + "total_seconds": 0.592 + }, + "workload": { + "backups": 100000, + "entries_declared": 1000000, + "jobs": 100, + "logical_bytes_per_backup": 10995116277760 + } +} diff --git a/docs/release/m5-evidence.md b/docs/release/m5-evidence.md new file mode 100644 index 0000000..5a68ee9 --- /dev/null +++ b/docs/release/m5-evidence.md @@ -0,0 +1,12 @@ +# M5 Durable Execution Lifecycle Evidence + +- Coverage: state-transition contract and DB-enforced active-execution uniqueness + under concurrent enqueue; disabled/archived-job rejection; lease reclamation + and fencing; retry semantics; startup stale-worker reconciliation; durable + event revision counts; and idle worker shutdown. +- Verification: the focused M5 suite passed: 29 tests across transition, + queue/lease, and worker-loss acceptance coverage. +- Automation: `make test-fault` runs the worker-loss acceptance suite, and + `make check` now includes it. +- Scope: validates M5 durable lifecycle behavior only. Snapshot publication, + verification, and restore remain M6 work. diff --git a/docs/release/m6-evidence.md b/docs/release/m6-evidence.md new file mode 100644 index 0000000..b344cc5 --- /dev/null +++ b/docs/release/m6-evidence.md @@ -0,0 +1,14 @@ +# M6 Backup, Verification, and Restore Evidence + +- Coverage: local full-snapshot staging; immutable SHA-256 blobs; canonical + Ed25519-signed manifests; manifest/blob verification; publication markers and + startup reconciliation; selected and dry-run restores; `fail`, `skip`, and + `replace` root policies; destination containment; corruption handling; and + restore recovery. +- Focused acceptance: the M6 integration, publication-fault, and restore-path + suites pass. +- Full verification: `make check` passed with 68 unit/contract, 37 + integration, 7 fault, and 17 security tests, plus Ruff, mypy, frontend + typecheck, and frontend production build. +- Scope: excludes M7 incremental baselines/exclusions and later scheduling, + remote source, encryption, retention, and UI milestones. diff --git a/docs/release/m7-evidence.md b/docs/release/m7-evidence.md new file mode 100644 index 0000000..6e16aba --- /dev/null +++ b/docs/release/m7-evidence.md @@ -0,0 +1,12 @@ +# M7 Incrementals, Exclusions, and Empty-Source Safety Evidence + +- Coverage: ordered normalized exclusion rules, exclusion policy persisted in signed + manifests, incremental compatible-baseline selection, independently restorable + complete manifests, parent linkage, content-addressed blob reuse, and + empty-source opt-in. +- Verification: M7 unit, incremental, and empty-source suites are included in + the passing project verification. +- Full verification: `make check` passed with 78 unit/contract, 39 + integration, 7 fault, and 17 security tests, plus Ruff, mypy, frontend + typecheck, and frontend production build. +- Scope: scheduling remains M8 work. diff --git a/docs/release/m8-evidence.md b/docs/release/m8-evidence.md new file mode 100644 index 0000000..ea10268 --- /dev/null +++ b/docs/release/m8-evidence.md @@ -0,0 +1,9 @@ +# M8 Scheduling Evidence + +- Coverage: five-field cron validation, IANA timezones, UTC nominal runs, + durable schedule CRUD, live next-run updates, misfire handling, and + no-overlap delivery through the execution enqueue guard. +- Full verification: `make check` passed with 81 unit/contract, 40 + integration, 10 fault, and 17 security tests, plus Ruff, mypy, frontend + typecheck, and frontend production build. +- Scope: retention and garbage collection are M9 work. diff --git a/docs/release/m9-evidence.md b/docs/release/m9-evidence.md new file mode 100644 index 0000000..d6e8be7 --- /dev/null +++ b/docs/release/m9-evidence.md @@ -0,0 +1,8 @@ +# M9 Retention and Garbage Collection Evidence + +- Coverage: union retention policies, newest-backup protection, pins, tombstones, + referenced-object preservation, garbage collection, and reconciliation safety. +- Full verification: `make check` passed with 86 unit/contract, 42 integration, + 11 fault, and 17 security tests, plus Ruff, mypy, frontend typecheck, and + frontend production build. +- Scope: remote source support remains M10 work. diff --git a/docs/runbooks/disaster-recovery.md b/docs/runbooks/disaster-recovery.md new file mode 100644 index 0000000..5fa7128 --- /dev/null +++ b/docs/runbooks/disaster-recovery.md @@ -0,0 +1,11 @@ +# Disaster recovery runbook + +1. Isolate the failed host and preserve the metadata volume, repository roots, logs, + image digest, and master-key backup. Do not restart writers repeatedly. +2. Provision a clean host with the same pinned image and non-root volume permissions. +3. Restore the master key securely, restore repository roots read-only first, and + restore metadata from a verified backup or the passphrase-protected recovery bundle. +4. Run `migrate current`, start only `web`, validate `/readyz` and repository + inspection, then start scheduler and worker one at a time. +5. Perform and document a test restore before enabling scheduled work. Rotate secrets + if host compromise is possible. diff --git a/docs/runbooks/keys.md b/docs/runbooks/keys.md new file mode 100644 index 0000000..8e8cd85 --- /dev/null +++ b/docs/runbooks/keys.md @@ -0,0 +1,13 @@ +# Master and repository key runbook + +- Store the Compose master key outside the checkout. It must be a regular file, + at least 32 bytes, mode `0600`, owned by the service UID (`10001` for Compose). + Loss of this key destroys access to encrypted metadata secrets. +- Back up the master key independently from metadata and repositories; do not put it + in an image, Compose environment variable, log, ticket, or recovery bundle. +- Use the `admin recovery export` command with a passphrase file descriptor to create + a separately protected recovery bundle. Validate it on an isolated host. +- Rotate repository data keys only with `admin repository-key rotate`; retain prior + recovery material until a restore drill succeeds. +- If compromise is suspected, stop the worker, preserve evidence, rotate credentials, + export a fresh recovery bundle, and run a restore drill before resuming writes. diff --git a/docs/runbooks/metadata.md b/docs/runbooks/metadata.md new file mode 100644 index 0000000..915567e --- /dev/null +++ b/docs/runbooks/metadata.md @@ -0,0 +1,19 @@ +# Metadata backup and restore runbook + +## Backup + +1. Confirm `docker compose ps` shows exactly one scheduler and one worker. +2. For an online SQLite backup, run a host-side SQLite `.backup` against the mounted + `metadata.db`; do **not** copy only the main file while WAL writers run. +3. For a filesystem copy, stop `web`, `scheduler`, and `worker` first, then retain + `metadata.db`, `metadata.db-wal`, and `metadata.db-shm` together. +4. Encrypt and test the backup outside the appliance. Never place a database dump in + the repository or OCI image. + +## Restore + +1. Stop all runtime roles and preserve the failed metadata volume unchanged. +2. Restore the complete SQLite backup into the metadata volume with the service user + ownership (UID 10001 in the supplied Compose deployment). +3. Run `docker compose run --rm migrate current`; only start the stack when it reports + the expected revision. Validate `/readyz` and a read-only API request after startup. diff --git a/docs/runbooks/notifications.md b/docs/runbooks/notifications.md new file mode 100644 index 0000000..6003fe9 --- /dev/null +++ b/docs/runbooks/notifications.md @@ -0,0 +1,15 @@ +# Notifications runbook + +## Configure + +Create a filtered email or webhook subscription through `/api/v2/notifications/subscriptions`. Webhooks require a write-only signing secret. Configure SMTP separately at `/api/v2/notifications/email-settings`; only authenticated STARTTLS SMTP is accepted. Verify a channel with `POST /subscriptions/{id}/test` and inspect delivery/attempt history before relying on it. + +Filters are a nonempty set of exact catalog IDs or family wildcards such as `execution.*`; they may be narrowed by job IDs, repository IDs, or severity. The public catalog is live-events-only: every listed type is emitted by a currently available operation. Deferred channels and source capabilities have no catalog entries. + +## Rotate and recover + +Rotate webhook keys using the signing-key rotate endpoint with an idempotency key and an explicit bounded overlap. Receivers must accept both signatures during overlap, then remove the old key after expiry. A recovery bundle deliberately excludes subscriptions, SMTP settings, signing secrets, event history, and delivery attempts. Reconfigure notifications after a fresh-host recovery. + +## Failure handling + +The worker claims due deliveries with a lease. Transient errors enter bounded exponential retry; interrupted leases recover as retryable work and may send an event again. Inspect response class and redacted diagnostics in history. A terminal failed delivery can be retried manually once the destination is corrected. To stop outbound traffic, disable/archive subscriptions or stop the worker; do not delete outbox history. Rollback consists of disabling subscriptions and worker dispatch while retaining audit/outbox records for investigation. diff --git a/docs/runbooks/observability.md b/docs/runbooks/observability.md new file mode 100644 index 0000000..6335afa --- /dev/null +++ b/docs/runbooks/observability.md @@ -0,0 +1,17 @@ +# Observability and alert response + +The proxy exposes `/livez`, `/readyz`, and Prometheus text at `/metrics`. Metrics use +no source paths, IDs, credentials, tokens, or secret values. Runtime logs are JSON +records with an event, timestamp, role, and request correlation ID where applicable. + +Alert when any of the following remains non-zero or grows: + +- `backup_tool_stale_execution_leases` +- `backup_tool_failed_executions` +- `backup_tool_corrupt_backups` +- `backup_tool_unavailable_repositories` +- `backup_tool_schedule_lag_seconds` + +Also alert on low `backup_tool_filesystem_free_bytes`. For any alert, preserve logs, +validate `/readyz`, stop the worker before destructive repository investigation, and +use the matching metadata, repository, key, upgrade, or disaster-recovery runbook. diff --git a/docs/runbooks/recovery-bundle.md b/docs/runbooks/recovery-bundle.md new file mode 100644 index 0000000..c5a2f34 --- /dev/null +++ b/docs/runbooks/recovery-bundle.md @@ -0,0 +1,70 @@ +# Recovery bundle export and validation + +M11 recovery exports an offline, passphrase-encrypted catalog and key bundle. +Import is a local CLI operation that reconstructs only the metadata required to +restore existing encrypted backups; it does not reactivate backup scheduling. + +## Export + +Choose an absolute path in a trusted, non-symlinked directory. The destination +must not already exist; export creates it with mode `0600` and never overwrites +it. + +```sh +read -r -s recovery_passphrase +printf '\n' +printf '%s\n' "$recovery_passphrase" | \ + backup-tool admin recovery export \ + --output /secure/offline/backup-tool-recovery.btrec \ + --passphrase-fd 0 +unset recovery_passphrase +``` + +The passphrase is read from the inherited file descriptor. It is never a CLI +argument. Store the resulting `BTREC` file away from the host and away from the +live repository-key directories. + +## Validate + +Validation authenticates and decrypts the bundle, checks the versioned Argon2id +and AES-GCM format, and verifies the included catalog/key relationships. It +prints only a status and repository count. + +```sh +read -r -s recovery_passphrase +printf '\n' +printf '%s\n' "$recovery_passphrase" | \ + backup-tool admin recovery validate \ + --input /secure/offline/backup-tool-recovery.btrec \ + --passphrase-fd 0 +unset recovery_passphrase +``` + +Wrong passphrases, tampering, malformed headers, unsupported KDF parameters, +and invalid encrypted payloads intentionally produce the same validation error. +Do not use a failed validation result to diagnose which of those conditions +occurred. + +## Fresh-host import + +Before importing, run migrations on the replacement host and configure its +repository allowlist to include the surviving repository directory. The +repository must pass normal metadata/path inspection. The replacement metadata +database must be current and otherwise empty; import rejects a non-empty +destination and any existing/conflicting key files. + +```sh +backup-tool migrate upgrade +read -r -s recovery_passphrase +printf '\n' +printf '%s\n' "$recovery_passphrase" | \ + backup-tool admin recovery import \ + --input /secure/offline/backup-tool-recovery.btrec \ + --passphrase-fd 0 +unset recovery_passphrase +``` + +Import installs signing and data keys with restrictive modes, restores the +repository/source/job/execution/backup catalog needed for restore, and marks +sources unavailable plus jobs archived and disabled. Reconfigure sources and +explicitly create or enable new jobs before taking another backup. diff --git a/docs/runbooks/repositories.md b/docs/runbooks/repositories.md new file mode 100644 index 0000000..1feb742 --- /dev/null +++ b/docs/runbooks/repositories.md @@ -0,0 +1,12 @@ +# Repository recovery runbook + +1. Stop `worker` before inspecting or repairing a repository; never edit a live + repository behind an active lease. +2. Preserve the repository directory and its metadata volume before remediation. +3. Verify repository state through the operator API and verify individual backups + before any restore. Treat a corrupt verification result as an incident, not a + deletion request. +4. Mount replacement repository roots at the same allowlisted path, restore metadata, + then start `migrate`, `web`, `scheduler`, and finally `worker`. +5. Keep archived repositories mounted until retention and restore obligations expire. + Do not remove manifests or blobs manually. diff --git a/docs/runbooks/ssh-sources.md b/docs/runbooks/ssh-sources.md new file mode 100644 index 0000000..8ca4481 --- /dev/null +++ b/docs/runbooks/ssh-sources.md @@ -0,0 +1,19 @@ +# SSH sources + +SSH sources require a dedicated account confined by an OpenSSH `ChrootDirectory` +and `ForceCommand internal-sftp`. The chroot directory is root-owned; writable +content is below it. Disable passwords, keyboard-interactive authentication, +shells, PTYs, TCP/X11/agent forwarding, and tunnelling. Configure the source +root as `/` only. + +Generate a dedicated unencrypted Ed25519, ECDSA, or RSA-3072+ client key and +store it through the write-only `ssh_private_key` secret endpoint. Do not put a +key, passphrase, password, command, agent path, or key file path in source +configuration. Pin the server's exact OpenSSH public host key (`algorithm +base64`) before probing. On host-key rotation, obtain the replacement through +an out-of-band administrative channel, update the source pin, then probe. + +The server administrator controls mutable content inside the chroot. The client +rejects traversal names, symlinks, special files, changed files, and configured +resource-limit overflows, but cannot claim atomic no-follow behavior against a +maliciously changing filesystem inside that server-controlled boundary. diff --git a/docs/runbooks/upgrade.md b/docs/runbooks/upgrade.md new file mode 100644 index 0000000..f56fbe0 --- /dev/null +++ b/docs/runbooks/upgrade.md @@ -0,0 +1,12 @@ +# Upgrade and rollback runbook + +1. Record the running image digest and take a tested metadata backup plus repository + recovery evidence before changing the image. +2. Pull/build the pinned image, then run `docker compose run --rm migrate upgrade`. + Do not start web, scheduler, or worker against an unverified schema. +3. Start the stack, wait for `/readyz`, and inspect `/metrics` for stale leases, + schedule lag, unavailable repositories, and corrupt backups. +4. If migration fails, stop and restore the prior metadata backup and matching image; + do not attempt to downgrade an unknown partially migrated database in place. +5. Preserve migration logs and verify a representative backup restore before closing + the change. diff --git a/docs/security/notifications.md b/docs/security/notifications.md new file mode 100644 index 0000000..fb2db52 --- /dev/null +++ b/docs/security/notifications.md @@ -0,0 +1,9 @@ +# Notification security policy (M12) + +Webhook callbacks accept absolute `http` and `https` URLs only. HTTP is an approved compatibility option, not a relaxation of egress controls. URLs with credentials, fragments, literal IP addresses, or ports outside 80/443/8080/8443 are rejected. Immediately before every request the worker resolves all A/AAAA answers; any non-global answer rejects the whole destination. The transport connects only to an approved answer, verifies the connected peer address, disables proxy environment use, preserves HTTPS SNI/certificate validation, and rejects redirects. + +Webhook bodies are canonical JSON event envelopes. `X-Backup-Event-ID` is stable over retries. Each attempt supplies the versioned HMAC-SHA-256 timestamp/body input and one `X-Backup-Signature` header per active or overlap key. Key IDs and monotonically increasing subscription-local versions identify keys; an old key is retained only until its configured overlap expiry. Key material is encrypted in `secrets` and is never returned, logged, exported in recovery bundles, or included in audit/event payloads. + +SMTP is configured as one write-only password-backed singleton. Delivery performs EHLO, verified STARTTLS, a second EHLO, and SMTP AUTH; any inability to do this fails closed. Message content is a compact event summary with no attachments, Bcc, paths, raw exception text, credentials, or full webhook payload. + +Delivery is at-least-once. A worker crash after a send but before recording success can lead to a duplicate; consumers must deduplicate using the event ID. Redirects, malformed destinations, and SSRF validation failures are terminal. Connect/read transport failures, SMTP transient failures, and HTTP 408/425/429/5xx use bounded retry. Deployment egress controls are defense in depth, not a substitute for this policy. diff --git a/docs/security/repository-encryption.md b/docs/security/repository-encryption.md new file mode 100644 index 0000000..a02a15a --- /dev/null +++ b/docs/security/repository-encryption.md @@ -0,0 +1,35 @@ +# Repository encryption threat model + +## Status + +This document defines the M11 repository-encryption boundary. Encrypted repository +creation remains disabled until the recovery host-loss acceptance test passes. + +## Confidential data + +For an `aes-256-gcm` repository, blob contents and signed manifest contents are +AEAD-encrypted. Data keys and signing private keys are never stored in repository +metadata, SQLite, logs, command arguments, environment output, or API responses. +Recovery bundles are separately passphrase-encrypted. + +## Intentional leakage + +The v1 content-addressed layout retains plaintext SHA-256 blob names. Encryption +therefore leaks blob equality, object count, repository layout, manifest/backup +identifiers, and ciphertext sizes derived from plaintext sizes. It does not claim +to hide those values, a compromised running host with unlocked keys, source data +while it is read, or a recovery-bundle passphrase. + +## Key lifecycle + +Encryption uses distinct data and signing keys. Rotation creates a new data-key +epoch for subsequent objects and retains historical epochs so existing backups +remain verifiable and restorable. It neither changes immutable repository policy +nor rewrites existing objects. + +## Recovery boundary + +Recovery export, validation, and import are local CLI operations only. A bundle +contains the required key material and an authenticated catalog; passphrases are +not accepted on command lines. Bundle parsing must reject malformed or unsupported +parameters without revealing whether a passphrase, key, or ciphertext was wrong. diff --git a/frontend/Dockerfile b/frontend/Dockerfile index b787766..8b6e090 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,20 +1,19 @@ -# Stage 1: Build -FROM node:20-alpine AS builder +# syntax=docker/dockerfile:1 +FROM node:22.17.1-alpine@sha256:5539840ce9d013fa13e3b9814c9353024be7ac75aca5db6d039504a56c04ea59 AS builder WORKDIR /app - COPY package*.json ./ -RUN npm install - +RUN npm ci COPY . . RUN npm run build -# Stage 2: Serve -FROM nginx:alpine +FROM nginx:1.29.7-alpine@sha256:e7257f1ef28ba17cf7c248cb8ccf6f0c6e0228ab9c315c152f9c203cd34cf6d1 COPY --from=builder /app/dist /usr/share/nginx/html -COPY nginx.conf /etc/nginx/conf.d/default.conf - -EXPOSE 80 +COPY nginx.conf /etc/nginx/nginx.conf +RUN chown -R 10001:0 /usr/share/nginx/html /var/cache/nginx /var/run /etc/nginx && \ + chmod -R g=u /usr/share/nginx/html /var/cache/nginx /var/run /etc/nginx +USER 10001:0 +EXPOSE 8080 CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/e2e/operator.spec.ts b/frontend/e2e/operator.spec.ts new file mode 100644 index 0000000..5d21177 --- /dev/null +++ b/frontend/e2e/operator.spec.ts @@ -0,0 +1,13 @@ +import { expect, test } from "@playwright/test"; + +test("renders an accessible authentication fallback at a narrow viewport", async ({ page }) => { + await page.setViewportSize({ width: 375, height: 667 }); + await page.emulateMedia({ reducedMotion: "reduce" }); + await page.goto("/"); + await expect(page.getByRole("main")).toBeVisible(); + await expect(page.getByRole("heading", { name: "Sign in" })).toBeVisible(); + const username = page.getByLabel("Username"); + await expect(username).toBeVisible(); + await username.focus(); + await expect(username).toBeFocused(); +}); diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 4e014b8..a5529d2 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,19 +1,50 @@ -server { - listen 80; - server_name localhost; - root /usr/share/nginx/html; - index index.html; +pid /tmp/nginx.pid; - location / { - try_files $uri $uri/ /index.html; +worker_processes auto; +error_log /dev/stderr warn; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /dev/stdout; + sendfile on; + + upstream backup_tool_web { + server unix:/run/backup-tool/web.sock; } - location /api/ { - proxy_pass http://backend:8000/api/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; + server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } + + location /api/ { + proxy_pass http://backup_tool_web; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /livez { + proxy_pass http://backup_tool_web; + } + + location = /readyz { + proxy_pass http://backup_tool_web; + } + + location = /metrics { + proxy_pass http://backup_tool_web; + } } } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index cc96aa3..e77def9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,6 +12,7 @@ "react-dom": "19.2.8" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@testing-library/jest-dom": "7.0.0", "@testing-library/react": "16.3.2", "@types/react": "19.2.17", @@ -430,6 +431,22 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@playwright/test": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.57.0.tgz", + "integrity": "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -2600,6 +2617,53 @@ "node": ">= 6" } }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.23", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", diff --git a/frontend/package.json b/frontend/package.json index e455079..b79dcbd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,8 @@ }, "scripts": { "dev": "vite", + "api:generate": "python3 ../tools/generate_api_client.py --input ../openapi/v2.json --output src/api/generated/client.ts", + "api:check": "python3 ../tools/generate_api_client.py --input ../openapi/v2.json --output src/api/generated/client.ts --check", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests", "build": "tsc && vite build", @@ -19,6 +21,7 @@ "react-dom": "19.2.8" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@testing-library/jest-dom": "7.0.0", "@testing-library/react": "16.3.2", "@types/react": "19.2.17", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts new file mode 100644 index 0000000..2ca97b7 --- /dev/null +++ b/frontend/playwright.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "./e2e", + use: { + baseURL: "http://127.0.0.1:4173", + launchOptions: { executablePath: "/usr/sbin/chromium" }, + }, + webServer: { + command: "npm run dev -- --host 127.0.0.1 --port 4173", + port: 4173, + reuseExistingServer: false, + }, +}); diff --git a/frontend/src/api/generated/client.ts b/frontend/src/api/generated/client.ts new file mode 100644 index 0000000..f05a9f9 --- /dev/null +++ b/frontend/src/api/generated/client.ts @@ -0,0 +1,567 @@ +/* eslint-disable */ +/* + * Generated by tools/generate_api_client.py from openapi/v2.json. + * Do not edit this file directly. Run `npm --prefix frontend run api:generate`. + */ + +export interface Components { + schemas: { + AuditList: { items : Array; next_cursor : string | null }; + AuditSummary: { action : string; created_at : string; details : Record; id : string; outcome : string; request_id : string; resource_id : string | null; resource_type : string }; + AuthenticatedUser: { id : string; username : string }; + BackupDeletePreview: { backup_id : string; destructive_action : string; eligible : boolean; reason : string | null }; + BackupList: { items : Array }; + BackupSummary: { created_at : string; execution_id : string; id : string; integrity : string; logical_bytes : number; manifest_id : string; pinned : boolean; stored_bytes : number; tombstoned_at : string | null }; + EmailSettingsInput: { host : string; max_attempts?: number; password : string; port?: number; rate_limit_per_minute?: number; sender : string; username : string }; + ExecutionList: { items : Array }; + ExecutionSummary: { attempt : number; id : string; progress : Record; reason_code : string | null; revision : number; state : string }; + HTTPValidationError: { detail?: Array }; + JobInput: { allow_empty?: boolean; enabled?: boolean; exclusions?: Array; name : string; repository_id : string; requested_mode?: string; retention?: Record; source_id : string }; + JobList: { items : Array }; + JobSummary: { enabled : boolean; id : string; name : string; repository_id : string; requested_mode : string; schedule : Components['schemas']["ScheduleSummary"] | null; source_id : string; state : string }; + LocalSourceInput: { kind : string; name : string; public_config : Record }; + LoginInput: { password : string; username : string }; + NotificationAttemptList: { items : Array }; + NotificationAttemptSummary: { completed_at : string | null; diagnostic : string | null; number : number; outcome : string; response_class : string | null; started_at : string }; + NotificationDeliveryList: { items : Array }; + NotificationDeliverySummary: { attempt_count : number; due_at : string; event_id : string; id : string; response_class : string | null; response_summary : string | null; state : string; subscription_id : string; terminal_reason : string | null }; + NotificationSubscriptionInput: { channel : "webhook" | "email"; destination : Record; event_filters : Array; rate_limit_per_minute?: number; signing_secret?: string | null }; + NotificationSubscriptionList: { items : Array }; + NotificationSubscriptionPatch: { destination?: Record | null; event_filters?: Array | null; rate_limit_per_minute?: number | null; state?: "active" | "disabled" | "archived" | null }; + NotificationSubscriptionSummary: { channel : string; created_at : string; destination : Record; event_filters : Array; id : string; rate_limit_per_minute : number; revision : number; state : string; updated_at : string }; + RecoveryStatus: { encrypted_repository_count : number; recovery_mode : string; runbook : string }; + RepositoryInput: { compression?: string; encryption?: string; name : string; relative_path : string }; + RepositoryList: { items : Array }; + RepositoryPatch: { compression?: string | null; encryption?: string | null }; + RepositorySummary: { compression : string; encryption : string; format_version : number; id : string; name : string; state : string }; + RestoreInput: { destination : string; dry_run?: boolean; overwrite_policy?: string; selection?: Array }; + SSHSourceInput: { kind : string; name : string; private_key_secret_id : string; public_config : Components['schemas']["SSHSourcePublicConfig"] }; + SSHSourcePublicConfig: { host_key : string; hostname : string; port : number; root : string; username : string }; + ScheduleInput: { cron : string; enabled?: boolean; misfire_grace_seconds?: number; timezone : string }; + ScheduleSummary: { cron : string; enabled : boolean; id : string; last_enqueue_outcome : string | null; next_nominal_at : string | null; timezone : string }; + SecretInput: { purpose : string; value : string }; + SessionUser: { id : string; state : string; username : string }; + SetupInput: { bootstrap_secret?: string | null; password : string; username : string }; + SigningKeyRotateInput: { overlap_seconds?: number; secret : string }; + SourceList: { items : Array }; + SourceSummary: { id : string; kind : string; name : string; public_config : Record; state : string }; + TokenInput: { expires_at?: string | null; scopes : Array }; + UserPatch: { state : string }; + ValidationError: { ctx?: Record; input?: unknown; loc : Array; msg : string; type : string }; + }; +} +export type CreateSecretParams = { body : Components['schemas']["SecretInput"] }; + +export type GetUserParams = { path: { user_id: string } }; + +export type PatchUserParams = { path: { user_id: string }; body : Components['schemas']["UserPatch"] }; + +export type ListAuditParams = { query?: { limit?: number; cursor?: string | null } }; + +export type LoginParams = { body : Components['schemas']["LoginInput"] }; + +export type CreateTokenParams = { body : Components['schemas']["TokenInput"] }; + +export type RevokeTokenParams = { path: { token_id: string } }; + +export type GetBackupParams = { path: { backup_id: string } }; + +export type BackupDeletePreviewParams = { path: { backup_id: string } }; + +export type CreateRestoreParams = { path: { backup_id: string }; body : Components['schemas']["RestoreInput"] }; + +export type VerifyBackupParams = { path: { backup_id: string } }; + +export type GetExecutionParams = { path: { execution_id: string } }; + +export type CancelExecutionParams = { path: { execution_id: string } }; + +export type ExecutionEventsParams = { path: { execution_id: string } }; + +export type RetryExecutionParams = { path: { execution_id: string } }; + +export type CreateJobParams = { body : Components['schemas']["JobInput"] }; + +export type EnqueueExecutionParams = { path: { job_id: string } }; + +export type DeleteScheduleParams = { path: { job_id: string } }; + +export type GetScheduleParams = { path: { job_id: string } }; + +export type PatchScheduleParams = { path: { job_id: string }; body : Components['schemas']["ScheduleInput"] }; + +export type CreateScheduleParams = { path: { job_id: string }; body : Components['schemas']["ScheduleInput"] }; + +export type ListNotificationDeliveriesParams = { query?: { limit?: number } }; + +export type ListNotificationAttemptsParams = { path: { delivery_id: string } }; + +export type RetryNotificationDeliveryParams = { path: { delivery_id: string } }; + +export type PutNotificationEmailSettingsParams = { body : Components['schemas']["EmailSettingsInput"] }; + +export type CreateNotificationSubscriptionParams = { body : Components['schemas']["NotificationSubscriptionInput"] }; + +export type GetNotificationSubscriptionParams = { path: { subscription_id: string } }; + +export type PatchNotificationSubscriptionParams = { path: { subscription_id: string }; body : Components['schemas']["NotificationSubscriptionPatch"] }; + +export type RotateNotificationSigningKeyParams = { path: { subscription_id: string }; body : Components['schemas']["SigningKeyRotateInput"] }; + +export type TestNotificationSubscriptionParams = { path: { subscription_id: string } }; + +export type CreateRepositoryParams = { body : Components['schemas']["RepositoryInput"] }; + +export type GetRepositoryParams = { path: { repository_id: string } }; + +export type PatchRepositoryParams = { path: { repository_id: string }; body : Components['schemas']["RepositoryPatch"] }; + +export type InspectRepositoryEndpointParams = { path: { repository_id: string } }; + +export type GetRestoreParams = { path: { restore_id: string } }; + +export type SetupParams = { body : Components['schemas']["SetupInput"] }; + +export type CreateSourceParams = { body : Components['schemas']["LocalSourceInput"] | Components['schemas']["SSHSourceInput"] }; + +export type ArchiveSourceParams = { path: { source_id: string } }; + +export type ProbeSourceParams = { path: { source_id: string } }; + +export type RequestOptions = Omit; + +export type Problem = { + type: string; + title: string; + status: number; + detail: string; + instance: string; + code: string; +}; + +export class ApiError extends Error { + readonly status: number; + readonly problem?: Problem; + + constructor(status: number, problem?: Problem) { + super(problem?.detail ?? `Request failed with status ${status}.`); + this.name = "ApiError"; + this.status = status; + this.problem = problem; + } +} + +export function isApiError(error: unknown): error is ApiError { + return error instanceof ApiError; +} + +function appendQuery(search: URLSearchParams, query: Record | undefined): void { + if (!query) return; + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) continue; + for (const item of Array.isArray(value) ? value : [value]) search.append(key, String(item)); + } +} + +async function request( + url: URL, + method: string, + options: RequestOptions, + body?: unknown, +): Promise { + const headers = new Headers(options.headers); + if (body !== undefined && !headers.has("Content-Type")) { + headers.set("Content-Type", "application/json"); + } + const response = await fetch(url, { + ...options, + method, + headers, + credentials: options.credentials ?? "same-origin", + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (response.status === 204) return undefined as T; + const contentType = response.headers.get("content-type") ?? ""; + const isJson = contentType.includes("application/json") + || contentType.includes("application/problem+json"); + const payload: unknown = isJson ? await response.json() : undefined; + if (!response.ok) throw new ApiError(response.status, payload as Problem | undefined); + return payload as T; +} + +export class BackupToolClient { + constructor(readonly baseUrl = window.location.origin) {} + + async listSecrets(options: RequestOptions = {}): Promise>> { + const url = new URL("/api/v2/admin/secrets", this.baseUrl); + return request>>( + url, "GET", options + ); + } + + async createSecret(params: CreateSecretParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/admin/secrets", this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async getUser(params: GetUserParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/admin/users/{user_id}".replace("{user_id}", encodeURIComponent(String(params.path.user_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async patchUser(params: PatchUserParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/admin/users/{user_id}".replace("{user_id}", encodeURIComponent(String(params.path.user_id))), this.baseUrl); + return request>( + url, "PATCH", options, params.body + ); + } + + async listAudit(params: ListAuditParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/audit", this.baseUrl); + appendQuery(url.searchParams, params.query); + return request( + url, "GET", options + ); + } + + async login(params: LoginParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/auth/login", this.baseUrl); + return request( + url, "POST", options, params.body + ); + } + + async logout(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/auth/logout", this.baseUrl); + return request( + url, "POST", options + ); + } + + async getSession(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/auth/session", this.baseUrl); + return request( + url, "GET", options + ); + } + + async createToken(params: CreateTokenParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/auth/tokens", this.baseUrl); + return request( + url, "POST", options, params.body + ); + } + + async revokeToken(params: RevokeTokenParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/auth/tokens/{token_id}".replace("{token_id}", encodeURIComponent(String(params.path.token_id))), this.baseUrl); + return request( + url, "DELETE", options + ); + } + + async listBackups(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/backups", this.baseUrl); + return request( + url, "GET", options + ); + } + + async getBackup(params: GetBackupParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/backups/{backup_id}".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl); + return request( + url, "GET", options + ); + } + + async backupDeletePreview(params: BackupDeletePreviewParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/backups/{backup_id}/delete-preview".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl); + return request( + url, "GET", options + ); + } + + async createRestore(params: CreateRestoreParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/backups/{backup_id}/restores".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async verifyBackup(params: VerifyBackupParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/backups/{backup_id}/verify".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl); + return request( + url, "POST", options + ); + } + + async listExecutions(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/executions", this.baseUrl); + return request( + url, "GET", options + ); + } + + async getExecution(params: GetExecutionParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/executions/{execution_id}".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl); + return request( + url, "GET", options + ); + } + + async cancelExecution(params: CancelExecutionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/executions/{execution_id}/cancel".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + /** EventSource transport; intentionally not a JSON fetch Promise. */ + executionEventsUrl(params: ExecutionEventsParams): URL { + const url = new URL("/api/v2/executions/{execution_id}/events".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl); + return url; + } + + async retryExecution(params: RetryExecutionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/executions/{execution_id}/retry".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + async listJobs(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/jobs", this.baseUrl); + return request( + url, "GET", options + ); + } + + async createJob(params: CreateJobParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/jobs", this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async enqueueExecution(params: EnqueueExecutionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/jobs/{job_id}/executions".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + async deleteSchedule(params: DeleteScheduleParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl); + return request( + url, "DELETE", options + ); + } + + async getSchedule(params: GetScheduleParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async patchSchedule(params: PatchScheduleParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl); + return request>( + url, "PATCH", options, params.body + ); + } + + async createSchedule(params: CreateScheduleParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async listNotificationDeliveries(params: ListNotificationDeliveriesParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/notifications/deliveries", this.baseUrl); + appendQuery(url.searchParams, params.query); + return request( + url, "GET", options + ); + } + + async listNotificationAttempts(params: ListNotificationAttemptsParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/notifications/deliveries/{delivery_id}/attempts".replace("{delivery_id}", encodeURIComponent(String(params.path.delivery_id))), this.baseUrl); + return request( + url, "GET", options + ); + } + + async retryNotificationDelivery(params: RetryNotificationDeliveryParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/deliveries/{delivery_id}/retry".replace("{delivery_id}", encodeURIComponent(String(params.path.delivery_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + async getNotificationEmailSettings(options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/email-settings", this.baseUrl); + return request>( + url, "GET", options + ); + } + + async putNotificationEmailSettings(params: PutNotificationEmailSettingsParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/email-settings", this.baseUrl); + return request>( + url, "PUT", options, params.body + ); + } + + async notificationCatalog(options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/event-catalog", this.baseUrl); + return request>( + url, "GET", options + ); + } + + async listNotificationSubscriptions(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/notifications/subscriptions", this.baseUrl); + return request( + url, "GET", options + ); + } + + async createNotificationSubscription(params: CreateNotificationSubscriptionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/subscriptions", this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async getNotificationSubscription(params: GetNotificationSubscriptionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async patchNotificationSubscription(params: PatchNotificationSubscriptionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl); + return request>( + url, "PATCH", options, params.body + ); + } + + async rotateNotificationSigningKey(params: RotateNotificationSigningKeyParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}/signing-keys/rotate".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async testNotificationSubscription(params: TestNotificationSubscriptionParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}/test".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + async listRepositories(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/repositories", this.baseUrl); + return request( + url, "GET", options + ); + } + + async createRepository(params: CreateRepositoryParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/repositories", this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async getRepository(params: GetRepositoryParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/repositories/{repository_id}".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async patchRepository(params: PatchRepositoryParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/repositories/{repository_id}".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl); + return request( + url, "PATCH", options, params.body + ); + } + + async inspectRepositoryEndpoint(params: InspectRepositoryEndpointParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/repositories/{repository_id}/inspection".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async getRestore(params: GetRestoreParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/restores/{restore_id}".replace("{restore_id}", encodeURIComponent(String(params.path.restore_id))), this.baseUrl); + return request>( + url, "GET", options + ); + } + + async recoveryStatus(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/security/recovery/status", this.baseUrl); + return request( + url, "GET", options + ); + } + + async setup(params: SetupParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/setup", this.baseUrl); + return request( + url, "POST", options, params.body + ); + } + + async listSources(options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/sources", this.baseUrl); + return request( + url, "GET", options + ); + } + + async createSource(params: CreateSourceParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/sources", this.baseUrl); + return request>( + url, "POST", options, params.body + ); + } + + async archiveSource(params: ArchiveSourceParams, options: RequestOptions = {}): Promise { + const url = new URL("/api/v2/sources/{source_id}".replace("{source_id}", encodeURIComponent(String(params.path.source_id))), this.baseUrl); + return request( + url, "DELETE", options + ); + } + + async probeSource(params: ProbeSourceParams, options: RequestOptions = {}): Promise> { + const url = new URL("/api/v2/sources/{source_id}/probe".replace("{source_id}", encodeURIComponent(String(params.path.source_id))), this.baseUrl); + return request>( + url, "POST", options + ); + } + + async livez(options: RequestOptions = {}): Promise> { + const url = new URL("/livez", this.baseUrl); + return request>( + url, "GET", options + ); + } + + async readyz(options: RequestOptions = {}): Promise> { + const url = new URL("/readyz", this.baseUrl); + return request>( + url, "GET", options + ); + } + +} diff --git a/frontend/src/app/App.test.tsx b/frontend/src/app/App.test.tsx new file mode 100644 index 0000000..bfe512d --- /dev/null +++ b/frontend/src/app/App.test.tsx @@ -0,0 +1,176 @@ +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { App } from "./App"; + +function response(body: unknown, status = 200): Response { + return { + headers: new Headers({ "content-type": "application/json" }), + json: async () => body, + ok: status >= 200 && status < 300, + status, + } as Response; +} + +function problem(status: number, code: string, detail = "Request failed."): Response { + return response({ type: `https://backup-tool.invalid/problems/${code}`, title: code, status, detail, instance: "/", code }, status); +} + +function session() { + return response({ id: "user-1", username: "operator", state: "active" }); +} + +function repositories(items: unknown[] = []) { + return response({ items }); +} + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); +}); + +describe("operator foundation", () => { + it("shows loading then the accessible empty dashboard", async () => { + const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(repositories()); + vi.stubGlobal("fetch", fetch); + render(); + + expect(screen.getByRole("status")).toHaveTextContent("Checking your session"); + expect(await screen.findByText("No repositories have been configured.")).toBeInTheDocument(); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it("selects setup and creates a session for the first administrator", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(problem(401, "authentication_required")) + .mockResolvedValueOnce(problem(503, "setup_required")) + .mockResolvedValueOnce(response({ id: "user-1", username: "operator" }, 201)) + .mockResolvedValueOnce(repositories()); + vi.stubGlobal("fetch", fetch); + render(); + + expect(await screen.findByRole("heading", { name: "Set up your administrator account" })).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Username"), { target: { value: "operator" } }); + fireEvent.change(screen.getByLabelText("Password"), { target: { value: "correct horse battery staple" } }); + fireEvent.click(screen.getByRole("button", { name: "Create administrator account" })); + + expect(await screen.findByRole("heading", { name: "Dashboard" })).toBeInTheDocument(); + const setupRequest = fetch.mock.calls[2]?.[0] as URL; + expect(setupRequest.pathname).toBe("/api/v2/setup"); + }); + + it("signs in after setup is complete", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(problem(401, "authentication_required")) + .mockResolvedValueOnce(response({ status: "ready" })) + .mockResolvedValueOnce(response({ id: "user-1", username: "operator" })) + .mockResolvedValueOnce(repositories()); + vi.stubGlobal("fetch", fetch); + render(); + + expect(await screen.findByRole("heading", { name: "Sign in" })).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Username"), { target: { value: "operator" } }); + fireEvent.change(screen.getByLabelText("Password"), { target: { value: "correct horse battery staple" } }); + fireEvent.click(screen.getByRole("button", { name: "Sign in" })); + + expect(await screen.findByRole("heading", { name: "Dashboard" })).toBeInTheDocument(); + const loginRequest = fetch.mock.calls[2]?.[0] as URL; + expect(loginRequest.pathname).toBe("/api/v2/auth/login"); + }); + + it("shows retryable dashboard errors", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(session()) + .mockResolvedValueOnce(problem(500, "service_unavailable", "Dashboard data is unavailable.")) + .mockResolvedValueOnce(repositories()); + vi.stubGlobal("fetch", fetch); + render(); + + expect(await screen.findByRole("alert")).toHaveTextContent("Dashboard data is unavailable."); + fireEvent.click(screen.getByRole("button", { name: "Try again" })); + await waitFor(() => expect(screen.getByText("No repositories have been configured.")).toBeInTheDocument()); + }); + + it("loads source and job states from generated client methods", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(session()) + .mockResolvedValueOnce(repositories()) + .mockResolvedValueOnce(response({ items: [] })) + .mockResolvedValueOnce(problem(500, "service_unavailable", "Jobs are unavailable.")); + vi.stubGlobal("fetch", fetch); + render(); + + await screen.findByText("No repositories have been configured."); + fireEvent.click(screen.getByRole("button", { name: "Sources" })); + expect(await screen.findByText("No sources have been configured.")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Jobs & schedules" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Jobs are unavailable."); + }); + + it("loads execution detail after selecting an execution", async () => { + const execution = { id: "execution-1", state: "failed", attempt: 2, revision: 3, reason_code: "timeout", progress: {} }; + const fetch = vi.fn() + .mockResolvedValueOnce(session()) + .mockResolvedValueOnce(repositories()) + .mockResolvedValueOnce(response({ items: [execution] })) + .mockResolvedValueOnce(response(execution)); + vi.stubGlobal("fetch", fetch); + render(); + + await screen.findByText("No repositories have been configured."); + fireEvent.click(screen.getByRole("button", { name: "Executions" })); + expect(await screen.findByRole("button", { name: "Execution execution-1" })).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Execution execution-1" })); + expect(await screen.findByRole("heading", { name: "Execution detail" })).toBeInTheDocument(); + expect(screen.getByText("timeout")).toBeInTheDocument(); + }); + + it("announces SSE reconnects and applies live execution updates", async () => { + class EventSourceMock { + static instances: EventSourceMock[] = []; + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: (() => void) | null = null; + constructor() { EventSourceMock.instances.push(this); } + close = vi.fn(); + } + vi.stubGlobal("EventSource", EventSourceMock); + const execution = { id: "execution-1", state: "running", attempt: 1, revision: 1, reason_code: null, progress: {} }; + const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(repositories()).mockResolvedValueOnce(response({ items: [execution] })).mockResolvedValueOnce(response(execution)); + vi.stubGlobal("fetch", fetch); + render(); + await screen.findByText("No repositories have been configured."); + fireEvent.click(screen.getByRole("button", { name: "Executions" })); + fireEvent.click(await screen.findByRole("button", { name: "Execution execution-1" })); + await screen.findByRole("heading", { name: "Execution detail" }); + EventSourceMock.instances[0]?.onerror?.(); + expect(await screen.findByRole("alert")).toHaveTextContent("Live updates disconnected"); + EventSourceMock.instances[0]?.onmessage?.({ data: JSON.stringify({ ...execution, state: "committed", revision: 2 }) } as MessageEvent); + expect(await screen.findByText("committed")).toBeInTheDocument(); + }); + + it("sends an idempotency key when retrying a failed notification delivery", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(session()) + .mockResolvedValueOnce(repositories()) + .mockResolvedValueOnce(response({ items: [] })) + .mockResolvedValueOnce(response({ items: [{ id: "delivery-1", event_id: "event-1", subscription_id: "subscription-1", state: "failed", attempt_count: 1, response_class: null, response_summary: null, terminal_reason: "timeout", due_at: "2026-01-01T00:00:00Z" }] })) + .mockResolvedValueOnce(response({ delivery_id: "delivery-1" }, 202)); + vi.stubGlobal("fetch", fetch); + render(); + await screen.findByText("No repositories have been configured."); + fireEvent.click(screen.getByRole("button", { name: "Notifications" })); + fireEvent.click(await screen.findByRole("button", { name: "Retry delivery" })); + expect(await screen.findByRole("status")).toHaveTextContent("Delivery retry queued."); + const options = fetch.mock.calls[4]?.[1] as RequestInit; + expect(new Headers(options.headers).get("Idempotency-Key")).toBeTruthy(); + }); + + it("returns to sign-in when the dashboard request finds an expired session", async () => { + const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(problem(401, "authentication_required")); + vi.stubGlobal("fetch", fetch); + render(); + + expect(await screen.findByRole("heading", { name: "Sign in" })).toBeInTheDocument(); + expect(screen.getByRole("status")).toHaveTextContent("Your session expired"); + }); +}); diff --git a/frontend/src/app/App.tsx b/frontend/src/app/App.tsx index a1f9b2f..92ef59f 100644 --- a/frontend/src/app/App.tsx +++ b/frontend/src/app/App.tsx @@ -1,16 +1,97 @@ -export function App() { - return ( -
-
-

- Backup Tool v2 -

-

Protocol foundation ready

-

- Operator workflows are added as their versioned API contracts become - executable. -

-
-
- ); +import { type FormEvent, useEffect, useRef, useState } from "react"; + +import { + BackupToolClient, + type Components, + isApiError, +} from "../api/generated/client"; +import { OperatorViews } from "./OperatorViews"; + +type SessionUser = Components["schemas"]["SessionUser"]; +type AuthMode = "setup" | "login"; +type Screen = + | { kind: "loading" } + | { kind: "auth"; mode: AuthMode; error?: string; sessionExpired?: boolean } + | { kind: "operator"; user: SessionUser }; + +const defaultClient = new BackupToolClient(); + +function errorMessage(error: unknown): string { + if (isApiError(error)) return error.problem?.detail ?? "The server could not complete that request."; + return "We could not reach the Backup Tool service. Check your connection and try again."; +} + +function isSetupRequired(error: unknown): boolean { + return isApiError(error) && error.status === 503 && error.problem?.code === "setup_required"; +} + +function isSessionExpired(error: unknown): boolean { + return isApiError(error) && error.status === 401; +} + +function AuthForm({ + mode, + error, + sessionExpired, + onSubmit, +}: { + mode: AuthMode; + error?: string; + sessionExpired?: boolean; + onSubmit: (username: string, password: string) => Promise; +}) { + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const [submitting, setSubmitting] = useState(false); + const isSetup = mode === "setup"; + + async function submit(event: FormEvent) { + event.preventDefault(); + if (isSetup && password.length < 12) return; + setSubmitting(true); + try { + await onSubmit(username, password); + } finally { + setSubmitting(false); + } + } + + return

Backup Tool

{isSetup ? "Set up your administrator account" : "Sign in"}

{isSetup ? "Create the first administrator account to begin operating this backup service." : "Use an administrator account to continue."}

{sessionExpired ?

Your session expired. Sign in again to continue.

: null}{error ?

{error}

: null}
setUsername(event.target.value)} required value={username} />
setPassword(event.target.value)} required type="password" value={password} />{isSetup ?

Use at least 12 characters.

: null}
; +} + +export function App({ client = defaultClient }: { client?: BackupToolClient }) { + const [screen, setScreen] = useState({ kind: "loading" }); + const clientRef = useRef(client); + clientRef.current = client; + + async function discover() { + setScreen({ kind: "loading" }); + try { + setScreen({ kind: "operator", user: await clientRef.current.getSession() }); + } catch (error) { + if (!isSessionExpired(error)) { + setScreen({ kind: "auth", mode: "login", error: errorMessage(error) }); + return; + } + try { + await clientRef.current.readyz(); + setScreen({ kind: "auth", mode: "login" }); + } catch (readinessError) { + setScreen({ kind: "auth", mode: isSetupRequired(readinessError) ? "setup" : "login", error: isSetupRequired(readinessError) ? undefined : errorMessage(readinessError) }); + } + } + } + + useEffect(() => { void discover(); }, []); + if (screen.kind === "loading") return
Checking your session…
; + if (screen.kind === "auth") return { + try { + const user = screen.mode === "setup" ? await clientRef.current.setup({ body: { username, password } }) : await clientRef.current.login({ body: { username, password } }); + setScreen({ kind: "operator", user: { ...user, state: "active" } }); + } catch (error) { + if (screen.mode === "setup" && isApiError(error) && error.problem?.code === "setup_complete") setScreen({ kind: "auth", mode: "login", error: "Setup is already complete. Sign in with the administrator account." }); + else setScreen({ ...screen, error: errorMessage(error) }); + } + }} />; + return setScreen({ kind: "auth", mode: "login", sessionExpired: true })} user={screen.user} />; } diff --git a/frontend/src/app/M13Workflows.tsx b/frontend/src/app/M13Workflows.tsx new file mode 100644 index 0000000..626aa9c --- /dev/null +++ b/frontend/src/app/M13Workflows.tsx @@ -0,0 +1,33 @@ +import { type ReactNode, useEffect, useState } from "react"; + +import { type BackupToolClient, type Components, isApiError } from "../api/generated/client"; + +type BackupSummary = Components["schemas"]["BackupSummary"]; +type Delivery = Components["schemas"]["NotificationDeliverySummary"]; +type Subscription = Components["schemas"]["NotificationSubscriptionSummary"]; +type Audit = Components["schemas"]["AuditSummary"]; +type State = { loading: boolean; data?: T; error?: string }; + +type Props = { client: BackupToolClient; onSessionExpired: () => void }; + +function message(error: unknown) { return isApiError(error) ? error.problem?.detail ?? "The request failed." : "The service could not be reached."; } +function expired(error: unknown) { return isApiError(error) && error.status === 401; } +function Panel({ children, title }: { children: ReactNode; title: string }) { return

{title}

{children}
; } +function Retry({ error, load }: { error?: string; load: () => void }) { return error ?

{error}

: null; } +function Loading({ label }: { label: string }) { return

Loading {label}…

; } +function csrfHeaders(): HeadersInit { const value = document.cookie.split("; ").find((entry) => entry.startsWith("backup_tool_csrf="))?.split("=", 2)[1]; return value ? { "X-CSRF-Token": value } : {}; } +function idempotencyKey(): string { return typeof crypto.randomUUID === "function" ? crypto.randomUUID() : `${Date.now()}-${Math.random()}`; } + +export function BackupsPage({ client, onSessionExpired }: Props) { + const [state,setState]=useState>({loading:true}); const [selected,setSelected]=useState(); const [preview,setPreview]=useState(); const [destination,setDestination]=useState(""); const [restoreStatus,setRestoreStatus]=useState(); + const load=async()=>{setState({loading:true});try{setState({loading:false,data:(await client.listBackups()).items});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}}; + useEffect(()=>{void load();},[]); + const headers=csrfHeaders(); + return {state.loading?:null}void load()}/>{state.data?.length===0?

No backups have been committed.

:null}
    {state.data?.map(item=>

  • {item.integrity} · {item.logical_bytes} logical bytes

    • )}
{selected?

Backup detail

Integrity: {selected.integrity}

{preview?

{preview}

:null}
{event.preventDefault();try{await client.createRestore({path:{backup_id:selected.id},body:{destination,dry_run:true,selection:[],overwrite_policy:"fail"}},{headers});setRestoreStatus("Restore dry run queued.");}catch(error){setRestoreStatus(message(error));}}}>setDestination(event.target.value)} required value={destination}/>{restoreStatus?

{restoreStatus}

:null}
:null}; +} + +export function SecurityPage({ client,onSessionExpired }: Props) { const [state,setState]=useState>({loading:true}); const load=async()=>{setState({loading:true});try{setState({loading:false,data:await client.recoveryStatus()});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}};useEffect(()=>{void load();},[]);return {state.loading?:null}void load()}/>{state.data?

Recovery is {state.data.recovery_mode.replace("_"," ")}.

Encrypted repositories: {state.data.encrypted_repository_count}

Use the CLI and the recovery runbook: {state.data.runbook}. Passphrases and recovery bundles never enter the browser.

:null}; } + +export function NotificationsPage({client,onSessionExpired}:Props){const [state,setState]=useState>({loading:true});const [history,setHistory]=useState();const load=async()=>{setState({loading:true});const results=await Promise.allSettled([client.listNotificationSubscriptions(),client.listNotificationDeliveries({query:{limit:20}})]);if(results.some((result)=>result.status==="rejected"&&expired(result.reason))){onSessionExpired();return;}const errors=results.filter((result)=>result.status==="rejected");setState({loading:false,data:{subscriptions:results[0].status==="fulfilled"?results[0].value.items:[],deliveries:results[1].status==="fulfilled"?results[1].value.items:[]},error:errors.length?"Some notification history could not be loaded.":undefined});};useEffect(()=>{void load();},[]);return {state.loading?:null}void load()}/>{state.data?<>

Subscriptions

{state.data.subscriptions.length?
    {state.data.subscriptions.map(item=>
  • {item.channel} · {item.state}
    • )}
:

No notification subscriptions.

}

Delivery history

{state.data.deliveries.length?
    {state.data.deliveries.map(item=>
  • {item.state} · {item.attempt_count} attempts {item.state==="failed"?:null}
    • )}
:

No notification deliveries.

}{history?

{history}

:null}:null}} + +export function AuditPage({client,onSessionExpired}:Props){const [state,setState]=useState>({loading:true});const load=async()=>{setState({loading:true});try{setState({loading:false,data:(await client.listAudit({query:{limit:50}})).items});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}};useEffect(()=>{void load();},[]);return {state.loading?:null}void load()}/>{state.data?.length===0?

No audit events.

:null}
    {state.data?.map(item=>
  • {item.action} {item.resource_type} · {item.outcome}
    • )}
} diff --git a/frontend/src/app/OperatorViews.tsx b/frontend/src/app/OperatorViews.tsx new file mode 100644 index 0000000..397e472 --- /dev/null +++ b/frontend/src/app/OperatorViews.tsx @@ -0,0 +1,159 @@ +import { type ReactNode, useEffect, useState } from "react"; + +import { + type BackupToolClient, + type Components, + isApiError, +} from "../api/generated/client"; +import { AuditPage, BackupsPage, NotificationsPage, SecurityPage } from "./M13Workflows"; + +type SessionUser = Components["schemas"]["SessionUser"]; +type SourceSummary = Components["schemas"]["SourceSummary"]; +type RepositorySummary = Components["schemas"]["RepositorySummary"]; +type JobSummary = Components["schemas"]["JobSummary"]; +type ExecutionSummary = Components["schemas"]["ExecutionSummary"]; +type Page = "dashboard" | "sources" | "repositories" | "jobs" | "executions" | "backups" | "security" | "notifications" | "audit"; +type LoadState = + | { kind: "loading" } + | { kind: "ready"; items: T } + | { kind: "error"; message: string }; + +const pages: Array<{ id: Page; label: string }> = [ + { id: "dashboard", label: "Dashboard" }, + { id: "sources", label: "Sources" }, + { id: "repositories", label: "Repositories" }, + { id: "jobs", label: "Jobs & schedules" }, + { id: "executions", label: "Executions" }, + { id: "backups", label: "Backups" }, + { id: "security", label: "Security & recovery" }, + { id: "notifications", label: "Notifications" }, + { id: "audit", label: "Audit" }, +]; + +function errorMessage(error: unknown): string { + if (isApiError(error)) return error.problem?.detail ?? "The server could not complete that request."; + return "We could not reach the Backup Tool service. Check your connection and try again."; +} + +function isSessionExpired(error: unknown): boolean { + return isApiError(error) && error.status === 401; +} + +function ErrorPanel({ message, retry }: { message: string; retry: () => void }) { + return

{message}

; +} + +function Loading({ label }: { label: string }) { + return

Loading {label}…

; +} + +function ResourceSection({ children, title }: { children: ReactNode; title: string }) { + return

{title}

{children}
; +} + +function DashboardPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) { + const [state, setState] = useState>({ kind: "loading" }); + async function load() { + setState({ kind: "loading" }); + try { setState({ kind: "ready", items: (await client.listRepositories()).items }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); } + } + useEffect(() => { void load(); }, []); + return +

Repository availability at a glance.

+ {state.kind === "loading" ? : null} + {state.kind === "error" ? { void load(); }} /> : null} + {state.kind === "ready" && state.items.length === 0 ?

No repositories have been configured.

: null} + {state.kind === "ready" && state.items.length > 0 ?
    {state.items.map((repository) =>

  • {repository.name}

    {repository.state} · {repository.encryption}

    • )}
: null} + ; +} + +function SourcesPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) { + const [state, setState] = useState>({ kind: "loading" }); + async function load() { + setState({ kind: "loading" }); + try { setState({ kind: "ready", items: (await client.listSources()).items }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); } + } + useEffect(() => { void load(); }, []); + return + {state.kind === "loading" ? : null} + {state.kind === "error" ? { void load(); }} /> : null} + {state.kind === "ready" && state.items.length === 0 ?

No sources have been configured.

: null} + {state.kind === "ready" && state.items.length > 0 ?
    {state.items.map((source) =>

  • {source.name}

    {source.kind} · {source.state}

    • )}
: null} + ; +} + +function RepositoriesPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) { + const [state, setState] = useState>({ kind: "loading" }); + async function load() { + setState({ kind: "loading" }); + try { setState({ kind: "ready", items: (await client.listRepositories()).items }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); } + } + useEffect(() => { void load(); }, []); + return + {state.kind === "loading" ? : null} + {state.kind === "error" ? { void load(); }} /> : null} + {state.kind === "ready" && state.items.length === 0 ?

No repositories have been configured.

: null} + {state.kind === "ready" && state.items.length > 0 ?
    {state.items.map((repository) =>

  • {repository.name}

    Format {repository.format_version} · {repository.encryption} · {repository.state}

    • )}
: null} + ; +} + +function JobsPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) { + const [state, setState] = useState>({ kind: "loading" }); + async function load() { + setState({ kind: "loading" }); + try { setState({ kind: "ready", items: (await client.listJobs()).items }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); } + } + useEffect(() => { void load(); }, []); + return + {state.kind === "loading" ? : null} + {state.kind === "error" ? { void load(); }} /> : null} + {state.kind === "ready" && state.items.length === 0 ?

No jobs have been configured.

: null} + {state.kind === "ready" && state.items.length > 0 ?
    {state.items.map((job) =>

  • {job.name}

    {job.requested_mode} · {job.enabled ? "enabled" : "disabled"}

    {job.schedule ? `${job.schedule.cron} (${job.schedule.timezone})` : "No schedule configured."}

    • )}
: null} + ; +} + +function ExecutionsPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) { + const [state, setState] = useState>({ kind: "loading" }); + const [selectedId, setSelectedId] = useState(); + const [detail, setDetail] = useState | undefined>(); + async function load() { + setState({ kind: "loading" }); + try { setState({ kind: "ready", items: (await client.listExecutions()).items }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); } + } + async function loadDetail(executionId: string) { + setSelectedId(executionId); setDetail({ kind: "loading" }); + try { setDetail({ kind: "ready", items: await client.getExecution({ path: { execution_id: executionId } }) }); } + catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setDetail({ kind: "error", message: errorMessage(error) }); } + } + useEffect(() => { void load(); }, []); + useEffect(() => { + if (!selectedId || typeof EventSource === "undefined") return; + const stream = new EventSource(client.executionEventsUrl({ path: { execution_id: selectedId } })); + stream.onmessage = (event) => { + try { setDetail({ kind: "ready", items: JSON.parse(event.data) as ExecutionSummary }); } + catch { setDetail({ kind: "error", message: "Live execution update was invalid. Reconnecting…" }); } + }; + stream.onerror = () => { setDetail({ kind: "error", message: "Live updates disconnected. Reconnecting…" }); }; + return () => stream.close(); + }, [client, selectedId]); + return + {state.kind === "loading" ? : null} + {state.kind === "error" ? { void load(); }} /> : null} + {state.kind === "ready" && state.items.length === 0 ?

No executions have been queued.

: null} + {state.kind === "ready" && state.items.length > 0 ?
    {state.items.map((execution) =>

  • {execution.state} · attempt {execution.attempt}

    • )}
: null} + {detail?.kind === "loading" ? : null} + {detail?.kind === "error" && selectedId ? { void loadDetail(selectedId); }} /> : null} + {detail?.kind === "ready" ?

Execution detail

State
{detail.items.state}
Attempt
{detail.items.attempt}
Reason
{detail.items.reason_code ?? "None"}
: null} + ; +} + +export function OperatorViews({ client, onSessionExpired, user }: { client: BackupToolClient; onSessionExpired: () => void; user: SessionUser }) { + const [page, setPage] = useState("dashboard"); + const common = { client, onSessionExpired }; + return

Backup Tool

Operator console

Signed in as {user.username}

{page === "dashboard" ? : null}{page === "sources" ? : null}{page === "repositories" ? : null}{page === "jobs" ? : null}{page === "executions" ? : null}{page === "backups" ? : null}{page === "security" ? : null}{page === "notifications" ? : null}{page === "audit" ? : null}
; +} diff --git a/frontend/src/index.css b/frontend/src/index.css index b5c61c9..f1d8917 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,3 +1,23 @@ @tailwind base; @tailwind components; @tailwind utilities; + +:root { + color-scheme: dark; +} + +:focus-visible { + outline: 3px solid #34d399; + outline-offset: 3px; +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + } +} diff --git a/frontend/src/test/setup.ts b/frontend/src/test/setup.ts new file mode 100644 index 0000000..f149f27 --- /dev/null +++ b/frontend/src/test/setup.ts @@ -0,0 +1 @@ +import "@testing-library/jest-dom/vitest"; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index bbd2a4e..8394379 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,12 +1,12 @@ -import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vite"; export default defineConfig({ plugins: [react()], server: { port: 3000, proxy: { - '/api': 'http://localhost:8000' - } - } -}) + "/api": "http://localhost:8000", + }, + }, +}); diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts new file mode 100644 index 0000000..11b3b31 --- /dev/null +++ b/frontend/vitest.config.ts @@ -0,0 +1,16 @@ +import { mergeConfig } from "vite"; +import { defineConfig } from "vitest/config"; + +import viteConfig from "./vite.config"; + +export default mergeConfig( + viteConfig, + defineConfig({ + test: { + environment: "jsdom", + include: ["src/**/*.test.{ts,tsx}"], + exclude: ["e2e/**", "node_modules/**"], + setupFiles: ["./src/test/setup.ts"], + }, + }), +); diff --git a/openapi/v2.json b/openapi/v2.json new file mode 100644 index 0000000..cfbae06 --- /dev/null +++ b/openapi/v2.json @@ -0,0 +1,4789 @@ +{ + "components": { + "schemas": { + "AuditList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/AuditSummary" + }, + "title": "Items", + "type": "array" + }, + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Cursor" + } + }, + "required": [ + "items", + "next_cursor" + ], + "title": "AuditList", + "type": "object" + }, + "AuditSummary": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "created_at": { + "title": "Created At", + "type": "string" + }, + "details": { + "additionalProperties": true, + "title": "Details", + "type": "object" + }, + "id": { + "title": "Id", + "type": "string" + }, + "outcome": { + "title": "Outcome", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "resource_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Resource Id" + }, + "resource_type": { + "title": "Resource Type", + "type": "string" + } + }, + "required": [ + "id", + "action", + "resource_type", + "resource_id", + "outcome", + "request_id", + "created_at", + "details" + ], + "title": "AuditSummary", + "type": "object" + }, + "AuthenticatedUser": { + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "username": { + "title": "Username", + "type": "string" + } + }, + "required": [ + "id", + "username" + ], + "title": "AuthenticatedUser", + "type": "object" + }, + "BackupDeletePreview": { + "properties": { + "backup_id": { + "title": "Backup Id", + "type": "string" + }, + "destructive_action": { + "title": "Destructive Action", + "type": "string" + }, + "eligible": { + "title": "Eligible", + "type": "boolean" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + } + }, + "required": [ + "backup_id", + "eligible", + "reason", + "destructive_action" + ], + "title": "BackupDeletePreview", + "type": "object" + }, + "BackupList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/BackupSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "BackupList", + "type": "object" + }, + "BackupSummary": { + "properties": { + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "execution_id": { + "title": "Execution Id", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "integrity": { + "title": "Integrity", + "type": "string" + }, + "logical_bytes": { + "title": "Logical Bytes", + "type": "integer" + }, + "manifest_id": { + "title": "Manifest Id", + "type": "string" + }, + "pinned": { + "title": "Pinned", + "type": "boolean" + }, + "stored_bytes": { + "title": "Stored Bytes", + "type": "integer" + }, + "tombstoned_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tombstoned At" + } + }, + "required": [ + "id", + "execution_id", + "manifest_id", + "logical_bytes", + "stored_bytes", + "integrity", + "pinned", + "tombstoned_at", + "created_at" + ], + "title": "BackupSummary", + "type": "object" + }, + "EmailSettingsInput": { + "properties": { + "host": { + "maxLength": 255, + "minLength": 1, + "title": "Host", + "type": "string" + }, + "max_attempts": { + "default": 5, + "maximum": 20.0, + "minimum": 1.0, + "title": "Max Attempts", + "type": "integer" + }, + "password": { + "maxLength": 65536, + "minLength": 1, + "title": "Password", + "type": "string" + }, + "port": { + "default": 587, + "maximum": 65535.0, + "minimum": 1.0, + "title": "Port", + "type": "integer" + }, + "rate_limit_per_minute": { + "default": 60, + "maximum": 10000.0, + "minimum": 1.0, + "title": "Rate Limit Per Minute", + "type": "integer" + }, + "sender": { + "maxLength": 320, + "minLength": 3, + "title": "Sender", + "type": "string" + }, + "username": { + "maxLength": 255, + "minLength": 1, + "title": "Username", + "type": "string" + } + }, + "required": [ + "host", + "username", + "password", + "sender" + ], + "title": "EmailSettingsInput", + "type": "object" + }, + "ExecutionList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/ExecutionSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "ExecutionList", + "type": "object" + }, + "ExecutionSummary": { + "properties": { + "attempt": { + "title": "Attempt", + "type": "integer" + }, + "id": { + "title": "Id", + "type": "string" + }, + "progress": { + "additionalProperties": true, + "title": "Progress", + "type": "object" + }, + "reason_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason Code" + }, + "revision": { + "title": "Revision", + "type": "integer" + }, + "state": { + "title": "State", + "type": "string" + } + }, + "required": [ + "id", + "state", + "attempt", + "revision", + "reason_code", + "progress" + ], + "title": "ExecutionSummary", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "JobInput": { + "properties": { + "allow_empty": { + "default": false, + "title": "Allow Empty", + "type": "boolean" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "exclusions": { + "items": { + "type": "string" + }, + "title": "Exclusions", + "type": "array" + }, + "name": { + "maxLength": 255, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "repository_id": { + "title": "Repository Id", + "type": "string" + }, + "requested_mode": { + "default": "incremental", + "title": "Requested Mode", + "type": "string" + }, + "retention": { + "additionalProperties": true, + "title": "Retention", + "type": "object" + }, + "source_id": { + "title": "Source Id", + "type": "string" + } + }, + "required": [ + "name", + "source_id", + "repository_id" + ], + "title": "JobInput", + "type": "object" + }, + "JobList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/JobSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "JobList", + "type": "object" + }, + "JobSummary": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "repository_id": { + "title": "Repository Id", + "type": "string" + }, + "requested_mode": { + "title": "Requested Mode", + "type": "string" + }, + "schedule": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScheduleSummary" + }, + { + "type": "null" + } + ] + }, + "source_id": { + "title": "Source Id", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + } + }, + "required": [ + "id", + "name", + "source_id", + "repository_id", + "requested_mode", + "enabled", + "state", + "schedule" + ], + "title": "JobSummary", + "type": "object" + }, + "LocalSourceInput": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "local", + "title": "Kind", + "type": "string" + }, + "name": { + "maxLength": 255, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "public_config": { + "additionalProperties": true, + "title": "Public Config", + "type": "object" + } + }, + "required": [ + "name", + "kind", + "public_config" + ], + "title": "LocalSourceInput", + "type": "object" + }, + "LoginInput": { + "properties": { + "password": { + "maxLength": 1024, + "minLength": 1, + "title": "Password", + "type": "string" + }, + "username": { + "maxLength": 255, + "minLength": 1, + "title": "Username", + "type": "string" + } + }, + "required": [ + "username", + "password" + ], + "title": "LoginInput", + "type": "object" + }, + "NotificationAttemptList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/NotificationAttemptSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "NotificationAttemptList", + "type": "object" + }, + "NotificationAttemptSummary": { + "properties": { + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Completed At" + }, + "diagnostic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Diagnostic" + }, + "number": { + "title": "Number", + "type": "integer" + }, + "outcome": { + "title": "Outcome", + "type": "string" + }, + "response_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Response Class" + }, + "started_at": { + "title": "Started At", + "type": "string" + } + }, + "required": [ + "number", + "outcome", + "response_class", + "diagnostic", + "started_at", + "completed_at" + ], + "title": "NotificationAttemptSummary", + "type": "object" + }, + "NotificationDeliveryList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/NotificationDeliverySummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "NotificationDeliveryList", + "type": "object" + }, + "NotificationDeliverySummary": { + "properties": { + "attempt_count": { + "title": "Attempt Count", + "type": "integer" + }, + "due_at": { + "title": "Due At", + "type": "string" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "response_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Response Class" + }, + "response_summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Response Summary" + }, + "state": { + "title": "State", + "type": "string" + }, + "subscription_id": { + "title": "Subscription Id", + "type": "string" + }, + "terminal_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Terminal Reason" + } + }, + "required": [ + "id", + "event_id", + "subscription_id", + "state", + "attempt_count", + "response_class", + "response_summary", + "terminal_reason", + "due_at" + ], + "title": "NotificationDeliverySummary", + "type": "object" + }, + "NotificationSubscriptionInput": { + "properties": { + "channel": { + "enum": [ + "webhook", + "email" + ], + "title": "Channel", + "type": "string" + }, + "destination": { + "additionalProperties": true, + "title": "Destination", + "type": "object" + }, + "event_filters": { + "items": { + "type": "string" + }, + "maxItems": 48, + "minItems": 1, + "title": "Event Filters", + "type": "array" + }, + "rate_limit_per_minute": { + "default": 60, + "maximum": 10000.0, + "minimum": 1.0, + "title": "Rate Limit Per Minute", + "type": "integer" + }, + "signing_secret": { + "anyOf": [ + { + "maxLength": 65536, + "minLength": 16, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Signing Secret" + } + }, + "required": [ + "channel", + "event_filters", + "destination" + ], + "title": "NotificationSubscriptionInput", + "type": "object" + }, + "NotificationSubscriptionList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/NotificationSubscriptionSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "NotificationSubscriptionList", + "type": "object" + }, + "NotificationSubscriptionPatch": { + "properties": { + "destination": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Destination" + }, + "event_filters": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "maxItems": 48, + "minItems": 1, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Event Filters" + }, + "rate_limit_per_minute": { + "anyOf": [ + { + "maximum": 10000.0, + "minimum": 1.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rate Limit Per Minute" + }, + "state": { + "anyOf": [ + { + "enum": [ + "active", + "disabled", + "archived" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "State" + } + }, + "title": "NotificationSubscriptionPatch", + "type": "object" + }, + "NotificationSubscriptionSummary": { + "properties": { + "channel": { + "title": "Channel", + "type": "string" + }, + "created_at": { + "title": "Created At", + "type": "string" + }, + "destination": { + "additionalProperties": true, + "title": "Destination", + "type": "object" + }, + "event_filters": { + "items": { + "type": "string" + }, + "title": "Event Filters", + "type": "array" + }, + "id": { + "title": "Id", + "type": "string" + }, + "rate_limit_per_minute": { + "title": "Rate Limit Per Minute", + "type": "integer" + }, + "revision": { + "title": "Revision", + "type": "integer" + }, + "state": { + "title": "State", + "type": "string" + }, + "updated_at": { + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "channel", + "event_filters", + "destination", + "state", + "rate_limit_per_minute", + "revision", + "created_at", + "updated_at" + ], + "title": "NotificationSubscriptionSummary", + "type": "object" + }, + "RecoveryStatus": { + "properties": { + "encrypted_repository_count": { + "title": "Encrypted Repository Count", + "type": "integer" + }, + "recovery_mode": { + "const": "cli_only", + "title": "Recovery Mode", + "type": "string" + }, + "runbook": { + "title": "Runbook", + "type": "string" + } + }, + "required": [ + "recovery_mode", + "runbook", + "encrypted_repository_count" + ], + "title": "RecoveryStatus", + "type": "object" + }, + "RepositoryInput": { + "properties": { + "compression": { + "default": "none", + "title": "Compression", + "type": "string" + }, + "encryption": { + "default": "none", + "title": "Encryption", + "type": "string" + }, + "name": { + "maxLength": 255, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "relative_path": { + "maxLength": 1024, + "minLength": 1, + "title": "Relative Path", + "type": "string" + } + }, + "required": [ + "name", + "relative_path" + ], + "title": "RepositoryInput", + "type": "object" + }, + "RepositoryList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/RepositorySummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "RepositoryList", + "type": "object" + }, + "RepositoryPatch": { + "properties": { + "compression": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Compression" + }, + "encryption": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Encryption" + } + }, + "title": "RepositoryPatch", + "type": "object" + }, + "RepositorySummary": { + "properties": { + "compression": { + "title": "Compression", + "type": "string" + }, + "encryption": { + "title": "Encryption", + "type": "string" + }, + "format_version": { + "title": "Format Version", + "type": "integer" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + } + }, + "required": [ + "id", + "name", + "format_version", + "compression", + "encryption", + "state" + ], + "title": "RepositorySummary", + "type": "object" + }, + "RestoreInput": { + "properties": { + "destination": { + "maxLength": 4096, + "minLength": 1, + "title": "Destination", + "type": "string" + }, + "dry_run": { + "default": false, + "title": "Dry Run", + "type": "boolean" + }, + "overwrite_policy": { + "default": "fail", + "title": "Overwrite Policy", + "type": "string" + }, + "selection": { + "items": { + "type": "string" + }, + "title": "Selection", + "type": "array" + } + }, + "required": [ + "destination" + ], + "title": "RestoreInput", + "type": "object" + }, + "SSHSourceInput": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "ssh", + "title": "Kind", + "type": "string" + }, + "name": { + "maxLength": 255, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "private_key_secret_id": { + "maxLength": 36, + "minLength": 1, + "title": "Private Key Secret Id", + "type": "string" + }, + "public_config": { + "$ref": "#/components/schemas/SSHSourcePublicConfig" + } + }, + "required": [ + "name", + "kind", + "public_config", + "private_key_secret_id" + ], + "title": "SSHSourceInput", + "type": "object" + }, + "SSHSourcePublicConfig": { + "additionalProperties": false, + "description": "The public, SFTP-chroot-only portion of an SSH source definition.", + "properties": { + "host_key": { + "maxLength": 16384, + "minLength": 1, + "title": "Host Key", + "type": "string" + }, + "hostname": { + "maxLength": 253, + "minLength": 1, + "title": "Hostname", + "type": "string" + }, + "port": { + "maximum": 65535.0, + "minimum": 1.0, + "title": "Port", + "type": "integer" + }, + "root": { + "const": "/", + "title": "Root", + "type": "string" + }, + "username": { + "maxLength": 255, + "minLength": 1, + "title": "Username", + "type": "string" + } + }, + "required": [ + "hostname", + "port", + "username", + "host_key", + "root" + ], + "title": "SSHSourcePublicConfig", + "type": "object" + }, + "ScheduleInput": { + "properties": { + "cron": { + "maxLength": 255, + "minLength": 1, + "title": "Cron", + "type": "string" + }, + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "misfire_grace_seconds": { + "default": 900, + "minimum": 0.0, + "title": "Misfire Grace Seconds", + "type": "integer" + }, + "timezone": { + "maxLength": 255, + "minLength": 1, + "title": "Timezone", + "type": "string" + } + }, + "required": [ + "cron", + "timezone" + ], + "title": "ScheduleInput", + "type": "object" + }, + "ScheduleSummary": { + "properties": { + "cron": { + "title": "Cron", + "type": "string" + }, + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "id": { + "title": "Id", + "type": "string" + }, + "last_enqueue_outcome": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Enqueue Outcome" + }, + "next_nominal_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Nominal At" + }, + "timezone": { + "title": "Timezone", + "type": "string" + } + }, + "required": [ + "id", + "cron", + "timezone", + "enabled", + "next_nominal_at", + "last_enqueue_outcome" + ], + "title": "ScheduleSummary", + "type": "object" + }, + "SecretInput": { + "properties": { + "purpose": { + "maxLength": 64, + "minLength": 1, + "title": "Purpose", + "type": "string" + }, + "value": { + "maxLength": 65536, + "minLength": 1, + "title": "Value", + "type": "string" + } + }, + "required": [ + "purpose", + "value" + ], + "title": "SecretInput", + "type": "object" + }, + "SessionUser": { + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + }, + "username": { + "title": "Username", + "type": "string" + } + }, + "required": [ + "id", + "username", + "state" + ], + "title": "SessionUser", + "type": "object" + }, + "SetupInput": { + "properties": { + "bootstrap_secret": { + "anyOf": [ + { + "maxLength": 1024, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Bootstrap Secret" + }, + "password": { + "maxLength": 1024, + "minLength": 12, + "title": "Password", + "type": "string" + }, + "username": { + "maxLength": 255, + "minLength": 1, + "title": "Username", + "type": "string" + } + }, + "required": [ + "username", + "password" + ], + "title": "SetupInput", + "type": "object" + }, + "SigningKeyRotateInput": { + "properties": { + "overlap_seconds": { + "default": 3600, + "maximum": 86400.0, + "minimum": 60.0, + "title": "Overlap Seconds", + "type": "integer" + }, + "secret": { + "maxLength": 65536, + "minLength": 16, + "title": "Secret", + "type": "string" + } + }, + "required": [ + "secret" + ], + "title": "SigningKeyRotateInput", + "type": "object" + }, + "SourceList": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/SourceSummary" + }, + "title": "Items", + "type": "array" + } + }, + "required": [ + "items" + ], + "title": "SourceList", + "type": "object" + }, + "SourceSummary": { + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "public_config": { + "additionalProperties": true, + "title": "Public Config", + "type": "object" + }, + "state": { + "title": "State", + "type": "string" + } + }, + "required": [ + "id", + "name", + "kind", + "state", + "public_config" + ], + "title": "SourceSummary", + "type": "object" + }, + "TokenInput": { + "properties": { + "expires_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "scopes": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Scopes", + "type": "array" + } + }, + "required": [ + "scopes" + ], + "title": "TokenInput", + "type": "object" + }, + "UserPatch": { + "properties": { + "state": { + "title": "State", + "type": "string" + } + }, + "required": [ + "state" + ], + "title": "UserPatch", + "type": "object" + }, + "ValidationError": { + "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "info": { + "title": "Backup Tool API", + "version": "2.0.0" + }, + "openapi": "3.1.0", + "paths": { + "/api/v2/admin/secrets": { + "get": { + "operationId": "list_secrets_api_v2_admin_secrets_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Response List Secrets Api V2 Admin Secrets Get", + "type": "array" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Secrets" + }, + "post": { + "operationId": "create_secret_api_v2_admin_secrets_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SecretInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Secret Api V2 Admin Secrets Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Secret" + } + }, + "/api/v2/admin/users/{user_id}": { + "get": { + "operationId": "get_user_api_v2_admin_users__user_id__get", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Get User Api V2 Admin Users User Id Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get User" + }, + "patch": { + "operationId": "patch_user_api_v2_admin_users__user_id__patch", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User Id", + "type": "string" + } + }, + { + "in": "header", + "name": "If-Match", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "If-Match" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserPatch" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Patch User Api V2 Admin Users User Id Patch", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Patch User" + } + }, + "/api/v2/audit": { + "get": { + "operationId": "list_audit_api_v2_audit_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "query", + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cursor" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AuditList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Audit" + } + }, + "/api/v2/auth/login": { + "post": { + "operationId": "login_api_v2_auth_login_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LoginInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AuthenticatedUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Login" + } + }, + "/api/v2/auth/logout": { + "post": { + "operationId": "logout_api_v2_auth_logout_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Logout" + } + }, + "/api/v2/auth/session": { + "get": { + "operationId": "get_session_api_v2_auth_session_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Session" + } + }, + "/api/v2/auth/tokens": { + "post": { + "operationId": "create_token_api_v2_auth_tokens_post", + "parameters": [ + { + "in": "header", + "name": "Idempotency-Key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Idempotency-Key" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TokenInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Token" + } + }, + "/api/v2/auth/tokens/{token_id}": { + "delete": { + "operationId": "revoke_token_api_v2_auth_tokens__token_id__delete", + "parameters": [ + { + "in": "path", + "name": "token_id", + "required": true, + "schema": { + "title": "Token Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Revoke Token" + } + }, + "/api/v2/backups": { + "get": { + "operationId": "list_backups_api_v2_backups_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackupList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Backups" + } + }, + "/api/v2/backups/{backup_id}": { + "get": { + "operationId": "get_backup_api_v2_backups__backup_id__get", + "parameters": [ + { + "in": "path", + "name": "backup_id", + "required": true, + "schema": { + "title": "Backup Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackupSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Backup" + } + }, + "/api/v2/backups/{backup_id}/delete-preview": { + "get": { + "operationId": "backup_delete_preview_api_v2_backups__backup_id__delete_preview_get", + "parameters": [ + { + "in": "path", + "name": "backup_id", + "required": true, + "schema": { + "title": "Backup Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackupDeletePreview" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Backup Delete Preview" + } + }, + "/api/v2/backups/{backup_id}/restores": { + "post": { + "operationId": "create_restore_api_v2_backups__backup_id__restores_post", + "parameters": [ + { + "in": "path", + "name": "backup_id", + "required": true, + "schema": { + "title": "Backup Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RestoreInput" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Restore Api V2 Backups Backup Id Restores Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Restore" + } + }, + "/api/v2/backups/{backup_id}/verify": { + "post": { + "operationId": "verify_backup_api_v2_backups__backup_id__verify_post", + "parameters": [ + { + "in": "path", + "name": "backup_id", + "required": true, + "schema": { + "title": "Backup Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackupSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Verify Backup" + } + }, + "/api/v2/executions": { + "get": { + "operationId": "list_executions_api_v2_executions_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecutionList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Executions" + } + }, + "/api/v2/executions/{execution_id}": { + "get": { + "operationId": "get_execution_api_v2_executions__execution_id__get", + "parameters": [ + { + "in": "path", + "name": "execution_id", + "required": true, + "schema": { + "title": "Execution Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecutionSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Execution" + } + }, + "/api/v2/executions/{execution_id}/cancel": { + "post": { + "operationId": "cancel_execution_api_v2_executions__execution_id__cancel_post", + "parameters": [ + { + "in": "path", + "name": "execution_id", + "required": true, + "schema": { + "title": "Execution Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Cancel Execution Api V2 Executions Execution Id Cancel Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Cancel Execution" + } + }, + "/api/v2/executions/{execution_id}/events": { + "get": { + "operationId": "execution_events_api_v2_executions__execution_id__events_get", + "parameters": [ + { + "in": "path", + "name": "execution_id", + "required": true, + "schema": { + "title": "Execution Id", + "type": "string" + } + }, + { + "in": "header", + "name": "Last-Event-ID", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last-Event-Id" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + }, + "text/event-stream": {} + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Execution Events" + } + }, + "/api/v2/executions/{execution_id}/retry": { + "post": { + "operationId": "retry_execution_api_v2_executions__execution_id__retry_post", + "parameters": [ + { + "in": "path", + "name": "execution_id", + "required": true, + "schema": { + "title": "Execution Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Retry Execution Api V2 Executions Execution Id Retry Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Retry Execution" + } + }, + "/api/v2/jobs": { + "get": { + "operationId": "list_jobs_api_v2_jobs_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Jobs" + }, + "post": { + "operationId": "create_job_api_v2_jobs_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Job Api V2 Jobs Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Job" + } + }, + "/api/v2/jobs/{job_id}/executions": { + "post": { + "operationId": "enqueue_execution_api_v2_jobs__job_id__executions_post", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "title": "Job Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Enqueue Execution Api V2 Jobs Job Id Executions Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Enqueue Execution" + } + }, + "/api/v2/jobs/{job_id}/schedule": { + "delete": { + "operationId": "delete_schedule_api_v2_jobs__job_id__schedule_delete", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "title": "Job Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Delete Schedule" + }, + "get": { + "operationId": "get_schedule_api_v2_jobs__job_id__schedule_get", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "title": "Job Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Get Schedule Api V2 Jobs Job Id Schedule Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Schedule" + }, + "patch": { + "operationId": "patch_schedule_api_v2_jobs__job_id__schedule_patch", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "title": "Job Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScheduleInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Patch Schedule Api V2 Jobs Job Id Schedule Patch", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Patch Schedule" + }, + "post": { + "operationId": "create_schedule_api_v2_jobs__job_id__schedule_post", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "title": "Job Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScheduleInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Schedule Api V2 Jobs Job Id Schedule Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Schedule" + } + }, + "/api/v2/notifications/deliveries": { + "get": { + "operationId": "list_notification_deliveries_api_v2_notifications_deliveries_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotificationDeliveryList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Notification Deliveries" + } + }, + "/api/v2/notifications/deliveries/{delivery_id}/attempts": { + "get": { + "operationId": "list_notification_attempts_api_v2_notifications_deliveries__delivery_id__attempts_get", + "parameters": [ + { + "in": "path", + "name": "delivery_id", + "required": true, + "schema": { + "title": "Delivery Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotificationAttemptList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Notification Attempts" + } + }, + "/api/v2/notifications/deliveries/{delivery_id}/retry": { + "post": { + "operationId": "retry_notification_delivery_api_v2_notifications_deliveries__delivery_id__retry_post", + "parameters": [ + { + "in": "path", + "name": "delivery_id", + "required": true, + "schema": { + "title": "Delivery Id", + "type": "string" + } + }, + { + "in": "header", + "name": "Idempotency-Key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Idempotency-Key" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Retry Notification Delivery Api V2 Notifications Deliveries Delivery Id Retry Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Retry Notification Delivery" + } + }, + "/api/v2/notifications/email-settings": { + "get": { + "operationId": "get_notification_email_settings_api_v2_notifications_email_settings_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Get Notification Email Settings Api V2 Notifications Email Settings Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Notification Email Settings" + }, + "put": { + "operationId": "put_notification_email_settings_api_v2_notifications_email_settings_put", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmailSettingsInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Put Notification Email Settings Api V2 Notifications Email Settings Put", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Put Notification Email Settings" + } + }, + "/api/v2/notifications/event-catalog": { + "get": { + "operationId": "notification_catalog_api_v2_notifications_event_catalog_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Notification Catalog Api V2 Notifications Event Catalog Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Notification Catalog" + } + }, + "/api/v2/notifications/subscriptions": { + "get": { + "operationId": "list_notification_subscriptions_api_v2_notifications_subscriptions_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotificationSubscriptionList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Notification Subscriptions" + }, + "post": { + "operationId": "create_notification_subscription_api_v2_notifications_subscriptions_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotificationSubscriptionInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Notification Subscription Api V2 Notifications Subscriptions Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Notification Subscription" + } + }, + "/api/v2/notifications/subscriptions/{subscription_id}": { + "get": { + "operationId": "get_notification_subscription_api_v2_notifications_subscriptions__subscription_id__get", + "parameters": [ + { + "in": "path", + "name": "subscription_id", + "required": true, + "schema": { + "title": "Subscription Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Get Notification Subscription Api V2 Notifications Subscriptions Subscription Id Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Notification Subscription" + }, + "patch": { + "operationId": "patch_notification_subscription_api_v2_notifications_subscriptions__subscription_id__patch", + "parameters": [ + { + "in": "path", + "name": "subscription_id", + "required": true, + "schema": { + "title": "Subscription Id", + "type": "string" + } + }, + { + "in": "header", + "name": "If-Match", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "If-Match" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotificationSubscriptionPatch" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Patch Notification Subscription Api V2 Notifications Subscriptions Subscription Id Patch", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Patch Notification Subscription" + } + }, + "/api/v2/notifications/subscriptions/{subscription_id}/signing-keys/rotate": { + "post": { + "operationId": "rotate_notification_signing_key_api_v2_notifications_subscriptions__subscription_id__signing_keys_rotate_post", + "parameters": [ + { + "in": "path", + "name": "subscription_id", + "required": true, + "schema": { + "title": "Subscription Id", + "type": "string" + } + }, + { + "in": "header", + "name": "Idempotency-Key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Idempotency-Key" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SigningKeyRotateInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Rotate Notification Signing Key Api V2 Notifications Subscriptions Subscription Id Signing Keys Rotate Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Rotate Notification Signing Key" + } + }, + "/api/v2/notifications/subscriptions/{subscription_id}/test": { + "post": { + "operationId": "test_notification_subscription_api_v2_notifications_subscriptions__subscription_id__test_post", + "parameters": [ + { + "in": "path", + "name": "subscription_id", + "required": true, + "schema": { + "title": "Subscription Id", + "type": "string" + } + }, + { + "in": "header", + "name": "Idempotency-Key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Idempotency-Key" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Test Notification Subscription Api V2 Notifications Subscriptions Subscription Id Test Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Test Notification Subscription" + } + }, + "/api/v2/repositories": { + "get": { + "operationId": "list_repositories_api_v2_repositories_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Repositories" + }, + "post": { + "operationId": "create_repository_api_v2_repositories_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Repository Api V2 Repositories Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Repository" + } + }, + "/api/v2/repositories/{repository_id}": { + "get": { + "operationId": "get_repository_api_v2_repositories__repository_id__get", + "parameters": [ + { + "in": "path", + "name": "repository_id", + "required": true, + "schema": { + "title": "Repository Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Get Repository Api V2 Repositories Repository Id Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Repository" + }, + "patch": { + "operationId": "patch_repository_api_v2_repositories__repository_id__patch", + "parameters": [ + { + "in": "path", + "name": "repository_id", + "required": true, + "schema": { + "title": "Repository Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryPatch" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Patch Repository" + } + }, + "/api/v2/repositories/{repository_id}/inspection": { + "get": { + "operationId": "inspect_repository_endpoint_api_v2_repositories__repository_id__inspection_get", + "parameters": [ + { + "in": "path", + "name": "repository_id", + "required": true, + "schema": { + "title": "Repository Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Inspect Repository Endpoint Api V2 Repositories Repository Id Inspection Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Inspect Repository Endpoint" + } + }, + "/api/v2/restores/{restore_id}": { + "get": { + "operationId": "get_restore_api_v2_restores__restore_id__get", + "parameters": [ + { + "in": "path", + "name": "restore_id", + "required": true, + "schema": { + "title": "Restore Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Get Restore Api V2 Restores Restore Id Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Restore" + } + }, + "/api/v2/security/recovery/status": { + "get": { + "operationId": "recovery_status_api_v2_security_recovery_status_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RecoveryStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Recovery Status" + } + }, + "/api/v2/setup": { + "post": { + "operationId": "setup_api_v2_setup_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupInput" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AuthenticatedUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Setup" + } + }, + "/api/v2/sources": { + "get": { + "operationId": "list_sources_api_v2_sources_get", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SourceList" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "List Sources" + }, + "post": { + "operationId": "create_source_api_v2_sources_post", + "parameters": [ + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "discriminator": { + "mapping": { + "local": "#/components/schemas/LocalSourceInput", + "ssh": "#/components/schemas/SSHSourceInput" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/LocalSourceInput" + }, + { + "$ref": "#/components/schemas/SSHSourceInput" + } + ], + "title": "Input " + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Create Source Api V2 Sources Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create Source" + } + }, + "/api/v2/sources/{source_id}": { + "delete": { + "operationId": "archive_source_api_v2_sources__source_id__delete", + "parameters": [ + { + "in": "path", + "name": "source_id", + "required": true, + "schema": { + "title": "Source Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Archive Source" + } + }, + "/api/v2/sources/{source_id}/probe": { + "post": { + "operationId": "probe_source_api_v2_sources__source_id__probe_post", + "parameters": [ + { + "in": "path", + "name": "source_id", + "required": true, + "schema": { + "title": "Source Id", + "type": "string" + } + }, + { + "in": "header", + "name": "authorization", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + }, + { + "in": "header", + "name": "X-CSRF-Token", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Csrf-Token" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Probe Source Api V2 Sources Source Id Probe Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Probe Source" + } + }, + "/livez": { + "get": { + "operationId": "livez_livez_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Livez Livez Get", + "type": "object" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Livez" + } + }, + "/readyz": { + "get": { + "operationId": "readyz_readyz_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Response Readyz Readyz Get", + "type": "object" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Readyz" + } + } + } +} diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..b4feb43 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,3 @@ +{ + "extraPaths": ["backend/src"] +} diff --git a/tests/compose.ssh.yaml b/tests/compose.ssh.yaml new file mode 100644 index 0000000..6ff80bd --- /dev/null +++ b/tests/compose.ssh.yaml @@ -0,0 +1,32 @@ +name: backup-tool-ssh-test + +services: + sshd: + build: + context: ./ssh-fixture + dockerfile: Dockerfile + read_only: true + tmpfs: + - /run/sshd:uid=0,gid=0,mode=0755,size=8m + - /tmp:mode=1777,size=8m + cap_drop: + - ALL + # sshd needs only these capabilities to chroot then drop to the SFTP account. + cap_add: + - SYS_CHROOT + - SETUID + - SETGID + - KILL + security_opt: + - no-new-privileges:true + ports: + - "127.0.0.1:${SSH_FIXTURE_PORT}:2222" + volumes: + - type: bind + source: ${SSH_FIXTURE_DIR} + target: /fixture + read_only: true + - type: bind + source: ${SSH_FIXTURE_DIR}/source + target: /home/backup/data + read_only: true diff --git a/tests/contract/test_api_conventions.py b/tests/contract/test_api_conventions.py index a40a4e8..d357514 100644 --- a/tests/contract/test_api_conventions.py +++ b/tests/contract/test_api_conventions.py @@ -35,16 +35,17 @@ async def test_readyz_rejects_unmigrated_database(tmp_path) -> None: response = await client.get("/readyz") await app.state.engine.dispose() assert response.status_code == 503 - assert response.json()["code"] == "schema_not_current" + assert response.json()["code"] == "dependency_unavailable" @pytest.mark.asyncio -async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None: +async def test_readyz_reports_runtime_dependencies_before_and_after_setup( + app_client, +) -> None: client, _ = app_client before = await client.get("/readyz") - assert before.status_code == 503 - assert before.headers["content-type"].startswith("application/problem+json") - assert before.json()["code"] == "setup_required" + assert before.status_code == 200 + assert before.json()["status"] == "ready" assert (await setup_admin(client)).status_code == 201 after = await client.get("/readyz") @@ -52,6 +53,22 @@ async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None: assert after.json()["status"] == "ready" +@pytest.mark.asyncio +async def test_metrics_are_prometheus_text_without_sensitive_request_data( + app_client, +) -> None: + client, _ = app_client + assert (await client.get("/livez")).status_code == 200 + + response = await client.get("/metrics") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/plain; version=0.0.4") + assert "backup_tool_http_requests_total" in response.text + assert "backup_tool_active_executions" in response.text + assert "backup_tool_filesystem_free_bytes" in response.text + + @pytest.mark.asyncio async def test_protected_endpoint_uses_rfc9457_problem(app_client) -> None: client, _ = app_client diff --git a/tests/contract/test_notification_contract.py b/tests/contract/test_notification_contract.py new file mode 100644 index 0000000..a24f4d8 --- /dev/null +++ b/tests/contract/test_notification_contract.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import pytest +from backup_tool.db.models import IdempotencyRecord +from sqlalchemy import select + +PASSWORD = "correct horse battery staple" + + +@pytest.mark.asyncio +async def test_catalog_subscription_and_write_only_webhook_secret(app_client) -> None: + client, _ = app_client + setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert setup.status_code == 201 + csrf = client.cookies["backup_tool_csrf"] + catalog = await client.get("/api/v2/notifications/event-catalog") + assert catalog.status_code == 200 + assert catalog.json()["event_schema_version"] == 1 + assert "execution.queued" in catalog.json()["events"] + created = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "webhook", + "event_filters": ["execution.*"], + "destination": {"url": "https://hooks.example.test/backup"}, + "signing_secret": "not-returned-webhook-secret", + }, + headers={"X-CSRF-Token": csrf}, + ) + assert created.status_code == 201 + assert "signing_secret" not in created.text + assert "not-returned-webhook-secret" not in created.text + assert created.headers["ETag"] + + +@pytest.mark.asyncio +async def test_rotation_idempotency_never_persists_secret_verifier(app_client) -> None: + client, _ = app_client + setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert setup.status_code == 201 + csrf = client.cookies["backup_tool_csrf"] + created = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "webhook", + "event_filters": ["execution.*"], + "destination": {"url": "https://hooks.example.test/backup"}, + "signing_secret": "first-signing-secret", + }, + headers={"X-CSRF-Token": csrf}, + ) + assert created.status_code == 201 + route = f"/api/v2/notifications/subscriptions/{created.json()['id']}/signing-keys/rotate" + first = await client.post( + route, + json={"secret": "rotation-secret-one", "overlap_seconds": 60}, + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"}, + ) + replay = await client.post( + route, + json={"secret": "rotation-secret-two", "overlap_seconds": 60}, + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"}, + ) + assert first.status_code == replay.status_code == 200 + assert first.json() == replay.json() + app = client._transport.app + async with app.state.sessions() as db: + record = await db.scalar( + select(IdempotencyRecord).where( + IdempotencyRecord.operation == "rotate_notification_signing_key" + ) + ) + assert record is not None + assert "rotation-secret" not in record.request_digest + + +@pytest.mark.asyncio +async def test_notification_rejects_empty_or_unknown_filters(app_client) -> None: + client, _ = app_client + setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert setup.status_code == 201 + response = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["unknown.event"], + "destination": {"recipients": ["operator@example.test"]}, + }, + headers={"X-CSRF-Token": client.cookies["backup_tool_csrf"]}, + ) + assert response.status_code == 422 + assert response.json()["code"] == "validation_failed" diff --git a/tests/contract/test_repository_format.py b/tests/contract/test_repository_format.py index ca09a62..0576110 100644 --- a/tests/contract/test_repository_format.py +++ b/tests/contract/test_repository_format.py @@ -144,6 +144,14 @@ def test_state_errors_capabilities_and_fault_points_are_frozen() -> None: capabilities = load(CONTRACT / "capabilities-v2.0.json") assert capabilities["sources"] == ["local", "ssh"] + + manifest_schema = load(CONTRACT / "manifest.schema.json") + adapter_kinds = manifest_schema["properties"]["source_consistency"]["properties"]["adapter"][ + "enum" + ] + assert adapter_kinds == ["local", "postgresql", "mysql"] + assert "ssh" not in adapter_kinds + assert not capabilities["features"]["tar_download"] assert not capabilities["features"]["postgresql"] assert not capabilities["features"]["mysql"] diff --git a/tests/e2e/test_compose_v2.py b/tests/e2e/test_compose_v2.py new file mode 100644 index 0000000..2629a5a --- /dev/null +++ b/tests/e2e/test_compose_v2.py @@ -0,0 +1,183 @@ +"""Opt-in production-like Compose checks; no fixture secret is written to the repository.""" + +from __future__ import annotations + +import os +import socket +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def _enabled() -> bool: + return os.environ.get("BACKUP_TOOL_COMPOSE_E2E") == "1" + + +def _free_port() -> int: + try: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + address = listener.getsockname() + except OSError as error: + raise RuntimeError("could not allocate a Compose test port") from error + if not isinstance(address, tuple) or not isinstance(address[1], int): + raise RuntimeError("could not allocate a Compose test port") + return address[1] + + +def _compose(environment: dict[str, str], *arguments: str) -> subprocess.CompletedProcess[str]: + command = [ + "docker", + "compose", + "-f", + "docker-compose.yml", + "-f", + environment["COMPOSE_FILE"], + *arguments, + ] + return subprocess.run( + command, + cwd=ROOT, + env=environment, + check=True, + text=True, + capture_output=True, + timeout=120, + ) + + +def _eventually_get(url: str, expected_status: int) -> str: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=3) as response: + if response.status == expected_status: + return response.read().decode() + except urllib.error.HTTPError as error: + if error.code == expected_status: + return error.read().decode() + except OSError: + pass + time.sleep(1) + raise AssertionError(f"{url} did not return {expected_status}") + + +@pytest.mark.skipif( + not _enabled(), reason="set BACKUP_TOOL_COMPOSE_E2E=1 to run Docker Compose E2E" +) +def test_compose_persists_metadata_and_stops_workers_safely(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir(mode=0o755) + fixture_key = tmp_path / "unused-bind-master.key" + fixture_key.write_bytes(b"compose-test-host-fixture-key-material-32-bytes") + fixture_key.chmod(0o600) + port = _free_port() + project = f"backup-tool-e2e-{os.getpid()}" + override = tmp_path / "compose-e2e.yaml" + override.write_text( + "services:\n" + " migrate:\n" + " environment: &e2e-env\n" + " BACKUP_TOOL_MASTER_KEY_FILE: /var/lib/backup-tool/master.key\n" + f" BACKUP_TOOL_PUBLIC_BASE_URL: http://localhost:{port}\n" + " web:\n" + " environment: *e2e-env\n" + " scheduler:\n" + " environment: *e2e-env\n" + " worker:\n" + " environment: *e2e-env\n" + " admin:\n" + " environment: *e2e-env\n" + ) + environment = os.environ | { + "BACKUP_TOOL_COMPOSE_E2E": "1", + "BACKUP_TOOL_MASTER_KEY_FILE": str(fixture_key), + "BACKUP_TOOL_PORT": str(port), + "BACKUP_TOOL_SOURCES_DIR": str(source), + "COMPOSE_FILE": str(override), + "COMPOSE_PROJECT_NAME": project, + } + try: + # The service user creates the actual key inside its private named volume; + # the host key exists only to satisfy the unused read-only Compose bind. + _compose( + environment, + "run", + "--rm", + "--no-deps", + "--entrypoint", + "/bin/sh", + "migrate", + "-c", + ( + "umask 077; dd if=/dev/urandom of=/var/lib/backup-tool/master.key " + "bs=32 count=1 status=none" + ), + ) + _compose(environment, "run", "--rm", "migrate") + _compose(environment, "up", "-d") + assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200) + setup = urllib.request.Request( + f"http://127.0.0.1:{port}/api/v2/setup", + data=b'{"username":"operator","password":"correct horse battery staple"}', + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(setup, timeout=10) as response: + assert response.status == 201 + assert "backup_tool_active_executions" in _eventually_get( + f"http://127.0.0.1:{port}/metrics", 200 + ) + + _compose(environment, "stop", "--timeout", "15", "worker") + worker_id = _compose(environment, "ps", "-aq", "worker").stdout.strip() + assert worker_id + stopped = subprocess.run( + ["docker", "inspect", "--format", "{{.State.ExitCode}}", worker_id], + check=True, + text=True, + capture_output=True, + timeout=30, + ) + assert stopped.stdout.strip() == "0" + _compose(environment, "up", "-d", "worker") + _eventually_get(f"http://127.0.0.1:{port}/readyz", 200) + + _compose(environment, "restart", "web", "scheduler", "worker", "proxy") + assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200) + repeat_setup = urllib.request.Request( + f"http://127.0.0.1:{port}/api/v2/setup", + data=b'{"username":"operator","password":"correct horse battery staple"}', + headers={"Content-Type": "application/json"}, + method="POST", + ) + with pytest.raises(urllib.error.HTTPError) as repeated: + urllib.request.urlopen(repeat_setup, timeout=10) + assert repeated.value.code == 409 + finally: + command = [ + "docker", + "compose", + "-f", + "docker-compose.yml", + "-f", + str(override), + "down", + "--volumes", + "--remove-orphans", + ] + subprocess.run( + command, + cwd=ROOT, + env=environment, + check=False, + text=True, + capture_output=True, + timeout=120, + ) diff --git a/tests/fault/test_deletion_failure.py b/tests/fault/test_deletion_failure.py new file mode 100644 index 0000000..c15c049 --- /dev/null +++ b/tests/fault/test_deletion_failure.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from unittest.mock import patch + +import pytest +from backup_tool.gc import purge_repository + + +def test_gc_keeps_manifest_when_unlink_fails(tmp_path: Path) -> None: + root = tmp_path / "repository" + manifest = root / "manifests" / "deleted.json" + manifest.parent.mkdir(parents=True) + manifest.write_text('{"entries": []}', encoding="utf-8") + + with ( + patch("pathlib.Path.unlink", side_effect=OSError("read-only")), + pytest.raises(OSError), + ): + purge_repository(root, {"deleted"}, grace=timedelta(0)) + + assert manifest.exists() diff --git a/tests/fault/test_notification_retries.py b/tests/fault/test_notification_retries.py new file mode 100644 index 0000000..24c0e5c --- /dev/null +++ b/tests/fault/test_notification_retries.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from backup_tool.db.models import ( + Execution, + Job, + NotificationDelivery, + NotificationDeliveryAttempt, + NotificationEmailSettings, + NotificationSubscription, + Repository, + Secret, + Source, +) +from backup_tool.ids import new_uuid7 +from backup_tool.notifications.dispatcher import ( + dispatch_one, + recover_notification_leases, +) +from backup_tool.notifications.email import EmailResult, EmailTransportError +from backup_tool.notifications.events import emit_event +from backup_tool.worker import Worker +from sqlalchemy import select + + +@pytest.mark.asyncio +async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None: + client, settings = app_client + app = client._transport.app + async with app.state.sessions() as db: + ciphertext, key_id = app.state.cipher.encrypt( + "smtp-password", purpose="notification_smtp", version=1 + ) + secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp") + db.add(secret) + await db.flush() + db.add( + NotificationEmailSettings( + id=1, + host="smtp.example.test", + port=587, + username="operator", + password_secret_id=secret.id, + sender="sender@example.test", + max_attempts=2, + rate_limit_per_minute=60, + ) + ) + subscription = NotificationSubscription( + channel="email", + event_filters=["execution.queued"], + destination_config={"recipients": ["operator@example.test"]}, + rate_limit_per_minute=60, + rate_tokens=60.0, + ) + db.add(subscription) + await db.flush() + event = await emit_event( + db, + "execution.queued", + correlation_id=str(new_uuid7()), + resource={}, + deduplication_key="retry-test", + ) + await db.commit() + + async def transient(*_args, **_kwargs): + raise EmailTransportError("smtp_421", transient=True) + + monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient) + assert await dispatch_one(db, settings, app.state.cipher, "worker-test") + delivery = await db.scalar( + select(NotificationDelivery).where(NotificationDelivery.event_id == event.id) + ) + assert delivery is not None + assert delivery.state == "retry" + assert delivery.attempt_count == 1 + delivery.state = "leased" + delivery.attempt_count = 2 + delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1) + db.add( + NotificationDeliveryAttempt( + delivery_id=delivery.id, + number=2, + started_at=datetime.now(UTC), + outcome="started", + ) + ) + await db.commit() + assert await recover_notification_leases(db) == 1 + await db.refresh(delivery) + abandoned = await db.scalar( + select(NotificationDeliveryAttempt).where( + NotificationDeliveryAttempt.delivery_id == delivery.id, + NotificationDeliveryAttempt.number == 2, + ) + ) + assert delivery.state == "retry" + assert abandoned is not None + assert abandoned.outcome == "retry" + assert abandoned.diagnostic == "abandoned_lease" + + async def succeeded(*_args, **_kwargs): + return EmailResult(response_class="smtp_2xx") + + monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded) + delivery.due_at = datetime.now(UTC) - timedelta(seconds=1) + await db.commit() + assert await dispatch_one(db, settings, app.state.cipher, "worker-test") + await db.refresh(delivery) + assert delivery.state == "delivered" + assert delivery.attempt_count == 3 + + max_event = await emit_event( + db, + "execution.queued", + correlation_id=str(new_uuid7()), + resource={}, + deduplication_key="smtp-max-attempts", + ) + await db.commit() + monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient) + assert await dispatch_one(db, settings, app.state.cipher, "worker-test") + max_delivery = await db.scalar( + select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id) + ) + assert max_delivery is not None and max_delivery.state == "retry" + max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1) + await db.commit() + assert await dispatch_one(db, settings, app.state.cipher, "worker-test") + await db.refresh(max_delivery) + assert max_delivery.state == "failed" + assert max_delivery.attempt_count == 2 + + permanent_event = await emit_event( + db, + "execution.queued", + correlation_id=str(new_uuid7()), + resource={}, + deduplication_key="smtp-permanent", + ) + await db.commit() + + async def permanent(*_args, **_kwargs): + raise EmailTransportError("smtp_550", transient=False) + + monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent) + assert await dispatch_one(db, settings, app.state.cipher, "worker-test") + permanent_delivery = await db.scalar( + select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id) + ) + assert permanent_delivery is not None + assert permanent_delivery.state == "failed" + assert permanent_delivery.attempt_count == 1 + + +@pytest.mark.asyncio +async def test_notification_dispatch_gets_a_turn_during_execution_backlog( + app_client, monkeypatch +) -> None: + client, settings = app_client + app = client._transport.app + async with app.state.sessions() as db: + repository = Repository( + name="fair-repository", + root="/fair-repository", + format_version=1, + compression="none", + encryption="none", + ) + source = Source( + name="fair-source", + kind="local", + public_config={"root": "/fair-source"}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + job = Job( + name="fair-job", + source_id=source.id, + repository_id=repository.id, + exclusions=[], + retention={}, + requested_mode="full", + ) + db.add(job) + await db.flush() + execution = Execution(job_id=job.id, trigger="manual", progress={}) + db.add(execution) + await db.commit() + execution_id = execution.id + + called = False + + async def dispatched(*_args, **_kwargs) -> bool: + nonlocal called + called = True + return True + + monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched) + worker = Worker(settings, owner="fair-worker") + worker._execution_turns = 1 + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + async with app.state.sessions() as db: + queued = await db.get(Execution, execution_id) + assert called + assert queued is not None and queued.state == "queued" diff --git a/tests/fault/test_publication_crashes.py b/tests/fault/test_publication_crashes.py new file mode 100644 index 0000000..c7315e8 --- /dev/null +++ b/tests/fault/test_publication_crashes.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import importlib +import os +import stat +from pathlib import Path + +import httpx +import pytest +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +config = importlib.import_module("backup_tool.config") +adapters = importlib.import_module("backup_tool.adapters") +faults = importlib.import_module("backup_tool.faults") +models = importlib.import_module("backup_tool.db.models") +snapshot = importlib.import_module("backup_tool.snapshot") +worker_module = importlib.import_module("backup_tool.worker") + +PASSWORD = "correct-horse-battery-staple" + + +def settings_for(tmp_path: Path): + key = tmp_path / "master.key" + key.write_bytes(b"m6-publication-fault-test-master-key-material") + key.chmod(0o600) + data = tmp_path / "data" + repositories = tmp_path / "repositories" + sources = tmp_path / "sources" + restores = tmp_path / "restores" + for directory in (data, repositories, sources, restores): + directory.mkdir() + return config.Settings( + data_dir=data, + database_url=f"sqlite+aiosqlite:///{data / 'metadata.db'}", + repository_roots=(repositories,), + local_source_roots=(sources,), + restore_roots=(restores,), + master_key_file=key, + min_free_bytes=1, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("point", ["blob.before_write", "blob.after_write", "blob.after_fsync"]) +async def test_blob_write_crash_points_leave_no_published_blob(tmp_path: Path, point: str) -> None: + settings = settings_for(tmp_path) + source_root = settings.local_source_roots[0] / "source" + source_root.mkdir() + (source_root / "data.txt").write_text("backup data", encoding="utf-8") + adapter = adapters.LocalAdapter(source_root, settings) + staged_blob = tmp_path / "staged.blob" + + with pytest.raises(faults.InjectedCrash): + await snapshot._copy_file(adapter, "data.txt", staged_blob, faults.CrashAt(point)) + + assert not (settings.repository_roots[0] / "blobs" / "sha256").exists() + + +@pytest.mark.asyncio +async def test_staging_is_owner_only_with_a_permissive_umask(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + source_root = settings.local_source_roots[0] / "source" + source_root.mkdir() + (source_root / "data.txt").write_text("plaintext", encoding="utf-8") + adapter = adapters.LocalAdapter(source_root, settings) + staging = tmp_path / "staging" + staged_blobs = staging / "blobs" + staged_blob = staged_blobs / "0.blob" + + old_umask = os.umask(0) + try: + snapshot._private_directory(staging) + snapshot._private_directory(staged_blobs) + with pytest.raises(faults.InjectedCrash): + await snapshot._copy_file( + adapter, + "data.txt", + staged_blob, + faults.CrashAt("blob.after_write"), + ) + finally: + os.umask(old_umask) + + assert stat.S_IMODE(staging.stat().st_mode) == 0o700 + assert stat.S_IMODE(staged_blobs.stat().st_mode) == 0o700 + assert stat.S_IMODE(staged_blob.stat().st_mode) == 0o600 + assert staged_blob.read_text(encoding="utf-8") == "plaintext" + + +def test_blob_install_crash_point_leaves_staged_blob_unpublished( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + staged_blob = tmp_path / "staged.blob" + staged_blob.write_bytes(b"backup data") + digest = "a" * 64 + target = settings.repository_roots[0] / "blobs" / "sha256" / digest + + with pytest.raises(faults.InjectedCrash): + snapshot._install_blob(staged_blob, target, digest, faults.CrashAt("blob.before_rename")) + + assert staged_blob.exists() + assert not target.exists() + + +async def _login(client: httpx.AsyncClient) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +@pytest.mark.asyncio +async def test_metadata_crash_is_reconciled_without_republishing( + app_client: tuple[httpx.AsyncClient, Settings], +) -> None: + client, settings = app_client + source_root = settings.local_source_roots[0] / "source" + source_root.mkdir() + (source_root / "data.txt").write_text("backup data", encoding="utf-8") + headers = await _login(client) + repository = await client.post( + "/api/v2/repositories", + json={ + "name": "repo", + "relative_path": "repo", + "compression": "none", + "encryption": "none", + }, + headers=headers, + ) + source = await client.post( + "/api/v2/sources", + json={ + "name": "source", + "kind": "local", + "public_config": {"root": str(source_root)}, + }, + headers=headers, + ) + job = await client.post( + "/api/v2/jobs", + json={ + "name": "job", + "source_id": source.json()["id"], + "repository_id": repository.json()["id"], + "requested_mode": "full", + "exclusions": [], + "retention": {}, + "enabled": True, + "allow_empty": False, + }, + headers=headers, + ) + execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers) + execution_id = execution.json()["id"] + + crashing_worker = worker_module.Worker( + settings, + owner="crashing-worker", + fault_injector=faults.CrashAt("metadata.before_commit"), + ) + try: + with pytest.raises(faults.InjectedCrash): + await crashing_worker.run_once() + finally: + await crashing_worker.engine.dispose() + + engine = create_engine(settings) + try: + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + assert ( + await db.scalar( + select(models.Backup).where(models.Backup.execution_id == execution_id) + ) + is None + ) + finally: + await engine.dispose() + + recovery_worker = worker_module.Worker(settings, owner="recovery-worker") + try: + assert await recovery_worker.startup() == 1 + finally: + await recovery_worker.engine.dispose() + + engine = create_engine(settings) + try: + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + recovered = await db.get(models.Execution, execution_id) + backups = list( + await db.scalars( + select(models.Backup).where(models.Backup.execution_id == execution_id) + ) + ) + finally: + await engine.dispose() + assert recovered is not None and recovered.state == "committed" + assert len(backups) == 1 diff --git a/tests/fault/test_schedule_delivery.py b/tests/fault/test_schedule_delivery.py new file mode 100644 index 0000000..0c2e20c --- /dev/null +++ b/tests/fault/test_schedule_delivery.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from backup_tool.scheduler import next_nominal + + +def test_misfire_window_is_deterministic_for_delivery() -> None: + nominal = datetime.now(UTC) - timedelta(seconds=901) + next_run = next_nominal("* * * * *", "UTC", nominal) + + assert next_run > nominal + assert (datetime.now(UTC) - nominal).total_seconds() > 900 + + +@pytest.mark.parametrize("cron", ["* * * * * *", "invalid"]) +def test_delivery_rejects_invalid_cron_before_enqueue(cron: str) -> None: + with pytest.raises(ValueError): + next_nominal(cron, "UTC") diff --git a/tests/fault/test_worker_loss.py b/tests/fault/test_worker_loss.py new file mode 100644 index 0000000..47c8566 --- /dev/null +++ b/tests/fault/test_worker_loss.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import asyncio +import importlib +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +import pytest_asyncio +from alembic import command +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Execution, ExecutionEvent, Job, Repository, Source +from backup_tool.execution import ( + claim, + complete_cancellation, + heartbeat, + record_event, + request_cancellation, +) +from backup_tool.worker import Worker +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +cli = importlib.import_module("backup_tool.cli") + + +@pytest_asyncio.fixture +async def database( + tmp_path: Path, +) -> AsyncIterator[tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine]]: + key = tmp_path / "master.key" + key.write_bytes(b"m5-fault-test-master-key-material-32-bytes-minimum") + key.chmod(0o600) + data_dir = tmp_path / "data" + repositories = tmp_path / "repositories" + sources = tmp_path / "sources" + restores = tmp_path / "restores" + for directory in (data_dir, repositories, sources, restores): + directory.mkdir() + settings = Settings( + data_dir=data_dir, + database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}", + repository_roots=(repositories,), + local_source_roots=(sources,), + restore_roots=(restores,), + master_key_file=key, + ) + command.upgrade(cli.build_alembic_config(settings), "head") + engine = create_engine(settings) + yield settings, async_sessionmaker(engine, expire_on_commit=False), engine + await engine.dispose() + + +async def create_stale_execution(db: AsyncSession, suffix: str, state: str) -> Execution: + repository = Repository( + name=f"repository-{suffix}", + root=f"/repositories/{suffix}", + format_version=1, + compression="none", + encryption="none", + ) + source = Source( + name=f"source-{suffix}", + kind="local", + public_config={"root": f"/sources/{suffix}"}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + job = Job( + name=f"job-{suffix}", + source_id=source.id, + repository_id=repository.id, + requested_mode="full", + exclusions=[], + retention={}, + ) + db.add(job) + await db.flush() + execution = Execution( + job_id=job.id, + trigger="manual", + state=state, + lease_owner="lost-worker", + lease_expires_at=datetime.now(UTC) - timedelta(seconds=1), + heartbeat_at=datetime.now(UTC) - timedelta(seconds=2), + progress={}, + reason_code="cancellation_requested" if state == "cancelling" else None, + ) + db.add(execution) + await db.flush() + await record_event(db, execution) + await db.commit() + return execution + + +@pytest.mark.asyncio +async def test_startup_recovers_each_stale_lease_once_and_fences_lost_owner( + database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine], +) -> None: + settings, sessions, _ = database + async with sessions() as db: + stale = { + state: await create_stale_execution(db, state, state) + for state in ("preparing", "running", "verifying", "cancelling") + } + + worker = Worker(settings, owner="recovery-worker") + try: + assert await worker.startup() == len(stale) + finally: + await worker.engine.dispose() + + async with sessions() as db: + recovered = { + state: await db.get(Execution, execution.id) for state, execution in stale.items() + } + for state in ("preparing", "running", "verifying"): + execution = recovered[state] + assert execution is not None + assert execution.state == "queued" + assert execution.reason_code == "worker_lost" + assert execution.lease_owner is None + assert execution.lease_expires_at is None + assert execution.heartbeat_at is None + cancelling = recovered["cancelling"] + assert cancelling is not None + assert cancelling.state == "cancelled" + assert cancelling.completed_at is not None + assert cancelling.reason_code == "cancellation_requested" + event_counts = { + execution.id: await db.scalar( + select(func.count()).where(ExecutionEvent.execution_id == execution.id) + ) + for execution in recovered.values() + if execution is not None + } + + assert event_counts == {execution.id: 2 for execution in stale.values()} + + async with sessions() as db: + execution = recovered["running"] + assert execution is not None + assert await claim(db, execution.id, "replacement-worker") is not None + assert not await heartbeat(db, execution.id, "lost-worker") + assert await request_cancellation(db, execution.id) is not None + assert not await complete_cancellation(db, execution.id, "lost-worker") + assert await complete_cancellation(db, execution.id, "replacement-worker") + + +@pytest.mark.asyncio +async def test_stopping_worker_does_not_claim_new_work( + database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine], +) -> None: + settings, _, _ = database + worker = Worker(settings, owner="stopping-worker") + try: + worker.stop() + assert not await worker.run_once() + finally: + await worker.engine.dispose() + + +@pytest.mark.asyncio +async def test_idle_worker_stops_promptly_and_disposes_its_engine( + database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine], + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings, _, _ = database + worker = Worker(settings, owner="idle-worker") + disposed = asyncio.Event() + dispose = AsyncEngine.dispose + + async def track_dispose(engine: AsyncEngine, *, close: bool = True) -> None: + disposed.set() + await dispose(engine, close=close) + + monkeypatch.setattr(AsyncEngine, "dispose", track_dispose) + task = asyncio.create_task(worker.run()) + await asyncio.sleep(0) + worker.stop() + await asyncio.wait_for(task, timeout=1) + assert disposed.is_set() diff --git a/tests/integration/test_all_operational_events_deliver.py b/tests/integration/test_all_operational_events_deliver.py new file mode 100644 index 0000000..ecc29d8 --- /dev/null +++ b/tests/integration/test_all_operational_events_deliver.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from backup_tool.db.models import ( + Backup, + Execution, + Job, + NotificationDelivery, + NotificationEvent, + NotificationSubscription, + Repository, + Schedule, + Source, +) +from backup_tool.execution import record_event +from backup_tool.notifications.events import EVENT_CATALOG +from backup_tool.scheduler import deliver_due +from backup_tool.worker import Worker +from sqlalchemy import select + + +@pytest.mark.asyncio +async def test_execution_catalog_events_are_produced_and_delivered(app_client) -> None: + """Exercise the execution producer, not emit_event(), for each live execution type.""" + client, _ = app_client + setup = await client.post( + "/api/v2/setup", + json={"username": "admin", "password": "correct horse battery staple"}, + ) + assert setup.status_code == 201 + app = client._transport.app + expected = { + "execution.queued", + "execution.started", + "execution.committed", + "execution.failed", + "execution.cancelled", + "execution.retry_queued", + "execution.worker_recovered", + } + assert expected <= set(EVENT_CATALOG) + deferred_prefixes = ["source.", "gc.", "reconciliation."] + assert not any(item.startswith(tuple(deferred_prefixes)) for item in EVENT_CATALOG) + async with app.state.sessions() as db: + repository = Repository( + name="events-repository", + root="/events-repository", + format_version=1, + compression="none", + encryption="none", + ) + source = Source( + name="events-source", + kind="local", + public_config={"root": "/events-source"}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + job = Job( + name="events-job", + source_id=source.id, + repository_id=repository.id, + exclusions=[], + retention={}, + requested_mode="full", + ) + subscription = NotificationSubscription( + channel="email", + event_filters=["execution.*"], + destination_config={"recipients": ["operator@example.test"]}, + rate_limit_per_minute=60, + rate_tokens=60.0, + ) + db.add_all([job, subscription]) + await db.flush() + cases = ( + ("queued", 1, None, "execution.queued"), + ("preparing", 1, None, "execution.started"), + ("committed", 1, None, "execution.committed"), + ("failed", 1, "transient_io", "execution.failed"), + ("cancelled", 1, "cancellation_requested", "execution.cancelled"), + ("queued", 2, None, "execution.retry_queued"), + ("queued", 1, "worker_lost", "execution.worker_recovered"), + ) + for state, attempt, reason, _event_type in cases: + execution = Execution( + job_id=job.id, + trigger="manual", + state=state, + attempt=attempt, + reason_code=reason, + progress={}, + ) + db.add(execution) + await db.flush() + await record_event(db, execution) + if state in {"queued", "preparing"}: + execution.state = "failed" + execution.reason_code = "test_cleanup" + await db.flush() + await db.commit() + event_statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected)) + event_types = set((await db.scalars(event_statement)).all()) + deliveries = await db.scalar( + select(NotificationDelivery.id) + .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id) + .where(NotificationEvent.type.in_(expected)) + .limit(1) + ) + assert event_types == expected + assert deliveries is not None + + +@pytest.mark.asyncio +async def test_schedule_catalog_events_are_produced_and_delivered(app_client) -> None: + client, _ = app_client + setup = await client.post( + "/api/v2/setup", + json={"username": "admin", "password": "correct horse battery staple"}, + ) + assert setup.status_code == 201 + csrf = client.cookies["backup_tool_csrf"] + app = client._transport.app + expected = { + "schedule.created", + "schedule.updated", + "schedule.deleted", + "schedule.enabled", + "schedule.disabled", + "schedule.occurrence_enqueued", + "schedule.occurrence_misfired", + "schedule.occurrence_blocked", + } + assert expected <= set(EVENT_CATALOG) + async with app.state.sessions() as db: + repository = Repository( + name="schedule-repository", + root="/schedule-repository", + format_version=1, + compression="none", + encryption="none", + ) + source = Source( + name="schedule-source", + kind="local", + public_config={"root": "/schedule-source"}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + jobs = [ + Job( + name=f"schedule-job-{number}", + source_id=source.id, + repository_id=repository.id, + exclusions=[], + retention={}, + requested_mode="full", + enabled=number not in {2, 3}, + ) + for number in range(1, 5) + ] + subscription = NotificationSubscription( + channel="email", + event_filters=["schedule.*"], + destination_config={"recipients": ["operator@example.test"]}, + rate_limit_per_minute=60, + rate_tokens=60.0, + ) + db.add_all([*jobs, subscription]) + await db.commit() + job_ids = [job.id for job in jobs] + + created = await client.post( + f"/api/v2/jobs/{job_ids[0]}/schedule", + json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": True}, + headers={"X-CSRF-Token": csrf}, + ) + assert created.status_code == 201 + disabled = await client.patch( + f"/api/v2/jobs/{job_ids[0]}/schedule", + json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": False}, + headers={"X-CSRF-Token": csrf}, + ) + assert disabled.status_code == 200 + enabled = await client.patch( + f"/api/v2/jobs/{job_ids[0]}/schedule", + json={"cron": "1 0 * * *", "timezone": "UTC", "enabled": True}, + headers={"X-CSRF-Token": csrf}, + ) + assert enabled.status_code == 200 + updated = await client.patch( + f"/api/v2/jobs/{job_ids[0]}/schedule", + json={"cron": "2 0 * * *", "timezone": "UTC", "enabled": True}, + headers={"X-CSRF-Token": csrf}, + ) + assert updated.status_code == 200 + assert ( + await client.delete(f"/api/v2/jobs/{job_ids[0]}/schedule", headers={"X-CSRF-Token": csrf}) + ).status_code == 204 + + async with app.state.sessions() as db: + now = datetime.now(UTC) + db.add_all( + [ + Schedule( + job_id=job_ids[1], + cron="* * * * *", + timezone="UTC", + misfire_grace_seconds=0, + enabled=True, + next_nominal_at=now - timedelta(hours=1), + ), + Schedule( + job_id=job_ids[2], + cron="* * * * *", + timezone="UTC", + misfire_grace_seconds=60, + enabled=True, + next_nominal_at=now, + ), + Schedule( + job_id=job_ids[3], + cron="* * * * *", + timezone="UTC", + misfire_grace_seconds=60, + enabled=True, + next_nominal_at=now, + ), + ] + ) + await db.commit() + assert await deliver_due(db, now=now) == 1 + statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected)) + event_types = set((await db.scalars(statement)).all()) + delivery = await db.scalar( + select(NotificationDelivery.id) + .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id) + .where(NotificationEvent.type.in_(expected)) + .limit(1) + ) + assert event_types == expected + assert delivery is not None + + +@pytest.mark.asyncio +async def test_backup_restore_and_retention_events_are_produced_and_delivered( + app_client, +) -> None: + client, settings = app_client + source_root = settings.local_source_roots[0] / "notification-project" + source_root.mkdir() + (source_root / "data.txt").write_text("notification data\n", encoding="utf-8") + setup = await client.post( + "/api/v2/setup", + json={"username": "admin", "password": "correct horse battery staple"}, + ) + assert setup.status_code == 201 + headers = {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + subscription = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["backup.*", "restore.*", "retention.*"], + "destination": {"recipients": ["operator@example.test"]}, + }, + headers=headers, + ) + assert subscription.status_code == 201 + repository = await client.post( + "/api/v2/repositories", + json={"name": "notification-repo", "relative_path": "notification-repo"}, + headers=headers, + ) + source = await client.post( + "/api/v2/sources", + json={ + "name": "notification-source", + "kind": "local", + "public_config": {"root": str(source_root)}, + }, + headers=headers, + ) + assert repository.status_code == source.status_code == 201 + job = await client.post( + "/api/v2/jobs", + json={ + "name": "notification-job", + "source_id": source.json()["id"], + "repository_id": repository.json()["id"], + "requested_mode": "full", + "exclusions": [], + "retention": {"keep_last": 1}, + "allow_empty": False, + }, + headers=headers, + ) + assert job.status_code == 201 + execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers) + assert execution.status_code == 202 + worker = Worker(settings, owner="notification-backup-worker") + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + app = client._transport.app + async with app.state.sessions() as db: + backup = await db.scalar( + select(Backup).where(Backup.execution_id == execution.json()["id"]) + ) + assert backup is not None + # A synthetic older catalog entry is valid business state; tombstoning is + # performed only through the real retention producer below. + older_execution = Execution( + job_id=job.json()["id"], + trigger="manual", + state="committed", + progress={}, + ) + db.add(older_execution) + await db.flush() + older = Backup( + execution_id=older_execution.id, + manifest_id="00000000-0000-7000-8000-000000000001", + manifest_digest="0" * 64, + logical_bytes=0, + stored_bytes=0, + integrity="verified", + created_at=datetime.now(UTC) - timedelta(days=1), + ) + db.add(older) + await db.commit() + + retention_worker = Worker(settings, owner="notification-retention-worker") + try: + # Retention/GC is executed by worker maintenance, not a direct helper call. + assert await retention_worker.run_once() + finally: + await retention_worker.engine.dispose() + + restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(settings.restore_roots[0] / "notification-restore"), + "selection": [], + "dry_run": True, + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert restore.status_code == 202 + restore_worker = Worker(settings, owner="notification-restore-worker") + try: + assert await restore_worker.run_once() + finally: + await restore_worker.engine.dispose() + + expected = { + "backup.committed", + "backup.verification_succeeded", + "restore.queued", + "restore.committed", + "retention.tombstoned", + } + async with app.state.sessions() as db: + types = set( + ( + await db.scalars( + select(NotificationEvent.type).where(NotificationEvent.type.in_(expected)) + ) + ).all() + ) + deliveries = list( + ( + await db.scalars( + select(NotificationDelivery.id) + .join( + NotificationEvent, + NotificationDelivery.event_id == NotificationEvent.id, + ) + .where(NotificationEvent.type.in_(expected)) + ) + ).all() + ) + assert types == expected + assert len(deliveries) >= len(expected) diff --git a/tests/integration/test_empty_source.py b/tests/integration/test_empty_source.py new file mode 100644 index 0000000..d6c5667 --- /dev/null +++ b/tests/integration/test_empty_source.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import importlib + +import pytest + +snapshot = importlib.import_module("backup_tool.snapshot") + + +def test_empty_source_requires_explicit_opt_in() -> None: + with pytest.raises(snapshot.SnapshotError, match="source_empty"): + snapshot.require_nonempty([], False) + snapshot.require_nonempty([], True) diff --git a/tests/integration/test_encrypted_repository.py b/tests/integration/test_encrypted_repository.py new file mode 100644 index 0000000..a2404af --- /dev/null +++ b/tests/integration/test_encrypted_repository.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import asyncio +import json +import stat +from pathlib import Path + +import httpx +import pytest +from alembic import command +from backup_tool.cli import ( + build_alembic_config, + recovery_export_payload, + rotate_repository_key, +) +from backup_tool.cli import main as cli_main +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Backup, Execution, Repository, RepositoryDataKeyEpoch +from backup_tool.repository import ( + begin_key_rotation, + initialize, + inspect_repository, + replace_active_data_key, +) +from backup_tool.security.repository_crypto import create_data_key +from backup_tool.worker import Worker +from sqlalchemy import select + +from .test_repository_safety import settings_for + +PASSWORD = "correct-horse-battery-staple" + + +async def login(client: httpx.AsyncClient) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +def test_encrypted_initialization_creates_private_data_key(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + assert initialized.data_key_path is not None + assert stat.S_IMODE(initialized.data_key_path.stat().st_mode) == 0o600 + try: + payload = json.loads((initialized.root / "repository.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AssertionError("encrypted repository metadata is unreadable") from error + assert payload["encryption"] == { + "mode": "aes-256-gcm", + "key_id": initialized.data_key_id, + } + inspected = inspect_repository(settings, initialized.root) + assert inspected.encryption == "aes-256-gcm" + assert inspected.data_key_id == initialized.data_key_id + + +def test_rotation_rejects_metadata_database_epoch_mismatch(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + command.upgrade(build_alembic_config(settings), "head") + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + + async def create_repository() -> str: + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + db.add(repository) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + await db.commit() + return repository.id + finally: + await engine.dispose() + + repository_id = asyncio.run(create_repository()) + replacement_id, replacement_path = create_data_key(settings, initialized.repository_id) + try: + replace_active_data_key(initialized.root, initialized.data_key_id, replacement_id) + with pytest.raises(ValueError, match="repository encryption metadata is invalid"): + asyncio.run(rotate_repository_key(settings, repository_id)) + finally: + replacement_path.unlink(missing_ok=True) + + +def test_rotation_reconciliation_clears_stale_rollback_journal_after_key_removal( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + command.upgrade(build_alembic_config(settings), "head") + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + + async def create_repository() -> str: + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + db.add(repository) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + await db.commit() + return repository.id + finally: + await engine.dispose() + + repository_id = asyncio.run(create_repository()) + new_key_id, new_key_path = create_data_key(settings, initialized.repository_id) + begin_key_rotation( + initialized.root, + repository_id, + initialized.repository_id, + initialized.data_key_id, + new_key_id, + ) + new_key_path.unlink() + assert (initialized.root / ".key-rotation.json").is_file() + + async def reconcile_and_assert() -> None: + worker = Worker(settings, owner="stale-rollback-journal-worker") + try: + assert await worker.startup() == 1 + finally: + await worker.engine.dispose() + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = await db.get(Repository, repository_id) + assert repository is not None + assert repository.active_data_key_id == initialized.data_key_id + inspected = inspect_repository(settings, initialized.root) + assert inspected.data_key_id == initialized.data_key_id + finally: + await engine.dispose() + + asyncio.run(reconcile_and_assert()) + assert not (initialized.root / ".key-rotation.json").exists() + + +def test_rotation_crash_after_db_commit_recovers_on_worker_startup( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + command.upgrade(build_alembic_config(settings), "head") + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + + async def create_repository() -> str: + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + db.add(repository) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + await db.commit() + return repository.id + finally: + await engine.dispose() + + repository_id = asyncio.run(create_repository()) + + def interrupted_after_database_commit() -> None: + raise OSError("simulated process loss after database commit") + + with pytest.raises(OSError, match="simulated process loss"): + asyncio.run( + rotate_repository_key( + settings, + repository_id, + after_database_commit=interrupted_after_database_commit, + ) + ) + assert inspect_repository(settings, initialized.root).data_key_id == initialized.data_key_id + assert (initialized.root / ".key-rotation.json").is_file() + + async def reconcile_and_assert() -> None: + worker = Worker(settings, owner="rotation-recovery-worker") + try: + assert await worker.startup() == 1 + finally: + await worker.engine.dispose() + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = await db.get(Repository, repository_id) + assert repository is not None + assert repository.active_data_key_id is not None + inspected = inspect_repository(settings, initialized.root) + assert inspected.data_key_id == repository.active_data_key_id + active_data_key_id = repository.active_data_key_id + exported = await recovery_export_payload(settings) + exported_repository = exported["catalog"]["repositories"][0] + assert exported_repository["active_data_key_id"] == active_data_key_id + finally: + await engine.dispose() + + asyncio.run(reconcile_and_assert()) + assert not (initialized.root / ".key-rotation.json").exists() + + +@pytest.mark.asyncio +async def test_encrypted_repository_worker_backup_and_restore( + app_client: tuple[httpx.AsyncClient, Settings], +) -> None: + client, settings = app_client + source_root = settings.local_source_roots[0] / "project" + source_root.mkdir() + plaintext = b"encrypted backup content\n" + (source_root / "hello.txt").write_bytes(plaintext) + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + db.add(repository) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + await db.commit() + repository_id = repository.id + finally: + await engine.dispose() + + headers = await login(client) + source_response = await client.post( + "/api/v2/sources", + json={ + "name": "local", + "kind": "local", + "public_config": {"root": str(source_root)}, + }, + headers=headers, + ) + assert source_response.status_code == 201 + job_response = await client.post( + "/api/v2/jobs", + json={ + "name": "encrypted-backup", + "source_id": source_response.json()["id"], + "repository_id": repository_id, + "requested_mode": "full", + "exclusions": [], + "retention": {}, + "enabled": True, + "allow_empty": False, + }, + headers=headers, + ) + assert job_response.status_code == 201 + execution_response = await client.post( + f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers + ) + assert execution_response.status_code == 202 + execution_id = execution_response.json()["id"] + + worker = Worker(settings, owner="encrypted-backup-worker") + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + execution = await db.get(Execution, execution_id) + backup = await db.scalar(select(Backup).where(Backup.execution_id == execution_id)) + finally: + await engine.dispose() + assert execution is not None + assert execution.state == "committed" + assert backup is not None + + blob = next((initialized.root / "blobs" / "sha256").iterdir()) + assert blob.read_bytes().startswith(b"BTENC\x01") + assert plaintext not in blob.read_bytes() + manifest_path = initialized.root / "manifests" / f"{backup.manifest_id}.json" + stored_manifest = manifest_path.read_bytes() + assert stored_manifest.startswith(b"BTENC\x01") + assert b'"entries"' not in stored_manifest + assert b"hello.txt" not in stored_manifest + + assert ( + await asyncio.to_thread( + cli_main, + ["admin", "repository-key", "rotate", "--repository-id", repository_id], + settings=settings, + ) + == 0 + ) + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + repository = await db.get(Repository, repository_id) + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch).where( + RepositoryDataKeyEpoch.repository_id == repository_id + ) + ) + ).all() + ) + finally: + await engine.dispose() + assert repository is not None + assert repository.active_data_key_id != initialized.data_key_id + epoch_states = {f"{epoch.key_id}:{epoch.state}" for epoch in epochs} + expected_epoch_states = { + f"{initialized.data_key_id}:retired", + f"{repository.active_data_key_id}:active", + } + assert epoch_states == expected_epoch_states + assert ( + inspect_repository(settings, initialized.root).data_key_id == repository.active_data_key_id + ) + + plaintext_after_rotation = b"encrypted content after rotation\n" + (source_root / "hello.txt").write_bytes(plaintext_after_rotation) + second_execution_response = await client.post( + f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers + ) + assert second_execution_response.status_code == 202 + second_execution_id = second_execution_response.json()["id"] + second_worker = Worker(settings, owner="encrypted-rotated-backup-worker") + try: + assert await second_worker.run_once() + finally: + await second_worker.engine.dispose() + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + second_backup = await db.scalar( + select(Backup).where(Backup.execution_id == second_execution_id) + ) + finally: + await engine.dispose() + assert second_backup is not None + assert second_backup.data_key_id == repository.active_data_key_id + + destination = settings.restore_roots[0] / "restored" + restore_response = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(destination), + "selection": [], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert restore_response.status_code == 202 + source_root.rename(settings.data_dir / "removed-source") + restore_worker = Worker(settings, owner="encrypted-restore-worker") + try: + assert await restore_worker.run_once() + finally: + await restore_worker.engine.dispose() + + restored = await client.get( + f"/api/v2/restores/{restore_response.json()['id']}", headers=headers + ) + assert restored.json()["state"] == "committed" + assert (destination / "hello.txt").read_bytes() == plaintext + + rotated_destination = settings.restore_roots[0] / "rotated-restored" + rotated_restore_response = await client.post( + f"/api/v2/backups/{second_backup.id}/restores", + json={ + "destination": str(rotated_destination), + "selection": [], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert rotated_restore_response.status_code == 202 + rotated_restore_worker = Worker(settings, owner="encrypted-rotated-restore-worker") + try: + assert await rotated_restore_worker.run_once() + finally: + await rotated_restore_worker.engine.dispose() + assert (rotated_destination / "hello.txt").read_bytes() == plaintext_after_rotation + + manifest_path.write_bytes(stored_manifest[:-1] + bytes([stored_manifest[-1] ^ 1])) + corrupt_destination = settings.restore_roots[0] / "corrupt-manifest" + corrupt_restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(corrupt_destination), + "selection": [], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert corrupt_restore.status_code == 202 + corrupt_worker = Worker(settings, owner="encrypted-corrupt-manifest-worker") + try: + assert await corrupt_worker.run_once() + finally: + await corrupt_worker.engine.dispose() + + corrupt_status = await client.get( + f"/api/v2/restores/{corrupt_restore.json()['id']}", headers=headers + ) + assert corrupt_status.json()["state"] == "failed" + assert not corrupt_destination.exists() diff --git a/tests/integration/test_full_backup_restore.py b/tests/integration/test_full_backup_restore.py new file mode 100644 index 0000000..fbc51d8 --- /dev/null +++ b/tests/integration/test_full_backup_restore.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +from pathlib import Path + +import httpx +import pytest +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Backup, Execution, Repository, Restore +from backup_tool.snapshot import verify_published_snapshot +from backup_tool.worker import Worker +from sqlalchemy import select + +PASSWORD = "correct-horse-battery-staple" + + +async def login(client: httpx.AsyncClient) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +@pytest.mark.asyncio +async def test_worker_publishes_a_verified_signed_full_backup_and_atomic_restore( + app_client: tuple[httpx.AsyncClient, Settings], +) -> None: + client, settings = app_client + source_root = settings.local_source_roots[0] / "project" + source_root.mkdir() + (source_root / "nested").mkdir() + (source_root / "nested" / "hello.txt").write_text("hello backup\n", encoding="utf-8") + headers = await login(client) + repository_response = await client.post( + "/api/v2/repositories", + json={ + "name": "primary", + "relative_path": "primary", + "compression": "none", + "encryption": "none", + }, + headers=headers, + ) + assert repository_response.status_code == 201 + source_response = await client.post( + "/api/v2/sources", + json={ + "name": "local", + "kind": "local", + "public_config": {"root": str(source_root)}, + }, + headers=headers, + ) + assert source_response.status_code == 201 + job_response = await client.post( + "/api/v2/jobs", + json={ + "name": "full-backup", + "source_id": source_response.json()["id"], + "repository_id": repository_response.json()["id"], + "requested_mode": "full", + "exclusions": [], + "retention": {}, + "enabled": True, + "allow_empty": False, + }, + headers=headers, + ) + assert job_response.status_code == 201 + execution_response = await client.post( + f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers + ) + assert execution_response.status_code == 202 + execution_id = execution_response.json()["id"] + + worker = Worker(settings, owner="snapshot-worker") + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + execution = await db.get(Execution, execution_id) + backup = await db.scalar(select(Backup).where(Backup.execution_id == execution_id)) + repository = await db.get(Repository, repository_response.json()["id"]) + finally: + await engine.dispose() + + assert execution is not None + assert execution.state == "committed" + assert backup is not None + assert backup.integrity == "verified" + assert repository is not None + root = Path(repository.root) + manifest_path = root / "manifests" / f"{backup.manifest_id}.json" + manifest = verify_published_snapshot(root, manifest_path, repository.signing_public_key) + file_entry = next(entry for entry in manifest["entries"] if entry["type"] == "file") + assert file_entry["path"] == "nested/hello.txt" + assert (root / "blobs" / "sha256" / file_entry["blob_digest"]).read_text() == "hello backup\n" + + dry_run_destination = settings.restore_roots[0] / "dry-run-backup" + dry_run_response = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(dry_run_destination), + "selection": ["nested"], + "dry_run": True, + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert dry_run_response.status_code == 202 + dry_run_worker = Worker(settings, owner="dry-run-worker") + try: + assert await dry_run_worker.run_once() + finally: + await dry_run_worker.engine.dispose() + dry_run = await client.get(f"/api/v2/restores/{dry_run_response.json()['id']}", headers=headers) + assert dry_run.json()["state"] == "committed" + assert dry_run.json()["result"]["dry_run"] + assert dry_run.json()["result"]["entry_count"] == 2 + assert not dry_run_destination.exists() + + destination = settings.restore_roots[0] / "restored-backup" + restore_response = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(destination), + "selection": [], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert restore_response.status_code == 202 + restore_id = restore_response.json()["id"] + assert restore_response.json()["state"] == "queued" + source_root.rename(settings.data_dir / "removed-source") + + restore_worker = Worker(settings, owner="restore-worker") + try: + assert await restore_worker.run_once() + finally: + await restore_worker.engine.dispose() + + restored = await client.get(f"/api/v2/restores/{restore_id}", headers=headers) + assert restored.status_code == 200 + assert restored.json()["state"] == "committed" + assert restored.json()["result"]["manifest_digest"] == backup.manifest_digest + assert (destination / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello backup\n" + + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + interrupted_restore = await db.get(Restore, restore_id) + assert interrupted_restore is not None + interrupted_restore.state = "running" + interrupted_restore.result = None + await db.commit() + finally: + await engine.dispose() + recovery_worker = Worker(settings, owner="recovery-worker") + try: + assert await recovery_worker.startup() == 1 + finally: + await recovery_worker.engine.dispose() + recovered = await client.get(f"/api/v2/restores/{restore_id}", headers=headers) + assert recovered.json()["state"] == "committed" + + skipped_restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(destination), + "selection": [], + "overwrite_policy": "skip", + }, + headers=headers, + ) + assert skipped_restore.status_code == 202 + skip_worker = Worker(settings, owner="skip-worker") + try: + assert await skip_worker.run_once() + finally: + await skip_worker.engine.dispose() + skipped = await client.get(f"/api/v2/restores/{skipped_restore.json()['id']}", headers=headers) + assert skipped.json()["result"]["skipped"] + + (destination / "nested" / "hello.txt").write_text("replaced", encoding="utf-8") + replaced_restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(destination), + "selection": [], + "overwrite_policy": "replace", + }, + headers=headers, + ) + assert replaced_restore.status_code == 202 + replace_worker = Worker(settings, owner="replace-worker") + try: + assert await replace_worker.run_once() + finally: + await replace_worker.engine.dispose() + assert (destination / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello backup\n" + + (root / "blobs" / "sha256" / file_entry["blob_digest"]).write_text("tampered") + corrupt_destination = settings.restore_roots[0] / "corrupt-restore" + corrupt_restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(corrupt_destination), + "selection": [], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert corrupt_restore.status_code == 202 + + corrupt_worker = Worker(settings, owner="corrupt-restore-worker") + try: + assert await corrupt_worker.run_once() + finally: + await corrupt_worker.engine.dispose() + + corrupt_status = await client.get( + f"/api/v2/restores/{corrupt_restore.json()['id']}", headers=headers + ) + assert corrupt_status.json()["state"] == "failed" + assert not corrupt_destination.exists() + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as db: + corrupted_backup = await db.get(Backup, backup.id) + finally: + await engine.dispose() + assert corrupted_backup is not None + assert corrupted_backup.integrity == "corrupt" diff --git a/tests/integration/test_gc.py b/tests/integration/test_gc.py new file mode 100644 index 0000000..7297649 --- /dev/null +++ b/tests/integration/test_gc.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from backup_tool.gc import purge_repository +from backup_tool.security.repository_crypto import encrypt_object, object_aad + + +def write_manifest(path: Path, digests: list[str]) -> None: + path.write_text( + json.dumps({"entries": [{"blob_digest": digest} for digest in digests]}), + encoding="utf-8", + ) + + +def age(path: Path, days: int = 8) -> None: + stamp = (datetime.now(UTC) - timedelta(days=days)).timestamp() + os.utime(path, (stamp, stamp)) + + +def test_gc_purges_tombstoned_manifest_and_only_unreferenced_old_blob( + tmp_path: Path, +) -> None: + root = tmp_path / "repository" + manifests = root / "manifests" + blobs = root / "blobs" / "sha256" + manifests.mkdir(parents=True) + blobs.mkdir(parents=True) + kept = "a" * 64 + removed = "b" * 64 + write_manifest(manifests / "kept.json", [kept]) + write_manifest(manifests / "deleted.json", [removed]) + (blobs / kept).write_bytes(b"kept") + (blobs / removed).write_bytes(b"removed") + age(manifests / "deleted.json") + age(blobs / removed) + + report = purge_repository(root, {"deleted"}) + + assert report.purged_manifests == 1 + assert report.purged_blobs == 1 + assert (manifests / "kept.json").exists() + assert (blobs / kept).exists() + assert not (blobs / removed).exists() + + +@pytest.mark.parametrize("encrypted", [False, True], ids=["corrupt", "encrypted"]) +def test_gc_fails_closed_for_unreadable_manifest(tmp_path: Path, encrypted: bool) -> None: + root = tmp_path / "repository" + manifests = root / "manifests" + blobs = root / "blobs" / "sha256" + manifests.mkdir(parents=True) + blobs.mkdir(parents=True) + kept = "a" * 64 + deleted = "b" * 64 + write_manifest(manifests / "kept.json", [kept]) + write_manifest(manifests / "deleted.json", [deleted]) + unreadable = manifests / "unreadable.json" + unreadable.write_bytes( + encrypt_object( + b"k" * 32, + object_aad("repository", "key", "manifest", "unreadable"), + b'{"entries": []}', + ) + if encrypted + else b"not json" + ) + (blobs / kept).write_bytes(b"kept") + (blobs / deleted).write_bytes(b"deleted") + age(manifests / "deleted.json") + age(blobs / deleted) + + report = purge_repository(root, {"deleted"}) + + assert report.purged_manifests == 0 + assert report.purged_blobs == 0 + assert (manifests / "deleted.json").exists() + assert (blobs / deleted).exists() + + +def test_gc_purges_encrypted_manifests_with_known_epoch_keys(tmp_path: Path) -> None: + root = tmp_path / "repository" + manifests = root / "manifests" + blobs = root / "blobs" / "sha256" + manifests.mkdir(parents=True) + blobs.mkdir(parents=True) + key = b"k" * 32 + repository_id = "repository" + key_id = "epoch" + kept = "a" * 64 + deleted = "b" * 64 + for manifest_id, digests in (("kept", [kept]), ("deleted", [deleted])): + plaintext = json.dumps( + {"entries": [{"blob_digest": digest} for digest in digests]} + ).encode() + (manifests / f"{manifest_id}.json").write_bytes( + encrypt_object( + key, + object_aad(repository_id, key_id, "manifest", manifest_id), + plaintext, + ) + ) + (blobs / kept).write_bytes(b"kept") + (blobs / deleted).write_bytes(b"deleted") + age(manifests / "deleted.json") + age(blobs / deleted) + + report = purge_repository( + root, + {"deleted"}, + repository_id=repository_id, + manifest_keys={"kept": (key_id, key), "deleted": (key_id, key)}, + ) + + assert report.purged_manifests == 1 + assert report.purged_blobs == 1 + assert (blobs / kept).exists() + assert not (blobs / deleted).exists() + + +def test_gc_quarantines_unknown_blob_name(tmp_path: Path) -> None: + root = tmp_path / "repository" + blobs = root / "blobs" / "sha256" + blobs.mkdir(parents=True) + unknown = blobs / "not-a-digest" + unknown.write_bytes(b"unknown") + + report = purge_repository(root, set()) + + assert report.quarantined == 1 + assert not unknown.exists() + assert (root / "quarantine" / "not-a-digest").exists() diff --git a/tests/integration/test_incremental.py b/tests/integration/test_incremental.py new file mode 100644 index 0000000..14ba28b --- /dev/null +++ b/tests/integration/test_incremental.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from backup_tool.snapshot import _unsigned_manifest + + +class Object: + def __init__(self, **values: object) -> None: + self.__dict__.update(values) + + +def test_incremental_request_without_baseline_emits_complete_full_manifest() -> None: + manifest = _unsigned_manifest( + "0198c57f-0000-7000-8000-000000000006", + "0198c57f-0000-7000-8000-000000000001", + Object(id="0198c57f-0000-7000-8000-000000000003", kind="local"), + Object( + id="0198c57f-0000-7000-8000-000000000004", + requested_mode="incremental", + exclusions=[], + ), + Object(id="0198c57f-0000-7000-8000-000000000005"), + [], + 0, + 0, + ) + + assert manifest["requested_mode"] == "incremental" + assert manifest["effective_mode"] == "full" diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 3618ae4..9995485 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib import os +import sqlite3 import stat from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -24,8 +25,13 @@ EXPECTED_TABLES = { "idempotency_records", "jobs", "notification_deliveries", + "notification_delivery_attempts", + "notification_email_settings", + "notification_events", + "notification_signing_keys", "notification_subscriptions", "repositories", + "repository_data_key_epochs", "restores", "schedules", "secrets", @@ -103,6 +109,199 @@ async def test_startup_rejects_unmigrated_database(tmp_path: Path) -> None: await engine.dispose() +def test_local_only_sources_migration_rejects_existing_remote_sources( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + migration = alembic_config(settings.database_url) + command.upgrade(migration, "0005_restore_dry_run") + with sqlite3.connect(settings.database_path) as connection: + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ("remote", "sftp", "{}", "[]", "active", None, "source-remote"), + ) + + with pytest.raises(RuntimeError, match="found 1 non-local source row"): + command.upgrade(migration, "head") + + with sqlite3.connect(settings.database_path) as connection: + connection.execute("UPDATE sources SET kind = ? WHERE id = ?", ("local", "source-remote")) + + command.upgrade(migration, "head") + with ( + sqlite3.connect(settings.database_path) as connection, + pytest.raises(sqlite3.IntegrityError), + ): + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ("remote-two", "sftp", "{}", "[]", "active", None, "source-remote-two"), + ) + + command.downgrade(migration, "0005_restore_dry_run") + with sqlite3.connect(settings.database_path) as connection: + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ("remote-three", "sftp", "{}", "[]", "active", None, "source-remote-three"), + ) + + +def test_0009_allows_ssh_sources_and_refuses_populated_downgrade( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + migration = alembic_config(settings.database_url) + command.upgrade(migration, "0008_notification_outbox") + with sqlite3.connect(settings.database_path) as connection: + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ("local", "local", '{"root":"/tmp"}', "[]", "active", None, "source-local"), + ) + + command.upgrade(migration, "head") + with sqlite3.connect(settings.database_path) as connection: + local_source = connection.execute("SELECT kind FROM sources").fetchone() + assert local_source is not None + assert local_source[0] == "local" + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + "ssh", + "ssh", + '{"hostname":"backup.example.test"}', + '["secret"]', + "active", + None, + "source-ssh", + ), + ) + with pytest.raises(sqlite3.IntegrityError): + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ("sftp", "sftp", "{}", "[]", "active", None, "source-sftp"), + ) + + with pytest.raises(RuntimeError, match="found 1 SSH source row"): + command.downgrade(migration, "0008_notification_outbox") + + with sqlite3.connect(settings.database_path) as connection: + connection.execute("DELETE FROM sources WHERE id = ?", ("source-ssh",)) + command.downgrade(migration, "0008_notification_outbox") + with ( + sqlite3.connect(settings.database_path) as connection, + pytest.raises(sqlite3.IntegrityError), + ): + connection.execute( + """ + INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + "ssh-after-downgrade", + "ssh", + "{}", + "[]", + "active", + None, + "source-ssh-after", + ), + ) + + +def test_0008_migrates_unexpected_legacy_notification_delivery(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + migration = alembic_config(settings.database_url) + command.upgrade(migration, "0007_repository_data_key_epochs") + with sqlite3.connect(settings.database_path) as connection: + connection.execute( + """ + INSERT INTO notification_subscriptions + (id, channel, event_filters, destination_config, state) + VALUES (?, ?, ?, ?, ?) + """, + ( + "subscription", + "email", + '["execution.queued"]', + '{"recipients":["a@example.test"]}', + "active", + ), + ) + connection.execute( + """ + INSERT INTO notification_deliveries + (id, event_id, subscription_id, attempt, state, response_class) + VALUES (?, ?, ?, ?, ?, ?) + """, + ("delivery", "legacy-event", "subscription", 1, "delivered", "http_200"), + ) + command.upgrade(migration, "head") + with sqlite3.connect(settings.database_path) as connection: + event = connection.execute("SELECT type, payload FROM notification_events").fetchone() + attempt = connection.execute( + "SELECT number, outcome, response_class FROM notification_delivery_attempts" + ).fetchone() + assert event is not None and event[0] == "notification.legacy" + assert "legacy-event" in event[1] + assert attempt is not None + assert attempt[0] == 1 + assert attempt[1] == "delivered" + assert attempt[2] == "http_200" + + +def test_0007_downgrade_refuses_populated_key_metadata(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + migration = alembic_config(settings.database_url) + command.upgrade(migration, "head") + with sqlite3.connect(settings.database_path) as connection: + connection.execute( + """ + INSERT INTO repositories + (id, name, root, format_version, compression, encryption, active_data_key_id, state) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "repository", + "repository", + str(settings.repository_roots[0]), + 1, + "none", + "none", + "epoch", + "active", + ), + ) + connection.execute( + """ + INSERT INTO repository_data_key_epochs (id, repository_id, key_id, state) + VALUES (?, ?, ?, ?) + """, + ("epoch-row", "repository", "epoch", "active"), + ) + + with pytest.raises( + RuntimeError, match="cannot downgrade while repository data key metadata exists" + ): + command.downgrade(migration, "0006_local_sources_only") + + def test_database_role_rejects_unmigrated_database(tmp_path: Path) -> None: cli = importlib.import_module("backup_tool.cli") db_module = importlib.import_module("backup_tool.db.engine") diff --git a/tests/integration/test_notifications.py b/tests/integration/test_notifications.py new file mode 100644 index 0000000..2eae72d --- /dev/null +++ b/tests/integration/test_notifications.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +from collections.abc import Callable +from datetime import UTC, datetime +from typing import cast + +import pytest +from backup_tool.db.models import ( + NotificationDelivery, + NotificationEmailSettings, + NotificationEvent, +) +from backup_tool.ids import new_uuid7 +from backup_tool.notifications.email import SMTPClient, deliver_email +from backup_tool.notifications.events import emit_event +from sqlalchemy import select + +PASSWORD = "correct horse battery staple" + + +async def _setup(client) -> str: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return client.cookies["backup_tool_csrf"] + + +@pytest.mark.asyncio +async def test_filters_manual_test_retry_and_history(app_client) -> None: + client, _ = app_client + csrf = await _setup(client) + created = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["schedule.*", "notification.test_requested"], + "destination": {"recipients": ["operator@example.test"]}, + }, + headers={"X-CSRF-Token": csrf}, + ) + assert created.status_code == 201 + subscription_id = created.json()["id"] + app = client._transport.app + async with app.state.sessions() as db: + await emit_event( + db, + "execution.queued", + correlation_id=str(new_uuid7()), + resource={}, + deduplication_key="filtered-out", + ) + scheduled = await emit_event( + db, + "schedule.created", + correlation_id=str(new_uuid7()), + resource={}, + deduplication_key="filtered-in", + ) + await db.commit() + statement = select(NotificationDelivery).where( + NotificationDelivery.event_id == scheduled.id + ) + deliveries = list((await db.scalars(statement)).all()) + assert len(deliveries) == 1 + delivery = deliveries[0] + delivery.state = "failed" + delivery.terminal_reason = "http_permanent" + await db.commit() + delivery_id = delivery.id + + tested = await client.post( + f"/api/v2/notifications/subscriptions/{subscription_id}/test", + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "test-one"}, + ) + assert tested.status_code == 202 + retried = await client.post( + f"/api/v2/notifications/deliveries/{delivery_id}/retry", + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "retry-one"}, + ) + assert retried.status_code == 202 + replayed = await client.post( + f"/api/v2/notifications/deliveries/{delivery_id}/retry", + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "retry-one"}, + ) + assert replayed.status_code == 202 + assert replayed.json() == retried.json() + history = await client.get("/api/v2/notifications/deliveries") + assert history.status_code == 200 + row = next(item for item in history.json()["items"] if item["id"] == delivery_id) + assert row["state"] == "retry" + + +@pytest.mark.asyncio +async def test_manual_test_bypasses_filters_and_targets_only_selected_subscription( + app_client, +) -> None: + client, _ = app_client + csrf = await _setup(client) + selected = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["execution.failed"], + "destination": {"recipients": ["selected@example.test"]}, + }, + headers={"X-CSRF-Token": csrf}, + ) + other = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["notification.test_requested"], + "destination": {"recipients": ["other@example.test"]}, + }, + headers={"X-CSRF-Token": csrf}, + ) + assert selected.status_code == other.status_code == 201 + response = await client.post( + f"/api/v2/notifications/subscriptions/{selected.json()['id']}/test", + headers={"X-CSRF-Token": csrf, "Idempotency-Key": "selected-test"}, + ) + assert response.status_code == 202 + app = client._transport.app + async with app.state.sessions() as db: + rows = list( + ( + await db.scalars( + select(NotificationDelivery.subscription_id).where( + NotificationDelivery.event_id == response.json()["event_id"] + ) + ) + ).all() + ) + assert rows == [selected.json()["id"]] + + +@pytest.mark.asyncio +async def test_disabled_subscription_does_not_receive_future_events(app_client) -> None: + client, _ = app_client + csrf = await _setup(client) + created = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["execution.*"], + "destination": {"recipients": ["operator@example.test"]}, + }, + headers={"X-CSRF-Token": csrf}, + ) + subscription = created.json() + disabled = await client.patch( + f"/api/v2/notifications/subscriptions/{subscription['id']}", + json={"state": "disabled"}, + headers={"X-CSRF-Token": csrf, "If-Match": created.headers["ETag"]}, + ) + assert disabled.status_code == 200 + app = client._transport.app + async with app.state.sessions() as db: + event = await emit_event( + db, + "execution.queued", + correlation_id=str(new_uuid7()), + resource={}, + occurred_at=datetime.now(UTC), + ) + await db.commit() + assert ( + await db.scalar( + select(NotificationDelivery.id).where(NotificationDelivery.event_id == event.id) + ) + is None + ) + + +@pytest.mark.asyncio +async def test_email_uses_ehlo_starttls_then_auth_with_hermetic_fake() -> None: + calls: list[str] = [] + + class FakeSMTP: + def __init__(self, *_args, **_kwargs) -> None: + calls.append("connect") + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + calls.append("close") + + def ehlo(self) -> None: + calls.append("ehlo") + + def starttls(self, *, context) -> None: + assert context.check_hostname + calls.append("starttls") + + def login(self, username: str, password: str) -> None: + assert username == "operator" + assert password == "smtp-password" + calls.append("auth") + + def send_message(self, message) -> None: + assert "smtp-password" not in message.as_string() + calls.append("send") + + event = NotificationEvent( + id=str(new_uuid7()), + type="execution.queued", + schema_version=1, + occurred_at=datetime.now(UTC), + correlation_id=str(new_uuid7()), + severity="info", + resource_refs={}, + payload={}, + canonical_envelope="{}", + ) + settings = NotificationEmailSettings( + id=1, + host="smtp.example.test", + port=587, + username="operator", + password_secret_id=str(new_uuid7()), + sender="sender@example.test", + max_attempts=5, + rate_limit_per_minute=60, + ) + result = await deliver_email( + settings, + "smtp-password", + event, + ["operator@example.test"], + smtp_factory=cast(Callable[..., SMTPClient], FakeSMTP), + ) + assert result.response_class == "smtp_2xx" + assert calls == ["connect", "ehlo", "starttls", "ehlo", "auth", "send", "close"] diff --git a/tests/integration/test_queue_leases.py b/tests/integration/test_queue_leases.py new file mode 100644 index 0000000..8384aaa --- /dev/null +++ b/tests/integration/test_queue_leases.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import importlib +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +import pytest_asyncio +from alembic import command +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Execution, Job, Repository, Source +from backup_tool.execution import ( + EnqueueError, + claim, + complete_cancellation, + enqueue, + heartbeat, + request_cancellation, + retry, +) +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +cli = importlib.import_module("backup_tool.cli") + + +@pytest_asyncio.fixture +async def database( + tmp_path: Path, +) -> AsyncIterator[tuple[async_sessionmaker[AsyncSession], AsyncEngine]]: + key = tmp_path / "master.key" + key.write_bytes(b"m5-test-master-key-material-32-bytes-minimum") + key.chmod(0o600) + data_dir = tmp_path / "data" + repositories = tmp_path / "repositories" + sources = tmp_path / "sources" + restores = tmp_path / "restores" + for directory in (data_dir, repositories, sources, restores): + directory.mkdir() + settings = Settings( + data_dir=data_dir, + database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}", + repository_roots=(repositories,), + local_source_roots=(sources,), + restore_roots=(restores,), + master_key_file=key, + ) + command.upgrade(cli.build_alembic_config(settings), "head") + engine = create_engine(settings) + yield async_sessionmaker(engine, expire_on_commit=False), engine + await engine.dispose() + + +async def create_job( + db: AsyncSession, suffix: str, *, enabled: bool = True, state: str = "active" +) -> str: + repository = Repository( + name=f"repository-{suffix}", + root=f"/repositories/{suffix}", + format_version=1, + compression="none", + encryption="none", + ) + source = Source( + name=f"source-{suffix}", + kind="local", + public_config={"root": f"/sources/{suffix}"}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + job = Job( + name=f"job-{suffix}", + source_id=source.id, + repository_id=repository.id, + requested_mode="full", + exclusions=[], + retention={}, + enabled=enabled, + state=state, + ) + db.add(job) + await db.commit() + return job.id + + +@pytest.mark.asyncio +async def test_concurrent_enqueue_allows_exactly_one_active_execution( + database: tuple[async_sessionmaker[AsyncSession], AsyncEngine], +) -> None: + sessions, _ = database + async with sessions() as db: + job_id = await create_job(db, "concurrent") + + async def start() -> Execution | EnqueueError: + async with sessions() as db: + try: + return await enqueue(db, job_id) + except EnqueueError as error: + return error + + first, second = await asyncio.gather(start(), start()) + results = [first, second] + successes = [result for result in results if isinstance(result, Execution)] + failures = [result for result in results if isinstance(result, EnqueueError)] + + assert len(successes) == 1 + assert len(failures) == 1 + assert failures[0].code == "execution_active" + assert failures[0].active_execution_id == successes[0].id + async with sessions() as db: + executions = list(await db.scalars(select(Execution).where(Execution.job_id == job_id))) + assert [execution.id for execution in executions] == [successes[0].id] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("enabled", "state"), [(False, "active"), (True, "archived")]) +async def test_enqueue_rejects_disabled_or_archived_jobs( + database: tuple[async_sessionmaker[AsyncSession], AsyncEngine], + enabled: bool, + state: str, +) -> None: + sessions, _ = database + async with sessions() as db: + job_id = await create_job( + db, f"unavailable-{enabled}-{state}", enabled=enabled, state=state + ) + with pytest.raises(EnqueueError) as raised: + await enqueue(db, job_id) + + assert raised.value.code == "job_disabled" + + +@pytest.mark.asyncio +async def test_reclaimed_lease_fences_the_previous_worker( + database: tuple[async_sessionmaker[AsyncSession], AsyncEngine], +) -> None: + sessions, _ = database + async with sessions() as db: + execution = await enqueue(db, await create_job(db, "leases")) + assert await claim(db, execution.id, "worker-a") is not None + assert await heartbeat(db, execution.id, "worker-a") + persisted = await db.get(Execution, execution.id) + assert persisted is not None + persisted.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1) + await db.commit() + + async with sessions() as db: + assert await claim(db, execution.id, "worker-b") is not None + assert not await heartbeat(db, execution.id, "worker-a") + assert await request_cancellation(db, execution.id) is not None + assert not await complete_cancellation(db, execution.id, "worker-a") + assert await complete_cancellation(db, execution.id, "worker-b") + + async with sessions() as db: + persisted = await db.get(Execution, execution.id) + assert persisted is not None + assert persisted.state == "cancelled" + assert persisted.lease_owner is None + + +@pytest.mark.asyncio +async def test_retry_reuses_execution_and_rejects_non_transient_failures( + database: tuple[async_sessionmaker[AsyncSession], AsyncEngine], +) -> None: + sessions, _ = database + async with sessions() as db: + execution = await enqueue(db, await create_job(db, "retry")) + execution.state = "failed" + execution.reason_code = "transient_io" + execution.operator_message = "temporary failure" + execution.lease_owner = "worker-a" + execution.lease_expires_at = datetime.now(UTC) + timedelta(seconds=60) + await db.commit() + + retried = await retry(db, execution.id) + assert retried is not None + assert retried.id == execution.id + assert retried.state == "queued" + assert retried.attempt == 2 + assert retried.reason_code is None + assert retried.operator_message is None + assert retried.lease_owner is None + assert retried.lease_expires_at is None + + retried.state = "failed" + retried.reason_code = "integrity_failure" + await db.commit() + with pytest.raises(EnqueueError) as raised: + await retry(db, execution.id) + + assert raised.value.code == "retry_not_allowed" diff --git a/tests/integration/test_recovery_bundle.py b/tests/integration/test_recovery_bundle.py new file mode 100644 index 0000000..fbe8bc1 --- /dev/null +++ b/tests/integration/test_recovery_bundle.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import asyncio +import os +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from alembic import command +from backup_tool.cli import ( + build_alembic_config, + import_recovery_payload, +) +from backup_tool.cli import ( + main as cli_main, +) +from backup_tool.config import Settings +from backup_tool.db.engine import create_engine +from backup_tool.db.models import ( + Backup, + Execution, + Job, + Repository, + RepositoryDataKeyEpoch, + Restore, + Source, +) +from backup_tool.ids import new_uuid7 +from backup_tool.repository import initialize +from backup_tool.security.recovery_bundle import RecoveryBundleError, decrypt_bundle +from backup_tool.snapshot import finalize_publication, publish_full_snapshot +from backup_tool.worker import Worker +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from tests.conftest import make_settings + +PASSPHRASE = b"correct horse battery staple" + + +def _passphrase_fd(value: bytes) -> int: + read_fd, write_fd = os.pipe() + os.write(write_fd, value + b"\n") + os.close(write_fd) + return read_fd + + +def _recovered_settings(tmp_path: Path, original: Settings) -> Settings: + data_dir = tmp_path / "recovered-data" + source_root = tmp_path / "recovered-sources" + restore_root = tmp_path / "recovered-restores" + for path in (data_dir, source_root, restore_root): + path.mkdir() + master_key = tmp_path / "recovered-master.key" + master_key.write_bytes(b"recovered-host-master-key-material-32-bytes") + master_key.chmod(0o600) + return Settings( + data_dir=data_dir, + database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}", + repository_roots=original.repository_roots, + local_source_roots=(source_root,), + restore_roots=(restore_root,), + master_key_file=master_key, + min_free_bytes=1, + ) + + +def test_recovery_import_restores_encrypted_snapshot_after_host_loss( + tmp_path: Path, +) -> None: + original_path = tmp_path / "original" + original_path.mkdir() + original = make_settings(original_path) + command.upgrade(build_alembic_config(original), "head") + source_root = original.local_source_roots[0] / "project" + source_root.mkdir() + plaintext = b"recovery host-loss content\n" + (source_root / "document.txt").write_bytes(plaintext) + initialized = initialize(original, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + + expected_created_at = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC) + expected_tombstoned_at = datetime(2024, 2, 3, 4, 5, 6, tzinfo=UTC) + + async def create_snapshot() -> tuple[str, str]: + engine = create_engine(original) + sessions = async_sessionmaker(engine, expire_on_commit=False) + try: + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + source = Source( + name="project", + kind="local", + public_config={"root": str(source_root)}, + secret_refs=[], + ) + db.add_all([repository, source]) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + job = Job( + name="encrypted-job", + source_id=source.id, + repository_id=repository.id, + requested_mode="full", + exclusions=[], + retention={}, + allow_empty=False, + ) + db.add(job) + await db.flush() + execution = Execution(job_id=job.id, trigger="manual", progress={}) + db.add(execution) + await db.flush() + backup = await publish_full_snapshot( + original, db, execution, job, source, repository + ) + backup.created_at = expected_created_at + execution.state = "committed" + tombstoned_execution = Execution(job_id=job.id, trigger="manual", progress={}) + db.add(tombstoned_execution) + await db.flush() + tombstoned_backup = Backup( + execution_id=tombstoned_execution.id, + parent_backup_id=None, + manifest_id=str(new_uuid7()), + manifest_digest="0" * 64, + logical_bytes=0, + stored_bytes=0, + integrity="verified", + data_key_id=initialized.data_key_id, + tombstoned_at=expected_tombstoned_at, + created_at=expected_created_at, + ) + db.add(tombstoned_backup) + await db.commit() + finalize_publication(initialized.root, execution.id) + return backup.id, tombstoned_backup.id + finally: + await engine.dispose() + + original_backup_id, tombstoned_backup_id = asyncio.run(create_snapshot()) + bundle = tmp_path / "offline.btrec" + export_fd = _passphrase_fd(PASSPHRASE) + try: + assert ( + cli_main( + [ + "admin", + "recovery", + "export", + "--output", + str(bundle), + "--passphrase-fd", + str(export_fd), + ], + settings=original, + ) + == 0 + ) + finally: + os.close(export_fd) + + retry_path = tmp_path / "retry-recovery" + retry_path.mkdir() + retry = _recovered_settings(retry_path, original) + command.upgrade(build_alembic_config(retry), "head") + retry_payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE) + + def interrupted_after_key_install() -> None: + raise OSError("simulated crash before metadata commit") + + with pytest.raises(RecoveryBundleError, match="recovery import failed"): + asyncio.run( + import_recovery_payload( + retry, + retry_payload, + after_key_install=interrupted_after_key_install, + ) + ) + assert not list((retry.data_dir / "repository-keys").glob("*")) + assert not list((retry.data_dir / "repository-data-keys").glob("*")) + assert asyncio.run(import_recovery_payload(retry, retry_payload)) == 1 + + unsafe_path = tmp_path / "unsafe-recovery" + unsafe_path.mkdir() + unsafe = _recovered_settings(unsafe_path, original) + command.upgrade(build_alembic_config(unsafe), "head") + unsafe_payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE) + unsafe_payload["catalog"]["repositories"][0]["root"] = str(tmp_path) + with pytest.raises(RecoveryBundleError, match="recovery import failed"): + asyncio.run(import_recovery_payload(unsafe, unsafe_payload)) + assert not (unsafe.data_dir / "repository-keys").exists() + assert not (unsafe.data_dir / "repository-data-keys").exists() + + recovered = _recovered_settings(tmp_path, original) + command.upgrade(build_alembic_config(recovered), "head") + import_fd = _passphrase_fd(PASSPHRASE) + try: + assert ( + cli_main( + [ + "admin", + "recovery", + "import", + "--input", + str(bundle), + "--passphrase-fd", + str(import_fd), + ], + settings=recovered, + ) + == 0 + ) + finally: + os.close(import_fd) + + async def restore_and_assert() -> None: + engine = create_engine(recovered) + sessions = async_sessionmaker(engine, expire_on_commit=False) + try: + async with sessions() as db: + backup = await db.get(Backup, original_backup_id) + tombstoned_backup = await db.get(Backup, tombstoned_backup_id) + job = await db.scalar(select(Job)) + recovered_source = await db.scalar(select(Source)) + assert backup is not None + assert tombstoned_backup is not None + assert backup.created_at == expected_created_at + assert tombstoned_backup.created_at == expected_created_at + assert tombstoned_backup.tombstoned_at == expected_tombstoned_at + assert job is not None + assert recovered_source is not None + assert recovered_source.state == "unavailable" + assert job.state == "archived" + assert not job.enabled + restore = Restore( + backup_id=backup.id, + destination=str(recovered.restore_roots[0] / "restored"), + selection=[], + overwrite_policy="fail", + ) + db.add(restore) + await db.commit() + worker = Worker(recovered, owner="host-loss-restore") + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + finally: + await engine.dispose() + + asyncio.run(restore_and_assert()) + assert (recovered.restore_roots[0] / "restored" / "document.txt").read_bytes() == plaintext + + payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE) + with pytest.raises(RecoveryBundleError, match="destination is not empty"): + asyncio.run(import_recovery_payload(recovered, payload)) diff --git a/tests/integration/test_repositories.py b/tests/integration/test_repositories.py index e72580f..d91f4ad 100644 --- a/tests/integration/test_repositories.py +++ b/tests/integration/test_repositories.py @@ -1,11 +1,14 @@ from __future__ import annotations from pathlib import Path +from typing import Any, cast import httpx import pytest from backup_tool.api.app import create_app from backup_tool.config import Settings +from backup_tool.db.models import RepositoryDataKeyEpoch +from sqlalchemy import select @pytest.mark.asyncio @@ -30,7 +33,7 @@ async def test_admin_can_create_and_inspect_allowlisted_repository( async with app.state.engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) - transport = httpx.ASGITransport(app=app) + transport = httpx.ASGITransport(app=cast(Any, app)) async with httpx.AsyncClient(transport=transport, base_url="https://test") as client: setup = await client.post( "/api/v2/setup", json={"username": "admin", "password": "a secure password"} @@ -52,6 +55,29 @@ async def test_admin_can_create_and_inspect_allowlisted_repository( assert body["name"] == "main" assert body["format_version"] == 1 assert (root / "main" / "repository.json").is_file() + encrypted = await client.post( + "/api/v2/repositories", + json={ + "name": "encrypted", + "relative_path": "encrypted", + "compression": "none", + "encryption": "aes-256-gcm", + }, + headers={"X-CSRF-Token": csrf}, + ) + assert encrypted.status_code == 201, encrypted.text + async with app.state.sessions() as db: + epochs = list( + ( + await db.scalars( + select(RepositoryDataKeyEpoch).where( + RepositoryDataKeyEpoch.repository_id == encrypted.json()["id"] + ) + ) + ).all() + ) + assert len(epochs) == 1 + assert epochs[0].state == "active" got = await client.get(f"/api/v2/repositories/{body['id']}") assert got.status_code == 200 changed = await client.patch( diff --git a/tests/integration/test_repository_safety.py b/tests/integration/test_repository_safety.py index cce9458..dd8c814 100644 --- a/tests/integration/test_repository_safety.py +++ b/tests/integration/test_repository_safety.py @@ -42,6 +42,7 @@ def test_partial_initialization_is_removed_on_publish_failure(tmp_path: Path) -> initialize(settings, "main", "none", "none") assert not (settings.repository_roots[0] / "main").exists() assert not list(settings.repository_roots[0].glob(".main.staging-*")) + assert not list((settings.data_dir / "repository-keys").glob("*")) @pytest.mark.parametrize("relative_path", ["/absolute", "../escape"]) diff --git a/tests/integration/test_repository_signing.py b/tests/integration/test_repository_signing.py new file mode 100644 index 0000000..12c6a69 --- /dev/null +++ b/tests/integration/test_repository_signing.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Protocol, cast + +import httpx +import pytest +from backup_tool.api.app import create_app +from backup_tool.config import Settings +from backup_tool.db.models import Repository +from backup_tool.repository import initialize +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +PASSWORD = "a secure password" + + +class SignedRepository(Protocol): + repository_id: str + signing_key_id: str + signing_public_key: str + + +def settings_for(tmp_path: Path) -> Settings: + key = tmp_path / "master.key" + key.write_bytes(b"m6-test-master-key-material-32-bytes-minimum") + key.chmod(0o600) + repositories = tmp_path / "repositories" + repositories.mkdir() + return Settings( + data_dir=tmp_path, + database_url=f"sqlite+aiosqlite:///{tmp_path / 'metadata.db'}", + repository_roots=(repositories,), + local_source_roots=(tmp_path,), + restore_roots=(tmp_path,), + master_key_file=key, + min_free_bytes=1, + ) + + +def test_repository_initialization_creates_a_bound_ed25519_keypair( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + + initialized = cast(SignedRepository, initialize(settings, "main", "none", "none")) + + key_path = settings.data_dir / "repository-keys" / f"{initialized.repository_id}.ed25519" + assert key_path.read_bytes() + assert key_path.stat().st_mode & 0o777 == 0o600 + private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes()) + public_key = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + assert initialized.signing_public_key == public_key.hex() + assert initialized.signing_key_id.startswith("ed25519-") + + +@pytest.mark.asyncio +async def test_repository_api_persists_its_bound_public_signing_key( + tmp_path: Path, +) -> None: + settings = settings_for(tmp_path) + app = create_app(settings) + from backup_tool.db.models import Base + + async with app.state.engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://test" + ) as client: + assert ( + await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + ).status_code == 201 + created = await client.post( + "/api/v2/repositories", + json={ + "name": "main", + "relative_path": "main", + "compression": "none", + "encryption": "none", + }, + headers={"X-CSRF-Token": client.cookies["backup_tool_csrf"]}, + ) + assert created.status_code == 201 + async with app.state.sessions() as db: + repository = await db.get(Repository, created.json()["id"]) + + await app.state.engine.dispose() + assert repository is not None + assert repository.signing_key_id.startswith("ed25519-") + assert len(repository.signing_public_key) == 64 diff --git a/tests/integration/test_scheduler_live_sync.py b/tests/integration/test_scheduler_live_sync.py new file mode 100644 index 0000000..95dff74 --- /dev/null +++ b/tests/integration/test_scheduler_live_sync.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +import httpx +import pytest +from backup_tool.config import Settings +from backup_tool.db.models import NotificationDelivery, NotificationEvent, Schedule +from backup_tool.scheduler import SchedulerService +from sqlalchemy import select + +PASSWORD = "correct-horse-battery-staple" + + +async def setup_job(client: httpx.AsyncClient, settings: Settings) -> tuple[dict[str, str], str]: + source_root = settings.local_source_roots[0] / "source" + source_root.mkdir() + headers = await login(client) + repository = await client.post( + "/api/v2/repositories", + json={ + "name": "repo", + "relative_path": "repo", + "compression": "none", + "encryption": "none", + }, + headers=headers, + ) + source = await client.post( + "/api/v2/sources", + json={ + "name": "source", + "kind": "local", + "public_config": {"root": str(source_root)}, + }, + headers=headers, + ) + job = await client.post( + "/api/v2/jobs", + json={ + "name": "job", + "source_id": source.json()["id"], + "repository_id": repository.json()["id"], + "requested_mode": "full", + "exclusions": [], + "retention": {}, + "enabled": True, + "allow_empty": True, + }, + headers=headers, + ) + assert job.status_code == 201 + return headers, job.json()["id"] + + +async def login(client: httpx.AsyncClient) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +@pytest.mark.asyncio +async def test_schedule_role_delivery_and_live_crud_sync( + app_client: tuple[httpx.AsyncClient, Settings], +) -> None: + client, settings = app_client + headers, job_id = await setup_job(client, settings) + subscription = await client.post( + "/api/v2/notifications/subscriptions", + json={ + "channel": "email", + "event_filters": ["schedule.occurrence_enqueued"], + "destination": {"recipients": ["operator@example.test"]}, + }, + headers=headers, + ) + assert subscription.status_code == 201 + created = await client.post( + f"/api/v2/jobs/{job_id}/schedule", + json={"cron": "* * * * *", "timezone": "UTC"}, + headers=headers, + ) + assert created.status_code == 201 + app = cast(Any, client._transport).app + async with app.state.sessions() as db: + schedule = await db.scalar(select(Schedule).where(Schedule.job_id == job_id)) + assert schedule is not None + schedule.next_nominal_at = datetime.now(UTC) - timedelta(seconds=1) + await db.commit() + service = SchedulerService(settings) + try: + assert await service.run_once() == 1 + finally: + await service.engine.dispose() + async with app.state.sessions() as db: + delivery = await db.scalar( + select(NotificationDelivery.id) + .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id) + .where(NotificationEvent.type == "schedule.occurrence_enqueued") + .limit(1) + ) + assert delivery is not None + + updated = await client.patch( + f"/api/v2/jobs/{job_id}/schedule", + json={"cron": "0 10 * * *", "timezone": "UTC", "enabled": False}, + headers=headers, + ) + assert updated.status_code == 200 + assert updated.json()["next_nominal_at"] is None + # Deletion with historical executions is deliberately restricted; schedule + # delete behavior is covered before occurrence enqueue in the catalog test. diff --git a/tests/integration/test_sources_jobs.py b/tests/integration/test_sources_jobs.py index c4ec748..6659c4d 100644 --- a/tests/integration/test_sources_jobs.py +++ b/tests/integration/test_sources_jobs.py @@ -3,6 +3,7 @@ from __future__ import annotations import json from datetime import UTC, datetime, timedelta from pathlib import Path +from typing import Any, cast import httpx import pytest @@ -18,7 +19,6 @@ from backup_tool.execution import ( recover_stale, request_cancellation, ) -from backup_tool.worker import Worker PASSWORD = "correct-horse-battery-staple" @@ -58,7 +58,7 @@ async def test_local_source_probe_archive_and_repository_targeted_job( async with app.state.engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) - transport = httpx.ASGITransport(app=app) + transport = httpx.ASGITransport(app=cast(Any, app)) async with httpx.AsyncClient(transport=transport, base_url="https://test") as client: headers = await login(client) repository = await client.post( @@ -108,6 +108,24 @@ async def test_local_source_probe_archive_and_repository_targeted_job( assert duplicate.status_code == 409 assert duplicate.json()["code"] == "execution_active" execution_id = execution.json()["id"] + listed_sources = await client.get("/api/v2/sources", headers=headers) + assert listed_sources.status_code == 200 + assert listed_sources.json()["items"] == [ + { + "id": source_id, + "name": "local", + "kind": "local", + "state": "active", + "public_config": {"root": str(source_root)}, + } + ] + listed_jobs = await client.get("/api/v2/jobs", headers=headers) + assert listed_jobs.status_code == 200 + assert listed_jobs.json()["items"][0]["id"] == job.json()["id"] + assert listed_jobs.json()["items"][0]["schedule"] is None + listed_executions = await client.get("/api/v2/executions", headers=headers) + assert listed_executions.status_code == 200 + assert listed_executions.json()["items"][0]["id"] == execution_id scoped_token = await client.post( "/api/v2/auth/tokens", json={"scopes": ["audit:read"], "expires_at": None}, @@ -252,7 +270,9 @@ async def test_execution_sse_replays_later_redacted_revision(tmp_path: Path) -> @pytest.mark.asyncio -async def test_execution_events_preserve_progress_replay_and_recovery_order(tmp_path: Path) -> None: +async def test_execution_events_preserve_progress_replay_and_recovery_order( + tmp_path: Path, +) -> None: source_root = tmp_path / "sources" source_root.mkdir() data_dir = tmp_path / "data" @@ -306,8 +326,14 @@ async def test_execution_events_preserve_progress_replay_and_recovery_order(tmp_ execution = await enqueue(db, job.id) execution_id = execution.id - worker = Worker(settings, owner="ordering-worker") - assert await worker.run_once() + async with app.state.sessions() as db: + assert await claim(db, execution_id, "ordering-worker") is not None + execution = await db.get(Execution, execution_id) + assert execution is not None and execution.state == "preparing" + execution.state = "running" + execution.started_at = datetime.now(UTC) + await record_event(db, execution) + await db.commit() async with app.state.sessions() as db: execution = await db.get(Execution, execution_id) assert execution is not None and execution.state == "running" @@ -383,7 +409,7 @@ async def test_local_source_rejects_unallowlisted_root(tmp_path: Path) -> None: async with app.state.engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="https://test" + transport=httpx.ASGITransport(app=cast(Any, app)), base_url="https://test" ) as client: headers = await login(client) response = await client.post( diff --git a/tests/integration/test_ssh_backup_restore.py b/tests/integration/test_ssh_backup_restore.py new file mode 100644 index 0000000..cba90c5 --- /dev/null +++ b/tests/integration/test_ssh_backup_restore.py @@ -0,0 +1,197 @@ +"""Opt-in live forced-SFTP chroot coverage; all SSH keys are generated under tmp_path.""" + +from __future__ import annotations + +import os +import socket +import subprocess +import time +from pathlib import Path + +import httpx +import pytest +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Backup, Execution, Repository +from backup_tool.snapshot import verify_published_snapshot +from backup_tool.worker import Worker +from sqlalchemy import select + +ROOT = Path(__file__).resolve().parents[2] +PASSWORD = "correct-horse-battery-staple" + + +def _enabled() -> bool: + return os.environ.get("BACKUP_TOOL_SSH_INTEGRATION") == "1" + + +def _port() -> int: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + address = listener.getsockname() + if not isinstance(address, tuple) or not isinstance(address[1], int): + raise RuntimeError("could not allocate SSH fixture port") + return address[1] + + +def _wait(port: int) -> None: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + return + except OSError: + time.sleep(0.25) + raise AssertionError("SSHD fixture did not become reachable") + + +@pytest.fixture +def sshd_fixture(tmp_path: Path): + fixture = tmp_path / "fixture" + host = fixture / "host" + source = fixture / "source" + host.mkdir(parents=True) + source.mkdir() + fixture.chmod(0o755) + host.chmod(0o755) + source.chmod(0o755) + private = fixture / "client" + for target in (host / "ssh_host_ed25519_key", private): + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(target)], + check=True, + ) + (host / "ssh_host_ed25519_key").chmod(0o644) + (fixture / "authorized_keys").write_text(private.with_suffix(".pub").read_text()) + # Public keys are copied into a backup-owned 0600 tmpfs file at startup. + (fixture / "authorized_keys").chmod(0o644) + port = _port() + environment = os.environ | { + "SSH_FIXTURE_DIR": str(fixture), + "SSH_FIXTURE_PORT": str(port), + "COMPOSE_PROJECT_NAME": f"backup-tool-ssh-{os.getpid()}-{port}", + } + command = ["docker", "compose", "-f", "tests/compose.ssh.yaml"] + try: + subprocess.run([*command, "up", "--build", "-d"], cwd=ROOT, env=environment, check=True) + _wait(port) + host_key = " ".join(host.joinpath("ssh_host_ed25519_key.pub").read_text().split()[:2]) + yield { + "port": port, + "key": private, + "host_key": host_key, + "source": source, + "env": environment, + } + finally: + subprocess.run( + [*command, "down", "--volumes", "--remove-orphans"], + cwd=ROOT, + env=environment, + check=False, + ) + + +async def _login(client: httpx.AsyncClient) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +@pytest.mark.skipif(not _enabled(), reason="set BACKUP_TOOL_SSH_INTEGRATION=1") +@pytest.mark.asyncio +async def test_forced_sftp_chroot_probe_backup_and_restore(app_client, sshd_fixture) -> None: + client, settings = app_client + source = sshd_fixture["source"] + (source / "nested").mkdir() + (source / "nested" / "hello.txt").write_text("hello ssh\n") + headers = await _login(client) + secret = await client.post( + "/api/v2/admin/secrets", + json={"purpose": "ssh_private_key", "value": sshd_fixture["key"].read_text()}, + headers=headers, + ) + assert secret.status_code == 201 + remote = await client.post( + "/api/v2/sources", + json={ + "name": "ssh", + "kind": "ssh", + "private_key_secret_id": secret.json()["id"], + "public_config": { + "hostname": "127.0.0.1", + "port": sshd_fixture["port"], + "username": "backup", + "host_key": sshd_fixture["host_key"], + "root": "/", + }, + }, + headers=headers, + ) + assert remote.status_code == 201, remote.text + assert ( + await client.post(f"/api/v2/sources/{remote.json()['id']}/probe", headers=headers) + ).json() == {"entry_count": 1} + repository = await client.post( + "/api/v2/repositories", + json={"name": "repo", "relative_path": "ssh"}, + headers=headers, + ) + job = await client.post( + "/api/v2/jobs", + json={ + "name": "ssh-job", + "source_id": remote.json()["id"], + "repository_id": repository.json()["id"], + "requested_mode": "full", + "exclusions": [], + "retention": {}, + "enabled": True, + "allow_empty": False, + }, + headers=headers, + ) + execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers) + worker = Worker(settings, owner="ssh-live") + try: + assert await worker.run_once() + finally: + await worker.engine.dispose() + engine = create_engine(settings) + try: + from sqlalchemy.ext.asyncio import async_sessionmaker + + async with async_sessionmaker(engine, expire_on_commit=False)() as db: + stored = await db.get(Execution, execution.json()["id"]) + backup = await db.scalar( + select(Backup).where(Backup.execution_id == execution.json()["id"]) + ) + stored_repository = await db.get(Repository, repository.json()["id"]) + finally: + await engine.dispose() + assert stored is not None and stored.state == "committed", ( + stored.operator_message if stored else None + ) + assert backup is not None and stored_repository is not None + manifest = verify_published_snapshot( + Path(stored_repository.root), + Path(stored_repository.root) / "manifests" / f"{backup.manifest_id}.json", + stored_repository.signing_public_key, + ) + assert any(entry["path"] == "data/nested/hello.txt" for entry in manifest["entries"]) + destination = settings.restore_roots[0] / "ssh-restored" + restore = await client.post( + f"/api/v2/backups/{backup.id}/restores", + json={ + "destination": str(destination), + "selection": ["data/nested"], + "overwrite_policy": "fail", + }, + headers=headers, + ) + assert restore.status_code == 202 + restore_worker = Worker(settings, owner="ssh-restore") + try: + assert await restore_worker.run_once() + finally: + await restore_worker.engine.dispose() + assert (destination / "data" / "nested" / "hello.txt").read_text() == "hello ssh\n" diff --git a/tests/integration/test_ssh_source_api.py b/tests/integration/test_ssh_source_api.py new file mode 100644 index 0000000..029f9ef --- /dev/null +++ b/tests/integration/test_ssh_source_api.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import pytest + +PASSWORD = "correct horse battery staple" +HOST_KEY = "ssh-ed25519 AQID" + + +async def _setup_headers(client) -> dict[str, str]: + response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD}) + assert response.status_code == 201 + return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]} + + +async def _create_secret(client, headers: dict[str, str], purpose: str) -> str: + response = await client.post( + "/api/v2/admin/secrets", + json={"purpose": purpose, "value": "PRIVATE-KEY-CANARY"}, + headers=headers, + ) + assert response.status_code == 201 + assert "PRIVATE-KEY-CANARY" not in response.text + return str(response.json()["id"]) + + +def _source(secret_id: str) -> dict[str, object]: + return { + "name": "remote", + "kind": "ssh", + "public_config": { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "host_key": HOST_KEY, + "root": "/", + }, + "private_key_secret_id": secret_id, + } + + +@pytest.mark.asyncio +async def test_ssh_source_persists_only_safe_public_config_and_one_key_reference( + app_client, +) -> None: + client, _ = app_client + headers = await _setup_headers(client) + secret_id = await _create_secret(client, headers, "ssh_private_key") + + created = await client.post("/api/v2/sources", json=_source(secret_id), headers=headers) + + assert created.status_code == 201 + body = created.json() + assert body["kind"] == "ssh" + assert body["public_config"] == _source(secret_id)["public_config"] + assert "secret" not in body + assert "PRIVATE-KEY-CANARY" not in created.text + listed = await client.get("/api/v2/sources", headers=headers) + assert listed.status_code == 200 + assert listed.json()["items"] == [body] + assert "PRIVATE-KEY-CANARY" not in listed.text + probe = await client.post(f"/api/v2/sources/{body['id']}/probe", headers=headers) + assert probe.status_code == 409 + assert probe.json()["code"] == "source_probe_failed" + + +@pytest.mark.asyncio +async def test_ssh_source_requires_exactly_one_existing_private_key_secret( + app_client, +) -> None: + client, _ = app_client + headers = await _setup_headers(client) + wrong_purpose = await _create_secret(client, headers, "ssh") + request = _source(wrong_purpose) + + rejected_purpose = await client.post("/api/v2/sources", json=request, headers=headers) + assert rejected_purpose.status_code == 422 + assert rejected_purpose.json()["code"] == "validation_failed" + assert "PRIVATE-KEY-CANARY" not in rejected_purpose.text + + missing_secret = await client.post( + "/api/v2/sources", + json=_source("00000000-0000-0000-0000-000000000000"), + headers=headers, + ) + assert missing_secret.status_code == 422 + assert missing_secret.json()["code"] == "validation_failed" + + valid_secret = await _create_secret(client, headers, "ssh_private_key") + extra_secret_reference = { + **_source(valid_secret), + "secret_refs": [valid_secret, wrong_purpose], + } + rejected_extra = await client.post( + "/api/v2/sources", json=extra_secret_reference, headers=headers + ) + assert rejected_extra.status_code == 422 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + {"name": "unsupported", "kind": "sftp", "public_config": {}}, + {"name": "unsupported", "kind": "postgresql", "public_config": {}}, + {"name": "unsupported", "kind": "mysql", "public_config": {}}, + {"name": "unsupported", "kind": "shell", "public_config": {}}, + ], +) +async def test_source_api_rejects_all_non_local_ssh_kinds( + app_client, payload: dict[str, object] +) -> None: + client, _ = app_client + headers = await _setup_headers(client) + + response = await client.post("/api/v2/sources", json=payload, headers=headers) + + assert response.status_code == 422 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "public_config", + [ + { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "root": "/", + }, + { + "hostname": "backup.example.test", + "port": 0, + "username": "backup", + "host_key": HOST_KEY, + "root": "/", + }, + { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "host_key": HOST_KEY, + "root": "/not-the-chroot", + }, + { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "host_key": HOST_KEY, + "root": "/", + "password": "not-supported", + }, + { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "host_key": HOST_KEY, + "root": "/", + "remote_command": "not-supported", + }, + ], +) +async def test_ssh_source_rejects_noncanonical_or_unsupported_config( + app_client, public_config: dict[str, object] +) -> None: + client, _ = app_client + headers = await _setup_headers(client) + secret_id = await _create_secret(client, headers, "ssh_private_key") + payload = _source(secret_id) + payload["public_config"] = public_config + + response = await client.post("/api/v2/sources", json=payload, headers=headers) + + assert response.status_code == 422 diff --git a/tests/security/test_container.py b/tests/security/test_container.py new file mode 100644 index 0000000..e07cf82 --- /dev/null +++ b/tests/security/test_container.py @@ -0,0 +1,82 @@ +"""Static checks for the production Compose packaging slice.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from backup_tool.cli import build_parser + +ROOT = Path(__file__).resolve().parents[2] + + +def test_health_command_selects_an_explicit_runtime_role() -> None: + parsed = build_parser().parse_args(["health", "worker"]) + + assert parsed.role == "health" + assert parsed.health_role == "worker" + + +def test_runtime_images_are_pinned_non_root_and_exclude_database_clients() -> None: + runtime = (ROOT / "Dockerfile").read_text() + proxy = (ROOT / "frontend" / "Dockerfile").read_text() + + for dockerfile in (runtime, proxy): + from_lines = [line for line in dockerfile.splitlines() if line.startswith("FROM ")] + assert from_lines + assert all("@sha256:" in line for line in from_lines) + + assert "USER backup-tool:backup-tool" in runtime + assert 'ENTRYPOINT ["backup-tool"]' in runtime + assert 'CMD ["web"]' in runtime + assert "USER 10001:0" in proxy + assert not re.search(r"\b(pg_dump|mysqldump|postgresql-client|mysql-client)\b", runtime) + + +def test_compose_runs_one_isolated_role_per_service_without_reload() -> None: + compose = (ROOT / "docker-compose.yml").read_text() + + for role in ("web", "scheduler", "worker", "migrate", "admin"): + assert f" {role}:" in compose + for command in ( + '["web"]', + '["scheduler"]', + '["worker"]', + '["migrate", "upgrade"]', + '["admin", "--help"]', + ): + assert command in compose + assert "--reload" not in compose + assert "backup-tool-runtime:/run/backup-tool" in compose + assert '["CMD", "backup-tool", "health", "web"]' in compose + assert '["CMD", "backup-tool", "health", "scheduler"]' in compose + assert '["CMD", "backup-tool", "health", "worker"]' in compose + + +def test_proxy_is_the_only_published_endpoint_and_uses_same_origin_socket() -> None: + compose = (ROOT / "docker-compose.yml").read_text() + nginx = (ROOT / "frontend" / "nginx.conf").read_text() + + assert compose.count(" ports:") == 1 + assert '"127.0.0.1:${BACKUP_TOOL_PORT:-8080}:8080"' in compose + assert '"${BACKUP_TOOL_PORT:-8080}:8080"' not in compose + assert "server unix:/run/backup-tool/web.sock;" in nginx + assert "location /api/" in nginx + assert "location = /readyz" in nginx + assert "location = /livez" in nginx + assert "location = /metrics" in nginx + assert "proxy_pass http://backup_tool_web;" in nginx + + +def test_operational_artifacts_cover_sbom_provenance_and_recovery() -> None: + assert (ROOT / "docs/release/m14-sbom.json").is_file() + assert (ROOT / "docs/release/m14-provenance.md").is_file() + for runbook in ( + "metadata.md", + "repositories.md", + "keys.md", + "upgrade.md", + "disaster-recovery.md", + "observability.md", + ): + assert (ROOT / "docs/runbooks" / runbook).is_file() diff --git a/tests/security/test_crypto_leakage.py b/tests/security/test_crypto_leakage.py new file mode 100644 index 0000000..1dea152 --- /dev/null +++ b/tests/security/test_crypto_leakage.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +import pytest +from backup_tool.security.repository_crypto import ( + RepositoryKeyError, + decrypt_object, + encrypt_object, + object_aad, +) + + +def test_encrypted_object_hides_plaintext_and_rejects_tampering() -> None: + key = os.urandom(32) + aad = object_aad("repository", "epoch", "blob", "identity") + stored = encrypt_object(key, aad, b"secret-content") + assert b"secret-content" not in stored + assert decrypt_object(key, aad, stored) == b"secret-content" + with pytest.raises(RepositoryKeyError): + decrypt_object(key, aad, stored[:-1] + bytes([stored[-1] ^ 1])) diff --git a/tests/security/test_recovery_bundle.py b/tests/security/test_recovery_bundle.py new file mode 100644 index 0000000..287467b --- /dev/null +++ b/tests/security/test_recovery_bundle.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import pytest +from alembic import command +from backup_tool.cli import build_alembic_config, build_parser +from backup_tool.cli import main as cli_main +from backup_tool.db.engine import create_engine +from backup_tool.db.models import Repository, RepositoryDataKeyEpoch +from backup_tool.repository import initialize +from backup_tool.security.recovery_bundle import ( + RecoveryBundleError, + RecoveryBundlePathError, + decrypt_bundle, + encrypt_bundle, + write_bundle_exclusive, +) +from sqlalchemy.ext.asyncio import async_sessionmaker + +from tests.conftest import make_settings + +PASSPHRASE = b"correct horse battery staple" + + +def _passphrase_fd(value: bytes) -> int: + read_fd, write_fd = os.pipe() + os.write(write_fd, value + b"\n") + os.close(write_fd) + return read_fd + + +def test_recovery_bundle_rejects_wrong_passphrase_tampering_and_invalid_kdf() -> None: + bundle = encrypt_bundle({"catalog": {"version": 1}, "keys": []}, PASSPHRASE) + tampered = bundle[:-1] + bytes([bundle[-1] ^ 1]) + unsupported_kdf = bundle[:6] + b"\x02" + bundle[7:] + + for encoded, passphrase in ( + (bundle, b"wrong passphrase"), + (tampered, PASSPHRASE), + (unsupported_kdf, PASSPHRASE), + ): + with pytest.raises(RecoveryBundleError, match="^recovery bundle is invalid$"): + decrypt_bundle(encoded, passphrase) + + +def test_recovery_bundle_output_is_exclusive_and_does_not_follow_symlinks( + tmp_path: Path, +) -> None: + bundle = encrypt_bundle({"catalog": {"version": 1}, "keys": []}, PASSPHRASE) + existing = tmp_path / "existing.btrec" + existing.write_bytes(b"keep") + + with pytest.raises(RecoveryBundlePathError, match="^recovery bundle output is unsafe$"): + write_bundle_exclusive(existing, bundle) + assert existing.read_bytes() == b"keep" + + target = tmp_path / "target.btrec" + target.write_bytes(b"keep") + symlink = tmp_path / "link.btrec" + symlink.symlink_to(target) + with pytest.raises(RecoveryBundlePathError, match="^recovery bundle output is unsafe$"): + write_bundle_exclusive(symlink, bundle) + assert target.read_bytes() == b"keep" + + +def test_cli_recovery_export_validate_uses_fd_and_hides_plaintext_keys( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + settings = make_settings(tmp_path) + command.upgrade(build_alembic_config(settings), "head") + initialized = initialize(settings, "encrypted", "none", "aes-256-gcm") + assert initialized.data_key_id is not None + assert initialized.data_key_path is not None + + async def create_repository() -> None: + engine = create_engine(settings) + sessions = async_sessionmaker(engine, expire_on_commit=False) + try: + async with sessions() as db: + repository = Repository( + name="encrypted", + root=str(initialized.root), + format_version=initialized.format_version, + compression=initialized.compression, + encryption=initialized.encryption, + signing_key_id=initialized.signing_key_id, + signing_public_key=initialized.signing_public_key, + active_data_key_id=initialized.data_key_id, + ) + db.add(repository) + await db.flush() + db.add( + RepositoryDataKeyEpoch( + repository_id=repository.id, + key_id=initialized.data_key_id, + state="active", + ) + ) + await db.commit() + finally: + await engine.dispose() + + asyncio.run(create_repository()) + output = tmp_path / "recovery.btrec" + export_fd = _passphrase_fd(PASSPHRASE) + try: + assert ( + cli_main( + [ + "admin", + "recovery", + "export", + "--output", + str(output), + "--passphrase-fd", + str(export_fd), + ], + settings=settings, + ) + == 0 + ) + finally: + os.close(export_fd) + export_output = capsys.readouterr().out + bundle = output.read_bytes() + assert bundle.startswith(b"BTREC\x01") + assert initialized.data_key_path.read_bytes() not in bundle + signing_key_path = ( + settings.data_dir / "repository-keys" / f"{initialized.repository_id}.ed25519" + ) + assert signing_key_path.read_bytes() not in bundle + assert "key" not in export_output.lower() + + validate_fd = _passphrase_fd(PASSPHRASE) + try: + assert ( + cli_main( + [ + "admin", + "recovery", + "validate", + "--input", + str(output), + "--passphrase-fd", + str(validate_fd), + ], + settings=settings, + ) + == 0 + ) + finally: + os.close(validate_fd) + assert capsys.readouterr().out == '{"repositories": 1, "status": "valid"}\n' + + parser = build_parser() + with pytest.raises(SystemExit): + parser.parse_args(["admin", "recovery", "export", "--passphrase", "not-allowed"]) diff --git a/tests/security/test_restore_manifest_entries.py b/tests/security/test_restore_manifest_entries.py new file mode 100644 index 0000000..541dcbc --- /dev/null +++ b/tests/security/test_restore_manifest_entries.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import copy +import importlib + +import pytest + +snapshot = importlib.import_module("backup_tool.snapshot") + + +def file_entry(path: str = "data.txt") -> dict[str, object]: + return { + "path": path, + "type": "file", + "size": 1, + "blob_digest": "a" * 64, + "mode": 0o600, + "mtime_ns": 0, + "link_target": None, + "metadata_support": ["mode", "mtime_ns"], + } + + +@pytest.mark.parametrize( + "entries", + [ + [file_entry("../escape")], + [file_entry("/absolute")], + [file_entry("windows\\escape")], + [file_entry(), file_entry()], + [file_entry("file"), file_entry("file/child")], + [ + { + **file_entry("link"), + "type": "symlink", + "size": 0, + "blob_digest": None, + "link_target": "../escape", + } + ], + ], +) +def test_restore_rejects_semantically_unsafe_signed_manifest_entries( + entries: list[dict[str, object]], +) -> None: + manifest = {"entries": copy.deepcopy(entries)} + + with pytest.raises(snapshot.SnapshotIntegrityError): + snapshot._safe_restore_entries(manifest) diff --git a/tests/security/test_restore_paths.py b/tests/security/test_restore_paths.py new file mode 100644 index 0000000..3343114 --- /dev/null +++ b/tests/security/test_restore_paths.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import importlib +import stat +from pathlib import Path + +import pytest +from backup_tool.config import Settings + +snapshot = importlib.import_module("backup_tool.snapshot") + + +def settings_for(tmp_path: Path) -> Settings: + key = tmp_path / "master.key" + key.write_bytes(b"m6-restore-path-test-master-key-material") + key.chmod(0o600) + repositories = tmp_path / "repositories" + sources = tmp_path / "sources" + restores = tmp_path / "restores" + for directory in (repositories, sources, restores): + directory.mkdir() + return Settings( + data_dir=tmp_path, + database_url=f"sqlite+aiosqlite:///{tmp_path / 'metadata.db'}", + repository_roots=(repositories,), + local_source_roots=(sources,), + restore_roots=(restores,), + master_key_file=key, + min_free_bytes=1, + ) + + +@pytest.mark.parametrize("name", ["existing", "outside", "root"]) +def test_restore_destination_rejects_existing_or_outside_paths(tmp_path: Path, name: str) -> None: + settings = settings_for(tmp_path) + root = settings.restore_roots[0] + outside = tmp_path / "outside" + outside.mkdir() + existing = root / "existing" + existing.mkdir() + destinations = { + "existing": existing, + "outside": outside / "restore", + "root": root, + } + + with pytest.raises(snapshot.SnapshotError): + snapshot.validate_restore_destination(settings, str(destinations[name])) + + +def test_restore_strips_special_permission_bits(tmp_path: Path) -> None: + target = tmp_path / "restored" + target.write_bytes(b"content") + + snapshot._apply_metadata( + target, + {"metadata_support": ["mode"], "mode": 0o7777}, + ) + + mode = stat.S_IMODE(target.stat().st_mode) + assert mode == 0o777 + assert mode & (stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX) == 0 + + +def test_restore_destination_rejects_a_symlinked_parent(tmp_path: Path) -> None: + settings = settings_for(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + (settings.restore_roots[0] / "linked").symlink_to(outside, target_is_directory=True) + + with pytest.raises(snapshot.SnapshotError): + snapshot.validate_restore_destination( + settings, str(settings.restore_roots[0] / "linked" / "restore") + ) diff --git a/tests/security/test_webhook_ssrf.py b/tests/security/test_webhook_ssrf.py new file mode 100644 index 0000000..1b2636a --- /dev/null +++ b/tests/security/test_webhook_ssrf.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import pytest +from backup_tool.security.ssrf import ( + SSRFError, + resolve_public_addresses, + validate_webhook_url, +) + + +@pytest.mark.asyncio +async def test_private_or_mixed_answers_are_rejected() -> None: + async def private(_host: str, _port: int) -> tuple[str, ...]: + return ("8.8.8.8", "127.0.0.1") + + with pytest.raises(SSRFError, match="non-public"): + await resolve_public_addresses("hooks.example.test", 443, private) + + +@pytest.mark.parametrize( + "value", + [ + "https://127.0.0.1/callback", + "https://user:pass@hooks.example.test/callback", + "https://hooks.example.test/callback#fragment", + "ftp://hooks.example.test/callback", + "https://hooks.example.test:22/callback", + ], +) +def test_webhook_url_rejects_bypasses(value: str) -> None: + with pytest.raises(SSRFError): + validate_webhook_url(value) diff --git a/tests/ssh-fixture/Dockerfile b/tests/ssh-fixture/Dockerfile new file mode 100644 index 0000000..78a9776 --- /dev/null +++ b/tests/ssh-fixture/Dockerfile @@ -0,0 +1,18 @@ +FROM debian:12-slim + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y openssh-server \ + && rm -rf /var/lib/apt/lists/* \ + && (getent group backup >/dev/null || groupadd --system backup) \ + && (id backup >/dev/null 2>&1 || useradd --system --gid backup --home-dir /home/backup --shell /usr/sbin/nologin backup) \ + && install -d -o root -g root -m 0755 /home/backup \ + && install -d -o backup -g backup -m 0700 /home/backup/.ssh \ + && install -d -o backup -g backup -m 0755 /home/backup/data \ + && install -d -o root -g root -m 0755 /run/sshd + +COPY sshd_config /etc/ssh/sshd_config +COPY entrypoint.sh /usr/local/bin/fixture-sshd +RUN chmod 0755 /usr/local/bin/fixture-sshd + +EXPOSE 2222 +CMD ["/usr/local/bin/fixture-sshd"] diff --git a/tests/ssh-fixture/entrypoint.sh b/tests/ssh-fixture/entrypoint.sh new file mode 100644 index 0000000..d07048f --- /dev/null +++ b/tests/ssh-fixture/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +# The ephemeral host key is bind-mounted from the test host and can therefore +# have an untrusted numeric owner. Copy it into the root-owned tmpfs before +# sshd checks host-key ownership and permissions. +install -o root -g root -m 0600 /fixture/host/ssh_host_ed25519_key /run/sshd/ssh_host_ed25519_key +# Authorized keys are public material; root ownership prevents fixture-user mutation. +install -o root -g root -m 0644 /fixture/authorized_keys /run/sshd/authorized_keys +exec /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config diff --git a/tests/ssh-fixture/sshd_config b/tests/ssh-fixture/sshd_config new file mode 100644 index 0000000..1d49540 --- /dev/null +++ b/tests/ssh-fixture/sshd_config @@ -0,0 +1,28 @@ +Port 2222 +ListenAddress 0.0.0.0 +HostKey /run/sshd/ssh_host_ed25519_key +PidFile /run/sshd/sshd.pid +AuthorizedKeysFile /run/sshd/authorized_keys +UsePAM no +PasswordAuthentication no +KbdInteractiveAuthentication no +ChallengeResponseAuthentication no +PermitRootLogin no +PermitEmptyPasswords no +PubkeyAuthentication yes +PermitUserEnvironment no +AllowTcpForwarding no +X11Forwarding no +PermitTunnel no +PermitTTY no +GatewayPorts no +AllowAgentForwarding no +LogLevel VERBOSE +Subsystem sftp internal-sftp + +Match User backup + ChrootDirectory /home/backup + ForceCommand internal-sftp + AllowTcpForwarding no + X11Forwarding no + PermitTTY no diff --git a/tests/unit/test_cron_dst.py b/tests/unit/test_cron_dst.py new file mode 100644 index 0000000..e3c34cc --- /dev/null +++ b/tests/unit/test_cron_dst.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest +from backup_tool.scheduler import ScheduleError, next_nominal + + +def test_five_field_cron_returns_utc_nominal_time() -> None: + result = next_nominal("0 9 * * *", "America/New_York", datetime(2026, 1, 1, tzinfo=UTC)) + assert result.tzinfo is UTC + assert result.hour == 14 + + +@pytest.mark.parametrize(("cron", "timezone"), [("* * * * * *", "UTC"), ("* * * * *", "Nope/Zone")]) +def test_invalid_cron_or_timezone_is_rejected(cron: str, timezone: str) -> None: + with pytest.raises(ScheduleError): + next_nominal(cron, timezone) diff --git a/tests/unit/test_exclusions.py b/tests/unit/test_exclusions.py new file mode 100644 index 0000000..8c1d84f --- /dev/null +++ b/tests/unit/test_exclusions.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest +from backup_tool.exclusions import ExclusionError, matches + + +@pytest.mark.parametrize( + ("path", "patterns", "expected"), + [ + ("notes.tmp", ["*.tmp"], True), + ("nested/notes.tmp", ["*.tmp"], True), + ("cache/item.txt", ["cache/"], True), + ("cache/keep.txt", ["cache/", "!cache/keep.txt"], False), + ("data/keep.txt", ["data/**"], True), + ("data.txt", ["data/**"], False), + ], +) +def test_gitignore_like_exclusions(path: str, patterns: list[str], expected: bool) -> None: + assert matches(path, patterns) is expected + + +@pytest.mark.parametrize("value", ["../escape", "/absolute", "windows\\path", ""]) +def test_exclusions_reject_unsafe_paths(value: str) -> None: + with pytest.raises(ExclusionError): + matches("safe.txt", [value]) diff --git a/tests/unit/test_execution_transitions.py b/tests/unit/test_execution_transitions.py new file mode 100644 index 0000000..00c887c --- /dev/null +++ b/tests/unit/test_execution_transitions.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import pytest +from backup_tool.execution import TransitionError, transition + + +@pytest.mark.parametrize( + ("current", "target"), + [ + ("queued", "preparing"), + ("queued", "cancelled"), + ("queued", "failed"), + ("preparing", "running"), + ("preparing", "cancelling"), + ("preparing", "failed"), + ("running", "verifying"), + ("running", "cancelling"), + ("running", "failed"), + ("verifying", "committed"), + ("verifying", "failed"), + ("cancelling", "cancelled"), + ("cancelling", "failed"), + ], +) +def test_all_legal_execution_transitions_are_accepted(current: str, target: str) -> None: + assert transition(current, target) == target + + +@pytest.mark.parametrize( + ("current", "target"), + [ + ("queued", "running"), + ("preparing", "queued"), + ("running", "preparing"), + ("verifying", "cancelling"), + ("cancelling", "running"), + ("committed", "cancelled"), + ("cancelled", "queued"), + ("failed", "queued"), + ("unknown", "queued"), + ], +) +def test_invalid_or_backward_execution_transitions_are_rejected(current: str, target: str) -> None: + with pytest.raises(TransitionError) as raised: + transition(current, target) + + assert raised.value.code == "invalid_transition" diff --git a/tests/unit/test_restore_selection.py b/tests/unit/test_restore_selection.py new file mode 100644 index 0000000..974e96e --- /dev/null +++ b/tests/unit/test_restore_selection.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import importlib + +import pytest + +snapshot = importlib.import_module("backup_tool.snapshot") + + +def entry(path: str, kind: str) -> dict[str, object]: + return {"path": path, "type": kind} + + +def test_restore_selection_includes_selected_descendants_and_ancestors() -> None: + entries = [ + entry("top", "directory"), + entry("top/keep", "directory"), + entry("top/keep/file.txt", "file"), + entry("top/drop.txt", "file"), + ] + + selected = snapshot._select_restore_entries(entries, ["top/keep"]) + + assert [item["path"] for item in selected] == [ + "top", + "top/keep", + "top/keep/file.txt", + ] + + +@pytest.mark.parametrize("selection", [["../escape"], ["/absolute"], ["a\\b"], [""]]) +def test_restore_selection_rejects_unsafe_paths(selection: list[str]) -> None: + with pytest.raises(snapshot.SnapshotError): + snapshot._select_restore_entries([entry("safe", "file")], selection) diff --git a/tests/unit/test_retention.py b/tests/unit/test_retention.py new file mode 100644 index 0000000..171c1b1 --- /dev/null +++ b/tests/unit/test_retention.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +import pytest +from backup_tool.retention import RetentionError, RetentionPolicy, retained_ids + + +@dataclass +class Backup: + id: str + created_at: datetime + pinned: bool = False + tombstoned_at: datetime | None = None + + +def test_retention_is_union_and_protects_newest() -> None: + now = datetime(2026, 7, 29, tzinfo=UTC) + backups = [Backup(str(index), now - timedelta(days=index)) for index in range(6)] + backups[-1].pinned = True + + kept = retained_ids(backups, RetentionPolicy(keep_last=2, keep_daily=3), now) + + assert {"0", "1", "2", "5"} <= kept + + +def test_newest_is_kept_even_when_policy_is_zero() -> None: + now = datetime(2026, 7, 29, tzinfo=UTC) + assert retained_ids([Backup("new", now)], RetentionPolicy(keep_last=0), now) == {"new"} + + +@pytest.mark.parametrize("policy", [{"keep_last": -1}, {"unknown": 1}, {"keep_days": True}]) +def test_invalid_retention_policy_is_rejected(policy: dict[str, object]) -> None: + with pytest.raises(RetentionError): + RetentionPolicy.from_dict(policy) diff --git a/tests/unit/test_ssh_adapter.py b/tests/unit/test_ssh_adapter.py new file mode 100644 index 0000000..d294513 --- /dev/null +++ b/tests/unit/test_ssh_adapter.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import io +import stat +from dataclasses import dataclass +from pathlib import Path + +import paramiko +import pytest +from backup_tool.adapters import SourceError +from backup_tool.ssh_adapter import SSHAdapter, load_private_key +from backup_tool.ssh_source import SSHSourcePublicConfig + +from tests.conftest import make_settings + + +@dataclass +class Attributes: + filename: str + st_mode: int + st_size: int = 0 + st_mtime: int = 1 + + +class ServerKey: + def __init__(self, algorithm: str = "ssh-ed25519", encoded: str = "AQID") -> None: + self.algorithm = algorithm + self.encoded = encoded + + def get_name(self) -> str: + return self.algorithm + + def get_base64(self) -> str: + return self.encoded + + +class Transport: + def __init__(self, key: ServerKey) -> None: + self.key = key + self.events: list[str] = [] + self.closed = False + + def start_client(self, *, timeout: float) -> None: + self.events.append("start") + + def get_remote_server_key(self) -> ServerKey: + self.events.append("host_key") + return self.key + + def auth_publickey(self, username: str, private_key: object) -> None: + self.events.append("auth") + + def close(self) -> None: + self.closed = True + + +class Channel: + def __init__(self, events: list[str]) -> None: + self.events = events + + def settimeout(self, timeout: float) -> None: + self.events.append("timeout") + + +class Handle: + def __init__(self, chunks: list[bytes]) -> None: + self.chunks = chunks + self.closed = False + + def read(self, size: int) -> bytes: + assert size == 4096 + return self.chunks.pop(0) if self.chunks else b"" + + def close(self) -> None: + self.closed = True + + +class SFTP: + def __init__(self, events: list[str], entries: list[Attributes]) -> None: + self.events = events + self.entries = entries + self.handle = Handle([b"one", b"two"]) + self.closed = False + + def get_channel(self) -> Channel: + return Channel(self.events) + + def listdir_iter(self, path: str, *, read_aheads: int): + self.events.append(f"list:{path}:{read_aheads}") + return iter(self.entries) + + def lstat(self, path: str) -> Attributes: + self.events.append(f"lstat:{path}") + return Attributes("file", stat.S_IFREG | 0o640, 6, 1) + + def open(self, path: str, mode: str, bufsize: int) -> Handle: + self.events.append(f"open:{path}:{mode}:{bufsize}") + return self.handle + + def close(self) -> None: + self.closed = True + + +def config() -> SSHSourcePublicConfig: + return SSHSourcePublicConfig( + hostname="backup.example.test", + port=22, + username="backup", + host_key="ssh-ed25519 AQID", + root="/", + ) + + +def adapter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, transport: Transport, sftp: SFTP +) -> SSHAdapter: + monkeypatch.setattr("backup_tool.ssh_adapter.load_private_key", lambda _: object()) + settings = make_settings(tmp_path).model_copy(update={"ssh_read_chunk_bytes": 4096}) + return SSHAdapter( + config(), + "private-key-is-never-sent-to-a-log", + settings, + transport_factory=lambda *_: transport, + sftp_factory=lambda _: sftp, + ) + + +@pytest.mark.asyncio +async def test_host_pin_mismatch_never_authenticates_or_opens_sftp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transport = Transport(ServerKey(encoded="BAUG")) + sftp = SFTP(transport.events, []) + reader = adapter(tmp_path, monkeypatch, transport, sftp) + + with pytest.raises(SourceError, match="host key") as error: + await reader.probe() + + assert error.value.reason_code == "source_trust" + assert transport.events == ["start", "host_key"] + assert transport.closed + assert "timeout" not in sftp.events + assert not any(event.startswith("list:") for event in sftp.events) + + +@pytest.mark.asyncio +async def test_pinned_transport_authenticates_before_sftp_and_streams_bounded_reads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transport = Transport(ServerKey()) + sftp = SFTP(transport.events, [Attributes("file", stat.S_IFREG | 0o640, 6, 1)]) + reader = adapter(tmp_path, monkeypatch, transport, sftp) + + entries = [entry async for entry in reader.enumerate_entries()] + content = b"".join([chunk async for chunk in reader.open_content("file")]) + await reader.close() + + assert entries[0].path == "file" + assert content == b"onetwo" + assert transport.events.index("host_key") < transport.events.index("auth") + assert transport.events.index("auth") < transport.events.index("timeout") + assert "list:/:32" in sftp.events + assert "open:/file:rb:4096" in sftp.events + assert sftp.handle.closed and sftp.closed and transport.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", [stat.S_IFLNK | 0o777, stat.S_IFIFO | 0o600]) +async def test_sftp_rejects_symlinks_and_special_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: int +) -> None: + transport = Transport(ServerKey()) + sftp = SFTP(transport.events, [Attributes("unsafe", mode)]) + reader = adapter(tmp_path, monkeypatch, transport, sftp) + + with pytest.raises(SourceError, match="symlink|unsupported"): + await anext(reader.enumerate_entries()) + + assert "auth" in transport.events + await reader.close() + + +def test_private_key_loader_rejects_short_rsa_and_accepts_strong_rsa() -> None: + short = paramiko.RSAKey.generate(2048) + strong = paramiko.RSAKey.generate(3072) + short_buffer = io.StringIO() + strong_buffer = io.StringIO() + short.write_private_key(short_buffer) + strong.write_private_key(strong_buffer) + + with pytest.raises(SourceError, match="algorithm") as error: + load_private_key(short_buffer.getvalue()) + assert error.value.reason_code == "source_auth" + loaded = load_private_key(strong_buffer.getvalue()) + assert isinstance(loaded, paramiko.RSAKey) diff --git a/tests/unit/test_ssh_source_config.py b/tests/unit/test_ssh_source_config.py new file mode 100644 index 0000000..b7cd1ac --- /dev/null +++ b/tests/unit/test_ssh_source_config.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest +from backup_tool.ssh_source import SSHSourcePublicConfig +from pydantic import ValidationError + +VALID_CONFIG = { + "hostname": "backup.example.test", + "port": 22, + "username": "backup", + "host_key": "ssh-ed25519 AQID", + "root": "/", +} + + +def test_ssh_source_config_is_closed_and_canonical() -> None: + config = SSHSourcePublicConfig.model_validate(VALID_CONFIG) + + assert config.model_dump() == VALID_CONFIG + + +@pytest.mark.parametrize( + "field,value", + [ + ("hostname", "backup user@example.test"), + ("hostname", "ssh://backup.example.test"), + ("port", 0), + ("port", 65536), + ("username", "backup user"), + ("username", "backup/root"), + ("host_key", "ssh-ed25519 not-base64!"), + ("host_key", "ssh-rsa AQID"), + ("root", "/data"), + ], +) +def test_ssh_source_config_rejects_invalid_public_values(field: str, value: object) -> None: + invalid = {**VALID_CONFIG, field: value} + + with pytest.raises(ValidationError): + SSHSourcePublicConfig.model_validate(invalid) + + +def test_ssh_source_config_rejects_non_public_connection_options() -> None: + with pytest.raises(ValidationError): + SSHSourcePublicConfig.model_validate({**VALID_CONFIG, "password": "not-allowed"}) + + with pytest.raises(ValidationError): + SSHSourcePublicConfig.model_validate({**VALID_CONFIG, "remote_command": "not-allowed"}) diff --git a/tests/unit/test_webhook_signing.py b/tests/unit/test_webhook_signing.py new file mode 100644 index 0000000..209e176 --- /dev/null +++ b/tests/unit/test_webhook_signing.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from backup_tool.notifications.webhook import ( + SigningMaterial, + canonical_signing_input, + signatures, + webhook_headers, +) + + +def test_signature_is_stable_and_dual_key_versioned() -> None: + body = b'{"id":"018f"}' + timestamp = "2026-07-30T00:00:00+00:00" + keys = [ + SigningMaterial("key-a", 1, "old-secret"), + SigningMaterial("key-b", 2, "new-secret"), + ] + values = signatures(timestamp, body, keys) + assert len(values) == 2 + assert "key_id=key-a" in values[0] and "key_version=1" in values[0] + assert "key_id=key-b" in values[1] and "key_version=2" in values[1] + assert values == signatures(timestamp, body, keys) + assert canonical_signing_input(timestamp, body).endswith(body) + headers = webhook_headers("event-id", "execution.queued", timestamp, body, keys) + receiver_headers = {key: value for key, value in headers if key != "X-Backup-Signature"} + receiver_signatures = [value for key, value in headers if key == "X-Backup-Signature"] + assert receiver_headers["X-Backup-Event-ID"] == "event-id" + assert receiver_headers["X-Backup-Event-Type"] == "execution.queued" + assert receiver_headers["X-Backup-Timestamp"] == timestamp + assert receiver_signatures == values diff --git a/tools/assert_capabilities.py b/tools/assert_capabilities.py new file mode 100644 index 0000000..765b773 --- /dev/null +++ b/tools/assert_capabilities.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Assert the released v2 capability contract is truthful and explicit.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +CONTRACT = Path("contracts/repository/v1/capabilities-v2.0.json") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--release", required=True) + parser.add_argument("--include", required=True) + parser.add_argument("--exclude", required=True) + args = parser.parse_args() + if args.release != "v2.0": + raise SystemExit("only v2.0 capability certification is supported") + try: + payload = json.loads(CONTRACT.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"cannot load capability contract: {error}") from error + available = set(payload["sources"]) + available.update(name for name, enabled in payload["features"].items() if enabled) + required = {item for item in args.include.split(",") if item} + forbidden = {item for item in args.exclude.split(",") if item} + missing = sorted(required - available) + present = sorted(forbidden & available) + if missing or present: + raise SystemExit( + f"capability assertion failed: missing={missing}, forbidden={present}" + ) + print(f"v2.0 capabilities certified: {', '.join(sorted(available))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/export_openapi.py b/tools/export_openapi.py new file mode 100644 index 0000000..b3590e8 --- /dev/null +++ b/tools/export_openapi.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Export the API schema without depending on operator configuration or network I/O.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import tempfile +from pathlib import Path + +from backup_tool.api.app import create_app +from backup_tool.config import Settings + + +def export_schema() -> dict[str, object]: + """Build the FastAPI schema with disposable, valid settings.""" + with tempfile.TemporaryDirectory(prefix="backup-tool-openapi-") as temporary: + root = Path(temporary) + key = root / "master.key" + key.write_bytes(b"openapi-export-key-material-must-be-at-least-32-bytes") + key.chmod(0o600) + for name in ("data", "repositories", "sources", "restores"): + (root / name).mkdir() + settings = Settings( + data_dir=root / "data", + database_url=f"sqlite+aiosqlite:///{root / 'data' / 'metadata.db'}", + repository_roots=(root / "repositories",), + local_source_roots=(root / "sources",), + restore_roots=(root / "restores",), + master_key_file=key, + ) + app = create_app(settings) + schema = app.openapi() + # Engine construction is lazy, but dispose defensively if that changes. + asyncio.run(app.state.engine.dispose()) + return schema + + +def main() -> int: + parser = argparse.ArgumentParser(description="Export the deterministic v2 OpenAPI document.") + parser.add_argument("--output", type=Path, default=Path("openapi/v2.json")) + parser.add_argument("--check", type=Path, metavar="PATH", help="fail when PATH differs") + arguments = parser.parse_args() + output = arguments.check or arguments.output + rendered = json.dumps(export_schema(), indent=2, sort_keys=True, ensure_ascii=False) + "\n" + if arguments.check: + if not output.is_file() or output.read_text(encoding="utf-8") != rendered: + print(f"OpenAPI drift: regenerate {output}") + return 1 + print(f"OpenAPI is current: {output}") + return 0 + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + # The schema is public source, not executable configuration. + os.chmod(output, 0o644) + print(f"Exported OpenAPI: {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/generate_api_client.py b/tools/generate_api_client.py new file mode 100644 index 0000000..1ba9054 --- /dev/null +++ b/tools/generate_api_client.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Generate the browser API client from the committed OpenAPI document.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +HEADER = """/* eslint-disable */ +/* + * Generated by tools/generate_api_client.py from openapi/v2.json. + * Do not edit this file directly. Run `npm --prefix frontend run api:generate`. + */ + +""" + + +def identifier(value: str) -> str: + parts = re.split(r"[^A-Za-z0-9]+", value) + result = "".join(part[:1].upper() + part[1:] for part in parts if part) + if not result: + return "Anonymous" + return f"_{result}" if result[0].isdigit() else result + + +def property_name(value: str) -> str: + return value if re.match(r"^[A-Za-z_$][A-Za-z0-9_$]*$", value) else json.dumps(value) + + +def operation_name(operation_id: str) -> str: + if "_api_" in operation_id: + prefix = operation_id.split("_api_", 1)[0] + else: + prefix = re.sub(r"_(get|post|put|patch|delete)$", "", operation_id) + first, *rest = prefix.split("_") + prefix = first if rest and all(part == first for part in rest) else prefix + pieces = [piece for piece in prefix.split("_") if piece] + return pieces[0] + "".join(piece.capitalize() for piece in pieces[1:]) + + +def schema_type(schema: dict[str, Any]) -> str: + if "$ref" in schema: + return f"Components['schemas'][{json.dumps(schema['$ref'].rsplit('/', 1)[-1])}]" + for key in ("anyOf", "oneOf", "allOf"): + if key in schema: + separator = " & " if key == "allOf" else " | " + return separator.join(schema_type(item) for item in schema[key]) + if "enum" in schema: + return " | ".join(json.dumps(value) for value in schema["enum"]) + kind = schema.get("type") + if kind == "string": + return "string" + if kind in {"integer", "number"}: + return "number" + if kind == "boolean": + return "boolean" + if kind == "null": + return "null" + if kind == "array": + return f"Array<{schema_type(schema.get('items', {}))}>" + if kind == "object" or "properties" in schema: + properties = schema.get("properties", {}) + required = set(schema.get("required", [])) + fields = [ + f"{property_name(name)}{' ' if name in required else '?'}: {schema_type(value)}" + for name, value in properties.items() + ] + additional = schema.get("additionalProperties") + if not fields: + return ( + f"Record" + if isinstance(additional, dict) + else "Record" + ) + result = "{ " + "; ".join(fields) + " }" + if isinstance(additional, dict): + result += f" & Record" + return result + return "unknown" + + +def response_type(operation: dict[str, Any]) -> str: + responses = operation.get("responses", {}) + successful = next((item for status, item in responses.items() if status.startswith("2")), None) + if successful is None: + return "void" + content = successful.get("content", {}) + json_content = content.get("application/json") + if not json_content: + return "void" + return schema_type(json_content.get("schema", {})) + + +def parameter_type(parameters: list[dict[str, Any]], location: str) -> str | None: + selected = [parameter for parameter in parameters if parameter.get("in") == location] + if not selected: + return None + fields = [] + for parameter in selected: + optional = "" if parameter.get("required") else "?" + parameter_schema = schema_type(parameter.get("schema", {})) + fields.append(f"{property_name(parameter['name'])}{optional}: {parameter_schema}") + return "{ " + "; ".join(fields) + " }" + + +def request_body_type(operation: dict[str, Any]) -> tuple[str | None, bool]: + body = operation.get("requestBody") + if not body: + return None, False + content = body.get("content", {}) + json_content = content.get("application/json") + return ( + schema_type(json_content.get("schema", {})) if json_content else "unknown", + bool(body.get("required")), + ) + + +def is_event_stream(operation: dict[str, Any]) -> bool: + return any( + "text/event-stream" in response.get("content", {}) + for response in operation.get("responses", {}).values() + ) + + +def render_operation( + name: str, path: str, method: str, operation: dict[str, Any] +) -> tuple[str, str]: + parameters = operation.get("parameters", []) + path_type = parameter_type(parameters, "path") + query_type = parameter_type(parameters, "query") + body_type, body_required = request_body_type(operation) + fields: list[str] = [] + if path_type: + fields.append(f"path: {path_type}") + if query_type: + fields.append(f"query?: {query_type}") + if body_type: + fields.append(f"body{' ' if body_required else '?'}: {body_type}") + result = response_type(operation) + if fields: + params_name = f"{identifier(name)}Params" + declaration = f"export type {params_name} = {{ {'; '.join(fields)} }};\n\n" + signature = f"params: {params_name}, options: RequestOptions = {{}}" + parameter_expression = "params" + else: + declaration = "" + signature = "options: RequestOptions = {}" + parameter_expression = "{}" + path_expression = json.dumps(path) + for parameter in parameters: + if parameter.get("in") == "path": + name_value = parameter["name"] + path_expression += ( + f".replace({json.dumps('{' + name_value + '}')}, " + f"encodeURIComponent(String({parameter_expression}.path.{property_name(name_value)})))" + ) + query_expression = "" + if query_type: + query_expression = f"\n\t\tappendQuery(url.searchParams, {parameter_expression}.query);" + body_expression = f", {parameter_expression}.body" if body_type else "" + if is_event_stream(operation): + return ( + declaration, + "\t/** EventSource transport; intentionally not a JSON fetch Promise. */\n" + + f"\t{name}Url({signature.split(', options')[0]}): URL {{\n" + + f"\t\tconst url = new URL({path_expression}, this.baseUrl);" + + query_expression + + "\n\t\treturn url;\n\t}\n\n", + ) + return ( + declaration, + f"\tasync {name}({signature}): Promise<{result}> {{\n" + + f"\t\tconst url = new URL({path_expression}, this.baseUrl);" + + query_expression + + f"\n\t\treturn request<{result}>(\n" + + f"\t\t\turl, {json.dumps(method.upper())}, options{body_expression}\n" + + "\t\t);\n\t}\n\n", + ) + + +def generate(spec: dict[str, Any]) -> str: + schemas = spec.get("components", {}).get("schemas", {}) + type_lines = ["export interface Components {", " schemas: {"] + for name, schema in schemas.items(): + type_lines.append(f" {property_name(name)}: {schema_type(schema)};") + type_lines += [" };", "}", ""] + operation_declarations: list[str] = [] + operations: list[str] = [] + for path, path_item in spec.get("paths", {}).items(): + for method, operation in path_item.items(): + if method not in {"get", "post", "put", "patch", "delete"}: + continue + declaration, implementation = render_operation( + operation_name(operation["operationId"]), path, method, operation + ) + operation_declarations.append(declaration) + operations.append(implementation) + runtime = r"""export type RequestOptions = Omit; + +export type Problem = { + type: string; + title: string; + status: number; + detail: string; + instance: string; + code: string; +}; + +export class ApiError extends Error { + readonly status: number; + readonly problem?: Problem; + + constructor(status: number, problem?: Problem) { + super(problem?.detail ?? `Request failed with status ${status}.`); + this.name = "ApiError"; + this.status = status; + this.problem = problem; + } +} + +export function isApiError(error: unknown): error is ApiError { + return error instanceof ApiError; +} + +function appendQuery(search: URLSearchParams, query: Record | undefined): void { + if (!query) return; + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) continue; + for (const item of Array.isArray(value) ? value : [value]) search.append(key, String(item)); + } +} + +async function request( + url: URL, + method: string, + options: RequestOptions, + body?: unknown, +): Promise { + const headers = new Headers(options.headers); + if (body !== undefined && !headers.has("Content-Type")) { + headers.set("Content-Type", "application/json"); + } + const response = await fetch(url, { + ...options, + method, + headers, + credentials: options.credentials ?? "same-origin", + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (response.status === 204) return undefined as T; + const contentType = response.headers.get("content-type") ?? ""; + const isJson = contentType.includes("application/json") + || contentType.includes("application/problem+json"); + const payload: unknown = isJson ? await response.json() : undefined; + if (!response.ok) throw new ApiError(response.status, payload as Problem | undefined); + return payload as T; +} + +export class BackupToolClient { + constructor(readonly baseUrl = window.location.origin) {} + +""" + return ( + HEADER + + "\n".join(type_lines) + + "".join(operation_declarations) + + runtime + + "".join(operations) + + "}\n" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate TypeScript API client from OpenAPI.") + parser.add_argument("--input", type=Path, default=Path("openapi/v2.json")) + parser.add_argument("--output", type=Path, default=Path("frontend/src/api/generated/client.ts")) + parser.add_argument("--check", action="store_true", help="fail when generated output differs") + arguments = parser.parse_args() + rendered = generate(json.loads(arguments.input.read_text(encoding="utf-8"))) + if arguments.check: + if ( + not arguments.output.is_file() + or arguments.output.read_text(encoding="utf-8") != rendered + ): + print(f"Generated client drift: regenerate {arguments.output}") + return 1 + print(f"Generated client is current: {arguments.output}") + return 0 + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered, encoding="utf-8") + print(f"Generated API client: {arguments.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/generate_sbom.py b/tools/generate_sbom.py new file mode 100644 index 0000000..83ada7d --- /dev/null +++ b/tools/generate_sbom.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Generate a deterministic, dependency-only CycloneDX SBOM for the M14 OCI images.""" + +from __future__ import annotations + +import json +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUTPUT = ROOT / "docs" / "release" / "m14-sbom.json" + + +def component(name: str, version: str, ecosystem: str) -> dict[str, str]: + return { + "name": name, + "type": "library", + "version": version, + "purl": f"pkg:{ecosystem}/{name}@{version}", + } + + +def main() -> None: + try: + project = tomllib.loads((ROOT / "backend" / "pyproject.toml").read_text()) + package_lock = json.loads((ROOT / "frontend" / "package-lock.json").read_text()) + except (OSError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error: + raise RuntimeError("could not load pinned dependency metadata") from error + components: list[dict[str, str]] = [] + for dependency in project["project"]["dependencies"]: + match = re.fullmatch( + r"([A-Za-z0-9_.-]+)(?:\[[A-Za-z0-9_,-]+\])?==([A-Za-z0-9_.+-]+)", dependency + ) + if match is None: + raise ValueError(f"un-pinned Python dependency: {dependency}") + components.append(component(match.group(1), match.group(2), "pypi")) + for path, item in package_lock.get("packages", {}).items(): + if not path.startswith("node_modules/") or "version" not in item: + continue + components.append(component(path.removeprefix("node_modules/"), item["version"], "npm")) + payload = { + "bomFormat": "CycloneDX", + "components": sorted(components, key=lambda item: (item["purl"], item["version"])), + "metadata": { + "component": component("backup-tool", project["project"]["version"], "generic") + }, + "specVersion": "1.5", + "version": 1, + } + OUTPUT.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/run_scale_certification.py b/tools/run_scale_certification.py new file mode 100644 index 0000000..bcb131c --- /dev/null +++ b/tools/run_scale_certification.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Run reproducible synthetic v2.0 metadata-scale certification on the reference host.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import sqlite3 +import tempfile +import time +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--jobs", type=int, required=True) + parser.add_argument("--entries", type=int, required=True) + parser.add_argument("--backups", type=int, required=True) + parser.add_argument("--logical-bytes", type=int, default=10 * 1024**4) + parser.add_argument( + "--report", type=Path, default=Path("docs/release/m15-scale-report.json") + ) + args = parser.parse_args() + if min(args.jobs, args.entries, args.backups, args.logical_bytes) <= 0: + raise SystemExit("scale inputs must be positive") + + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="backup-tool-scale-") as directory: + database = Path(directory) / "scale.db" + connection = sqlite3.connect(database) + connection.executescript( + "CREATE TABLE jobs(id INTEGER PRIMARY KEY, name TEXT NOT NULL);" + "CREATE TABLE backups(" + "id INTEGER PRIMARY KEY, job_id INTEGER NOT NULL, " + "logical_bytes INTEGER NOT NULL, manifest_id TEXT NOT NULL);" + "CREATE INDEX backups_job_created ON backups(job_id, id DESC);" + ) + connection.executemany( + "INSERT INTO jobs(name) VALUES (?)", + ((f"job-{i}",) for i in range(args.jobs)), + ) + connection.commit() + write_start = time.monotonic() + batch = 10_000 + for first in range(0, args.backups, batch): + last = min(first + batch, args.backups) + connection.executemany( + "INSERT INTO backups(job_id, logical_bytes, manifest_id) VALUES (?, ?, ?)", + ( + ( + index % args.jobs + 1, + args.logical_bytes, + f"synthetic-{index:08d}", + ) + for index in range(first, last) + ), + ) + connection.commit() + write_seconds = time.monotonic() - write_start + page_start = time.monotonic() + rows = connection.execute( + "SELECT id FROM backups ORDER BY id DESC LIMIT 100 OFFSET 99_900" + ).fetchall() + page_seconds = time.monotonic() - page_start + backup_count = connection.execute("SELECT COUNT(*) FROM backups").fetchone()[0] + connection.close() + database_bytes = database.stat().st_size + + report = { + "reference_host": { + "platform": platform.platform(), + "python": platform.python_version(), + "cpus": os.cpu_count(), + }, + "method": ( + "synthetic metadata certification; logical bytes are sparse and no physical " + "10 TiB payload is allocated" + ), + "workload": { + "jobs": args.jobs, + "entries_declared": args.entries, + "backups": backup_count, + "logical_bytes_per_backup": args.logical_bytes, + }, + "results": { + "backup_insert_seconds": round(write_seconds, 3), + "pagination_seconds": round(page_seconds, 6), + "pagination_rows": len(rows), + "database_bytes": database_bytes, + "total_seconds": round(time.monotonic() - started, 3), + }, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"scale certification report: {args.report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())