feat(v2): complete v2 reimplementation

.dockerignore

@@ -22,6 +22,7 @@ wheels/
 
 # Virtual environments
 venv/
+.venv/
 env/
 ENV/
 
@@ -64,6 +65,9 @@ logs/
 
 # Tests
 tests/
+!tests/ssh-fixture/
+!tests/ssh-fixture/Dockerfile
+!tests/ssh-fixture/sshd_config
 .pytest_cache/
 .coverage
 

.gitignore

@@ -50,6 +50,7 @@ Thumbs.db
 # Frontend
 frontend/node_modules/
 frontend/dist/
+frontend/test-results/
 frontend/yarn.lock
 frontend/pnpm-lock.yaml
 

AGENTS.md

@@ -0,0 +1,25 @@
+# Agent Operating Instructions
+
+## Continuous milestone execution
+
+When the user asks to continue, work autonomously through the active milestone
+and its verification. Do **not** send progress-only, acknowledgement, empty, or
+status final responses. Reply only when:
+
+1. the user asks for status;
+2. a real product, security, credentials, or destructive-action decision needs
+   the user's input; or
+3. the requested milestone is fully implemented and verified.
+
+Use one foreground milestone batch where possible. If work must run in a
+background subagent, avoid notifying the user manually; inspect and verify the
+result before replying at the same completion boundary.
+
+A chat turn still necessarily ends after a model response. This file prevents
+unnecessary model-generated completion messages; it cannot suppress
+harness-generated tool or subagent notifications.
+
+## Verification
+
+Before claiming a milestone boundary, run `make check` and record focused
+acceptance evidence under `docs/release/`.

Dockerfile

@@ -0,0 +1,41 @@
+# syntax=docker/dockerfile:1
+# Pinned by manifest digest; update tag and digest together.
+FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS builder
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /build
+COPY backend/pyproject.toml ./pyproject.toml
+COPY backend/src ./src
+RUN python -m pip install --upgrade pip==25.3 && \
+    python -m pip wheel --wheel-dir /wheels .
+
+FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS runtime
+
+ENV PATH="/opt/venv/bin:${PATH}" \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    BACKUP_TOOL_ALEMBIC_ROOT=/app
+
+RUN groupadd --gid 10001 backup-tool && \
+    useradd --uid 10001 --gid backup-tool --create-home --home-dir /home/backup-tool \
+        --shell /usr/sbin/nologin backup-tool && \
+    python -m venv /opt/venv && \
+    install --directory --owner=backup-tool --group=backup-tool --mode=0750 \
+        /var/lib/backup-tool \
+        /var/lib/backup-tool/repositories \
+        /var/lib/backup-tool/restores \
+        /run/backup-tool
+
+COPY --from=builder /wheels /wheels
+RUN python -m pip install --no-index --find-links=/wheels backup-tool==2.0.0.dev0 && \
+    rm -rf /wheels
+COPY --chown=backup-tool:backup-tool backend/alembic.ini /app/alembic.ini
+COPY --chown=backup-tool:backup-tool backend/alembic /app/alembic
+
+WORKDIR /app
+USER backup-tool:backup-tool
+
+ENTRYPOINT ["backup-tool"]
+CMD ["web"]

Makefile

@@ -2,7 +2,7 @@ PYTHON ?= .venv/bin/python
 BOOTSTRAP_PYTHON ?= python3
 NPM ?= npm
 
-.PHONY: setup install test-fast test-integration test-security lint typecheck frontend-build check-v1-absent check
+.PHONY: setup install test-fast test-integration test-fault test-security test-e2e test-ssh-integration lint typecheck frontend-build check-v1-absent check-openapi check
 
 setup:
 	test -x $(PYTHON) || $(BOOTSTRAP_PYTHON) -m venv .venv
@@ -18,9 +18,18 @@ test-fast:
 test-integration:
 	$(PYTHON) -m pytest tests/integration -q
 
+test-fault:
+	$(PYTHON) -m pytest tests/fault -q
+
 test-security:
 	$(PYTHON) -m pytest tests/security -q
 
+test-e2e:
+	BACKUP_TOOL_COMPOSE_E2E=1 $(PYTHON) -m pytest tests/e2e/test_compose_v2.py -q
+
+test-ssh-integration:
+	BACKUP_TOOL_SSH_INTEGRATION=1 $(PYTHON) -m pytest tests/integration/test_ssh_backup_restore.py -q
+
 lint:
 	$(PYTHON) -m ruff check --config backend/pyproject.toml backend/src backend/alembic tests tools
 	$(PYTHON) -m ruff format --check --config backend/pyproject.toml backend/src backend/alembic tests tools
@@ -32,7 +41,12 @@ typecheck:
 frontend-build:
 	$(NPM) --prefix frontend run build
 
+check-openapi:
+	$(PYTHON) tools/export_openapi.py
+	$(NPM) --prefix frontend run api:generate
+	git diff --exit-code -- openapi/v2.json frontend/src/api/generated
+
 check-v1-absent:
 	$(PYTHON) tools/forbidden_v1_scan.py .
 
-check: check-v1-absent test-fast test-integration test-security lint typecheck frontend-build
+check: check-v1-absent check-openapi test-fast test-integration test-fault test-security lint typecheck frontend-build

README.md

@@ -4,7 +4,7 @@ A modern backup management application with a FastAPI backend and React frontend
 
 ## Features
 
-- **Multiple Source Types**: Local filesystem, SSH/SFTP, and database (PostgreSQL, MySQL)
+- **Source Type**: Local filesystem
 - **Backup Strategies**: Full and incremental backups
 - **Scheduled Backups**: Cron-based scheduling with APScheduler
 - **Retention Policies**: Count-based and days-based backup retention
@@ -22,7 +22,7 @@ backup-tool/
 │   │   ├── schemas.py   # Pydantic schemas
 │   │   └── main.py      # Application entry point
 │   ├── backup/          # Backup engine
-│   │   ├── adapters/    # Source adapters (local, SSH, database)
+│   │   ├── adapters/    # Local source adapter
 │   │   ├── engine.py    # Backup execution engine
 │   │   ├── scheduler.py # Job scheduler
 │   │   └── retention.py # Retention policies
@@ -51,9 +51,10 @@ docker compose --profile dev up -d
 ```
 
 Access the application:
-- Backend API: http://localhost:8000
+
-- Frontend: http://localhost:3000
+- Backend API: <http://localhost:8000>
-- API Docs: http://localhost:8000/docs
+- Frontend: <http://localhost:3000>
+- API Docs: <http://localhost:8000/docs>
 
 ### Option 2: Manual Setup
 
@@ -96,15 +97,16 @@ uvicorn app.main:app --host 0.0.0.0 --port 8000
 ## API Documentation
 
 Once the backend is running, visit:
-- Swagger UI: http://localhost:8000/docs
+
-- ReDoc: http://localhost:8000/redoc
+- Swagger UI: <http://localhost:8000/docs>
+- ReDoc: <http://localhost:8000/redoc>
 
 ## Configuration
 
 ### Environment Variables
 
 | Variable | Description | Default |
-|----------|-------------|---------|
+| ---------- | ------------- | --------- |
 | `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///./backup_tool.db` |
 | `CORS_ORIGINS` | Comma-separated allowed CORS origins | `http://localhost:3000` |
 | `SQL_ECHO` | Enable SQL query logging | `false` |
@@ -113,17 +115,20 @@ Once the backend is running, visit:
 ### Docker-Specific Configuration
 
 When running with Docker Compose, the following volumes are mounted:
+
 - `backup-data`: Persisted SQLite database at `/app/data`
 - `backup-storage`: Backup files at `/app/backups`
 
 ### Development vs Production
 
 **Development Mode** (`docker compose --profile dev up -d`):
+
 - Backend hot reload enabled
 - Frontend Vite dev server with HMR
 - Source code mounted as volumes
 
 **Production Mode** (`docker compose --profile prod up -d`):
+
 - Optimized frontend build served via nginx
 - Backend without reload
 - Static assets compiled
@@ -133,6 +138,7 @@ When running with Docker Compose, the following volumes are mounted:
 ### Docker Issues
 
 **Port already in use**
+
 ```bash
 # Check what's using port 8000
 lsof -i :8000
@@ -141,6 +147,7 @@ lsof -i :8000
 ```
 
 **Container fails to start**
+
 ```bash
 # Check logs
 docker logs backup-tool-backend
@@ -150,6 +157,7 @@ docker compose build --no-cache
 ```
 
 **Permission denied on data directory**
+
 ```bash
 # Fix permissions
 docker compose exec backend chown -R backup-tool:backup-tool /app/data
@@ -157,6 +165,7 @@ docker compose exec backend chown -R backup-tool:backup-tool /app/data
 
 **Tests fail in Docker**
 Tests require development dependencies. Install with:
+
 ```bash
 docker compose exec backend pip install -e ".[dev]"
 ```
@@ -165,11 +174,13 @@ docker compose exec backend pip install -e ".[dev]"
 
 **Python version incompatibility**
 Ensure Python 3.11+ is installed:
+
 ```bash
 python --version
 ```
 
 **Node modules conflicts**
+
 ```bash
 cd frontend
 rm -rf node_modules package-lock.json
@@ -182,7 +193,7 @@ npm install
 
 1. Clone the repository
 2. Run `docker compose --profile prod up -d`
-3. Access at http://localhost:3000
+3. Access at <http://localhost:3000>
 
 ### Manual Deployment
 

backend/alembic/versions/0004_repository_signing_keys.py

@@ -0,0 +1,29 @@
+"""bind repositories to manifest signing public keys
+
+Revision ID: 0004_repository_signing_keys
+Revises: 0003_execution_events
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0004_repository_signing_keys"
+down_revision = "0003_execution_events"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("repositories") as batch:
+        batch.add_column(
+            sa.Column("signing_key_id", sa.String(length=64), nullable=False, server_default="")
+        )
+        batch.add_column(
+            sa.Column("signing_public_key", sa.String(length=64), nullable=False, server_default="")
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("repositories") as batch:
+        batch.drop_column("signing_public_key")
+        batch.drop_column("signing_key_id")

backend/alembic/versions/0005_restore_dry_run.py

@@ -0,0 +1,23 @@
+"""persist restore dry-run intent
+
+Revision ID: 0005_restore_dry_run
+Revises: 0004_repository_signing_keys
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0005_restore_dry_run"
+down_revision = "0004_repository_signing_keys"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("restores") as batch:
+        batch.add_column(sa.Column("dry_run", sa.Boolean(), nullable=False, server_default="0"))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("restores") as batch:
+        batch.drop_column("dry_run")

backend/alembic/versions/0006_local_sources_only.py

@@ -0,0 +1,43 @@
+"""restrict persisted sources to local
+
+Revision ID: 0006_local_sources_only
+Revises: 0005_restore_dry_run
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0006_local_sources_only"
+down_revision = "0005_restore_dry_run"
+branch_labels = None
+depends_on = None
+
+
+def _reject_nonlocal_sources() -> None:
+    connection = op.get_bind()
+    sources = sa.table("sources", sa.column("kind"))
+    count = connection.scalar(
+        sa.select(sa.func.count()).select_from(sources).where(sources.c.kind != "local")
+    )
+    if count is None:
+        raise RuntimeError("Cannot inspect persisted source kinds before migration.")
+    if count:
+        raise RuntimeError(
+            "Cannot restrict sources to local: "
+            f"found {count} non-local source row(s). Remove or migrate them before upgrading."
+        )
+
+
+def upgrade() -> None:
+    _reject_nonlocal_sources()
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(
+            op.f("ck_sources_kind"), "kind IN ('local','sftp','postgresql','mysql')"
+        )

backend/alembic/versions/0007_repository_data_key_epochs.py

@@ -0,0 +1,87 @@
+"""add repository data key epochs
+
+Revision ID: 0007_repository_data_key_epochs
+Revises: 0006_local_sources_only
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0007_repository_data_key_epochs"
+down_revision = "0006_local_sources_only"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("repositories", sa.Column("active_data_key_id", sa.String(36), nullable=True))
+    op.add_column("backups", sa.Column("data_key_id", sa.String(36), nullable=True))
+    op.create_table(
+        "repository_data_key_epochs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "repository_id",
+            sa.String(36),
+            sa.ForeignKey("repositories.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("key_id", sa.String(36), nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("retired_at", sa.DateTime(timezone=True)),
+        sa.UniqueConstraint(
+            "repository_id", "key_id", name="uq_repository_data_key_epochs_repository_key_epoch"
+        ),
+        sa.CheckConstraint(
+            "state IN ('active','retired')",
+            name="ck_repository_data_key_epochs_repository_data_key_epoch_state",
+        ),
+    )
+    op.create_index(
+        "ix_repository_data_key_epochs_repository_id",
+        "repository_data_key_epochs",
+        ["repository_id"],
+    )
+    op.create_index(
+        "uq_repository_data_key_epochs_active",
+        "repository_data_key_epochs",
+        ["repository_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'active'"),
+    )
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    epochs = sa.table("repository_data_key_epochs")
+    repositories = sa.table("repositories", sa.column("active_data_key_id"))
+    backups = sa.table("backups", sa.column("data_key_id"))
+    epoch_count = connection.scalar(sa.select(sa.func.count()).select_from(epochs))
+    active_key_count = connection.scalar(
+        sa.select(sa.func.count())
+        .select_from(repositories)
+        .where(repositories.c.active_data_key_id.is_not(None))
+    )
+    backup_key_count = connection.scalar(
+        sa.select(sa.func.count()).select_from(backups).where(backups.c.data_key_id.is_not(None))
+    )
+    if epoch_count or active_key_count or backup_key_count:
+        raise RuntimeError("cannot downgrade while repository data key metadata exists")
+    op.drop_index("uq_repository_data_key_epochs_active", table_name="repository_data_key_epochs")
+    op.drop_index(
+        "ix_repository_data_key_epochs_repository_id", table_name="repository_data_key_epochs"
+    )
+    op.drop_table("repository_data_key_epochs")
+    op.drop_column("backups", "data_key_id")
+    op.drop_column("repositories", "active_data_key_id")

backend/alembic/versions/0008_notification_outbox.py

@@ -0,0 +1,445 @@
+"""replace notification attempt stub with a durable M12 outbox
+
+Revision ID: 0008_notification_outbox
+Revises: 0007_repository_data_key_epochs
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+import sqlalchemy as sa
+from alembic import op
+from backup_tool.ids import new_uuid7
+
+revision = "0008_notification_outbox"
+down_revision = "0007_repository_data_key_epochs"
+branch_labels = None
+depends_on = None
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _parameterized_execute(connection: sa.Connection, statement: object) -> Any:
+    """Execute only SQLAlchemy Core statements, never dynamic SQL strings."""
+    database = connection.execution_options()
+    return database.execute(statement)  # type: ignore[arg-type]
+
+
+def upgrade() -> None:
+    connection = op.get_bind()
+    # SQLite batch-rebuilds notification_subscriptions. Its old delivery table
+    # references this table, so enforcement must be suspended for this migration
+    # only while the legacy rows are copied into the replacement outbox shape.
+    if connection.dialect.name == "sqlite":
+        connection.exec_driver_sql("PRAGMA foreign_keys=OFF")
+    # Extend the existing subscription rows first: pre-M12 rows stay disabled until
+    # an operator explicitly configures a new credential/key.
+    with op.batch_alter_table("notification_subscriptions") as batch:
+        batch.add_column(
+            sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60")
+        )
+        batch.add_column(sa.Column("rate_tokens", sa.Float(), nullable=False, server_default="60"))
+        batch.add_column(sa.Column("rate_updated_at", sa.DateTime(timezone=True), nullable=True))
+        batch.add_column(sa.Column("revision", sa.Integer(), nullable=False, server_default="1"))
+        batch.create_check_constraint("rate_positive", "rate_limit_per_minute > 0")
+        batch.create_check_constraint("rate_tokens_nonnegative", "rate_tokens >= 0")
+        batch.create_check_constraint("revision_positive", "revision > 0")
+
+    op.create_table(
+        "notification_events",
+        sa.Column("type", sa.String(96), nullable=False),
+        sa.Column("schema_version", sa.Integer(), nullable=False, server_default="1"),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("correlation_id", sa.String(36), nullable=False),
+        sa.Column("severity", sa.String(16), nullable=False),
+        sa.Column("resource_refs", sa.JSON(), nullable=False),
+        sa.Column("payload", sa.JSON(), nullable=False),
+        sa.Column("canonical_envelope", sa.Text(), nullable=False),
+        sa.Column("deduplication_key", sa.String(255), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.CheckConstraint("schema_version = 1", name="ck_notification_events_schema_version"),
+        sa.CheckConstraint(
+            "severity IN ('info','warning','error','security')",
+            name="ck_notification_events_severity",
+        ),
+        sa.UniqueConstraint("deduplication_key", name="uq_notification_events_deduplication_key"),
+    )
+    op.create_index(
+        "ix_notification_events_type_occurred", "notification_events", ["type", "occurred_at"]
+    )
+
+    # Preserve unexpected rows from the unused baseline shape.  Renaming first
+    # keeps the original data intact if an upgrade is interrupted before copy.
+    op.rename_table("notification_deliveries", "notification_deliveries_legacy")
+    op.create_table(
+        "notification_deliveries",
+        sa.Column(
+            "event_id",
+            sa.String(36),
+            sa.ForeignKey("notification_events.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "subscription_id",
+            sa.String(36),
+            sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("state", sa.String(32), nullable=False, server_default="pending"),
+        sa.Column("due_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_owner", sa.String(255), nullable=True),
+        sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("terminal_reason", sa.String(96), nullable=True),
+        sa.Column("response_class", sa.String(64), nullable=True),
+        sa.Column("response_summary", sa.String(512), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint(
+            "attempt_count >= 0", name="ck_notification_deliveries_attempt_count_nonnegative"
+        ),
+        sa.CheckConstraint(
+            "state IN ('pending','leased','delivered','retry','failed')",
+            name="ck_notification_deliveries_state",
+        ),
+        sa.UniqueConstraint(
+            "event_id", "subscription_id", name="uq_notification_deliveries_event_subscription"
+        ),
+    )
+    op.create_index(
+        "ix_notification_deliveries_due", "notification_deliveries", ["state", "due_at"]
+    )
+    op.create_index(
+        "ix_notification_deliveries_lease", "notification_deliveries", ["state", "lease_expires_at"]
+    )
+    op.create_table(
+        "notification_delivery_attempts",
+        sa.Column(
+            "delivery_id",
+            sa.String(36),
+            sa.ForeignKey("notification_deliveries.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("number", sa.Integer(), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("outcome", sa.String(32), nullable=False, server_default="started"),
+        sa.Column("response_class", sa.String(64), nullable=True),
+        sa.Column("diagnostic", sa.String(512), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.CheckConstraint("number > 0", name="ck_notification_delivery_attempts_number_positive"),
+        sa.CheckConstraint(
+            "outcome IN ('started','delivered','retry','failed')",
+            name="ck_notification_delivery_attempts_outcome",
+        ),
+        sa.UniqueConstraint(
+            "delivery_id", "number", name="uq_notification_delivery_attempts_delivery_number"
+        ),
+    )
+    op.create_index(
+        "ix_notification_attempts_delivery",
+        "notification_delivery_attempts",
+        ["delivery_id", "number"],
+    )
+
+    legacy = sa.table(
+        "notification_deliveries_legacy",
+        sa.column("id"),
+        sa.column("event_id"),
+        sa.column("subscription_id"),
+        sa.column("attempt"),
+        sa.column("state"),
+        sa.column("response_class"),
+        sa.column("next_attempt_at"),
+        sa.column("created_at"),
+        sa.column("updated_at"),
+    )
+    rows = _parameterized_execute(connection, sa.select(legacy)).mappings().all()
+    events = sa.table(
+        "notification_events",
+        sa.column("id"),
+        sa.column("type"),
+        sa.column("schema_version"),
+        sa.column("occurred_at"),
+        sa.column("correlation_id"),
+        sa.column("severity"),
+        sa.column("resource_refs", sa.JSON()),
+        sa.column("payload", sa.JSON()),
+        sa.column("canonical_envelope"),
+        sa.column("deduplication_key"),
+    )
+    deliveries = sa.table(
+        "notification_deliveries",
+        *[
+            sa.column(name)
+            for name in (
+                "id",
+                "event_id",
+                "subscription_id",
+                "state",
+                "due_at",
+                "attempt_count",
+                "terminal_reason",
+                "response_class",
+                "response_summary",
+                "created_at",
+                "updated_at",
+            )
+        ],
+    )
+    attempts = sa.table(
+        "notification_delivery_attempts",
+        *[
+            sa.column(name)
+            for name in (
+                "id",
+                "delivery_id",
+                "number",
+                "started_at",
+                "completed_at",
+                "outcome",
+                "response_class",
+                "diagnostic",
+            )
+        ],
+    )
+    for row in rows:
+        occurred = row["created_at"] or _now()
+        event_id, delivery_id, attempt_id = (str(new_uuid7()), str(new_uuid7()), str(new_uuid7()))
+        payload = {"legacy_event_id": str(row["event_id"]), "legacy_delivery_id": str(row["id"])}
+        envelope = {
+            "event_schema_version": 1,
+            "id": event_id,
+            "type": "notification.legacy",
+            "occurred_at": occurred.isoformat()
+            if hasattr(occurred, "isoformat")
+            else str(occurred),
+            "correlation_id": event_id,
+            "severity": "warning",
+            "resource": {"subscription_id": str(row["subscription_id"])},
+            "payload": payload,
+        }
+        _parameterized_execute(
+            connection,
+            events.insert().values(
+                id=event_id,
+                type="notification.legacy",
+                schema_version=1,
+                occurred_at=occurred,
+                correlation_id=event_id,
+                severity="warning",
+                resource_refs=envelope["resource"],
+                payload=payload,
+                canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
+                deduplication_key=f"legacy:{row['id']}",
+            ),
+        )
+        old_state = str(row["state"])
+        new_state = "retry" if old_state == "pending" else old_state
+        try:
+            legacy_attempt = max(1, int(row["attempt"]))
+        except (TypeError, ValueError) as error:
+            raise RuntimeError("legacy notification delivery has an invalid attempt") from error
+        _parameterized_execute(
+            connection,
+            deliveries.insert().values(
+                id=delivery_id,
+                event_id=event_id,
+                subscription_id=row["subscription_id"],
+                state=new_state,
+                due_at=row["next_attempt_at"] or occurred,
+                attempt_count=legacy_attempt,
+                terminal_reason="legacy_migrated" if new_state == "failed" else None,
+                response_class=row["response_class"],
+                response_summary="legacy delivery migrated",
+                created_at=occurred,
+                updated_at=row["updated_at"] or occurred,
+            ),
+        )
+        _parameterized_execute(
+            connection,
+            attempts.insert().values(
+                id=attempt_id,
+                delivery_id=delivery_id,
+                number=legacy_attempt,
+                started_at=occurred,
+                completed_at=row["updated_at"] if new_state in {"delivered", "failed"} else None,
+                outcome="delivered"
+                if new_state == "delivered"
+                else ("failed" if new_state == "failed" else "retry"),
+                response_class=row["response_class"],
+                diagnostic="legacy delivery migrated",
+            ),
+        )
+    op.drop_table("notification_deliveries_legacy")
+
+    op.create_table(
+        "notification_signing_keys",
+        sa.Column(
+            "subscription_id",
+            sa.String(36),
+            sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("version", sa.Integer(), nullable=False),
+        sa.Column(
+            "secret_id",
+            sa.String(36),
+            sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("state", sa.String(16), nullable=False, server_default="active"),
+        sa.Column("overlap_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint("version > 0", name="ck_notification_signing_keys_version_positive"),
+        sa.CheckConstraint(
+            "state IN ('active','overlap','retired')", name="ck_notification_signing_keys_state"
+        ),
+        sa.UniqueConstraint(
+            "subscription_id", "version", name="uq_notification_signing_keys_subscription_version"
+        ),
+    )
+    op.create_index(
+        "ix_notification_signing_keys_subscription",
+        "notification_signing_keys",
+        ["subscription_id", "state"],
+    )
+    op.create_index(
+        "uq_notification_signing_keys_active",
+        "notification_signing_keys",
+        ["subscription_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'active'"),
+    )
+    op.create_index(
+        "uq_notification_signing_keys_overlap",
+        "notification_signing_keys",
+        ["subscription_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'overlap'"),
+    )
+    op.create_table(
+        "notification_email_settings",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("host", sa.String(255), nullable=False),
+        sa.Column("port", sa.Integer(), nullable=False, server_default="587"),
+        sa.Column("username", sa.String(255), nullable=False),
+        sa.Column(
+            "password_secret_id",
+            sa.String(36),
+            sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("sender", sa.String(320), nullable=False),
+        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="5"),
+        sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60"),
+        sa.CheckConstraint("id = 1", name="ck_notification_email_settings_singleton"),
+        sa.CheckConstraint("port > 0 AND port < 65536", name="ck_notification_email_settings_port"),
+        sa.CheckConstraint("max_attempts > 0", name="ck_notification_email_settings_max_attempts"),
+        sa.CheckConstraint(
+            "rate_limit_per_minute > 0", name="ck_notification_email_settings_rate_positive"
+        ),
+    )
+    if connection.dialect.name == "sqlite":
+        connection.exec_driver_sql("PRAGMA foreign_keys=ON")
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    for table in (
+        "notification_events",
+        "notification_delivery_attempts",
+        "notification_signing_keys",
+        "notification_email_settings",
+    ):
+        if bind.scalar(sa.text(f"SELECT count(*) FROM {table}")):
+            raise RuntimeError(
+                "cannot downgrade while M12 notification history or configuration exists"
+            )
+    # A fresh M12 schema can safely return to the historical stub shape.
+    op.drop_table("notification_email_settings")
+    op.drop_index("uq_notification_signing_keys_overlap", table_name="notification_signing_keys")
+    op.drop_index("uq_notification_signing_keys_active", table_name="notification_signing_keys")
+    op.drop_index(
+        "ix_notification_signing_keys_subscription", table_name="notification_signing_keys"
+    )
+    op.drop_table("notification_signing_keys")
+    op.drop_index("ix_notification_attempts_delivery", table_name="notification_delivery_attempts")
+    op.drop_table("notification_delivery_attempts")
+    op.drop_index("ix_notification_deliveries_lease", table_name="notification_deliveries")
+    op.drop_index("ix_notification_deliveries_due", table_name="notification_deliveries")
+    op.drop_table("notification_deliveries")
+    op.drop_index("ix_notification_events_type_occurred", table_name="notification_events")
+    op.drop_table("notification_events")
+    op.create_table(
+        "notification_deliveries",
+        sa.Column("event_id", sa.String(36), nullable=False),
+        sa.Column("subscription_id", sa.String(36), nullable=False),
+        sa.Column("attempt", sa.Integer(), nullable=False),
+        sa.Column("state", sa.String(32), nullable=False),
+        sa.Column("response_class", sa.String(64)),
+        sa.Column("next_attempt_at", sa.DateTime(timezone=True)),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint("attempt > 0", name="ck_notification_deliveries_attempt_positive"),
+        sa.CheckConstraint(
+            "state IN ('pending','delivered','retry','failed')",
+            name="ck_notification_deliveries_state",
+        ),
+        sa.ForeignKeyConstraint(
+            ["subscription_id"], ["notification_subscriptions.id"], ondelete="RESTRICT"
+        ),
+        sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
+    )
+    op.create_index(
+        "ix_notification_deliveries_state_next",
+        "notification_deliveries",
+        ["state", "next_attempt_at"],
+    )
+    with op.batch_alter_table("notification_subscriptions") as batch:
+        batch.drop_constraint("revision_positive", type_="check")
+        batch.drop_constraint("rate_tokens_nonnegative", type_="check")
+        batch.drop_constraint("rate_positive", type_="check")
+        batch.drop_column("revision")
+        batch.drop_column("rate_updated_at")
+        batch.drop_column("rate_tokens")
+        batch.drop_column("rate_limit_per_minute")

backend/alembic/versions/0009_allow_ssh_sources.py

@@ -0,0 +1,43 @@
+"""allow staged SSH source definitions
+
+Revision ID: 0009_allow_ssh_sources
+Revises: 0008_notification_outbox
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0009_allow_ssh_sources"
+down_revision = "0008_notification_outbox"
+branch_labels = None
+depends_on = None
+
+
+def _reject_ssh_sources() -> None:
+    connection = op.get_bind()
+    sources = sa.table("sources", sa.column("kind"))
+    count = connection.scalar(
+        sa.select(sa.func.count()).select_from(sources).where(sources.c.kind == "ssh")
+    )
+    if count is None:
+        raise RuntimeError("Cannot inspect persisted SSH sources before migration.")
+    if count:
+        raise RuntimeError(
+            "Cannot restrict sources to local: "
+            f"found {count} SSH source row(s). Remove them before downgrading."
+        )
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind IN ('local','ssh')")
+
+
+def downgrade() -> None:
+    _reject_ssh_sources()
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")

backend/pyproject.toml

@@ -18,9 +18,9 @@ dependencies = [
   "cryptography==49.0.0",
   "fastapi==0.136.1",
   "httpx==0.28.1",
-  "paramiko==5.0.0",
   "pydantic==2.13.4",
   "pydantic-settings==2.14.2",
+  "paramiko==5.0.0",
   "sqlalchemy[asyncio]==2.0.49",
   "uvicorn[standard]==0.51.0",
 ]

backend/src/backup_tool/adapters.py

@@ -1,21 +1,43 @@
 from __future__ import annotations
 
+import os
+import stat
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Protocol
 
 from backup_tool.config import Settings
 
 
 class SourceError(ValueError):
-    pass
+    """A redacted source failure with a stable execution reason."""
+
+    def __init__(self, message: str, *, reason_code: str = "source_invalid") -> None:
+        super().__init__(message)
+        self.reason_code = reason_code
 
 
 @dataclass(frozen=True)
 class Entry:
     path: str
     kind: str
-    size: int | None = None
+    size: int
+    mode: int
+    mtime_ns: int
+    link_target: str | None = None
+
+
+class SourceReader(Protocol):
+    def validate_config(self) -> None: ...
+
+    async def probe(self) -> dict[str, int]: ...
+
+    def enumerate_entries(self) -> AsyncIterator[Entry]: ...
+
+    def open_content(self, path: str) -> AsyncIterator[bytes]: ...
+
+    async def close(self) -> None: ...
 
 
 class LocalAdapter:
@@ -36,22 +58,59 @@ class LocalAdapter:
 
     async def enumerate_entries(self) -> AsyncIterator[Entry]:
         self.validate_config()
-        for item in self.root.rglob("*"):
+        for item in sorted(self.root.rglob("*"), key=lambda candidate: candidate.as_posix()):
             relative = item.relative_to(self.root).as_posix()
-            if item.is_symlink():
+            metadata = item.lstat()
-                yield Entry(relative, "symlink")
+            if stat.S_ISLNK(metadata.st_mode):
-            elif item.is_file():
+                target = os.readlink(item)
-                yield Entry(relative, "file", item.stat().st_size)
+                target_path = Path(target)
-            elif item.is_dir():
+                if (
-                yield Entry(relative, "directory")
+                    target_path.is_absolute()
+                    or "\\" in target
+                    or ".." in target_path.parts
+                    or not target
+                ):
+                    raise SourceError(f"symlink target for {relative!r} is unsafe")
+                yield Entry(
+                    relative,
+                    "symlink",
+                    0,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                    target,
+                )
+            elif stat.S_ISREG(metadata.st_mode):
+                yield Entry(
+                    relative,
+                    "file",
+                    metadata.st_size,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                )
+            elif stat.S_ISDIR(metadata.st_mode):
+                yield Entry(
+                    relative,
+                    "directory",
+                    0,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                )
+            else:
+                raise SourceError(f"local source entry {relative!r} has an unsupported type")
+
+    async def close(self) -> None:
+        return None
 
     async def open_content(self, path: str) -> AsyncIterator[bytes]:
-        candidate = (self.root / path).resolve()
+        requested = Path(path)
-        if (
+        if not path or requested.is_absolute() or "\\" in path or ".." in requested.parts:
-            not candidate.is_relative_to(self.root)
+            raise SourceError("invalid local source entry")
-            or not candidate.is_file()
+        candidate = self.root / requested
-            or candidate.is_symlink()
+        metadata = candidate.lstat()
-        ):
+        if not stat.S_ISREG(metadata.st_mode) or candidate.is_symlink():
+            raise SourceError("invalid local source entry")
+        resolved = candidate.resolve()
+        if not resolved.is_relative_to(self.root):
             raise SourceError("invalid local source entry")
         with candidate.open("rb") as handle:
             while chunk := handle.read(1024 * 1024):

backend/src/backup_tool/cli.py

@@ -4,16 +4,62 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import base64
+import binascii
+import json
+import os
+import socket
 from collections.abc import Callable, Sequence
+from contextlib import suppress
+from datetime import UTC, datetime
 from pathlib import Path
+from typing import Any
 
 from alembic import command
 from alembic.config import Config
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from backup_tool import __version__
 from backup_tool.config import Settings
 
 from .db.engine import assert_schema_current, create_engine
+from .db.models import (
+    Backup,
+    Base,
+    Execution,
+    Job,
+    Repository,
+    RepositoryDataKeyEpoch,
+    Source,
+)
+from .repository import (
+    begin_key_rotation,
+    finish_key_rotation,
+    inspect_repository,
+    install_signing_key,
+    load_signing_key,
+    reconcile_key_rotations,
+    replace_active_data_key,
+)
+from .scheduler import run_scheduler
+from .security.recovery_bundle import (
+    RecoveryBundleError,
+    decrypt_bundle,
+    encrypt_bundle,
+    read_bundle_file,
+    read_passphrase_fd,
+    write_bundle_exclusive,
+)
+from .security.repository_crypto import (
+    RepositoryKeyError,
+    create_data_key,
+    install_data_key,
+    load_data_key,
+)
+from .web import run_web
 from .worker import run_worker
 
 RoleHandler = Callable[[Settings], int]
@@ -25,8 +71,8 @@ def _placeholder_role(_settings: Settings) -> int:
 
 
 ROLE_HANDLERS: dict[str, RoleHandler] = {
-    "web": _placeholder_role,
+    "web": run_web,
-    "scheduler": _placeholder_role,
+    "scheduler": run_scheduler,
     "worker": run_worker,
     "admin": _placeholder_role,
 }
@@ -37,14 +83,34 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--version", action="version", version=__version__)
     subparsers = parser.add_subparsers(dest="role", required=True)
     for role in DATABASE_ROLES:
-        subparsers.add_parser(role, help=f"run the {role} role")
+        role_parser = subparsers.add_parser(role, help=f"run the {role} role")
+        if role == "admin":
+            admin_parsers = role_parser.add_subparsers(dest="admin_command")
+            repository_key = admin_parsers.add_parser("repository-key")
+            repository_key_parsers = repository_key.add_subparsers(dest="repository_key_command")
+            rotate = repository_key_parsers.add_parser("rotate")
+            rotate.add_argument("--repository-id", required=True)
+            recovery = admin_parsers.add_parser("recovery")
+            recovery_parsers = recovery.add_subparsers(dest="recovery_command", required=True)
+            export = recovery_parsers.add_parser("export")
+            export.add_argument("--output", required=True, type=Path)
+            export.add_argument("--passphrase-fd", required=True, type=int)
+            validate = recovery_parsers.add_parser("validate")
+            validate.add_argument("--input", required=True, type=Path)
+            validate.add_argument("--passphrase-fd", required=True, type=int)
+            import_ = recovery_parsers.add_parser("import")
+            import_.add_argument("--input", required=True, type=Path)
+            import_.add_argument("--passphrase-fd", required=True, type=int)
     migrate = subparsers.add_parser("migrate", help="manage the metadata schema")
     migrate.add_argument("action", choices=("upgrade", "downgrade", "current"))
+    health = subparsers.add_parser("health", help="check runtime role readiness")
+    health.add_argument("health_role", choices=("web", "scheduler", "worker"))
     return parser
 
 
 def build_alembic_config(settings: Settings) -> Config:
-    backend_root = Path(__file__).resolve().parents[2]
+    default_root = Path(__file__).resolve().parents[2]
+    backend_root = Path(os.environ.get("BACKUP_TOOL_ALEMBIC_ROOT", default_root))
     migration = Config(backend_root / "alembic.ini")
     migration.set_main_option("sqlalchemy.url", settings.database_url)
     return migration
@@ -61,6 +127,24 @@ def require_current_schema(settings: Settings) -> None:
     asyncio.run(check())
 
 
+def run_role_healthcheck(settings: Settings, role: str) -> int:
+    """Verify the selected isolated runtime role is ready to serve work."""
+    if role != "web":
+        from backup_tool.observability.health import check_role_readiness
+
+        asyncio.run(check_role_readiness(settings, role))
+        return 0
+    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+        client.settimeout(2)
+        client.connect(str(settings.web_socket_path))
+        request = b"GET /readyz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
+        client.sendall(request)
+        status_line = client.recv(64).split(b"\r\n", 1)[0]
+    if status_line != b"HTTP/1.1 200 OK":
+        raise RuntimeError("web role is not ready")
+    return 0
+
+
 def run_migration(action: str, settings: Settings) -> int:
     migration = build_alembic_config(settings)
     if action == "upgrade":
@@ -76,12 +160,792 @@ def load_settings() -> Settings:
     return Settings()  # type: ignore[call-arg]
 
 
+async def rotate_repository_key(
+    settings: Settings,
+    repository_id: str,
+    *,
+    after_database_commit: Callable[[], None] | None = None,
+) -> dict[str, str]:
+    """Journal a key rotation so a crash always has a deterministic recovery path."""
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    new_key_path: Path | None = None
+    database_committed = False
+    journal_started = False
+    root: Path | None = None
+    old_key_id = ""
+    new_key_id = ""
+    try:
+        async with sessions() as db:
+            await reconcile_key_rotations(settings, db)
+            await db.rollback()
+            async with db.begin():
+                repository = await db.scalar(
+                    select(Repository).where(Repository.id == repository_id).with_for_update()
+                )
+                if repository is None:
+                    raise ValueError("repository was not found")
+                if repository.encryption != "aes-256-gcm" or repository.active_data_key_id is None:
+                    raise ValueError("repository encryption is not enabled")
+                inspected = inspect_repository(settings, Path(repository.root))
+                root = inspected.root
+                if (
+                    inspected.encryption != "aes-256-gcm"
+                    or inspected.data_key_id != repository.active_data_key_id
+                ):
+                    raise ValueError("repository encryption metadata is invalid")
+                epochs = list(
+                    (
+                        await db.scalars(
+                            select(RepositoryDataKeyEpoch).where(
+                                RepositoryDataKeyEpoch.repository_id == repository.id
+                            )
+                        )
+                    ).all()
+                )
+                active = [epoch for epoch in epochs if epoch.state == "active"]
+                if len(active) != 1 or active[0].key_id != repository.active_data_key_id:
+                    raise ValueError("repository key epochs are invalid")
+                old_key_id = active[0].key_id
+                new_key_id, new_key_path = create_data_key(settings, inspected.repository_id)
+                begin_key_rotation(
+                    inspected.root,
+                    repository.id,
+                    inspected.repository_id,
+                    old_key_id,
+                    new_key_id,
+                )
+                journal_started = True
+                active[0].state = "retired"
+                active[0].retired_at = datetime.now(UTC)
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=new_key_id,
+                        state="active",
+                    )
+                )
+                repository.active_data_key_id = new_key_id
+            database_committed = True
+            if after_database_commit is not None:
+                after_database_commit()
+            if root is None:
+                raise ValueError("repository rotation metadata is invalid")
+            replace_active_data_key(root, old_key_id, new_key_id)
+            finish_key_rotation(root)
+            journal_started = False
+            return {
+                "repository_id": repository_id,
+                "retired_key_id": old_key_id,
+                "active_key_id": new_key_id,
+            }
+    finally:
+        if not database_committed:
+            if journal_started and root is not None:
+                with suppress(ValueError):
+                    finish_key_rotation(root)
+            if new_key_path is not None:
+                new_key_path.unlink(missing_ok=True)
+        await engine.dispose()
+
+
+async def recovery_export_payload(settings: Settings) -> dict[str, Any]:
+    """Collect the current repository trust catalog and offline key material."""
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    try:
+        async with sessions() as db:
+            await reconcile_key_rotations(settings, db)
+            await db.rollback()
+            repositories = list(
+                (await db.scalars(select(Repository).order_by(Repository.id))).all()
+            )
+            catalog_repositories: list[dict[str, Any]] = []
+            key_records: list[dict[str, Any]] = []
+            for repository in repositories:
+                inspected = inspect_repository(settings, Path(repository.root))
+                if inspected.encryption != repository.encryption or (
+                    repository.encryption == "aes-256-gcm"
+                    and repository.active_data_key_id != inspected.data_key_id
+                ):
+                    raise RecoveryBundleError("recovery bundle export is unavailable")
+                private_key = load_signing_key(
+                    settings,
+                    inspected.repository_id,
+                    repository.signing_key_id,
+                    repository.signing_public_key,
+                )
+                epochs = list(
+                    (
+                        await db.scalars(
+                            select(RepositoryDataKeyEpoch)
+                            .where(RepositoryDataKeyEpoch.repository_id == repository.id)
+                            .order_by(RepositoryDataKeyEpoch.key_id)
+                        )
+                    ).all()
+                )
+                active = [epoch for epoch in epochs if epoch.state == "active"]
+                if repository.encryption == "aes-256-gcm":
+                    if (
+                        repository.active_data_key_id is None
+                        or len(active) != 1
+                        or active[0].key_id != repository.active_data_key_id
+                    ):
+                        raise RecoveryBundleError("recovery bundle export is unavailable")
+                elif epochs:
+                    raise RecoveryBundleError("recovery bundle export is unavailable")
+                data_keys: list[dict[str, str]] = []
+                for epoch in epochs:
+                    data_keys.append(
+                        {
+                            "key_id": epoch.key_id,
+                            "key": base64.b64encode(
+                                load_data_key(settings, inspected.repository_id, epoch.key_id)
+                            ).decode("ascii"),
+                        }
+                    )
+                catalog_repositories.append(
+                    {
+                        "active_data_key_id": repository.active_data_key_id,
+                        "compression": repository.compression,
+                        "data_key_epochs": [
+                            {
+                                "key_id": epoch.key_id,
+                                "retired_at": (
+                                    epoch.retired_at.isoformat()
+                                    if epoch.retired_at is not None
+                                    else None
+                                ),
+                                "state": epoch.state,
+                            }
+                            for epoch in epochs
+                        ],
+                        "database_id": repository.id,
+                        "encryption": repository.encryption,
+                        "format_version": repository.format_version,
+                        "name": repository.name,
+                        "repository_id": inspected.repository_id,
+                        "root": repository.root,
+                        "signing_key_id": repository.signing_key_id,
+                        "signing_public_key": repository.signing_public_key,
+                        "state": repository.state,
+                    }
+                )
+                key_records.append(
+                    {
+                        "data_keys": data_keys,
+                        "repository_id": inspected.repository_id,
+                        "signing_private_key": base64.b64encode(
+                            private_key.private_bytes(
+                                serialization.Encoding.Raw,
+                                serialization.PrivateFormat.Raw,
+                                serialization.NoEncryption(),
+                            )
+                        ).decode("ascii"),
+                    }
+                )
+            sources = list((await db.scalars(select(Source).order_by(Source.id))).all())
+            jobs = list((await db.scalars(select(Job).order_by(Job.id))).all())
+            executions = list((await db.scalars(select(Execution).order_by(Execution.id))).all())
+            backups = list((await db.scalars(select(Backup).order_by(Backup.id))).all())
+            if any(source.secret_refs for source in sources):
+                raise RecoveryBundleError("recovery bundle export is unavailable")
+            return {
+                "catalog": {
+                    "backups": [
+                        {
+                            "created_at": backup.created_at.isoformat(),
+                            "data_key_id": backup.data_key_id,
+                            "execution_id": backup.execution_id,
+                            "id": backup.id,
+                            "integrity": backup.integrity,
+                            "logical_bytes": backup.logical_bytes,
+                            "manifest_digest": backup.manifest_digest,
+                            "manifest_id": backup.manifest_id,
+                            "parent_backup_id": backup.parent_backup_id,
+                            "pinned": backup.pinned,
+                            "stored_bytes": backup.stored_bytes,
+                            "tombstoned_at": (
+                                backup.tombstoned_at.isoformat()
+                                if backup.tombstoned_at is not None
+                                else None
+                            ),
+                        }
+                        for backup in backups
+                    ],
+                    "executions": [
+                        {
+                            "attempt": execution.attempt,
+                            "id": execution.id,
+                            "job_id": execution.job_id,
+                            "progress": execution.progress,
+                            "revision": execution.revision,
+                            "trigger": execution.trigger,
+                        }
+                        for execution in executions
+                    ],
+                    "format": "backup-tool-recovery-catalog",
+                    "jobs": [
+                        {
+                            "allow_empty": job.allow_empty,
+                            "exclusions": job.exclusions,
+                            "id": job.id,
+                            "name": job.name,
+                            "repository_id": job.repository_id,
+                            "requested_mode": job.requested_mode,
+                            "retention": job.retention,
+                            "source_id": job.source_id,
+                        }
+                        for job in jobs
+                    ],
+                    "repositories": catalog_repositories,
+                    "sources": [
+                        {
+                            "id": source.id,
+                            "kind": source.kind,
+                            "name": source.name,
+                            "public_config": source.public_config,
+                        }
+                        for source in sources
+                    ],
+                    "version": 2,
+                },
+                "keys": key_records,
+            }
+    except (OSError, RepositoryKeyError, ValueError) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise
+        raise RecoveryBundleError("recovery bundle export is unavailable") from error
+    finally:
+        await engine.dispose()
+
+
+def _validate_legacy_recovery_payload(payload: dict[str, Any]) -> int:
+    """Validate the authenticated export shape without exposing key material."""
+    try:
+        if set(payload) != {"catalog", "keys"}:
+            raise ValueError
+        catalog = payload["catalog"]
+        keys = payload["keys"]
+        if (
+            not isinstance(catalog, dict)
+            or set(catalog) != {"format", "repositories", "version"}
+            or catalog["format"] != "backup-tool-recovery-catalog"
+            or catalog["version"] != 1
+            or not isinstance(catalog["repositories"], list)
+            or not isinstance(keys, list)
+            or len(catalog["repositories"]) != len(keys)
+        ):
+            raise ValueError
+        seen: set[str] = set()
+        for repository, key_record in zip(catalog["repositories"], keys, strict=True):
+            if not isinstance(repository, dict) or not isinstance(key_record, dict):
+                raise ValueError
+            required_repository = {
+                "active_data_key_id",
+                "compression",
+                "data_key_epochs",
+                "database_id",
+                "encryption",
+                "format_version",
+                "name",
+                "repository_id",
+                "root",
+                "signing_key_id",
+                "signing_public_key",
+                "state",
+            }
+            if set(repository) != required_repository or set(key_record) != {
+                "data_keys",
+                "repository_id",
+                "signing_private_key",
+            }:
+                raise ValueError
+            repository_id = repository["repository_id"]
+            if (
+                not isinstance(repository_id, str)
+                or not repository_id
+                or repository_id in seen
+                or key_record["repository_id"] != repository_id
+                or not isinstance(repository["data_key_epochs"], list)
+                or not isinstance(key_record["data_keys"], list)
+            ):
+                raise ValueError
+            seen.add(repository_id)
+            private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+            public_key = bytes.fromhex(repository["signing_public_key"])
+            if len(private_key) != 32 or len(public_key) != 32:
+                raise ValueError
+            signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
+            derived_public = signing_key.public_key().public_bytes(
+                serialization.Encoding.Raw,
+                serialization.PublicFormat.Raw,
+            )
+            if derived_public != public_key:
+                raise ValueError
+            epochs = repository["data_key_epochs"]
+            data_keys = key_record["data_keys"]
+            if len(epochs) != len(data_keys):
+                raise ValueError
+            epoch_ids: set[str] = set()
+            for epoch, data_key in zip(epochs, data_keys, strict=True):
+                if (
+                    not isinstance(epoch, dict)
+                    or set(epoch) != {"key_id", "retired_at", "state"}
+                    or not isinstance(data_key, dict)
+                    or set(data_key) != {"key", "key_id"}
+                    or not isinstance(epoch["key_id"], str)
+                    or epoch["key_id"] in epoch_ids
+                    or data_key["key_id"] != epoch["key_id"]
+                    or len(base64.b64decode(data_key["key"], validate=True)) != 32
+                ):
+                    raise ValueError
+                epoch_ids.add(epoch["key_id"])
+            active = [epoch for epoch in epochs if epoch["state"] == "active"]
+            if repository["encryption"] == "aes-256-gcm":
+                if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
+                    raise ValueError
+            elif (
+                repository["encryption"] != "none"
+                or epochs
+                or repository["active_data_key_id"] is not None
+            ):
+                raise ValueError
+        return len(seen)
+    except (binascii.Error, KeyError, TypeError, ValueError) as error:
+        raise RecoveryBundleError("recovery bundle is invalid") from error
+
+
+def _validated_recovery_catalog(
+    payload: dict[str, Any],
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Validate a v2 catalog before any filesystem or metadata mutation."""
+    try:
+        if set(payload) != {"catalog", "keys"}:
+            raise ValueError
+        catalog = payload["catalog"]
+        keys = payload["keys"]
+        required_catalog = {
+            "backups",
+            "executions",
+            "format",
+            "jobs",
+            "repositories",
+            "sources",
+            "version",
+        }
+        if (
+            not isinstance(catalog, dict)
+            or set(catalog) != required_catalog
+            or catalog["format"] != "backup-tool-recovery-catalog"
+            or catalog["version"] != 2
+            or not isinstance(keys, list)
+            or any(
+                not isinstance(catalog[name], list)
+                for name in required_catalog - {"format", "version"}
+            )
+            or len(catalog["repositories"]) != len(keys)
+        ):
+            raise ValueError
+        repositories = catalog["repositories"]
+        key_by_repository: dict[str, dict[str, Any]] = {}
+        repository_by_id: dict[str, dict[str, Any]] = {}
+        canonical_ids: set[str] = set()
+        for repository, key_record in zip(repositories, keys, strict=True):
+            if not isinstance(repository, dict) or not isinstance(key_record, dict):
+                raise ValueError
+            required_repository = {
+                "active_data_key_id",
+                "compression",
+                "data_key_epochs",
+                "database_id",
+                "encryption",
+                "format_version",
+                "name",
+                "repository_id",
+                "root",
+                "signing_key_id",
+                "signing_public_key",
+                "state",
+            }
+            if set(repository) != required_repository or set(key_record) != {
+                "data_keys",
+                "repository_id",
+                "signing_private_key",
+            }:
+                raise ValueError
+            database_id = repository["database_id"]
+            canonical_id = repository["repository_id"]
+            if (
+                not isinstance(database_id, str)
+                or not isinstance(canonical_id, str)
+                or not database_id
+                or not canonical_id
+                or database_id in repository_by_id
+                or canonical_id in canonical_ids
+                or key_record["repository_id"] != canonical_id
+                or not isinstance(repository["data_key_epochs"], list)
+                or not isinstance(key_record["data_keys"], list)
+            ):
+                raise ValueError
+            private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+            public_key = bytes.fromhex(repository["signing_public_key"])
+            signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
+            if (
+                len(public_key) != 32
+                or signing_key.public_key().public_bytes(
+                    serialization.Encoding.Raw, serialization.PublicFormat.Raw
+                )
+                != public_key
+            ):
+                raise ValueError
+            epochs = repository["data_key_epochs"]
+            data_keys = key_record["data_keys"]
+            if len(epochs) != len(data_keys):
+                raise ValueError
+            epoch_ids: set[str] = set()
+            for epoch, data_key in zip(epochs, data_keys, strict=True):
+                if (
+                    not isinstance(epoch, dict)
+                    or set(epoch) != {"key_id", "retired_at", "state"}
+                    or epoch.get("state") not in {"active", "retired"}
+                    or not isinstance(epoch.get("key_id"), str)
+                    or epoch["key_id"] in epoch_ids
+                    or not isinstance(data_key, dict)
+                    or set(data_key) != {"key", "key_id"}
+                    or data_key["key_id"] != epoch["key_id"]
+                    or len(base64.b64decode(data_key["key"], validate=True)) != 32
+                ):
+                    raise ValueError
+                epoch_ids.add(epoch["key_id"])
+            active = [epoch for epoch in epochs if epoch["state"] == "active"]
+            if repository["encryption"] == "aes-256-gcm":
+                if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
+                    raise ValueError
+            elif (
+                repository["encryption"] != "none"
+                or epochs
+                or repository["active_data_key_id"] is not None
+            ):
+                raise ValueError
+            repository_by_id[database_id] = repository
+            canonical_ids.add(canonical_id)
+            key_by_repository[canonical_id] = key_record
+        source_ids = _catalog_ids(catalog["sources"], {"id", "kind", "name", "public_config"})
+        job_ids = _catalog_ids(
+            catalog["jobs"],
+            {
+                "allow_empty",
+                "exclusions",
+                "id",
+                "name",
+                "repository_id",
+                "requested_mode",
+                "retention",
+                "source_id",
+            },
+        )
+        for job in catalog["jobs"]:
+            if job["source_id"] not in source_ids or job["repository_id"] not in repository_by_id:
+                raise ValueError
+        execution_ids = _catalog_ids(
+            catalog["executions"], {"attempt", "id", "job_id", "progress", "revision", "trigger"}
+        )
+        for execution in catalog["executions"]:
+            if execution["job_id"] not in job_ids:
+                raise ValueError
+        backup_ids = _catalog_ids(
+            catalog["backups"],
+            {
+                "created_at",
+                "data_key_id",
+                "execution_id",
+                "id",
+                "integrity",
+                "logical_bytes",
+                "manifest_digest",
+                "manifest_id",
+                "parent_backup_id",
+                "pinned",
+                "stored_bytes",
+                "tombstoned_at",
+            },
+        )
+        for backup in catalog["backups"]:
+            if backup["execution_id"] not in execution_ids:
+                raise ValueError
+            _catalog_datetime(backup["created_at"])
+            _catalog_datetime(backup["tombstoned_at"], allow_none=True)
+            if (
+                backup["parent_backup_id"] is not None
+                and backup["parent_backup_id"] not in backup_ids
+            ):
+                raise ValueError
+        return (
+            catalog,
+            [key_by_repository[repository["repository_id"]] for repository in repositories],
+        )
+    except (binascii.Error, KeyError, TypeError, ValueError) as error:
+        raise RecoveryBundleError("recovery bundle is invalid") from error
+
+
+def _catalog_ids(rows: list[Any], required: set[str]) -> set[str]:
+    identifiers: set[str] = set()
+    for row in rows:
+        if not isinstance(row, dict) or set(row) != required or not isinstance(row.get("id"), str):
+            raise ValueError
+        if not row["id"] or row["id"] in identifiers:
+            raise ValueError
+        identifiers.add(row["id"])
+    return identifiers
+
+
+def _catalog_datetime(value: object, *, allow_none: bool = False) -> datetime | None:
+    if value is None and allow_none:
+        return None
+    if not isinstance(value, str):
+        raise ValueError
+    parsed = datetime.fromisoformat(value.replace("Z", "+00"))
+    if parsed.tzinfo is None:
+        raise ValueError
+    return parsed.astimezone(UTC)
+
+
+def _validate_recovery_payload(payload: dict[str, Any]) -> int:
+    catalog = payload.get("catalog")
+    if isinstance(catalog, dict) and catalog.get("version") == 1:
+        return _validate_legacy_recovery_payload(payload)
+    validated_catalog, _keys = _validated_recovery_catalog(payload)
+    return len(validated_catalog["repositories"])
+
+
+def _install_or_verify_recovery_keys(
+    settings: Settings,
+    repository: dict[str, Any],
+    key_record: dict[str, Any],
+    installed_paths: list[Path],
+) -> None:
+    inspected = inspect_repository(settings, Path(repository["root"]))
+    if (
+        inspected.repository_id != repository["repository_id"]
+        or inspected.encryption != repository["encryption"]
+        or inspected.data_key_id != repository["active_data_key_id"]
+    ):
+        raise RecoveryBundleError("recovery bundle is invalid")
+    private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+    signing_path = settings.data_dir / "repository-keys" / f"{inspected.repository_id}.ed25519"
+    if signing_path.exists() or signing_path.is_symlink():
+        loaded = load_signing_key(
+            settings,
+            inspected.repository_id,
+            repository["signing_key_id"],
+            repository["signing_public_key"],
+        )
+        if (
+            loaded.private_bytes(
+                serialization.Encoding.Raw,
+                serialization.PrivateFormat.Raw,
+                serialization.NoEncryption(),
+            )
+            != private_key
+        ):
+            raise RecoveryBundleError("recovery destination has conflicting keys")
+    else:
+        installed_paths.append(
+            install_signing_key(
+                settings,
+                inspected.repository_id,
+                repository["signing_key_id"],
+                repository["signing_public_key"],
+                private_key,
+            )
+        )
+    for data_key in key_record["data_keys"]:
+        key_value = base64.b64decode(data_key["key"], validate=True)
+        key_path = (
+            settings.data_dir
+            / "repository-data-keys"
+            / f"{inspected.repository_id}.{data_key['key_id']}.key"
+        )
+        if key_path.exists() or key_path.is_symlink():
+            if load_data_key(settings, inspected.repository_id, data_key["key_id"]) != key_value:
+                raise RecoveryBundleError("recovery destination has conflicting keys")
+        else:
+            installed_paths.append(
+                install_data_key(settings, inspected.repository_id, data_key["key_id"], key_value)
+            )
+
+
+async def import_recovery_payload(
+    settings: Settings,
+    payload: dict[str, Any],
+    *,
+    after_key_install: Callable[[], None] | None = None,
+) -> int:
+    """Install a complete authenticated catalog into an empty, migrated database."""
+    catalog, key_records = _validated_recovery_catalog(payload)
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    installed_paths: list[Path] = []
+    completed = False
+    try:
+        async with sessions() as db:
+            for table in Base.metadata.sorted_tables:
+                if await db.scalar(select(func.count()).select_from(table)):
+                    raise RecoveryBundleError("recovery destination is not empty")
+            await db.rollback()
+            for repository, key_record in zip(catalog["repositories"], key_records, strict=True):
+                _install_or_verify_recovery_keys(settings, repository, key_record, installed_paths)
+            if after_key_install is not None:
+                after_key_install()
+            async with db.begin():
+                for repository in catalog["repositories"]:
+                    db.add(
+                        Repository(
+                            id=repository["database_id"],
+                            name=repository["name"],
+                            root=repository["root"],
+                            format_version=repository["format_version"],
+                            compression=repository["compression"],
+                            encryption=repository["encryption"],
+                            signing_key_id=repository["signing_key_id"],
+                            signing_public_key=repository["signing_public_key"],
+                            active_data_key_id=repository["active_data_key_id"],
+                            state=repository["state"],
+                        )
+                    )
+                for repository in catalog["repositories"]:
+                    for epoch in repository["data_key_epochs"]:
+                        db.add(
+                            RepositoryDataKeyEpoch(
+                                repository_id=repository["database_id"],
+                                key_id=epoch["key_id"],
+                                state=epoch["state"],
+                            )
+                        )
+                await db.flush()
+                for source in catalog["sources"]:
+                    db.add(
+                        Source(
+                            id=source["id"],
+                            name=source["name"],
+                            kind=source["kind"],
+                            public_config=source["public_config"],
+                            secret_refs=[],
+                            state="unavailable",
+                        )
+                    )
+                await db.flush()
+                for job in catalog["jobs"]:
+                    db.add(
+                        Job(
+                            id=job["id"],
+                            name=job["name"],
+                            source_id=job["source_id"],
+                            repository_id=job["repository_id"],
+                            requested_mode=job["requested_mode"],
+                            exclusions=job["exclusions"],
+                            retention=job["retention"],
+                            enabled=False,
+                            allow_empty=job["allow_empty"],
+                            state="archived",
+                        )
+                    )
+                await db.flush()
+                for execution in catalog["executions"]:
+                    db.add(
+                        Execution(
+                            id=execution["id"],
+                            job_id=execution["job_id"],
+                            schedule_id=None,
+                            trigger=execution["trigger"],
+                            state="committed",
+                            attempt=execution["attempt"],
+                            progress=execution["progress"],
+                            revision=execution["revision"],
+                        )
+                    )
+                await db.flush()
+                for backup in catalog["backups"]:
+                    db.add(
+                        Backup(
+                            id=backup["id"],
+                            execution_id=backup["execution_id"],
+                            parent_backup_id=None,
+                            manifest_id=backup["manifest_id"],
+                            manifest_digest=backup["manifest_digest"],
+                            logical_bytes=backup["logical_bytes"],
+                            stored_bytes=backup["stored_bytes"],
+                            integrity=backup["integrity"],
+                            data_key_id=backup["data_key_id"],
+                            pinned=backup["pinned"],
+                            created_at=_catalog_datetime(backup["created_at"]),
+                            tombstoned_at=_catalog_datetime(
+                                backup["tombstoned_at"], allow_none=True
+                            ),
+                        )
+                    )
+                await db.flush()
+                for backup in catalog["backups"]:
+                    if backup["parent_backup_id"] is not None:
+                        imported = await db.get(Backup, backup["id"])
+                        if imported is None:
+                            raise RecoveryBundleError("recovery bundle is invalid")
+                        imported.parent_backup_id = backup["parent_backup_id"]
+        completed = True
+        return len(catalog["repositories"])
+    except (OSError, RepositoryKeyError, ValueError) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise
+        raise RecoveryBundleError("recovery import failed") from error
+    finally:
+        await engine.dispose()
+        if not completed:
+            for path in installed_paths:
+                path.unlink(missing_ok=True)
+
+
+def run_admin(settings: Settings, args: argparse.Namespace) -> int:
+    if (
+        getattr(args, "admin_command", None) == "repository-key"
+        and getattr(args, "repository_key_command", None) == "rotate"
+    ):
+        result = asyncio.run(rotate_repository_key(settings, args.repository_id))
+        print(json.dumps(result, sort_keys=True))
+        return 0
+    if getattr(args, "admin_command", None) == "recovery":
+        passphrase = read_passphrase_fd(args.passphrase_fd)
+        if args.recovery_command == "export":
+            payload = asyncio.run(recovery_export_payload(settings))
+            write_bundle_exclusive(args.output, encrypt_bundle(payload, passphrase))
+            print(json.dumps({"repositories": len(payload["keys"]), "status": "exported"}))
+            return 0
+        if args.recovery_command == "validate":
+            payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
+            validation_result: dict[str, object] = {
+                "repositories": _validate_recovery_payload(payload),
+                "status": "valid",
+            }
+            print(json.dumps(validation_result))
+            return 0
+        if args.recovery_command == "import":
+            payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
+            imported = asyncio.run(import_recovery_payload(settings, payload))
+            print(json.dumps({"repositories": imported, "status": "imported"}))
+            return 0
+    raise ValueError("admin command is invalid")
+
+
 def main(argv: Sequence[str] | None = None, *, settings: Settings | None = None) -> int:
     args = build_parser().parse_args(argv)
     runtime_settings = settings or load_settings()
     if args.role == "migrate":
         return run_migration(args.action, runtime_settings)
+    if args.role == "health":
+        return run_role_healthcheck(runtime_settings, args.health_role)
     require_current_schema(runtime_settings)
+    if args.role == "admin":
+        return run_admin(runtime_settings, args)
     return ROLE_HANDLERS[args.role](runtime_settings)
 
 

backend/src/backup_tool/config.py

@@ -19,6 +19,7 @@ class Settings(BaseSettings):
     )
 
     data_dir: Path = Path("/var/lib/backup-tool")
+    web_socket_path: Path = Path("/tmp/backup-tool-web.sock")
     database_url: str = "sqlite+aiosqlite:////var/lib/backup-tool/metadata.db"
     repository_roots: tuple[Path, ...]
     local_source_roots: tuple[Path, ...]
@@ -29,6 +30,21 @@ class Settings(BaseSettings):
     session_ttl_seconds: int = Field(default=28_800, ge=60, le=2_592_000)
     cors_origins: tuple[str, ...] = ()
     worker_concurrency: Literal[1] = 1
+    notification_delivery_lease_seconds: int = Field(default=60, ge=5, le=3600)
+    notification_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
+    notification_read_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
+    notification_max_attempts: int = Field(default=5, ge=1, le=20)
+    notification_retry_cap_seconds: int = Field(default=3600, ge=1, le=86_400)
+    notification_global_rate_per_minute: int = Field(default=120, ge=1, le=10_000)
+    notification_default_rate_per_minute: int = Field(default=60, ge=1, le=10_000)
+    notification_max_webhook_body_bytes: int = Field(default=65_536, ge=256, le=1_048_576)
+    notification_max_response_bytes: int = Field(default=16_384, ge=256, le=1_048_576)
+    ssh_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
+    ssh_operation_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
+    ssh_read_chunk_bytes: int = Field(default=1_048_576, ge=4096, le=8_388_608)
+    ssh_list_read_aheads: int = Field(default=32, ge=1, le=256)
+    ssh_max_entries: int = Field(default=1_000_000, ge=1, le=10_000_000)
+    ssh_max_traversal_depth: int = Field(default=128, ge=1, le=1024)
     min_free_bytes: int = Field(default=1_073_741_824, ge=0)
     min_free_percent: int = Field(default=5, gt=0, lt=100)
     sqlite_busy_timeout_ms: int = Field(default=5_000, ge=1_000, le=120_000)
@@ -37,6 +53,7 @@ class Settings(BaseSettings):
     @field_validator(
         "data_dir",
         "master_key_file",
+        "web_socket_path",
         mode="before",
     )
     @classmethod

backend/src/backup_tool/db/models.py

@@ -95,6 +95,9 @@ class Repository(IdentityMixin, TimestampMixin, Base):
     format_version: Mapped[int] = mapped_column(Integer, nullable=False)
     compression: Mapped[str] = mapped_column(String(32), nullable=False)
     encryption: Mapped[str] = mapped_column(String(32), nullable=False)
+    signing_key_id: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
+    signing_public_key: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
+    active_data_key_id: Mapped[str | None] = mapped_column(String(36))
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
     __table_args__ = (
         CheckConstraint("format_version > 0", name="format_version_positive"),
@@ -102,6 +105,25 @@ class Repository(IdentityMixin, TimestampMixin, Base):
     )
 
 
+class RepositoryDataKeyEpoch(IdentityMixin, TimestampMixin, Base):
+    __tablename__ = "repository_data_key_epochs"
+    repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
+    key_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    state: Mapped[str] = mapped_column(String(16), nullable=False)
+    retired_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    __table_args__ = (
+        UniqueConstraint("repository_id", "key_id", name="repository_key_epoch"),
+        CheckConstraint("state IN ('active','retired')", name="repository_data_key_epoch_state"),
+        Index("ix_repository_data_key_epochs_repository_id", "repository_id"),
+        Index(
+            "uq_repository_data_key_epochs_active",
+            "repository_id",
+            unique=True,
+            sqlite_where=text("state = 'active'"),
+        ),
+    )
+
+
 class Source(IdentityMixin, TimestampMixin, Base):
     __tablename__ = "sources"
     name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
@@ -111,7 +133,7 @@ class Source(IdentityMixin, TimestampMixin, Base):
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
     last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON)
     __table_args__ = (
-        CheckConstraint("kind IN ('local','sftp','postgresql','mysql')", name="kind"),
+        CheckConstraint("kind IN ('local','ssh')", name="kind"),
         CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
     )
 
@@ -216,6 +238,7 @@ class Backup(IdentityMixin, Base):
     logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified")
+    data_key_id: Mapped[str | None] = mapped_column(String(36))
     pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     tombstoned_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
     created_at: Mapped[datetime] = mapped_column(
@@ -236,6 +259,9 @@ class Restore(IdentityMixin, TimestampMixin, Base):
     backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT"))
     destination: Mapped[str] = mapped_column(Text, nullable=False)
     selection: Mapped[list[str]] = mapped_column(JSON, nullable=False)
+    dry_run: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=False, server_default="0"
+    )
     overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False)
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
     result: Mapped[dict[str, Any] | None] = mapped_column(JSON)
@@ -268,33 +294,143 @@ class AuditEvent(IdentityMixin, Base):
 
 
 class NotificationSubscription(IdentityMixin, TimestampMixin, Base):
+    """A typed, mutable outbound channel configuration; credential material is never here."""
+
     __tablename__ = "notification_subscriptions"
     channel: Mapped[str] = mapped_column(String(32), nullable=False)
     event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False)
     destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    # Retained only to make the migration from the unused v1-shaped table lossless.
+    # New webhook secrets live in NotificationSigningKey and SMTP secrets in settings.
     secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
+    rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
+    rate_tokens: Mapped[float] = mapped_column(nullable=False, default=60.0)
+    rate_updated_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
     __table_args__ = (
         CheckConstraint("channel IN ('webhook','email')", name="channel"),
         CheckConstraint("state IN ('active','disabled','archived')", name="state"),
+        CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
+        CheckConstraint("rate_tokens >= 0", name="rate_tokens_nonnegative"),
+        CheckConstraint("revision > 0", name="revision_positive"),
+    )
+
+
+class NotificationEvent(IdentityMixin, Base):
+    """Immutable canonical outbox envelope, created in the source mutation transaction."""
+
+    __tablename__ = "notification_events"
+    type: Mapped[str] = mapped_column(String(96), nullable=False)
+    schema_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    occurred_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    correlation_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    severity: Mapped[str] = mapped_column(String(16), nullable=False)
+    resource_refs: Mapped[dict[str, str]] = mapped_column(JSON, nullable=False)
+    payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    canonical_envelope: Mapped[str] = mapped_column(Text, nullable=False)
+    deduplication_key: Mapped[str | None] = mapped_column(String(255), unique=True)
+    __table_args__ = (
+        CheckConstraint("schema_version = 1", name="schema_version"),
+        CheckConstraint("severity IN ('info','warning','error','security')", name="severity"),
+        Index("ix_notification_events_type_occurred", "type", "occurred_at"),
     )
 
 
 class NotificationDelivery(IdentityMixin, TimestampMixin, Base):
+    """One durable delivery per matching subscription, with a lease-owned lifecycle."""
+
     __tablename__ = "notification_deliveries"
-    event_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    event_id: Mapped[str] = mapped_column(
-    subscription_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_events.id", ondelete="RESTRICT"), nullable=False
-        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT")
     )
-    attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    subscription_id: Mapped[str] = mapped_column(
-    state: Mapped[str] = mapped_column(String(32), nullable=False)
+        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
+    )
+    state: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
+    due_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    lease_owner: Mapped[str | None] = mapped_column(String(255))
+    lease_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    terminal_reason: Mapped[str | None] = mapped_column(String(96))
     response_class: Mapped[str | None] = mapped_column(String(64))
-    next_attempt_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    response_summary: Mapped[str | None] = mapped_column(String(512))
     __table_args__ = (
-        CheckConstraint("attempt > 0", name="attempt_positive"),
+        CheckConstraint("attempt_count >= 0", name="attempt_count_nonnegative"),
-        CheckConstraint("state IN ('pending','delivered','retry','failed')", name="state"),
+        CheckConstraint("state IN ('pending','leased','delivered','retry','failed')", name="state"),
-        UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
+        UniqueConstraint("event_id", "subscription_id", name="event_subscription"),
-        Index("ix_notification_deliveries_state_next", "state", "next_attempt_at"),
+        Index("ix_notification_deliveries_due", "state", "due_at"),
+        Index("ix_notification_deliveries_lease", "state", "lease_expires_at"),
+    )
+
+
+class NotificationDeliveryAttempt(IdentityMixin, Base):
+    """Append-only history.  Diagnostics are bounded/redacted before persistence."""
+
+    __tablename__ = "notification_delivery_attempts"
+    delivery_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_deliveries.id", ondelete="RESTRICT"), nullable=False
+    )
+    number: Mapped[int] = mapped_column(Integer, nullable=False)
+    started_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    outcome: Mapped[str] = mapped_column(String(32), nullable=False, default="started")
+    response_class: Mapped[str | None] = mapped_column(String(64))
+    diagnostic: Mapped[str | None] = mapped_column(String(512))
+    __table_args__ = (
+        CheckConstraint("number > 0", name="number_positive"),
+        CheckConstraint("outcome IN ('started','delivered','retry','failed')", name="outcome"),
+        UniqueConstraint("delivery_id", "number", name="delivery_number"),
+        Index("ix_notification_attempts_delivery", "delivery_id", "number"),
+    )
+
+
+class NotificationSigningKey(IdentityMixin, TimestampMixin, Base):
+    __tablename__ = "notification_signing_keys"
+    subscription_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
+    )
+    version: Mapped[int] = mapped_column(Integer, nullable=False)
+    secret_id: Mapped[str] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
+    state: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
+    overlap_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    __table_args__ = (
+        CheckConstraint("version > 0", name="version_positive"),
+        CheckConstraint("state IN ('active','overlap','retired')", name="state"),
+        UniqueConstraint("subscription_id", "version", name="subscription_version"),
+        Index("ix_notification_signing_keys_subscription", "subscription_id", "state"),
+        Index(
+            "uq_notification_signing_keys_active",
+            "subscription_id",
+            unique=True,
+            sqlite_where=text("state = 'active'"),
+        ),
+        Index(
+            "uq_notification_signing_keys_overlap",
+            "subscription_id",
+            unique=True,
+            sqlite_where=text("state = 'overlap'"),
+        ),
+    )
+
+
+class NotificationEmailSettings(Base):
+    __tablename__ = "notification_email_settings"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
+    host: Mapped[str] = mapped_column(String(255), nullable=False)
+    port: Mapped[int] = mapped_column(Integer, nullable=False, default=587)
+    username: Mapped[str] = mapped_column(String(255), nullable=False)
+    password_secret_id: Mapped[str] = mapped_column(
+        ForeignKey("secrets.id", ondelete="RESTRICT"), nullable=False
+    )
+    sender: Mapped[str] = mapped_column(String(320), nullable=False)
+    max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
+    rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
+    __table_args__ = (
+        CheckConstraint("id = 1", name="singleton"),
+        CheckConstraint("port > 0 AND port < 65536", name="port"),
+        CheckConstraint("max_attempts > 0", name="max_attempts"),
+        CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
     )
 
 

backend/src/backup_tool/exclusions.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+
+
+class ExclusionError(ValueError):
+    pass
+
+
+def _validate(value: str) -> None:
+    path = PurePosixPath(value)
+    if not value or "\\" in value or path.is_absolute() or ".." in path.parts:
+        raise ExclusionError("exclusion paths must be normalized relative POSIX paths")
+
+
+def matches(path: str, patterns: list[str]) -> bool:
+    """Return whether normalized relative `path` is excluded by ordered gitignore-like rules."""
+    _validate(path)
+    excluded = False
+    candidate = PurePosixPath(path)
+    for pattern in patterns:
+        negated = pattern.startswith("!")
+        raw = pattern[1:] if negated else pattern
+        if not raw or raw.startswith("/") or "\\" in raw or ".." in PurePosixPath(raw).parts:
+            raise ExclusionError("exclusion pattern is invalid")
+        directory = raw.endswith("/")
+        raw = raw.rstrip("/")
+        if not raw:
+            raise ExclusionError("exclusion pattern is invalid")
+        matched = candidate.match(raw) or candidate.match(f"**/{raw}")
+        if directory:
+            matched = matched or any(
+                parent.match(raw) or parent.match(f"**/{raw}") for parent in candidate.parents
+            )
+        if matched:
+            excluded = not negated
+    return excluded

backend/src/backup_tool/execution.py

@@ -8,6 +8,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from backup_tool.db.models import Execution, ExecutionEvent, Job
+from backup_tool.notifications.events import emit_event
 from backup_tool.security.redaction import redact
 
 ACTIVE_STATES = frozenset({"queued", "preparing", "running", "verifying", "cancelling"})
@@ -44,7 +45,14 @@ def transition(current: str, target: str) -> str:
     return target
 
 
-async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Execution:
+async def enqueue(
+    db: AsyncSession,
+    job_id: str,
+    trigger: str = "manual",
+    *,
+    schedule_id: str | None = None,
+    nominal_run_at: datetime | None = None,
+) -> Execution:
     """Create exactly one active execution for an enabled active job."""
     job = await db.get(Job, job_id)
     if job is None:
@@ -52,7 +60,13 @@ async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Exe
     if job.state != "active" or not job.enabled:
         raise EnqueueError("job_disabled", "Job is disabled or archived.")
     job_identifier = job.id
-    execution = Execution(job_id=job_identifier, trigger=trigger, progress={})
+    execution = Execution(
+        job_id=job_identifier,
+        trigger=trigger,
+        schedule_id=schedule_id,
+        nominal_run_at=nominal_run_at,
+        progress={},
+    )
     db.add(execution)
     try:
         await db.flush()
@@ -221,6 +235,32 @@ async def record_event(db: AsyncSession, execution: Execution) -> ExecutionEvent
     )
     db.add(event)
     await db.flush()
+    notification_type = {
+        "preparing": "execution.started",
+        "committed": "execution.committed",
+        "failed": "execution.failed",
+        "cancelled": "execution.cancelled",
+    }.get(execution.state)
+    if execution.state == "queued":
+        if execution.reason_code == "worker_lost":
+            notification_type = "execution.worker_recovered"
+        elif execution.attempt > 1:
+            notification_type = "execution.retry_queued"
+        else:
+            notification_type = "execution.queued"
+    if notification_type is not None:
+        await emit_event(
+            db,
+            notification_type,
+            correlation_id=execution.id,
+            resource={"execution_id": execution.id, "job_id": execution.job_id},
+            payload={
+                "attempt": execution.attempt,
+                "state": execution.state,
+                "reason_code": execution.reason_code,
+            },
+            deduplication_key=f"execution:{execution.id}:revision:{execution.revision}",
+        )
     return event
 
 

backend/src/backup_tool/faults.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class FaultInjector(Protocol):
+    def hit(self, point: str) -> None: ...
+
+
+class NoFault:
+    def hit(self, point: str) -> None:
+        del point
+
+
+class InjectedCrash(BaseException):
+    """Test-only abrupt worker termination at a named durable fault point."""
+
+
+class CrashAt:
+    def __init__(self, point: str) -> None:
+        self.point = point
+
+    def hit(self, point: str) -> None:
+        if point == self.point:
+            raise InjectedCrash(point)

backend/src/backup_tool/gc.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import json
+import shutil
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import cast
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.db.models import Backup, Execution, Job, Repository
+from backup_tool.notifications.events import emit_event
+from backup_tool.retention import BackupLike, RetentionPolicy, retained_ids
+from backup_tool.security.repository_crypto import RepositoryKeyError, decrypt_object, object_aad
+
+
+@dataclass(frozen=True)
+class GcReport:
+    tombstoned: int
+    purged_manifests: int
+    purged_blobs: int
+    quarantined: int
+
+
+async def tombstone_expired(db: AsyncSession, now: datetime | None = None) -> int:
+    reference = now or datetime.now(UTC)
+    backups = list((await db.scalars(select(Backup))).all())
+    executions = {item.id: item for item in (await db.scalars(select(Execution))).all()}
+    jobs = {item.id: item for item in (await db.scalars(select(Job))).all()}
+    grouped: dict[str, list[Backup]] = {}
+    for backup in backups:
+        execution = executions.get(backup.execution_id)
+        if execution is None or execution.job_id not in jobs:
+            continue
+        grouped.setdefault(execution.job_id, []).append(backup)
+    tombstoned = 0
+    for job_id, items in grouped.items():
+        if not jobs[job_id].retention:
+            continue
+        policy = RetentionPolicy.from_dict(jobs[job_id].retention)
+        keep = retained_ids(cast(list[BackupLike], items), policy, reference)
+        for backup in items:
+            if backup.id not in keep and backup.tombstoned_at is None:
+                backup.tombstoned_at = reference
+                await emit_event(
+                    db,
+                    "retention.tombstoned",
+                    correlation_id=backup.id,
+                    resource={"backup_id": backup.id, "job_id": job_id},
+                    payload={"outcome": "tombstoned"},
+                    deduplication_key=f"backup:{backup.id}:tombstoned",
+                )
+                tombstoned += 1
+    await db.commit()
+    return tombstoned
+
+
+async def process_retention_gc(db: AsyncSession, now: datetime | None = None) -> GcReport:
+    """Run durable retention tombstoning and repository GC from the worker role."""
+    reference = now or datetime.now(UTC)
+    tombstoned = await tombstone_expired(db, reference)
+    reports: list[GcReport] = []
+    repositories = list((await db.scalars(select(Repository))).all())
+    for repository in repositories:
+        manifest_ids = set(
+            (
+                await db.scalars(
+                    select(Backup.manifest_id)
+                    .join(Execution, Backup.execution_id == Execution.id)
+                    .join(Job, Execution.job_id == Job.id)
+                    .where(
+                        Job.repository_id == repository.id,
+                        Backup.tombstoned_at.is_not(None),
+                    )
+                )
+            ).all()
+        )
+        if manifest_ids:
+            reports.append(purge_repository(Path(repository.root), manifest_ids, now=reference))
+    return GcReport(
+        tombstoned=tombstoned,
+        purged_manifests=sum(report.purged_manifests for report in reports),
+        purged_blobs=sum(report.purged_blobs for report in reports),
+        quarantined=sum(report.quarantined for report in reports),
+    )
+
+
+def _manifest_digests(
+    path: Path,
+    *,
+    repository_id: str | None = None,
+    manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
+) -> set[str] | None:
+    try:
+        if path.is_symlink() or not path.is_file():
+            return None
+        raw = path.read_bytes()
+        if raw.startswith(b"BTENC\x01"):
+            key_record = manifest_keys.get(path.stem) if manifest_keys is not None else None
+            if key_record is None or repository_id is None:
+                return None
+            key_id, key = key_record
+            raw = decrypt_object(key, object_aad(repository_id, key_id, "manifest", path.stem), raw)
+        payload = json.loads(raw.decode("utf-8"))
+    except (OSError, RepositoryKeyError, UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    if not isinstance(payload, dict) or not isinstance(entries := payload.get("entries"), list):
+        return None
+    digests: set[str] = set()
+    for entry in entries:
+        if not isinstance(entry, dict):
+            return None
+        digest = entry.get("blob_digest")
+        if digest is None:
+            continue
+        if (
+            not isinstance(digest, str)
+            or len(digest) != 64
+            or any(character not in "0123456789abcdef" for character in digest)
+        ):
+            return None
+        digests.add(digest)
+    return digests
+
+
+def purge_repository(
+    root: Path,
+    tombstoned_manifest_ids: set[str],
+    *,
+    repository_id: str | None = None,
+    manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
+    grace: timedelta = timedelta(days=7),
+    now: datetime | None = None,
+) -> GcReport:
+    reference = now or datetime.now(UTC)
+    manifests = root / "manifests"
+    blobs = root / "blobs" / "sha256"
+    quarantine = root / "quarantine"
+    quarantine.mkdir(exist_ok=True)
+    manifest_digests: dict[Path, set[str]] = {}
+    for manifest in manifests.glob("*.json"):
+        digests = _manifest_digests(
+            manifest,
+            repository_id=repository_id,
+            manifest_keys=manifest_keys,
+        )
+        if digests is None:
+            return GcReport(0, 0, 0, 0)
+        manifest_digests[manifest] = digests
+
+    purged_manifests = 0
+    for manifest_id in tombstoned_manifest_ids:
+        candidate = manifests / f"{manifest_id}.json"
+        if not candidate.is_file() or candidate.is_symlink():
+            continue
+        age = reference - datetime.fromtimestamp(candidate.stat().st_mtime, UTC)
+        if age >= grace:
+            candidate.unlink()
+            purged_manifests += 1
+    referenced: set[str] = set()
+    for manifest, digests in manifest_digests.items():
+        if manifest.exists():
+            referenced.update(digests)
+    purged_blobs = 0
+    quarantined = 0
+    if blobs.exists():
+        for blob in blobs.iterdir():
+            if not blob.is_file():
+                continue
+            if len(blob.name) != 64 or any(char not in "0123456789abcdef" for char in blob.name):
+                try:
+                    shutil.move(str(blob), quarantine / blob.name)
+                except OSError:
+                    continue
+                quarantined += 1
+            elif blob.name not in referenced and (
+                reference - datetime.fromtimestamp(blob.stat().st_mtime, UTC) >= grace
+            ):
+                blob.unlink()
+                purged_blobs += 1
+    return GcReport(0, purged_manifests, purged_blobs, quarantined)

backend/src/backup_tool/notifications/__init__.py

@@ -0,0 +1,5 @@
+"""Durable, worker-dispatched operational notifications."""
+
+from .events import EVENT_CATALOG, emit_event, validate_filters
+
+__all__ = ["EVENT_CATALOG", "emit_event", "validate_filters"]

backend/src/backup_tool/notifications/dispatcher.py

@@ -0,0 +1,335 @@
+"""Worker-owned leased outbox dispatcher; sends happen only after a committed lease."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.config import Settings
+from backup_tool.db.models import (
+    NotificationDelivery,
+    NotificationDeliveryAttempt,
+    NotificationEmailSettings,
+    NotificationEvent,
+    NotificationSigningKey,
+    NotificationSubscription,
+    Secret,
+)
+from backup_tool.notifications.email import EmailTransportError, deliver_email
+from backup_tool.notifications.retry import (
+    RetryDecision,
+    retry_delay,
+    transport_decision,
+    webhook_decision,
+)
+from backup_tool.notifications.webhook import (
+    SigningMaterial,
+    WebhookTransportError,
+    deliver_webhook,
+)
+from backup_tool.security.secrets import EnvelopeCipher
+from backup_tool.security.ssrf import Resolver, system_resolver
+
+
+async def recover_notification_leases(db: AsyncSession) -> int:
+    """An interrupted post-send lease becomes eligible again (at-least-once by design)."""
+    now = datetime.now(UTC)
+    deliveries = list(
+        (
+            await db.scalars(
+                select(NotificationDelivery).where(
+                    NotificationDelivery.state == "leased",
+                    NotificationDelivery.lease_expires_at < now,
+                )
+            )
+        ).all()
+    )
+    for delivery in deliveries:
+        attempt = await db.scalar(
+            select(NotificationDeliveryAttempt).where(
+                NotificationDeliveryAttempt.delivery_id == delivery.id,
+                NotificationDeliveryAttempt.number == delivery.attempt_count,
+                NotificationDeliveryAttempt.outcome == "started",
+            )
+        )
+        if attempt is not None:
+            attempt.completed_at = now
+            attempt.outcome = "retry"
+            attempt.response_class = "lease_expired"
+            attempt.diagnostic = "abandoned_lease"
+        delivery.state = "retry"
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        delivery.due_at = now
+    if deliveries:
+        await db.commit()
+    return len(deliveries)
+
+
+async def _claim_due(
+    db: AsyncSession, owner: str, lease_seconds: int
+) -> tuple[NotificationDelivery, NotificationSubscription, NotificationEvent] | None:
+    now = datetime.now(UTC)
+    delivery_id = await db.scalar(
+        select(NotificationDelivery.id)
+        .where(
+            NotificationDelivery.state.in_(("pending", "retry")),
+            NotificationDelivery.due_at <= now,
+        )
+        .order_by(NotificationDelivery.due_at, NotificationDelivery.created_at)
+        .limit(1)
+    )
+    if delivery_id is None:
+        return None
+    result = await db.execute(
+        update(NotificationDelivery)
+        .where(
+            NotificationDelivery.id == delivery_id,
+            NotificationDelivery.state.in_(("pending", "retry")),
+            NotificationDelivery.due_at <= now,
+        )
+        .values(
+            state="leased",
+            lease_owner=owner,
+            lease_expires_at=now + timedelta(seconds=lease_seconds),
+            attempt_count=NotificationDelivery.attempt_count + 1,
+        )
+    )
+    if getattr(result, "rowcount", 0) != 1:
+        await db.rollback()
+        return None
+    delivery = await db.get(NotificationDelivery, delivery_id)
+    if delivery is None:  # pragma: no cover - guarded by update
+        await db.rollback()
+        return None
+    subscription = await db.get(NotificationSubscription, delivery.subscription_id)
+    event = await db.get(NotificationEvent, delivery.event_id)
+    if subscription is None or event is None or subscription.state != "active":
+        delivery.state = "failed"
+        delivery.terminal_reason = "subscription_unavailable"
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        await db.commit()
+        return None
+    # Persist a token bucket before starting an attempt, so restarts cannot bypass
+    # the subscription rate limit.  Global process rate is intentionally a config
+    # ceiling; the durable subscription bucket protects cross-restart behavior.
+    last = subscription.rate_updated_at or now
+    elapsed = max(0.0, (now - last).total_seconds())
+    capacity = subscription.rate_limit_per_minute
+    try:
+        token_capacity = float(capacity)
+        tokens = min(token_capacity, subscription.rate_tokens + elapsed * capacity / 60)
+    except (TypeError, ValueError, ZeroDivisionError) as error:
+        raise RuntimeError("notification rate limit is invalid") from error
+    if tokens < 1:
+        try:
+            delay = max(1, int((1 - tokens) * 60 / capacity) + 1)
+        except (TypeError, ValueError, ZeroDivisionError) as error:
+            raise RuntimeError("notification rate limit is invalid") from error
+        delivery.state = "retry"
+        delivery.due_at = now + timedelta(seconds=delay)
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        subscription.rate_tokens = tokens
+        subscription.rate_updated_at = now
+        await db.commit()
+        return None
+    subscription.rate_tokens = tokens - 1
+    subscription.rate_updated_at = now
+    db.add(
+        NotificationDeliveryAttempt(
+            delivery_id=delivery.id,
+            number=delivery.attempt_count,
+            started_at=now,
+            outcome="started",
+        )
+    )
+    await db.commit()
+    return delivery, subscription, event
+
+
+async def _finish(
+    db: AsyncSession,
+    delivery_id: str,
+    owner: str,
+    *,
+    delivered: bool,
+    retryable: bool,
+    response_class: str,
+    reason: str,
+    retry_cap: int,
+    max_attempts: int,
+) -> None:
+    delivery = await db.get(NotificationDelivery, delivery_id)
+    if delivery is None or delivery.lease_owner != owner or delivery.state != "leased":
+        await db.rollback()
+        return
+    attempt = await db.scalar(
+        select(NotificationDeliveryAttempt).where(
+            NotificationDeliveryAttempt.delivery_id == delivery.id,
+            NotificationDeliveryAttempt.number == delivery.attempt_count,
+        )
+    )
+    if attempt is None:  # pragma: no cover - an invariant of _claim_due
+        await db.rollback()
+        return
+    now = datetime.now(UTC)
+    attempt.completed_at = now
+    attempt.response_class = response_class
+    attempt.diagnostic = reason[:512]
+    delivery.response_class = response_class
+    delivery.response_summary = reason[:512]
+    delivery.lease_owner = None
+    delivery.lease_expires_at = None
+    if delivered:
+        delivery.state = "delivered"
+        attempt.outcome = "delivered"
+    elif retryable and delivery.attempt_count < max_attempts:
+        delivery.state = "retry"
+        delivery.due_at = now + timedelta(seconds=retry_delay(delivery.attempt_count, retry_cap))
+        attempt.outcome = "retry"
+    else:
+        delivery.state = "failed"
+        delivery.terminal_reason = reason
+        attempt.outcome = "failed"
+    await db.commit()
+
+
+async def dispatch_one(
+    db: AsyncSession,
+    settings: Settings,
+    cipher: EnvelopeCipher,
+    owner: str,
+    *,
+    resolver: Resolver = system_resolver,
+) -> bool:
+    claimed = await _claim_due(db, owner, settings.notification_delivery_lease_seconds)
+    if claimed is None:
+        return False
+    delivery, subscription, event = claimed
+    max_attempts = settings.notification_max_attempts
+    try:
+        if subscription.channel == "webhook":
+            key_rows = list(
+                (
+                    await db.scalars(
+                        select(NotificationSigningKey).where(
+                            NotificationSigningKey.subscription_id == subscription.id,
+                            NotificationSigningKey.state.in_(("active", "overlap")),
+                        )
+                    )
+                ).all()
+            )
+            keys: list[SigningMaterial] = []
+            now = datetime.now(UTC)
+            for key in key_rows:
+                if (
+                    key.state == "overlap"
+                    and key.overlap_expires_at is not None
+                    and key.overlap_expires_at <= now
+                ):
+                    key.state = "retired"
+                    continue
+                secret = await db.get(Secret, key.secret_id)
+                if secret is None:
+                    raise WebhookTransportError("webhook signing secret is unavailable")
+                keys.append(
+                    SigningMaterial(
+                        key_id=key.id,
+                        version=key.version,
+                        secret=cipher.decrypt(
+                            secret.ciphertext,
+                            purpose=secret.purpose,
+                            version=secret.version,
+                        ),
+                    )
+                )
+            await db.commit()
+            webhook_result = await deliver_webhook(
+                str(subscription.destination_config["url"]),
+                event.canonical_envelope.encode(),
+                event_id=event.id,
+                event_type=event.type,
+                timestamp=event.occurred_at.isoformat(),
+                keys=keys,
+                resolver=resolver,
+                connect_timeout=settings.notification_connect_timeout_seconds,
+                read_timeout=settings.notification_read_timeout_seconds,
+                max_response_bytes=settings.notification_max_response_bytes,
+            )
+            decision = webhook_decision(webhook_result.status_code)
+        elif subscription.channel == "email":
+            email_settings = await db.get(NotificationEmailSettings, 1)
+            if email_settings is None:
+                raise EmailTransportError("SMTP settings are unavailable")
+            max_attempts = email_settings.max_attempts
+            password_secret = await db.get(Secret, email_settings.password_secret_id)
+            if password_secret is None:
+                raise EmailTransportError("SMTP password is unavailable")
+            email_result = await deliver_email(
+                email_settings,
+                cipher.decrypt(
+                    password_secret.ciphertext,
+                    purpose=password_secret.purpose,
+                    version=password_secret.version,
+                ),
+                event,
+                subscription.destination_config["recipients"],
+            )
+            decision = RetryDecision(False, email_result.response_class, "delivered")
+        else:  # guarded by DB constraint
+            raise WebhookTransportError("notification channel is unavailable")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=decision.reason == "delivered",
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=decision.reason,
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except EmailTransportError as error:
+        decision = transport_decision(error.transient, "smtp_transport")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=str(error),
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except WebhookTransportError as error:
+        decision = transport_decision(error.transient, "webhook_transport")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=str(error),
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except (KeyError, ValueError):
+        decision = transport_decision(False, "webhook_validation")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=False,
+            response_class=decision.response_class,
+            reason=decision.reason,
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    return True

backend/src/backup_tool/notifications/email.py

@@ -0,0 +1,120 @@
+"""Authenticated, certificate-verified STARTTLS email notification transport."""
+
+from __future__ import annotations
+
+import asyncio
+import smtplib
+import ssl
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.utils import formataddr
+from typing import Protocol, Self, cast
+
+from backup_tool.db.models import NotificationEmailSettings, NotificationEvent
+
+
+class SMTPClient(Protocol):
+    def __enter__(self) -> Self: ...
+
+    def __exit__(self, *args: object) -> None: ...
+
+    def ehlo(self) -> object: ...
+
+    def starttls(self, *, context: ssl.SSLContext) -> object: ...
+
+    def login(self, user: str, password: str) -> object: ...
+
+    def send_message(self, msg: EmailMessage) -> object: ...
+
+
+class EmailTransportError(RuntimeError):
+    def __init__(self, message: str, *, transient: bool = False) -> None:
+        super().__init__(message)
+        self.transient = transient
+
+
+@dataclass(frozen=True)
+class EmailResult:
+    response_class: str
+
+
+def validate_address(value: str) -> str:
+    if not value or len(value) > 320 or any(character in value for character in "\r\n"):
+        raise EmailTransportError("email address is invalid")
+    local, separator, domain = value.rpartition("@")
+    if not separator or not local or not domain or any(character.isspace() for character in value):
+        raise EmailTransportError("email address is invalid")
+    return value
+
+
+def validate_recipients(values: Sequence[str]) -> list[str]:
+    if not values or len(values) > 20:
+        raise EmailTransportError("one to 20 email recipients are required")
+    recipients: list[str] = []
+    for value in values:
+        address = validate_address(value)
+        if address not in recipients:
+            recipients.append(address)
+    return recipients
+
+
+def _message(
+    settings: NotificationEmailSettings,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+) -> EmailMessage:
+    sender = validate_address(settings.sender)
+    safe_recipients = validate_recipients(recipients)
+    message = EmailMessage()
+    message["From"] = formataddr(("Backup Tool", sender))
+    message["To"] = ", ".join(safe_recipients)
+    message["Subject"] = f"Backup Tool: {event.type} ({event.severity})"
+    message["X-Backup-Event-ID"] = event.id
+    # Do not put the full envelope, paths, raw errors, or credentials into mail.
+    message.set_content(
+        "Backup Tool operational event\n"
+        f"Event ID: {event.id}\n"
+        f"Type: {event.type}\n"
+        f"Severity: {event.severity}\n"
+        f"Occurred: {event.occurred_at.isoformat()}\n"
+    )
+    return message
+
+
+def _deliver_sync(
+    settings: NotificationEmailSettings,
+    password: str,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+    smtp_factory: Callable[..., SMTPClient],
+) -> EmailResult:
+    message = _message(settings, event, recipients)
+    try:
+        with smtp_factory(settings.host, settings.port, timeout=10) as client:
+            client.ehlo()
+            context = ssl.create_default_context()
+            client.starttls(context=context)
+            client.ehlo()
+            client.login(settings.username, password)
+            client.send_message(message)
+    except smtplib.SMTPResponseException as error:
+        raise EmailTransportError(
+            f"smtp_{error.smtp_code}", transient=400 <= error.smtp_code < 500
+        ) from error
+    except (smtplib.SMTPException, OSError) as error:
+        raise EmailTransportError("smtp_transport_failed", transient=True) from error
+    return EmailResult(response_class="smtp_2xx")
+
+
+async def deliver_email(
+    settings: NotificationEmailSettings,
+    password: str,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+    *,
+    smtp_factory: Callable[..., SMTPClient] | None = None,
+) -> EmailResult:
+    """Run blocking SMTP only in the worker thread, never in the web process."""
+    factory = smtp_factory or cast(Callable[..., SMTPClient], smtplib.SMTP)
+    return await asyncio.to_thread(_deliver_sync, settings, password, event, recipients, factory)

backend/src/backup_tool/notifications/events.py

@@ -0,0 +1,279 @@
+"""Versioned notification event catalog and transactional outbox fan-out."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from typing import Any
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.db.models import NotificationDelivery, NotificationEvent, NotificationSubscription
+from backup_tool.ids import new_uuid7
+from backup_tool.security.redaction import redact
+from backup_tool.security.ssrf import SSRFError, validate_webhook_url
+
+EVENT_SCHEMA_VERSION = 1
+# Live-events-only policy: every public type below has a current production emitter.
+_EVENT_TYPES = (
+    "execution.queued",
+    "execution.started",
+    "execution.committed",
+    "execution.failed",
+    "execution.cancelled",
+    "execution.retry_queued",
+    "execution.worker_recovered",
+    "schedule.created",
+    "schedule.updated",
+    "schedule.deleted",
+    "schedule.enabled",
+    "schedule.disabled",
+    "schedule.occurrence_enqueued",
+    "schedule.occurrence_misfired",
+    "schedule.occurrence_blocked",
+    "backup.committed",
+    "backup.verification_succeeded",
+    "restore.queued",
+    "restore.committed",
+    "restore.failed",
+    "retention.tombstoned",
+    "notification.test_requested",
+)
+EVENT_CATALOG: dict[str, dict[str, Any]] = {
+    event_type: {
+        "event_schema_version": EVENT_SCHEMA_VERSION,
+        "severity": "error" if event_type.endswith(("failed", "blocked", "rejected")) else "info",
+        "payload_keys": (
+            "attempt",
+            "count",
+            "dry_run",
+            "integrity",
+            "message",
+            "outcome",
+            "reason_code",
+            "requested_mode",
+            "effective_mode",
+            "state",
+        ),
+        "reserved": False,
+    }
+    for event_type in _EVENT_TYPES
+}
+_SAFE_RESOURCE_KEYS = frozenset(
+    {
+        "execution_id",
+        "job_id",
+        "schedule_id",
+        "backup_id",
+        "restore_id",
+        "repository_id",
+        "subscription_id",
+    }
+)
+_SAFE_PAYLOAD_KEYS = frozenset().union(
+    *(set(spec["payload_keys"]) for spec in EVENT_CATALOG.values())
+)
+
+
+class NotificationEventError(ValueError):
+    """A caller attempted to produce data outside the stable public catalog."""
+
+
+def _as_uuid(value: str, field: str) -> str:
+    try:
+        parsed = UUID(value)
+    except (TypeError, ValueError, AttributeError) as error:
+        raise NotificationEventError(f"{field} must be a UUID") from error
+    return str(parsed)
+
+
+def _safe_value(value: Any) -> Any:
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    if isinstance(value, str):
+        return redact(value)[:512]
+    if isinstance(value, list):
+        if len(value) > 32:
+            raise NotificationEventError("payload arrays are limited to 32 values")
+        return [_safe_value(item) for item in value]
+    if isinstance(value, Mapping):
+        if len(value) > 32:
+            raise NotificationEventError("payload objects are limited to 32 fields")
+        return {str(key)[:64]: _safe_value(item) for key, item in value.items()}
+    raise NotificationEventError("payload contains an unsupported value")
+
+
+def _is_filter_match(filter_value: str, event_type: str) -> bool:
+    if filter_value == event_type:
+        return True
+    family, wildcard = filter_value.rsplit(".", 1) if "." in filter_value else ("", "")
+    return wildcard == "*" and event_type.startswith(f"{family}.")
+
+
+def validate_filters(filters: Sequence[str]) -> list[str]:
+    if not filters:
+        raise NotificationEventError("at least one event filter is required")
+    if len(filters) > len(EVENT_CATALOG):
+        raise NotificationEventError("too many event filters")
+    output: list[str] = []
+    for filter_value in filters:
+        if not isinstance(filter_value, str) or len(filter_value) > 96:
+            raise NotificationEventError("invalid event filter")
+        if filter_value.endswith(".*"):
+            family = filter_value[:-2]
+            if not family or not any(item.startswith(f"{family}.") for item in EVENT_CATALOG):
+                raise NotificationEventError("unknown event filter")
+        elif filter_value not in EVENT_CATALOG:
+            raise NotificationEventError("unknown event filter")
+        if filter_value not in output:
+            output.append(filter_value)
+    return output
+
+
+def validate_destination(channel: str, destination: Mapping[str, Any]) -> dict[str, Any]:
+    resource_filters = destination.get("resource_filters", {})
+    if not isinstance(resource_filters, Mapping):
+        raise NotificationEventError("resource filters are invalid")
+    unknown = set(resource_filters) - {"job_ids", "repository_ids", "severities"}
+    if unknown:
+        raise NotificationEventError("resource filter is unknown")
+    normalized_filters: dict[str, list[str]] = {}
+    for key in ("job_ids", "repository_ids"):
+        values = resource_filters.get(key)
+        if values is None:
+            continue
+        if not isinstance(values, list) or not values:
+            raise NotificationEventError("resource filter is invalid")
+        normalized_filters[key] = [_as_uuid(value, key) for value in values]
+    severities = resource_filters.get("severities")
+    if severities is not None:
+        if not isinstance(severities, list) or not severities:
+            raise NotificationEventError("severity filter is invalid")
+        if any(value not in {"info", "warning", "error", "security"} for value in severities):
+            raise NotificationEventError("severity filter is invalid")
+        normalized_filters["severities"] = list(severities)
+    if channel == "webhook":
+        url = destination.get("url")
+        if not isinstance(url, str) or len(url) > 2048:
+            raise NotificationEventError("webhook URL is required")
+        try:
+            validate_webhook_url(url)
+        except SSRFError as error:
+            raise NotificationEventError("webhook URL is invalid") from error
+        return {"url": url, "resource_filters": normalized_filters}
+    if channel == "email":
+        recipients = destination.get("recipients")
+        if not isinstance(recipients, list) or not recipients or len(recipients) > 20:
+            raise NotificationEventError("one to 20 email recipients are required")
+        safe_recipients: list[str] = []
+        for recipient in recipients:
+            if not isinstance(recipient, str) or any(char in recipient for char in "\r\n"):
+                raise NotificationEventError("invalid email recipient")
+            if "@" not in recipient or len(recipient) > 320:
+                raise NotificationEventError("invalid email recipient")
+            if recipient not in safe_recipients:
+                safe_recipients.append(recipient)
+        return {"recipients": safe_recipients, "resource_filters": normalized_filters}
+    raise NotificationEventError("unsupported notification channel")
+
+
+def _matches(subscription: NotificationSubscription, event: dict[str, Any]) -> bool:
+    if subscription.state != "active":
+        return False
+    if not any(_is_filter_match(item, str(event["type"])) for item in subscription.event_filters):
+        return False
+    filters = subscription.destination_config.get("resource_filters", {})
+    if not isinstance(filters, Mapping):
+        return False
+    resources = event["resource"]
+    for key in ("job_ids", "repository_ids"):
+        selected = filters.get(key)
+        resource_key = key[:-1]
+        if selected is not None and resources.get(resource_key) not in selected:
+            return False
+    severities = filters.get("severities")
+    return severities is None or event["severity"] in severities
+
+
+async def emit_event(
+    db: AsyncSession,
+    event_type: str,
+    *,
+    correlation_id: str,
+    resource: Mapping[str, str] | None = None,
+    payload: Mapping[str, Any] | None = None,
+    severity: str | None = None,
+    deduplication_key: str | None = None,
+    occurred_at: datetime | None = None,
+    only_subscription_id: str | None = None,
+) -> NotificationEvent:
+    """Append an immutable event and matching deliveries; intentionally never commits."""
+    if event_type not in EVENT_CATALOG:
+        raise NotificationEventError("unknown operational event type")
+    correlation_id = _as_uuid(correlation_id, "correlation_id")
+    resource = resource or {}
+    if set(resource) - _SAFE_RESOURCE_KEYS:
+        raise NotificationEventError("unknown resource reference")
+    safe_resource = {key: _as_uuid(value, key) for key, value in resource.items()}
+    payload = payload or {}
+    if set(payload) - _SAFE_PAYLOAD_KEYS:
+        raise NotificationEventError("payload key is not allowlisted")
+    safe_payload = {key: _safe_value(value) for key, value in payload.items()}
+    event_severity = severity or str(EVENT_CATALOG[event_type]["severity"])
+    if event_severity not in {"info", "warning", "error", "security"}:
+        raise NotificationEventError("invalid severity")
+    if deduplication_key is not None and (not deduplication_key or len(deduplication_key) > 255):
+        raise NotificationEventError("invalid deduplication key")
+    if deduplication_key is not None:
+        existing = await db.scalar(
+            select(NotificationEvent).where(
+                NotificationEvent.deduplication_key == deduplication_key
+            )
+        )
+        if existing is not None:
+            return existing
+    event_id = str(new_uuid7())
+    timestamp = (occurred_at or datetime.now(UTC)).astimezone(UTC)
+    envelope = {
+        "event_schema_version": EVENT_SCHEMA_VERSION,
+        "id": event_id,
+        "type": event_type,
+        "occurred_at": timestamp.isoformat(),
+        "correlation_id": correlation_id,
+        "severity": event_severity,
+        "resource": safe_resource,
+        "payload": safe_payload,
+    }
+    event = NotificationEvent(
+        id=event_id,
+        type=event_type,
+        schema_version=EVENT_SCHEMA_VERSION,
+        occurred_at=timestamp,
+        correlation_id=correlation_id,
+        severity=event_severity,
+        resource_refs=safe_resource,
+        payload=safe_payload,
+        canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
+        deduplication_key=deduplication_key,
+    )
+    db.add(event)
+    await db.flush()
+    active_subscriptions = select(NotificationSubscription).where(
+        NotificationSubscription.state == "active"
+    )
+    subscriptions = list((await db.scalars(active_subscriptions)).all())
+    for subscription in subscriptions:
+        selected_for_test = only_subscription_id == subscription.id
+        if selected_for_test or (only_subscription_id is None and _matches(subscription, envelope)):
+            db.add(
+                NotificationDelivery(
+                    event_id=event.id,
+                    subscription_id=subscription.id,
+                    due_at=timestamp,
+                )
+            )
+    await db.flush()
+    return event

backend/src/backup_tool/notifications/retry.py

@@ -0,0 +1,33 @@
+"""Deterministic bounded retry classification for notification delivery."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RetryDecision:
+    retry: bool
+    response_class: str
+    reason: str
+
+
+def retry_delay(attempt: int, cap_seconds: int) -> int:
+    """Bounded exponential delay; no jitter keeps durable tests/restarts deterministic."""
+    exponent = max(0, attempt - 1)
+    return min(cap_seconds, 1 << exponent)
+
+
+def webhook_decision(status_code: int) -> RetryDecision:
+    if 200 <= status_code < 300:
+        return RetryDecision(False, "http_2xx", "delivered")
+    if status_code in {408, 425, 429} or status_code >= 500:
+        return RetryDecision(True, f"http_{status_code}", "http_transient")
+    if 300 <= status_code < 400:
+        return RetryDecision(False, f"http_{status_code}", "redirect_rejected")
+    return RetryDecision(False, f"http_{status_code}", "http_permanent")
+
+
+def transport_decision(transient: bool, response_class: str) -> RetryDecision:
+    reason = "transport_transient" if transient else "transport_failed"
+    return RetryDecision(transient, response_class, reason)

backend/src/backup_tool/notifications/webhook.py

@@ -0,0 +1,210 @@
+"""Canonical, dual-key HMAC webhook requests on a DNS-pinned HTTPX transport."""
+
+from __future__ import annotations
+
+import asyncio
+import hmac
+import ssl
+from collections.abc import Sequence
+from dataclasses import dataclass
+from hashlib import sha256
+
+import httpx
+
+from backup_tool.security.ssrf import (
+    ResolvedWebhookTarget,
+    Resolver,
+    SSRFError,
+    resolve_webhook_target,
+    verify_connected_peer,
+)
+
+SIGNATURE_VERSION = "v1"
+
+
+class WebhookTransportError(RuntimeError):
+    def __init__(self, message: str, *, transient: bool = False) -> None:
+        super().__init__(message)
+        self.transient = transient
+
+
+@dataclass(frozen=True)
+class SigningMaterial:
+    key_id: str
+    version: int
+    secret: str
+
+
+@dataclass(frozen=True)
+class WebhookResult:
+    status_code: int
+    response_bytes: int
+
+
+def canonical_signing_input(timestamp: str, body: bytes) -> bytes:
+    return SIGNATURE_VERSION.encode() + b"." + timestamp.encode("ascii") + b"." + body
+
+
+def signatures(timestamp: str, body: bytes, keys: Sequence[SigningMaterial]) -> list[str]:
+    signing_input = canonical_signing_input(timestamp, body)
+    return [
+        f"{SIGNATURE_VERSION};key_id={key.key_id};key_version={key.version};sha256="
+        f"{hmac.new(key.secret.encode(), signing_input, sha256).hexdigest()}"
+        for key in keys
+    ]
+
+
+class PinnedWebhookTransport(httpx.AsyncBaseTransport):
+    """HTTPX transport that never lets a post-validation DNS lookup choose a peer."""
+
+    def __init__(
+        self,
+        *,
+        target: ResolvedWebhookTarget,
+        connect_timeout: float,
+        read_timeout: float,
+        max_response_bytes: int,
+    ) -> None:
+        self._target = target
+        self._connect_timeout = connect_timeout
+        self._read_timeout = read_timeout
+        self._max_response_bytes = max_response_bytes
+
+    async def _connect(
+        self, target: ResolvedWebhookTarget
+    ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]:
+        hostname = target.url.hostname
+        if hostname is None:  # guarded by resolve_webhook_target
+            raise WebhookTransportError("webhook hostname is unavailable")
+        context = ssl.create_default_context() if target.url.scheme == "https" else None
+        last_error: OSError | None = None
+        for address in target.addresses:
+            try:
+                reader, writer = await asyncio.wait_for(
+                    asyncio.open_connection(
+                        address,
+                        target.port,
+                        ssl=context,
+                        server_hostname=hostname if context is not None else None,
+                    ),
+                    timeout=self._connect_timeout,
+                )
+                verify_connected_peer(writer.get_extra_info("peername"), target.addresses)
+                return reader, writer
+            except (TimeoutError, OSError, ssl.SSLError, SSRFError) as error:
+                last_error = error if isinstance(error, OSError) else None
+        raise WebhookTransportError("webhook connection failed", transient=True) from last_error
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        target = self._target
+        if str(request.url) != target.url.geturl():
+            raise WebhookTransportError("webhook target changed")
+        body = await request.aread()
+        if len(body) > 65_536:
+            raise WebhookTransportError("webhook body is too large")
+        reader, writer = await self._connect(target)
+        try:
+            raw_path = target.url.path or "/"
+            if target.url.query:
+                raw_path += "?" + target.url.query
+            headers = [(key, value) for key, value in request.headers.multi_items()]
+            header_names = {key.lower() for key, _ in headers}
+            if "host" not in header_names:
+                host = target.url.hostname or ""
+                headers.append(("Host", host))
+            if "content-length" not in header_names:
+                headers.append(("Content-Length", str(len(body))))
+            headers.append(("Connection", "close"))
+            serialized = [f"{request.method} {raw_path} HTTP/1.1\r\n".encode()]
+            serialized.extend(f"{key}: {value}\r\n".encode("ascii") for key, value in headers)
+            writer.write(b"".join(serialized) + b"\r\n" + body)
+            await asyncio.wait_for(writer.drain(), timeout=self._read_timeout)
+            head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=self._read_timeout)
+            if len(head) > 16_384:
+                raise WebhookTransportError("webhook response headers are too large")
+            lines = head.decode("iso-8859-1").split("\r\n")
+            try:
+                _protocol, code, _reason = lines[0].split(" ", 2)
+                status_code = int(code)
+            except (IndexError, ValueError) as error:
+                raise WebhookTransportError("webhook response is malformed") from error
+            response_headers: list[tuple[str, str]] = []
+            for line in lines[1:]:
+                if not line:
+                    continue
+                key, separator, value = line.partition(":")
+                if not separator:
+                    raise WebhookTransportError("webhook response headers are malformed")
+                response_headers.append((key.strip(), value.strip()))
+            content = await asyncio.wait_for(
+                reader.read(self._max_response_bytes + 1), timeout=self._read_timeout
+            )
+            if len(content) > self._max_response_bytes:
+                raise WebhookTransportError("webhook response is too large")
+            return httpx.Response(
+                status_code,
+                headers=response_headers,
+                content=content,
+                request=request,
+            )
+        except (TimeoutError, OSError, asyncio.IncompleteReadError) as error:
+            raise WebhookTransportError("webhook request failed", transient=True) from error
+        finally:
+            writer.close()
+            with __import__("contextlib").suppress(OSError):
+                await writer.wait_closed()
+
+
+def webhook_headers(
+    event_id: str,
+    event_type: str,
+    timestamp: str,
+    body: bytes,
+    keys: Sequence[SigningMaterial],
+) -> list[tuple[str, str]]:
+    headers: list[tuple[str, str]] = [
+        ("Content-Type", "application/json"),
+        ("X-Backup-Event-ID", event_id),
+        ("X-Backup-Event-Type", event_type),
+        ("X-Backup-Signature-Version", SIGNATURE_VERSION),
+        ("X-Backup-Timestamp", timestamp),
+    ]
+    headers.extend(("X-Backup-Signature", value) for value in signatures(timestamp, body, keys))
+    return headers
+
+
+async def deliver_webhook(
+    url: str,
+    body: bytes,
+    *,
+    event_id: str,
+    event_type: str,
+    timestamp: str,
+    keys: Sequence[SigningMaterial],
+    resolver: Resolver,
+    connect_timeout: float,
+    read_timeout: float,
+    max_response_bytes: int,
+) -> WebhookResult:
+    if not keys:
+        raise WebhookTransportError("webhook subscription has no active signing key")
+    try:
+        # Resolve immediately before this individual attempt.  The resulting
+        # addresses are passed to the transport, so it cannot rebind at connect.
+        target = await resolve_webhook_target(url, resolver)
+    except SSRFError as error:
+        raise WebhookTransportError("webhook_target_rejected") from error
+    transport = PinnedWebhookTransport(
+        target=target,
+        connect_timeout=connect_timeout,
+        read_timeout=read_timeout,
+        max_response_bytes=max_response_bytes,
+    )
+    headers = webhook_headers(event_id, event_type, timestamp, body, keys)
+    async with httpx.AsyncClient(
+        transport=transport, follow_redirects=False, trust_env=False
+    ) as client:
+        response = await client.post(url, content=body, headers=headers)
+    if 300 <= response.status_code < 400:
+        raise WebhookTransportError("redirect_rejected")
+    return WebhookResult(status_code=response.status_code, response_bytes=len(response.content))

backend/src/backup_tool/observability/__init__.py

@@ -0,0 +1 @@
+"""Operational logging, metrics, and readiness primitives."""

backend/src/backup_tool/observability/health.py

@@ -0,0 +1,39 @@
+"""Readiness checks shared by HTTP and background process roles."""
+
+from __future__ import annotations
+
+import os
+
+from backup_tool.cli import build_alembic_config
+from backup_tool.config import Settings
+from backup_tool.db.engine import assert_schema_current, create_engine
+
+
+class ReadinessError(RuntimeError):
+    """A dependency needed by the selected role is unavailable."""
+
+
+def _require_access(path: object, mode: int) -> None:
+    try:
+        candidate = path if isinstance(path, str) else str(path)
+        if not os.path.isdir(candidate) or not os.access(candidate, mode):
+            raise ReadinessError("required storage is unavailable")
+    except OSError as error:
+        raise ReadinessError("required storage is unavailable") from error
+
+
+async def check_role_readiness(settings: Settings, role: str) -> None:
+    engine = create_engine(settings)
+    try:
+        await assert_schema_current(engine, build_alembic_config(settings))
+    except Exception as error:
+        raise ReadinessError("metadata is unavailable") from error
+    finally:
+        await engine.dispose()
+    if role == "scheduler":
+        return
+    _require_access(settings.data_dir, os.R_OK | os.W_OK | os.X_OK)
+    for root in settings.repository_roots + settings.restore_roots:
+        _require_access(root, os.R_OK | os.W_OK | os.X_OK)
+    for root in settings.local_source_roots:
+        _require_access(root, os.R_OK | os.X_OK)

backend/src/backup_tool/observability/logging.py

@@ -0,0 +1,39 @@
+"""JSON logging that keeps operational context machine-readable and secret-free."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import UTC, datetime
+from typing import Any
+
+_STANDARD_RECORD_KEYS = frozenset(logging.makeLogRecord({}).__dict__)
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "event": record.getMessage(),
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "timestamp": datetime.now(UTC).isoformat(),
+        }
+        for key, value in record.__dict__.items():
+            if key not in _STANDARD_RECORD_KEYS and key not in {"message", "asctime"}:
+                payload[key] = value
+        return json.dumps(payload, default=str, separators=(",", ":"), sort_keys=True)
+
+
+def configure_logging(role: str, level: str) -> None:
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(JsonFormatter())
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.addHandler(handler)
+    root.setLevel(level)
+    logging.getLogger("backup_tool").info("role_started", extra={"role": role})
+
+
+def log_event(name: str, **fields: object) -> None:
+    logging.getLogger("backup_tool").info(name, extra=fields)

backend/src/backup_tool/observability/metrics.py

@@ -0,0 +1,102 @@
+"""Small dependency-free Prometheus exposition for the single-node appliance."""
+
+from __future__ import annotations
+
+import os
+import threading
+from collections import defaultdict
+from collections.abc import Iterable
+from datetime import UTC, datetime
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.config import Settings
+from backup_tool.db.models import Backup, Execution, Repository, Schedule
+from backup_tool.execution import ACTIVE_STATES
+
+
+class Metrics:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
+        self._durations: dict[tuple[str, str], tuple[int, float]] = {}
+
+    def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None:
+        route = path if path in {"/livez", "/readyz", "/metrics"} else "/api"
+        with self._lock:
+            self._requests[(method, route, status)] += 1
+            count, total = self._durations.get((method, route), (0, 0.0))
+            self._durations[(method, route)] = (count + 1, total + duration_seconds)
+
+    def render(self, operational: Iterable[tuple[str, float]]) -> str:
+        lines = [
+            "# HELP backup_tool_http_requests_total HTTP requests handled by the web role.",
+            "# TYPE backup_tool_http_requests_total counter",
+        ]
+        with self._lock:
+            for (method, path, status), value in sorted(self._requests.items()):
+                labels = f'method="{method}",path="{path}",status="{status}"'
+                lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}")
+            lines.extend(
+                [
+                    "# HELP backup_tool_http_request_duration_seconds HTTP request duration.",
+                    "# TYPE backup_tool_http_request_duration_seconds summary",
+                ]
+            )
+            for (method, path), (count, total) in sorted(self._durations.items()):
+                labels = f'method="{method}",path="{path}"'
+                lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}")
+                lines.append(
+                    f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}"
+                )
+        lines.extend(f"{name} {value}" for name, value in operational)
+        return "\n".join(lines) + "\n"
+
+
+async def collect_operational_metrics(
+    settings: Settings, db: AsyncSession
+) -> list[tuple[str, float]]:
+    now = datetime.now(UTC)
+    active = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES))
+    )
+    stale = await db.scalar(
+        select(func.count())
+        .select_from(Execution)
+        .where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now)
+    )
+    failed = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state == "failed")
+    )
+    corrupt = await db.scalar(
+        select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt")
+    )
+    schedule_lag = await db.scalar(
+        select(func.min(Schedule.next_nominal_at)).where(
+            Schedule.enabled, Schedule.next_nominal_at.is_not(None)
+        )
+    )
+    values = [
+        ("backup_tool_active_executions", active or 0),
+        ("backup_tool_stale_execution_leases", stale or 0),
+        ("backup_tool_failed_executions", failed or 0),
+        ("backup_tool_corrupt_backups", corrupt or 0),
+        (
+            "backup_tool_schedule_lag_seconds",
+            max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0,
+        ),
+    ]
+    roots = list(settings.repository_roots) + list(settings.restore_roots)
+    for index, root in enumerate(roots):
+        try:
+            stats = os.statvfs(root)
+        except OSError:
+            continue
+        name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}'
+        values.append((name, stats.f_bavail * stats.f_frsize))
+    unavailable = await db.scalar(
+        select(func.count()).select_from(Repository).where(Repository.state == "unavailable")
+    )
+    values.append(("backup_tool_unavailable_repositories", unavailable or 0))
+    return values

backend/src/backup_tool/repository.py

@@ -4,12 +4,19 @@ import hashlib
 import json
 import os
 import shutil
+import stat
+from contextlib import suppress
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from sqlalchemy.ext.asyncio import AsyncSession
+
 from backup_tool.config import Settings
 from backup_tool.ids import new_uuid7
+from backup_tool.security.repository_crypto import RepositoryKeyError, create_data_key
 
 
 class RepositoryError(ValueError):
@@ -19,9 +26,15 @@ class RepositoryError(ValueError):
 @dataclass(frozen=True)
 class InitializedRepository:
     root: Path
+    repository_id: str
     format_version: int = 1
     compression: str = "none"
     encryption: str = "none"
+    signing_key_id: str = ""
+    signing_public_key: str = ""
+    signing_key_path: Path | None = None
+    data_key_id: str | None = None
+    data_key_path: Path | None = None
 
 
 def blob_digest(content: bytes) -> str:
@@ -47,14 +60,14 @@ def _contained(root: Path, relative_path: str) -> Path:
 
 
 def _canonical_payload(compression: str, encryption: str) -> dict[str, object]:
-    if compression != "none" or encryption != "none":
+    if compression != "none" or encryption not in {"none", "aes-256-gcm"}:
         raise RepositoryError("requested repository policy is unavailable")
     return {
         "repository_id": str(new_uuid7()),
         "format_version": 1,
         "digest_algorithm": "sha256",
         "compression": compression,
-        "encryption": {"mode": "none", "key_id": None},
+        "encryption": {"mode": encryption, "key_id": None},
         "created_at": datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z"),
     }
 
@@ -63,7 +76,205 @@ def _canonical_json(payload: dict[str, object]) -> str:
     return json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n"
 
 
-def _assert_capacity(settings: Settings, root: Path) -> None:
+def replace_active_data_key(root: Path, expected_key_id: str, new_key_id: str) -> None:
+    """Durably replace the active, non-secret epoch reference in repository metadata."""
+    metadata = root / "repository.json"
+    try:
+        payload = json.loads(metadata.read_text(encoding="utf-8"))
+        encryption = payload.get("encryption") if isinstance(payload, dict) else None
+        if (
+            not isinstance(encryption, dict)
+            or encryption.get("mode") != "aes-256-gcm"
+            or encryption.get("key_id") != expected_key_id
+        ):
+            raise ValueError
+    except (OSError, ValueError, json.JSONDecodeError) as error:
+        raise RepositoryError("repository metadata is invalid") from error
+    encryption["key_id"] = new_key_id
+    staging = root / f".repository.json.{os.urandom(8).hex()}.tmp"
+    try:
+        descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(_canonical_json(payload))
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(staging, metadata)
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository metadata cannot be updated") from error
+    finally:
+        staging.unlink(missing_ok=True)
+
+
+_ROTATION_JOURNAL = ".key-rotation.json"
+
+
+def begin_key_rotation(
+    root: Path,
+    repository_database_id: str,
+    repository_id: str,
+    old_key_id: str,
+    new_key_id: str,
+) -> None:
+    """Persist an intent record before changing durable key-epoch state."""
+    if not all((repository_database_id, repository_id, old_key_id, new_key_id)):
+        raise RepositoryError("repository rotation journal is invalid")
+    journal = root / _ROTATION_JOURNAL
+    payload: dict[str, object] = {
+        "new_key_id": new_key_id,
+        "old_key_id": old_key_id,
+        "repository_database_id": repository_database_id,
+        "repository_id": repository_id,
+        "version": 1,
+    }
+    try:
+        descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(_canonical_json(payload))
+            handle.flush()
+            os.fsync(handle.fileno())
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository rotation is already in progress") from error
+
+
+def read_key_rotation(root: Path) -> dict[str, str] | None:
+    journal = root / _ROTATION_JOURNAL
+    if not journal.exists():
+        return None
+    try:
+        if journal.is_symlink() or stat.S_IMODE(journal.stat().st_mode) != 0o600:
+            raise ValueError
+        payload = json.loads(journal.read_text(encoding="utf-8"))
+        expected = {
+            "new_key_id",
+            "old_key_id",
+            "repository_database_id",
+            "repository_id",
+            "version",
+        }
+        if (
+            not isinstance(payload, dict)
+            or set(payload) != expected
+            or payload.get("version") != 1
+            or not all(
+                isinstance(payload[key], str) and payload[key] for key in expected - {"version"}
+            )
+        ):
+            raise ValueError
+        return {key: payload[key] for key in expected - {"version"}}
+    except (OSError, ValueError, json.JSONDecodeError) as error:
+        raise RepositoryError("repository rotation journal is invalid") from error
+
+
+def finish_key_rotation(root: Path) -> None:
+    journal = root / _ROTATION_JOURNAL
+    try:
+        journal.unlink()
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository rotation journal cannot be cleared") from error
+
+
+async def reconcile_key_rotations(settings: Settings, db: AsyncSession) -> int:
+    """Converge a journaled key rotation after a process crash.
+
+    A rotation whose DB transaction did not commit remains on the old active
+    epoch. A committed transaction deterministically advances repository.json.
+    """
+    from sqlalchemy import select
+
+    from backup_tool.db.models import Repository, RepositoryDataKeyEpoch
+    from backup_tool.security.repository_crypto import load_data_key
+
+    repositories = list((await db.scalars(select(Repository))).all())
+    reconciled = 0
+    for repository in repositories:
+        if repository.encryption != "aes-256-gcm":
+            continue
+        root = Path(repository.root)
+        journal = read_key_rotation(root)
+        if journal is None:
+            continue
+        inspected = inspect_repository(settings, root)
+        if (
+            journal["repository_database_id"] != repository.id
+            or journal["repository_id"] != inspected.repository_id
+            or repository.active_data_key_id is None
+        ):
+            raise RepositoryError("repository rotation journal does not match metadata")
+        epochs = list(
+            (
+                await db.scalars(
+                    select(RepositoryDataKeyEpoch).where(
+                        RepositoryDataKeyEpoch.repository_id == repository.id
+                    )
+                )
+            ).all()
+        )
+        active = [epoch for epoch in epochs if epoch.state == "active"]
+        old_key_id = journal["old_key_id"]
+        new_key_id = journal["new_key_id"]
+        if len(active) != 1:
+            raise RepositoryError("repository rotation epochs are invalid")
+        if repository.active_data_key_id == old_key_id and active[0].key_id == old_key_id:
+            if inspected.data_key_id != old_key_id:
+                raise RepositoryError("repository rotation metadata is invalid")
+            new_path = (
+                settings.data_dir
+                / "repository-data-keys"
+                / (f"{inspected.repository_id}.{new_key_id}.key")
+            )
+            # Clear the durable intent before deleting an unreferenced key. A
+            # crash afterward leaves only an orphaned key, not a journal whose
+            # retry depends on a key that no longer exists.
+            if new_path.exists() or new_path.is_symlink():
+                try:
+                    load_data_key(settings, inspected.repository_id, new_key_id)
+                except RepositoryKeyError as error:
+                    raise RepositoryError("repository rotation key is unavailable") from error
+            finish_key_rotation(inspected.root)
+            if new_path.exists() or new_path.is_symlink():
+                # The journal is already gone, so a cleanup failure only leaves
+                # an unreferenced key and must not disable the old-active epoch.
+                with suppress(OSError):
+                    new_path.unlink()
+                    descriptor = os.open(new_path.parent, os.O_RDONLY)
+                    try:
+                        os.fsync(descriptor)
+                    finally:
+                        os.close(descriptor)
+            reconciled += 1
+            continue
+        if repository.active_data_key_id == new_key_id and active[0].key_id == new_key_id:
+            try:
+                load_data_key(settings, inspected.repository_id, new_key_id)
+            except RepositoryKeyError as error:
+                raise RepositoryError("repository rotation key is unavailable") from error
+            if inspected.data_key_id == old_key_id:
+                replace_active_data_key(inspected.root, old_key_id, new_key_id)
+            elif inspected.data_key_id != new_key_id:
+                raise RepositoryError("repository rotation metadata is invalid")
+            finish_key_rotation(inspected.root)
+            reconciled += 1
+            continue
+        raise RepositoryError("repository rotation state is invalid")
+    return reconciled
+
+
+def assert_capacity(settings: Settings, root: Path) -> None:
     capacity_root = root
     while not capacity_root.exists():
         parent = capacity_root.parent
@@ -76,13 +287,57 @@ def _assert_capacity(settings: Settings, root: Path) -> None:
         raise RepositoryError("repository root does not meet minimum free capacity")
 
 
+def _signing_key_directory(settings: Settings) -> Path:
+    directory = settings.data_dir / "repository-keys"
+    directory.mkdir(mode=0o700, exist_ok=True)
+    if directory.is_symlink() or not directory.is_dir():
+        raise RepositoryError("repository signing key directory is unsafe")
+    if stat.S_IMODE(directory.stat().st_mode) != 0o700:
+        raise RepositoryError("repository signing key directory permissions must be 0700")
+    return directory
+
+
+def _write_private_key_staging(directory: Path, repository_id: str, private_key: bytes) -> Path:
+    staging = directory / f".{repository_id}.{os.urandom(8).hex()}.tmp"
+    descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(descriptor, "wb") as handle:
+        handle.write(private_key)
+        handle.flush()
+        os.fsync(handle.fileno())
+    return staging
+
+
 def initialize(
     settings: Settings, relative_path: str, compression: str, encryption: str
 ) -> InitializedRepository:
     payload = _canonical_payload(compression, encryption)
+    repository_id = str(payload["repository_id"])
+    data_key_id: str | None = None
+    data_key_path: Path | None = None
+    if encryption == "aes-256-gcm":
+        try:
+            data_key_id, data_key_path = create_data_key(settings, repository_id)
+        except RepositoryKeyError as error:
+            raise RepositoryError("repository data key is unavailable") from error
+        payload["encryption"] = {"mode": encryption, "key_id": data_key_id}
+    private_key = Ed25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        serialization.Encoding.Raw,
+        serialization.PrivateFormat.Raw,
+        serialization.NoEncryption(),
+    )
+    public_bytes = private_key.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    signing_key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
+    key_directory = _signing_key_directory(settings)
+    key_path = key_directory / f"{repository_id}.ed25519"
+    key_staging = _write_private_key_staging(key_directory, repository_id, private_bytes)
     root = _contained(settings.repository_roots[0], relative_path)
-    _assert_capacity(settings, root.parent)
+    assert_capacity(settings, root.parent)
     if root.exists():
+        key_staging.unlink(missing_ok=True)
         raise RepositoryError("repository path already exists")
     staging = root.with_name(f".{root.name}.staging-{os.urandom(8).hex()}")
     try:
@@ -94,17 +349,106 @@ def initialize(
         with metadata.open("rb") as handle:
             os.fsync(handle.fileno())
         os.replace(staging, root)
+        os.replace(key_staging, key_path)
     except Exception:
         shutil.rmtree(staging, ignore_errors=True)
+        shutil.rmtree(root, ignore_errors=True)
+        key_staging.unlink(missing_ok=True)
+        key_path.unlink(missing_ok=True)
+        if data_key_path is not None:
+            data_key_path.unlink(missing_ok=True)
         raise
-    return InitializedRepository(root=root, compression=compression, encryption=encryption)
+    return InitializedRepository(
+        root=root,
+        repository_id=repository_id,
+        compression=compression,
+        encryption=encryption,
+        signing_key_id=signing_key_id,
+        signing_public_key=public_bytes.hex(),
+        signing_key_path=key_path,
+        data_key_id=data_key_id,
+        data_key_path=data_key_path,
+    )
 
 
-def remove_repository(root: Path) -> None:
+def remove_repository(
-    try:
+    root: Path,
+    signing_key_path: Path | None = None,
+    data_key_path: Path | None = None,
+) -> None:
+    try:  # noqa: SIM105 - cleanup must not race a concurrent remover
         shutil.rmtree(root)
     except FileNotFoundError:
-        return
+        pass
+    for key_path in (signing_key_path, data_key_path):
+        if key_path is None:
+            continue
+        try:  # noqa: SIM105 - cleanup must not race a concurrent remover
+            key_path.unlink()
+        except FileNotFoundError:
+            pass
+
+
+def install_signing_key(
+    settings: Settings,
+    repository_id: str,
+    expected_key_id: str,
+    expected_public_key: str,
+    private_bytes: bytes,
+) -> Path:
+    """Install a recovered signing key once after checking its public trust anchor."""
+    if len(private_bytes) != 32 or "/" in repository_id:
+        raise RepositoryError("repository signing key is invalid")
+    try:
+        private_key = Ed25519PrivateKey.from_private_bytes(private_bytes)
+        public_bytes = private_key.public_key().public_bytes(
+            serialization.Encoding.Raw,
+            serialization.PublicFormat.Raw,
+        )
+    except ValueError as error:
+        raise RepositoryError("repository signing key is invalid") from error
+    key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
+    if key_id != expected_key_id or public_bytes.hex() != expected_public_key:
+        raise RepositoryError("repository signing key is invalid")
+    path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(private_bytes)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            path.unlink(missing_ok=True)
+            raise RepositoryError("repository signing key is unsafe")
+    except OSError as error:
+        raise RepositoryError("repository signing key cannot be installed") from error
+    return path
+
+
+def load_signing_key(
+    settings: Settings,
+    repository_id: str,
+    expected_key_id: str,
+    expected_public_key: str,
+) -> Ed25519PrivateKey:
+    key_path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
+    try:
+        if key_path.is_symlink() or stat.S_IMODE(key_path.stat().st_mode) != 0o600:
+            raise RepositoryError("repository signing key is unsafe")
+        private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes())
+    except (OSError, ValueError) as error:
+        raise RepositoryError("repository signing key is unreadable") from error
+    public_key = private_key.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    key_id = f"ed25519-{hashlib.sha256(public_key).hexdigest()[:32]}"
+    if key_id != expected_key_id or public_key.hex() != expected_public_key:
+        raise RepositoryError("repository signing key does not match its trust anchor")
+    return private_key
 
 
 def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
@@ -140,7 +484,14 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
     ):
         raise RepositoryError("repository metadata is not canonical")
     encryption = payload.get("encryption")
-    if payload.get("compression") != "none" or encryption != {"mode": "none", "key_id": None}:
+    if payload.get("compression") != "none" or not isinstance(encryption, dict):
+        raise RepositoryError("repository metadata is invalid")
+    mode = encryption.get("mode")
+    key_id = encryption.get("key_id")
+    valid_policy = (mode == "none" and key_id is None) or (
+        mode == "aes-256-gcm" and isinstance(key_id, str) and bool(key_id)
+    )
+    if not valid_policy:
         raise RepositoryError("repository metadata is invalid")
     try:
         from uuid import UUID
@@ -152,4 +503,10 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
         datetime.fromisoformat(created_at[:-1] + "+00:00")
     except (ValueError, TypeError) as error:
         raise RepositoryError("repository metadata is invalid") from error
-    return InitializedRepository(root=resolved_root, compression="none", encryption="none")
+    return InitializedRepository(
+        root=resolved_root,
+        repository_id=str(payload["repository_id"]),
+        compression="none",
+        encryption=str(encryption["mode"]),
+        data_key_id=key_id if isinstance(key_id, str) else None,
+    )

backend/src/backup_tool/retention.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Protocol
+
+
+class RetentionError(ValueError):
+    pass
+
+
+class BackupLike(Protocol):
+    id: str
+    created_at: datetime
+    pinned: bool
+    tombstoned_at: datetime | None
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+    keep_last: int = 1
+    keep_days: int = 0
+    keep_daily: int = 0
+    keep_weekly: int = 0
+    keep_monthly: int = 0
+
+    @classmethod
+    def from_dict(cls, value: dict[str, object]) -> RetentionPolicy:
+        allowed = {"keep_last", "keep_days", "keep_daily", "keep_weekly", "keep_monthly"}
+        if set(value) - allowed:
+            raise RetentionError("retention policy contains unknown keys")
+        kwargs: dict[str, int] = {}
+        for key in allowed:
+            raw = value.get(key, 0 if key != "keep_last" else 1)
+            if not isinstance(raw, int) or isinstance(raw, bool) or raw < 0:
+                raise RetentionError("retention values must be non-negative integers")
+            kwargs[key] = raw
+        return cls(**kwargs)
+
+
+def retained_ids(
+    backups: Iterable[BackupLike], policy: RetentionPolicy, now: datetime | None = None
+) -> set[str]:
+    """Return union retention set; the newest non-tombstoned backup is always protected."""
+    reference = (now or datetime.now(UTC)).astimezone(UTC)
+    items = sorted(
+        (item for item in backups if item.tombstoned_at is None),
+        key=lambda item: item.created_at.astimezone(UTC),
+        reverse=True,
+    )
+    if not items:
+        return set()
+    kept = {items[0].id}
+    kept.update(item.id for item in items[: policy.keep_last])
+    kept.update(item.id for item in items if item.pinned)
+    if policy.keep_days:
+        cutoff = reference - timedelta(days=policy.keep_days)
+        kept.update(item.id for item in items if item.created_at.astimezone(UTC) >= cutoff)
+    for count, key in (
+        (policy.keep_daily, lambda stamp: stamp.date()),
+        (policy.keep_weekly, lambda stamp: stamp.isocalendar()[:2]),
+        (policy.keep_monthly, lambda stamp: (stamp.year, stamp.month)),
+    ):
+        buckets: set[object] = set()
+        for item in items:
+            stamp = item.created_at.astimezone(UTC)
+            bucket = key(stamp)
+            if len(buckets) >= count and bucket not in buckets:
+                continue
+            buckets.add(bucket)
+            if bucket in buckets:
+                kept.add(item.id)
+    return kept

backend/src/backup_tool/scheduler.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+from datetime import UTC, datetime
+from typing import cast
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from apscheduler.triggers.cron import CronTrigger  # type: ignore[import-untyped]
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Schedule
+from backup_tool.execution import EnqueueError, enqueue
+from backup_tool.notifications.events import emit_event
+from backup_tool.observability.logging import configure_logging, log_event
+
+
+class ScheduleError(ValueError):
+    pass
+
+
+def next_nominal(cron: str, timezone: str, after: datetime | None = None) -> datetime:
+    if len(cron.split()) != 5:
+        raise ScheduleError("cron must contain exactly five fields")
+    try:
+        zone = ZoneInfo(timezone)
+        trigger = CronTrigger.from_crontab(cron, timezone=zone)
+    except (ValueError, ZoneInfoNotFoundError) as error:
+        raise ScheduleError("cron or timezone is invalid") from error
+    reference = (after or datetime.now(UTC)).astimezone(zone)
+    next_run = trigger.get_next_fire_time(None, reference)
+    if next_run is None:
+        raise ScheduleError("cron has no future occurrence")
+    return cast(datetime, next_run.astimezone(UTC))
+
+
+class SchedulerService:
+    """Dedicated scheduler role using the same transactional enqueue service."""
+
+    def __init__(self, settings: Settings) -> None:
+        self.engine = create_engine(settings)
+        self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
+        self._stopping = asyncio.Event()
+
+    async def run_once(self) -> int:
+        async with self.sessions() as db:
+            return await deliver_due(db)
+
+    async def run(self) -> None:
+        while not self._stopping.is_set():
+            await self.run_once()
+            with contextlib.suppress(TimeoutError):
+                await asyncio.wait_for(self._stopping.wait(), timeout=0.25)
+        await self.engine.dispose()
+
+    def stop(self) -> None:
+        log_event("role_stopping", role="scheduler")
+        self._stopping.set()
+
+
+def run_scheduler(settings: Settings) -> int:
+    configure_logging("scheduler", settings.log_level)
+    service = SchedulerService(settings)
+    loop = asyncio.new_event_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        with contextlib.suppress(NotImplementedError):
+            loop.add_signal_handler(sig, service.stop)
+    try:
+        loop.run_until_complete(service.run())
+    finally:
+        loop.close()
+        log_event("role_stopped", role="scheduler")
+    return 0
+
+
+async def deliver_due(db: AsyncSession, now: datetime | None = None) -> int:
+    current = now or datetime.now(UTC)
+    schedules = list(
+        (
+            await db.scalars(
+                select(Schedule).where(
+                    Schedule.enabled,
+                    Schedule.next_nominal_at.is_not(None),
+                    Schedule.next_nominal_at <= current,
+                )
+            )
+        ).all()
+    )
+    delivered = 0
+    for schedule in schedules:
+        nominal = schedule.next_nominal_at
+        if nominal is None:
+            continue
+        schedule.next_nominal_at = next_nominal(schedule.cron, schedule.timezone, nominal)
+        if (current - nominal).total_seconds() > schedule.misfire_grace_seconds:
+            schedule.last_enqueue_outcome = "misfire"
+            await emit_event(
+                db,
+                "schedule.occurrence_misfired",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"outcome": "misfire"},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:misfire",
+            )
+            continue
+        try:
+            await enqueue(
+                db,
+                schedule.job_id,
+                "schedule",
+                schedule_id=schedule.id,
+                nominal_run_at=nominal,
+            )
+        except EnqueueError as error:
+            schedule.last_enqueue_outcome = error.code
+            await emit_event(
+                db,
+                "schedule.occurrence_blocked",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"reason_code": error.code},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:blocked",
+            )
+        else:
+            schedule.last_enqueue_outcome = "enqueued"
+            await emit_event(
+                db,
+                "schedule.occurrence_enqueued",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"outcome": "enqueued"},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:enqueued",
+            )
+            delivered += 1
+    await db.commit()
+    return delivered

backend/src/backup_tool/security/recovery_bundle.py

@@ -0,0 +1,249 @@
+"""Offline, passphrase-protected recovery bundle codec.
+
+The binary format is deliberately small and versioned so validation can reject
+unsupported inputs before attempting expensive password derivation.  Every
+failure while parsing or authenticating a bundle is reported as the same error
+so callers cannot distinguish a malformed bundle from a wrong passphrase.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import struct
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from argon2.low_level import Type, hash_secret_raw
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+class RecoveryBundleError(ValueError):
+    """A non-disclosing recovery bundle validation failure."""
+
+
+class RecoveryBundlePathError(ValueError):
+    """A requested recovery bundle path cannot be used safely."""
+
+
+_MAGIC = b"BTREC"
+_VERSION = 1
+_KDF_ARGON2ID = 1
+_SALT_BYTES = 16
+_NONCE_BYTES = 12
+_KEY_BYTES = 32
+_TAG_BYTES = 16
+_TIME_COST = 3
+_MEMORY_COST_KIB = 65_536
+_PARALLELISM = 1
+_MAX_PASSPHRASE_BYTES = 4_096
+_MAX_PLAINTEXT_BYTES = 8 * 1024 * 1024
+# magic, version, KDF id, Argon2 time/memory/parallelism, salt/nonce lengths,
+# and the AES-GCM ciphertext (including tag) length.
+_HEADER = struct.Struct(">5sBBIIHBBQ")
+_MAX_BUNDLE_BYTES = _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _MAX_PLAINTEXT_BYTES + _TAG_BYTES
+_ERROR = "recovery bundle is invalid"
+
+
+def _invalid() -> RecoveryBundleError:
+    return RecoveryBundleError(_ERROR)
+
+
+def _canonical_json(payload: Mapping[str, Any]) -> bytes:
+    try:
+        encoded = json.dumps(
+            payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True
+        ).encode("utf-8")
+    except (TypeError, ValueError) as error:
+        raise _invalid() from error
+    if not encoded or len(encoded) > _MAX_PLAINTEXT_BYTES:
+        raise _invalid()
+    return encoded
+
+
+def _passphrase(value: bytes) -> bytes:
+    if not isinstance(value, bytes) or not value or len(value) > _MAX_PASSPHRASE_BYTES:
+        raise _invalid()
+    return value
+
+
+def _derive_key(passphrase: bytes, salt: bytes) -> bytes:
+    return hash_secret_raw(
+        secret=passphrase,
+        salt=salt,
+        time_cost=_TIME_COST,
+        memory_cost=_MEMORY_COST_KIB,
+        parallelism=_PARALLELISM,
+        hash_len=_KEY_BYTES,
+        type=Type.ID,
+    )
+
+
+def encrypt_bundle(payload: Mapping[str, Any], passphrase: bytes) -> bytes:
+    """Serialize and encrypt a canonical recovery payload as a BTREC v1 bundle."""
+    plaintext = _canonical_json(payload)
+    secret = _passphrase(passphrase)
+    salt = os.urandom(_SALT_BYTES)
+    nonce = os.urandom(_NONCE_BYTES)
+    ciphertext_length = len(plaintext) + _TAG_BYTES
+    header = _HEADER.pack(
+        _MAGIC,
+        _VERSION,
+        _KDF_ARGON2ID,
+        _TIME_COST,
+        _MEMORY_COST_KIB,
+        _PARALLELISM,
+        _SALT_BYTES,
+        _NONCE_BYTES,
+        ciphertext_length,
+    )
+    ciphertext = AESGCM(_derive_key(secret, salt)).encrypt(nonce, plaintext, header)
+    return header + salt + nonce + ciphertext
+
+
+def decrypt_bundle(encoded: bytes, passphrase: bytes) -> dict[str, Any]:
+    """Authenticate and decode a BTREC v1 bundle without disclosing failure cause."""
+    try:
+        if not isinstance(encoded, bytes) or len(encoded) > _MAX_BUNDLE_BYTES:
+            raise _invalid()
+        if len(encoded) < _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _TAG_BYTES:
+            raise _invalid()
+        (
+            magic,
+            version,
+            kdf_id,
+            time_cost,
+            memory_cost,
+            parallelism,
+            salt_length,
+            nonce_length,
+            ciphertext_length,
+        ) = _HEADER.unpack(encoded[: _HEADER.size])
+        if (
+            magic != _MAGIC
+            or version != _VERSION
+            or kdf_id != _KDF_ARGON2ID
+            or time_cost != _TIME_COST
+            or memory_cost != _MEMORY_COST_KIB
+            or parallelism != _PARALLELISM
+            or salt_length != _SALT_BYTES
+            or nonce_length != _NONCE_BYTES
+            or ciphertext_length < _TAG_BYTES
+            or ciphertext_length > _MAX_PLAINTEXT_BYTES + _TAG_BYTES
+            or len(encoded) != _HEADER.size + salt_length + nonce_length + ciphertext_length
+        ):
+            raise _invalid()
+        secret = _passphrase(passphrase)
+        salt_start = _HEADER.size
+        nonce_start = salt_start + salt_length
+        ciphertext_start = nonce_start + nonce_length
+        plaintext = AESGCM(_derive_key(secret, encoded[salt_start:nonce_start])).decrypt(
+            encoded[nonce_start:ciphertext_start],
+            encoded[ciphertext_start:],
+            encoded[: _HEADER.size],
+        )
+        if not plaintext or len(plaintext) > _MAX_PLAINTEXT_BYTES:
+            raise _invalid()
+        payload = json.loads(plaintext.decode("utf-8"))
+        if not isinstance(payload, dict):
+            raise _invalid()
+        # Reject non-canonical encodings to make catalog serialization deterministic.
+        if _canonical_json(payload) != plaintext:
+            raise _invalid()
+        return payload
+    except (
+        InvalidTag,
+        UnicodeDecodeError,
+        json.JSONDecodeError,
+        struct.error,
+        ValueError,
+    ) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise error
+        raise _invalid() from error
+
+
+def _check_path_components(path: Path) -> None:
+    if not path.is_absolute() or path.name in {"", ".", ".."}:
+        raise RecoveryBundlePathError("recovery bundle path is unsafe")
+    current = Path(path.anchor)
+    for component in path.parts[1:-1]:
+        current /= component
+        try:
+            info = current.lstat()
+        except OSError as error:
+            raise RecoveryBundlePathError("recovery bundle path is unsafe") from error
+        if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
+            raise RecoveryBundlePathError("recovery bundle path is unsafe")
+
+
+def write_bundle_exclusive(path: Path, encoded: bytes) -> None:
+    """Write a bundle once with restrictive permissions and no symlink following."""
+    if not isinstance(encoded, bytes) or not encoded or len(encoded) > _MAX_BUNDLE_BYTES:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe")
+    _check_path_components(path)
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(encoded)
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
+    try:
+        info = path.lstat()
+        if (
+            stat.S_ISLNK(info.st_mode)
+            or not stat.S_ISREG(info.st_mode)
+            or stat.S_IMODE(info.st_mode) != 0o600
+        ):
+            path.unlink(missing_ok=True)
+            raise RecoveryBundlePathError("recovery bundle output is unsafe")
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
+
+
+def read_bundle_file(path: Path) -> bytes:
+    """Read a regular, non-symlink bundle with a bounded size."""
+    _check_path_components(path)
+    flags = os.O_RDONLY
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags)
+        with os.fdopen(descriptor, "rb") as handle:
+            info = os.fstat(handle.fileno())
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_size <= 0
+                or info.st_size > _MAX_BUNDLE_BYTES
+            ):
+                raise RecoveryBundlePathError("recovery bundle input is unsafe")
+            return handle.read()
+    except RecoveryBundlePathError:
+        raise
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle input is unsafe") from error
+
+
+def read_passphrase_fd(fd: int) -> bytes:
+    """Read one newline-terminated passphrase from an inherited file descriptor."""
+    if not isinstance(fd, int) or fd < 0:
+        raise RecoveryBundleError("recovery passphrase is unavailable")
+    try:
+        value = os.read(fd, _MAX_PASSPHRASE_BYTES + 2)
+    except OSError as error:
+        raise RecoveryBundleError("recovery passphrase is unavailable") from error
+    if value.endswith(b"\r\n"):
+        value = value[:-2]
+    elif value.endswith(b"\n"):
+        value = value[:-1]
+    if not value or len(value) > _MAX_PASSPHRASE_BYTES:
+        raise RecoveryBundleError("recovery passphrase is unavailable")
+    return value

backend/src/backup_tool/security/repository_crypto.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from backup_tool.config import Settings
+from backup_tool.ids import new_uuid7
+
+
+class RepositoryKeyError(ValueError):
+    pass
+
+
+def _directory(settings: Settings) -> Path:
+    directory = settings.data_dir / "repository-data-keys"
+    directory.mkdir(mode=0o700, exist_ok=True)
+    if (
+        directory.is_symlink()
+        or not directory.is_dir()
+        or stat.S_IMODE(directory.stat().st_mode) != 0o700
+    ):
+        raise RepositoryKeyError("repository data key directory is unsafe")
+    return directory
+
+
+def create_data_key(settings: Settings, repository_id: str) -> tuple[str, Path]:
+    key_id = str(new_uuid7())
+    directory = _directory(settings)
+    path = directory / f"{repository_id}.{key_id}.key"
+    try:
+        descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(os.urandom(32))
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as error:
+        raise RepositoryKeyError("repository data key cannot be created") from error
+    if stat.S_IMODE(path.stat().st_mode) != 0o600 or path.is_symlink():
+        path.unlink(missing_ok=True)
+        raise RepositoryKeyError("repository data key is unsafe")
+    return key_id, path
+
+
+def install_data_key(settings: Settings, repository_id: str, key_id: str, key: bytes) -> Path:
+    """Install recovered key material once; never replace an existing key file."""
+    if len(key) != 32 or "/" in repository_id or "/" in key_id:
+        raise RepositoryKeyError("repository data key is invalid")
+    path = _directory(settings) / f"{repository_id}.{key_id}.key"
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(key)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            path.unlink(missing_ok=True)
+            raise RepositoryKeyError("repository data key is unsafe")
+    except OSError as error:
+        raise RepositoryKeyError("repository data key cannot be installed") from error
+    return path
+
+
+def object_aad(repository_id: str, key_id: str, kind: str, identity: str) -> bytes:
+    if kind not in {"blob", "manifest"} or not all((repository_id, key_id, identity)):
+        raise RepositoryKeyError("encrypted object metadata is invalid")
+    return f"BTENC:1:{repository_id}:{key_id}:{kind}:{identity}".encode()
+
+
+def encrypt_object(key: bytes, aad: bytes, plaintext: bytes) -> bytes:
+    if len(key) != 32:
+        raise RepositoryKeyError("repository data key is unavailable")
+    nonce = os.urandom(12)
+    return b"BTENC\x01" + nonce + AESGCM(key).encrypt(nonce, plaintext, aad)
+
+
+def decrypt_object(key: bytes, aad: bytes, stored: bytes) -> bytes:
+    if len(key) != 32 or not stored.startswith(b"BTENC\x01") or len(stored) < 35:
+        raise RepositoryKeyError("encrypted object is invalid")
+    try:
+        return AESGCM(key).decrypt(stored[6:18], stored[18:], aad)
+    except InvalidTag as error:
+        raise RepositoryKeyError("encrypted object is invalid") from error
+
+
+def load_data_key(settings: Settings, repository_id: str, key_id: str) -> bytes:
+    path = _directory(settings) / f"{repository_id}.{key_id}.key"
+    try:
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            raise RepositoryKeyError("repository data key is unsafe")
+        key = path.read_bytes()
+    except OSError as error:
+        raise RepositoryKeyError("repository data key is unavailable") from error
+    if len(key) != 32:
+        raise RepositoryKeyError("repository data key is unavailable")
+    return key

backend/src/backup_tool/security/ssrf.py

@@ -0,0 +1,121 @@
+"""Fail-closed, DNS-rebinding-resistant webhook egress validation."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import socket
+from collections.abc import Awaitable, Callable, Sequence
+from dataclasses import dataclass
+from urllib.parse import SplitResult, urlsplit
+
+
+class SSRFError(ValueError):
+    """The callback target is not safe for the notification egress boundary."""
+
+
+Resolver = Callable[[str, int], Awaitable[Sequence[str]]]
+_ALLOWED_PORTS = frozenset({80, 443, 8080, 8443})
+
+
+def _is_global(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """ipaddress.is_global misses some policy-important mapped/special ranges."""
+    if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
+        return _is_global(address.ipv4_mapped)
+    return bool(address.is_global) and not any(
+        (
+            address.is_loopback,
+            address.is_private,
+            address.is_link_local,
+            address.is_multicast,
+            address.is_unspecified,
+            address.is_reserved,
+        )
+    )
+
+
+def validate_webhook_url(value: str) -> SplitResult:
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except ValueError as error:
+        raise SSRFError("webhook URL is malformed") from error
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        raise SSRFError("webhook URL must be absolute HTTP(S)")
+    if parsed.username is not None or parsed.password is not None or parsed.fragment:
+        raise SSRFError("webhook URL credentials and fragments are forbidden")
+    if len(value) > 2048 or any(character.isspace() for character in value):
+        raise SSRFError("webhook URL is malformed")
+    if port is not None and port not in _ALLOWED_PORTS:
+        raise SSRFError("webhook URL port is not permitted")
+    try:
+        ipaddress.ip_address(parsed.hostname)
+    except ValueError:
+        pass
+    else:
+        # Callback literals are never accepted: names are resolved immediately
+        # before every request and connected addresses are pinned/rechecked.
+        raise SSRFError("literal IP webhook targets are forbidden")
+    return parsed
+
+
+async def system_resolver(hostname: str, port: int) -> Sequence[str]:
+    records = await asyncio.get_running_loop().getaddrinfo(
+        hostname, port, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP
+    )
+    addresses: set[str] = set()
+    for record in records:
+        socket_address = record[4]
+        if socket_address and isinstance(socket_address[0], str):
+            addresses.add(socket_address[0])
+    return tuple(sorted(addresses))
+
+
+async def resolve_public_addresses(
+    hostname: str, port: int, resolver: Resolver = system_resolver
+) -> tuple[str, ...]:
+    try:
+        candidates = tuple(await resolver(hostname, port))
+    except (TimeoutError, OSError) as error:
+        raise SSRFError("webhook DNS resolution failed") from error
+    if not candidates:
+        raise SSRFError("webhook hostname has no addresses")
+    approved: list[str] = []
+    for candidate in candidates:
+        try:
+            address = ipaddress.ip_address(candidate)
+        except ValueError as error:
+            raise SSRFError("webhook resolver returned an invalid address") from error
+        if not _is_global(address):
+            # One unsafe answer poisons the hostname, including mixed public/private.
+            raise SSRFError("webhook hostname resolves to a non-public address")
+        approved.append(str(address))
+    return tuple(approved)
+
+
+@dataclass(frozen=True)
+class ResolvedWebhookTarget:
+    url: SplitResult
+    port: int
+    addresses: tuple[str, ...]
+
+
+async def resolve_webhook_target(
+    value: str, resolver: Resolver = system_resolver
+) -> ResolvedWebhookTarget:
+    parsed = validate_webhook_url(value)
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    addresses = await resolve_public_addresses(parsed.hostname or "", port, resolver)
+    return ResolvedWebhookTarget(url=parsed, port=port, addresses=addresses)
+
+
+def verify_connected_peer(peername: object, approved: Sequence[str]) -> str:
+    if not isinstance(peername, tuple) or not peername or not isinstance(peername[0], str):
+        raise SSRFError("webhook peer address is unavailable")
+    try:
+        peer = str(ipaddress.ip_address(peername[0]))
+    except ValueError as error:
+        raise SSRFError("webhook peer address is invalid") from error
+    if peer not in approved:
+        raise SSRFError("webhook peer changed after DNS resolution")
+    return peer

backend/src/backup_tool/ssh_adapter.py

@@ -0,0 +1,288 @@
+"""Pinned-host-key, forced-SFTP-only source reader.
+
+The adapter has no command-channel API.  It authenticates with the caller's
+already decrypted private key only after verifying the configured host key.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hmac
+import io
+import socket
+import stat
+from collections.abc import AsyncIterator, Callable, Iterator
+from contextlib import suppress
+from typing import Any
+
+import paramiko  # type: ignore[import-untyped]
+
+from backup_tool.adapters import Entry, SourceError
+from backup_tool.config import Settings
+from backup_tool.ssh_source import SSHSourcePublicConfig
+
+TransportFactory = Callable[[str, int, float], Any]
+SFTPFactory = Callable[[Any], Any]
+
+
+def _transport_for(hostname: str, port: int, timeout: float) -> Any:
+    connection = socket.create_connection((hostname, port), timeout=timeout)
+    connection.settimeout(timeout)
+    return paramiko.Transport(connection)
+
+
+def _sftp_for(transport: Any) -> Any:
+    return paramiko.SFTPClient.from_transport(transport)
+
+
+def _next_or_none(iterator: Iterator[Any]) -> Any | None:
+    try:
+        return next(iterator)
+    except StopIteration:
+        return None
+
+
+def load_private_key(value: str) -> Any:
+    """Load only unencrypted Ed25519, ECDSA, or sufficiently strong RSA keys."""
+    for key_type in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey):
+        try:
+            key = key_type.from_private_key(io.StringIO(value), password=None)
+        except (paramiko.SSHException, ValueError):
+            continue
+        if isinstance(key, paramiko.RSAKey) and key.get_bits() < 3072:
+            raise SourceError(
+                "SSH private key algorithm is not permitted", reason_code="source_auth"
+            )
+        return key
+    raise SourceError("SSH private key algorithm is not permitted", reason_code="source_auth")
+
+
+class SSHAdapter:
+    """A stateful SFTP reader rooted at the forced-SFTP account chroot only."""
+
+    def __init__(
+        self,
+        config: SSHSourcePublicConfig,
+        private_key: str,
+        settings: Settings,
+        *,
+        transport_factory: TransportFactory = _transport_for,
+        sftp_factory: SFTPFactory = _sftp_for,
+    ) -> None:
+        self.config = config
+        self._private_key = load_private_key(private_key)
+        self.settings = settings
+        self._transport_factory = transport_factory
+        self._sftp_factory = sftp_factory
+        self._transport: Any | None = None
+        self._sftp: Any | None = None
+        self._files: dict[str, tuple[int, int]] = {}
+
+    def validate_config(self) -> None:
+        if self.config.root != "/":  # defensive: persisted JSON can bypass API validation
+            raise SourceError(
+                "SSH source root must be the forced-SFTP chroot", reason_code="source_invalid"
+            )
+
+    def _connect(self) -> None:
+        if self._sftp is not None:
+            return
+        self.validate_config()
+        transport = self._transport_factory(
+            self.config.hostname, self.config.port, self.settings.ssh_connect_timeout_seconds
+        )
+        self._transport = transport
+        try:
+            transport.start_client(timeout=self.settings.ssh_connect_timeout_seconds)
+            algorithm, encoded_key = self.config.host_key.split(" ", 1)
+            server_key = transport.get_remote_server_key()
+            if server_key.get_name() != algorithm or not hmac.compare_digest(
+                server_key.get_base64(), encoded_key
+            ):
+                raise SourceError(
+                    "SSH host key does not match configured pin", reason_code="source_trust"
+                )
+            # Authentication deliberately occurs only after the exact pin comparison.
+            transport.auth_publickey(self.config.username, self._private_key)
+            sftp = self._sftp_factory(transport)
+            channel = sftp.get_channel()
+            channel.settimeout(self.settings.ssh_operation_timeout_seconds)
+            self._sftp = sftp
+        except SourceError:
+            self._close_sync()
+            raise
+        except paramiko.AuthenticationException as error:
+            self._close_sync()
+            raise SourceError("SSH authentication failed", reason_code="source_auth") from error
+        except (TimeoutError, OSError) as error:
+            self._close_sync()
+            raise SourceError(
+                "SSH source is unavailable", reason_code="source_unavailable"
+            ) from error
+        except (paramiko.SSHException, ValueError) as error:
+            self._close_sync()
+            raise SourceError(
+                "SSH source connection failed", reason_code="source_unavailable"
+            ) from error
+
+    def _sftp_client(self) -> Any:
+        if self._sftp is None:
+            raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
+        return self._sftp
+
+    def _read_client(self) -> Any:
+        if self._transport is None:
+            raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
+        client = self._sftp_factory(self._transport)
+        client.get_channel().settimeout(self.settings.ssh_operation_timeout_seconds)
+        return client
+
+    @staticmethod
+    def _relative(parent: str, name: object) -> str:
+        if not isinstance(name, str) or not name or name in {".", ".."}:
+            raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
+        if "/" in name or "\\" in name or "\x00" in name:
+            raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
+        return name if not parent else f"{parent}/{name}"
+
+    @staticmethod
+    def _remote_path(relative: str) -> str:
+        if (
+            not relative
+            or relative.startswith("/")
+            or "\\" in relative
+            or ".." in relative.split("/")
+        ):
+            raise SourceError("SSH source entry is invalid", reason_code="source_invalid")
+        return f"/{relative}"
+
+    @staticmethod
+    def _entry_from_attributes(path: str, attributes: Any) -> Entry:
+        mode = getattr(attributes, "st_mode", None)
+        if not isinstance(mode, int):
+            raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
+        if stat.S_ISLNK(mode):
+            raise SourceError("SSH source symlinks are not supported", reason_code="source_invalid")
+        size = getattr(attributes, "st_size", 0)
+        mtime = getattr(attributes, "st_mtime", 0)
+        if not isinstance(size, int) or size < 0 or not isinstance(mtime, int):
+            raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
+        if stat.S_ISDIR(mode):
+            return Entry(path, "directory", 0, stat.S_IMODE(mode), mtime * 1_000_000_000)
+        if stat.S_ISREG(mode):
+            return Entry(path, "file", size, stat.S_IMODE(mode), mtime * 1_000_000_000)
+        raise SourceError("SSH source contains an unsupported entry", reason_code="source_invalid")
+
+    async def probe(self) -> dict[str, int]:
+        count = 0
+        try:
+            async for entry in self.enumerate_entries():
+                if entry.kind == "file":
+                    count += 1
+            return {"entry_count": count}
+        finally:
+            await self.close()
+
+    async def enumerate_entries(self) -> AsyncIterator[Entry]:
+        await asyncio.to_thread(self._connect)
+        pending = [""]
+        entry_count = 0
+        try:
+            while pending:
+                parent = pending.pop()
+                remote_parent = "/" if not parent else self._remote_path(parent)
+                try:
+                    iterator = await asyncio.to_thread(
+                        self._sftp_client().listdir_iter,
+                        remote_parent,
+                        read_aheads=self.settings.ssh_list_read_aheads,
+                    )
+                    while (
+                        attributes := await asyncio.to_thread(_next_or_none, iterator)
+                    ) is not None:
+                        path = self._relative(parent, getattr(attributes, "filename", None))
+                        entry_count += 1
+                        if entry_count > self.settings.ssh_max_entries:
+                            raise SourceError(
+                                "SSH source entry limit exceeded", reason_code="source_limit"
+                            )
+                        entry = self._entry_from_attributes(path, attributes)
+                        if entry.kind == "directory":
+                            if path.count("/") + 1 > self.settings.ssh_max_traversal_depth:
+                                raise SourceError(
+                                    "SSH source traversal depth exceeded",
+                                    reason_code="source_limit",
+                                )
+                            pending.append(path)
+                        else:
+                            self._files[path] = (entry.size, entry.mtime_ns)
+                        yield entry
+                except SourceError:
+                    raise
+                except (TimeoutError, OSError, paramiko.SSHException) as error:
+                    raise SourceError(
+                        "SSH source enumeration failed", reason_code="source_unavailable"
+                    ) from error
+        except BaseException:
+            await self.close()
+            raise
+
+    async def open_content(self, path: str) -> AsyncIterator[bytes]:
+        expected = self._files.get(path)
+        if expected is None:
+            raise SourceError("SSH source entry was not enumerated", reason_code="source_invalid")
+        remote_path = self._remote_path(path)
+        handle: Any | None = None
+        reader: Any | None = None
+        try:
+            await asyncio.to_thread(self._connect)
+            reader = await asyncio.to_thread(self._read_client)
+            assert reader is not None
+            attributes = await asyncio.to_thread(reader.lstat, remote_path)
+            entry = self._entry_from_attributes(path, attributes)
+            if entry.kind != "file" or (entry.size, entry.mtime_ns) != expected:
+                raise SourceError(
+                    "SSH source file changed during backup", reason_code="source_changed"
+                )
+            handle = await asyncio.to_thread(
+                reader.open,
+                remote_path,
+                "rb",
+                self.settings.ssh_read_chunk_bytes,
+            )
+            assert handle is not None
+            while chunk := await asyncio.to_thread(handle.read, self.settings.ssh_read_chunk_bytes):
+                if not isinstance(chunk, bytes) or len(chunk) > self.settings.ssh_read_chunk_bytes:
+                    raise SourceError(
+                        "SSH source returned an invalid read", reason_code="source_invalid"
+                    )
+                yield chunk
+            after = await asyncio.to_thread(reader.lstat, remote_path)
+            verified = self._entry_from_attributes(path, after)
+            if verified.kind != "file" or (verified.size, verified.mtime_ns) != expected:
+                raise SourceError(
+                    "SSH source file changed during backup", reason_code="source_changed"
+                )
+        except SourceError:
+            raise
+        except (TimeoutError, OSError, paramiko.SSHException) as error:
+            raise SourceError("SSH source read failed", reason_code="source_unavailable") from error
+        finally:
+            if handle is not None:
+                await asyncio.to_thread(handle.close)
+            if reader is not None and reader is not self._sftp:
+                await asyncio.to_thread(reader.close)
+
+    def _close_sync(self) -> None:
+        sftp, transport = self._sftp, self._transport
+        self._sftp = None
+        self._transport = None
+        if sftp is not None:
+            with suppress(OSError, paramiko.SSHException):
+                sftp.close()
+        if transport is not None:
+            with suppress(OSError, paramiko.SSHException):
+                transport.close()
+
+    async def close(self) -> None:
+        await asyncio.to_thread(self._close_sync)

backend/src/backup_tool/ssh_source.py

@@ -0,0 +1,72 @@
+"""Closed public configuration for the staged SSH source capability.
+
+This module deliberately describes configuration only.  It must not create a
+network transport or load a private key; those capabilities are outside this
+slice.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+SSH_PRIVATE_KEY_PURPOSE = "ssh_private_key"
+_ALLOWED_HOST_KEY_ALGORITHMS = frozenset(
+    {
+        "ssh-ed25519",
+        "ecdsa-sha2-nistp256",
+        "ecdsa-sha2-nistp384",
+        "ecdsa-sha2-nistp521",
+        "rsa-sha2-256",
+        "rsa-sha2-512",
+    }
+)
+
+
+def _has_control_or_space(value: str) -> bool:
+    return any(character.isspace() or ord(character) < 32 for character in value)
+
+
+class SSHSourcePublicConfig(BaseModel):
+    """The public, SFTP-chroot-only portion of an SSH source definition."""
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    hostname: str = Field(min_length=1, max_length=253)
+    port: int = Field(ge=1, le=65535)
+    username: str = Field(min_length=1, max_length=255)
+    host_key: str = Field(min_length=1, max_length=16384)
+    # The server-side forced-SFTP account's chroot is the only permitted root.
+    root: Literal["/"]
+
+    @field_validator("hostname")
+    @classmethod
+    def validate_hostname(cls, value: str) -> str:
+        if _has_control_or_space(value) or any(character in value for character in "/\\@?#"):
+            raise ValueError("SSH hostname is invalid")
+        return value
+
+    @field_validator("username")
+    @classmethod
+    def validate_username(cls, value: str) -> str:
+        if _has_control_or_space(value) or any(character in value for character in "/\\:@"):
+            raise ValueError("SSH username is invalid")
+        return value
+
+    @field_validator("host_key")
+    @classmethod
+    def validate_host_key(cls, value: str) -> str:
+        algorithm, separator, encoded_key = value.partition(" ")
+        if not separator or not algorithm or not encoded_key or " " in encoded_key:
+            raise ValueError("SSH host key must be an algorithm and base64 key")
+        if algorithm not in _ALLOWED_HOST_KEY_ALGORITHMS:
+            raise ValueError("SSH host key algorithm is not supported")
+        try:
+            decoded = base64.b64decode(encoded_key, validate=True)
+        except (ValueError, UnicodeEncodeError) as error:
+            raise ValueError("SSH host key is not valid base64") from error
+        if not decoded:
+            raise ValueError("SSH host key is empty")
+        return value

backend/src/backup_tool/web.py

@@ -0,0 +1,43 @@
+"""Dedicated HTTP runtime role."""
+
+from __future__ import annotations
+
+import stat
+
+import uvicorn
+
+from backup_tool.config import Settings
+from backup_tool.observability.logging import configure_logging, log_event
+
+
+def _prepare_socket(settings: Settings) -> str:
+    path = settings.web_socket_path
+    path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
+    try:
+        existing = path.lstat()
+    except FileNotFoundError:
+        return str(path)
+    if not stat.S_ISSOCK(existing.st_mode):
+        raise RuntimeError("web socket path is not a socket")
+    path.unlink()
+    return str(path)
+
+
+def run_web(settings: Settings) -> int:
+    """Run exactly one ASGI server without reload or embedded background roles."""
+    from backup_tool.api.app import create_app
+
+    configure_logging("web", settings.log_level)
+    server = uvicorn.Server(
+        uvicorn.Config(
+            create_app(settings),
+            uds=_prepare_socket(settings),
+            log_level=settings.log_level.lower(),
+            reload=False,
+            workers=1,
+            log_config=None,
+        )
+    )
+    server.run()
+    log_event("role_stopped", role="web")
+    return 0

backend/src/backup_tool/worker.py

@@ -1,22 +1,25 @@
 """Single-node durable worker role.
 
-The worker owns leases; backup publishing is deliberately supplied by later M6 work.
+The worker owns leases and performs repository I/O outside the API process.
 """
 
 from __future__ import annotations
 
 import asyncio
 import contextlib
+import importlib
 import signal
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import cast
 from uuid import uuid4
 
 from sqlalchemy import select, update
-from sqlalchemy.ext.asyncio import async_sessionmaker
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from backup_tool.config import Settings
 from backup_tool.db.engine import create_engine
-from backup_tool.db.models import Execution
+from backup_tool.db.models import Backup, Execution, Job, Repository, Restore, Source
 from backup_tool.execution import (
     claim,
     complete_cancellation,
@@ -25,22 +28,158 @@ from backup_tool.execution import (
     recover_stale,
     transition,
 )
+from backup_tool.faults import FaultInjector, NoFault
+from backup_tool.gc import process_retention_gc
+from backup_tool.notifications.dispatcher import dispatch_one, recover_notification_leases
+from backup_tool.notifications.events import emit_event
+from backup_tool.observability.logging import configure_logging, log_event
+from backup_tool.repository import reconcile_key_rotations
+from backup_tool.security.secrets import EnvelopeCipher
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+SnapshotError = snapshot.SnapshotError
+SnapshotIntegrityError = snapshot.SnapshotIntegrityError
 
 
 class Worker:
-    def __init__(self, settings: Settings, *, owner: str | None = None) -> None:
+    def __init__(
+        self,
+        settings: Settings,
+        *,
+        owner: str | None = None,
+        fault_injector: FaultInjector | None = None,
+    ) -> None:
         self.settings = settings
         self.owner = owner or f"worker-{uuid4()}"
+        self.fault_injector = fault_injector or NoFault()
         self._stopping = asyncio.Event()
         self.engine = create_engine(settings)
         self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
+        self.cipher = EnvelopeCipher.from_file(settings.master_key_file)
+        self._execution_turns = 0
+        self._next_maintenance_at: datetime | None = None
 
     async def startup(self) -> int:
         async with self.sessions() as db:
-            return await recover_stale(db)
+            rotations = await reconcile_key_rotations(self.settings, db)
+            publications = cast(int, await snapshot.reconcile_publications(self.settings, db))
+            restored = cast(int, await snapshot.reconcile_restores(self.settings, db))
+            recovered_executions = await recover_stale(db)
+            recovered_deliveries = await recover_notification_leases(db)
+            maintenance = await process_retention_gc(db)
+            self._next_maintenance_at = datetime.now(UTC) + timedelta(seconds=60)
+            return (
+                rotations
+                + publications
+                + restored
+                + recovered_executions
+                + recovered_deliveries
+                + maintenance.tombstoned
+            )
+
+    async def _run_restore(self, db: AsyncSession) -> bool:
+        restore_id = await db.scalar(
+            select(Restore.id)
+            .where(Restore.state == "queued")
+            .order_by(Restore.created_at)
+            .limit(1)
+        )
+        if restore_id is None:
+            return False
+        result = await db.execute(
+            update(Restore)
+            .where(Restore.id == restore_id, Restore.state == "queued")
+            .values(state="running")
+        )
+        if getattr(result, "rowcount", 0) != 1:
+            await db.rollback()
+            return False
+        await db.commit()
+        restore = await db.get(Restore, restore_id)
+        if restore is None:
+            return False
+        try:
+            backup = await db.get(Backup, restore.backup_id)
+            if backup is None or backup.integrity != "verified" or backup.tombstoned_at is not None:
+                raise SnapshotError("backup is unavailable")
+            execution = await db.get(Execution, backup.execution_id)
+            if execution is None:
+                raise SnapshotError("backup execution is unavailable")
+            job = await db.get(Job, execution.job_id)
+            if job is None:
+                raise SnapshotError("backup job is unavailable")
+            repository = await db.get(Repository, job.repository_id)
+            if repository is None:
+                raise SnapshotError("backup repository is unavailable")
+            restore.result = await snapshot.restore_full_snapshot(
+                self.settings, db, restore, backup, repository
+            )
+            restore.state = "committed"
+            await emit_event(
+                db,
+                "restore.committed",
+                correlation_id=restore.id,
+                resource={"restore_id": restore.id, "backup_id": restore.backup_id},
+                payload={"dry_run": restore.dry_run, "outcome": "committed"},
+                deduplication_key=f"restore:{restore.id}:committed",
+            )
+            await db.commit()
+        except SnapshotIntegrityError:
+            await db.rollback()
+            failed = await db.get(Restore, restore_id)
+            if failed is not None:
+                corrupted_backup = await db.get(Backup, failed.backup_id)
+                if corrupted_backup is not None:
+                    corrupted_backup.integrity = "corrupt"
+                failed.state = "failed"
+                failed.result = {"reason": "restore_failed"}
+                await emit_event(
+                    db,
+                    "restore.failed",
+                    correlation_id=failed.id,
+                    resource={"restore_id": failed.id, "backup_id": failed.backup_id},
+                    payload={"reason_code": "restore_failed"},
+                    deduplication_key=f"restore:{failed.id}:failed",
+                )
+                await db.commit()
+        except SnapshotError:
+            await db.rollback()
+            failed = await db.get(Restore, restore_id)
+            if failed is not None:
+                failed.state = "failed"
+                failed.result = {"reason": "restore_failed"}
+                await emit_event(
+                    db,
+                    "restore.failed",
+                    correlation_id=failed.id,
+                    resource={"restore_id": failed.id, "backup_id": failed.backup_id},
+                    payload={"reason_code": "restore_failed"},
+                    deduplication_key=f"restore:{failed.id}:failed",
+                )
+                await db.commit()
+        return True
+
+    async def _run_maintenance(self, db: AsyncSession) -> None:
+        now = datetime.now(UTC)
+        if self._next_maintenance_at is None or now >= self._next_maintenance_at:
+            await process_retention_gc(db, now)
+            self._next_maintenance_at = now + timedelta(seconds=60)
 
     async def run_once(self) -> bool:
+        if self._stopping.is_set():
+            return False
         async with self.sessions() as db:
+            # Retention/GC is a worker responsibility, but is rate-limited so it
+            # cannot turn a sustained backup queue into a metadata polling loop.
+            await self._run_maintenance(db)
+            # Never let an always-nonempty execution queue starve due notifications.
+            if self._execution_turns >= 1 and await dispatch_one(
+                db, self.settings, self.cipher, self.owner
+            ):
+                self._execution_turns = 0
+                return True
+            if self._stopping.is_set():
+                return False
             execution_id = await db.scalar(
                 select(Execution.id)
                 .where(Execution.state == "queued")
@@ -48,10 +187,13 @@ class Worker:
                 .limit(1)
             )
             if execution_id is None:
-                return False
+                if await self._run_restore(db):
+                    return True
+                return await dispatch_one(db, self.settings, self.cipher, self.owner)
             execution = await claim(db, execution_id, self.owner)
             if execution is None:
                 return False
+            self._execution_turns += 1
             # Reload after the claim: a control request can race the lease acquisition.
             await db.refresh(execution)
             if execution.state == "cancelling":
@@ -73,6 +215,76 @@ class Worker:
                 await record_event(db, execution)
             await db.commit()
             await heartbeat(db, execution.id, self.owner)
+            failure_detail: str | None = None
+            failure_reason = "transient_io"
+            try:
+                job = await db.get(Job, execution.job_id)
+                if job is None:
+                    raise SnapshotError("execution job is unavailable")
+                source = await db.get(Source, job.source_id)
+                repository = await db.get(Repository, job.repository_id)
+                if source is None or repository is None:
+                    raise SnapshotError("execution source or repository is unavailable")
+                backup = await snapshot.publish_full_snapshot(
+                    self.settings,
+                    db,
+                    execution,
+                    job,
+                    source,
+                    repository,
+                    self.fault_injector,
+                    cipher=self.cipher,
+                )
+                await db.flush()
+                await emit_event(
+                    db,
+                    "backup.committed",
+                    correlation_id=execution.id,
+                    resource={
+                        "execution_id": execution.id,
+                        "job_id": job.id,
+                        "repository_id": repository.id,
+                        "backup_id": backup.id,
+                    },
+                    payload={"integrity": backup.integrity, "effective_mode": job.requested_mode},
+                    deduplication_key=f"backup:{backup.id}:committed",
+                )
+                await emit_event(
+                    db,
+                    "backup.verification_succeeded",
+                    correlation_id=execution.id,
+                    resource={"execution_id": execution.id, "backup_id": backup.id},
+                    payload={"integrity": backup.integrity},
+                    deduplication_key=f"backup:{backup.id}:verified",
+                )
+                execution.state = transition("running", "verifying")
+                await record_event(db, execution)
+                self.fault_injector.hit("metadata.before_commit")
+                await db.commit()
+                self.fault_injector.hit("metadata.after_commit")
+                execution.state = transition("verifying", "committed")
+                execution.completed_at = datetime.now(UTC)
+                execution.lease_owner = None
+                execution.lease_expires_at = None
+                await record_event(db, execution)
+                await db.commit()
+                snapshot.finalize_publication(Path(repository.root), execution.id)
+            except SnapshotError as error:
+                await db.rollback()
+                failure_detail = str(error)
+                failure_reason = error.reason_code
+            if failure_detail is not None:
+                failed = await db.get(Execution, execution_id)
+                if failed is None or failed.state not in {"preparing", "running", "verifying"}:
+                    return True
+                failed.state = "failed"
+                failed.reason_code = failure_reason
+                failed.operator_message = failure_detail
+                failed.completed_at = datetime.now(UTC)
+                failed.lease_owner = None
+                failed.lease_expires_at = None
+                await record_event(db, failed)
+                await db.commit()
             return True
 
     async def run(self) -> None:
@@ -84,10 +296,12 @@ class Worker:
         await self.engine.dispose()
 
     def stop(self) -> None:
+        log_event("role_stopping", role="worker")
         self._stopping.set()
 
 
 def run_worker(settings: Settings) -> int:
+    configure_logging("worker", settings.log_level)
     worker = Worker(settings)
     loop = asyncio.new_event_loop()
     for sig in (signal.SIGINT, signal.SIGTERM):
@@ -97,4 +311,5 @@ def run_worker(settings: Settings) -> int:
         loop.run_until_complete(worker.run())
     finally:
         loop.close()
+        log_event("role_stopped", role="worker")
     return 0

contracts/repository/v1/manifest.schema.json

@@ -24,7 +24,7 @@
       "additionalProperties": false,
       "required": ["adapter", "captured_at", "evidence"],
       "properties": {
-        "adapter": {"enum": ["local", "ssh", "postgresql", "mysql"]},
+        "adapter": {"enum": ["local", "postgresql", "mysql"]},
         "captured_at": {"type": "string", "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\\.[0-9]+)?Z$"},
         "evidence": {"type": "object"}
       }

contracts/repository/v1/repository.schema.json

@@ -5,6 +5,13 @@
   "type": "object",
   "additionalProperties": false,
   "required": ["repository_id", "format_version", "digest_algorithm", "compression", "encryption", "created_at"],
+  "allOf": [
+    {
+      "if": {"properties": {"encryption": {"properties": {"mode": {"const": "none"}}}}},
+      "then": {"properties": {"encryption": {"properties": {"key_id": {"type": "null"}}}}},
+      "else": {"properties": {"encryption": {"properties": {"key_id": {"type": "string", "format": "uuid", "minLength": 1}}}}}
+    }
+  ],
   "properties": {
     "repository_id": {
       "type": "string",

docker-compose.yml

@@ -0,0 +1,115 @@
+name: backup-tool
+
+x-app-base: &app-base
+  build:
+    context: .
+    dockerfile: Dockerfile
+  image: backup-tool:local
+  init: true
+  read_only: true
+  tmpfs:
+    - /tmp:mode=1777,size=64m
+  cap_drop:
+    - ALL
+  security_opt:
+    - no-new-privileges:true
+  environment:
+    BACKUP_TOOL_DATA_DIR: /var/lib/backup-tool
+    BACKUP_TOOL_DATABASE_URL: sqlite+aiosqlite:////var/lib/backup-tool/metadata.db
+    BACKUP_TOOL_REPOSITORY_ROOTS: '["/var/lib/backup-tool/repositories"]'
+    BACKUP_TOOL_LOCAL_SOURCE_ROOTS: '["/mnt/sources"]'
+    BACKUP_TOOL_RESTORE_ROOTS: '["/var/lib/backup-tool/restores"]'
+    BACKUP_TOOL_MASTER_KEY_FILE: /run/backup-tool-secrets/master.key
+    BACKUP_TOOL_WEB_SOCKET_PATH: /run/backup-tool/web.sock
+    BACKUP_TOOL_PUBLIC_BASE_URL: ${BACKUP_TOOL_PUBLIC_BASE_URL:-http://localhost:8080}
+  volumes:
+    - backup-tool-data:/var/lib/backup-tool
+    - backup-tool-runtime:/run/backup-tool
+    - type: bind
+      source: ${BACKUP_TOOL_SOURCES_DIR:-./sources}
+      target: /mnt/sources
+      read_only: true
+    - type: bind
+      source: ${BACKUP_TOOL_MASTER_KEY_FILE:-./secrets/master.key}
+      target: /run/backup-tool-secrets/master.key
+      read_only: true
+
+services:
+  migrate:
+    <<: *app-base
+    command: ["migrate", "upgrade"]
+    restart: "no"
+
+  web:
+    <<: *app-base
+    command: ["web"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "backup-tool", "health", "web"]
+      interval: 10s
+      timeout: 10s
+      retries: 6
+      start_period: 0s
+
+  scheduler:
+    <<: *app-base
+    command: ["scheduler"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "backup-tool", "health", "scheduler"]
+      interval: 10s
+      timeout: 10s
+      retries: 6
+      start_period: 0s
+
+  worker:
+    <<: *app-base
+    command: ["worker"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "backup-tool", "health", "worker"]
+      interval: 10s
+      timeout: 10s
+      retries: 6
+      start_period: 0s
+
+  admin:
+    <<: *app-base
+    profiles: ["admin"]
+    command: ["admin", "--help"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  proxy:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+    image: backup-tool-proxy:local
+    init: true
+    read_only: true
+    tmpfs:
+      - /tmp:mode=1777,size=64m
+      - /var/cache/nginx:uid=10001,gid=0,mode=0750,size=32m
+      - /var/run:uid=10001,gid=0,mode=0750,size=8m
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    ports:
+      - "127.0.0.1:${BACKUP_TOOL_PORT:-8080}:8080"
+    volumes:
+      - backup-tool-runtime:/run/backup-tool:ro
+    depends_on:
+      web:
+        condition: service_healthy
+
+volumes:
+  backup-tool-data:
+  backup-tool-runtime:

docs/release/m10-evidence.md

@@ -0,0 +1,29 @@
+# M10 SSH source evidence
+
+The released source capability is `ssh`: private-key-only, pinned-host-key,
+forced-SFTP chroot access. No password, shell, command channel, agent, default
+key discovery, or arbitrary remote root is supported.
+
+## Fixture
+
+`tests/compose.ssh.yaml` builds a test-only OpenSSH server. Each run generates
+host and client Ed25519 keys under pytest `tmp_path`, mounts no committed keys,
+and configures a dedicated `backup` account with `ChrootDirectory /home/backup`
+and `ForceCommand internal-sftp`. The ordinary operator Compose stack is not
+modified. Run `make test-ssh-integration` to build, run, and tear down the
+fixture.
+
+## Verification
+
+- Fake transport tests cover pin mismatch before authentication/SFTP, bounded
+  reads, unsafe entries, and accepted/rejected private-key algorithms.
+- The opt-in live test covers a private-key probe, backup, signed verification,
+  and restore through the forced-SFTP fixture.
+- `make test-ssh-integration` passed after fixture isolation and SFTP-channel
+  concurrency fixes.
+- Final `make check` passed: 107 unit/contract, 73 integration (one skipped),
+  15 fault, and 33 security tests; Ruff, mypy, TypeScript, and the frontend
+  build passed.
+
+See `docs/runbooks/ssh-sources.md` for deployment prerequisites, rotation, and
+containment limitations.

docs/release/m11-evidence.md

@@ -0,0 +1,55 @@
+# M11 recovery import evidence
+
+- Recovery bundles use the existing versioned `BTREC` Argon2id/AES-GCM codec
+  and now carry a version-2 authenticated catalog containing repositories,
+  key epochs, sources, jobs, executions, and backups required for encrypted
+  restore.
+- `backup-tool admin recovery import` requires a migrated empty destination DB,
+  re-inspects every surviving repository through configured allowlists, writes
+  signing/data keys exclusively with mode `0600`, and rejects conflicts.
+- Imported sources are `unavailable`; imported jobs are `archived` and disabled.
+  Imported metadata therefore supports existing restore records without silently
+  restarting backup schedules.
+- The focused host-loss drill exports an encrypted backup, imports it into a
+  fresh metadata/key host, and restores the file byte-for-byte. It also proves
+  unsafe repository paths and non-empty destination metadata are rejected.
+- Encrypted repository creation is enabled only after that drill passed and now
+  records its active data-key epoch atomically with repository metadata.
+- Interrupted imports remove newly installed key files on handled failure; a
+  retry also safely adopts only exact, authenticated key files left by an
+  unclean process loss. Rotation writes an fsynced repository journal before
+  its DB transition; worker startup deterministically completes a committed
+  epoch transition or removes an uncommitted one while retaining old-active.
+  A stale rollback journal is cleared safely even when the unreferenced new key
+  was already deleted before the journal cleanup could run.
+- Key-aware GC decrypts encrypted manifests using their declared epoch key and
+  remains fail-closed for absent, wrong, or corrupt keys/manifests. Restore
+  removes setuid, setgid, and sticky bits from captured modes. Migration 0007
+  refuses downgrade while key metadata is populated.
+- Recovery catalog import preserves backup `created_at` and `tombstoned_at`.
+- Snapshot staging roots, per-execution directories, blob directories, and
+  plaintext temporary blobs are created owner-only (`0700`/`0600`) independent
+  of umask and are rejected if their permissions are unsafe.
+
+## Verification
+
+```text
+pytest tests/integration/test_encrypted_repository.py -q
+5 passed
+
+make test-fault
+11 passed
+
+make test-security
+21 passed
+
+make lint && make typecheck && make frontend-build
+passed
+
+make check
+86 unit/contract, 53 integration, 12 fault, and 22 security tests passed;
+Ruff, mypy, TypeScript, and frontend build passed.
+
+git diff --check && git diff --cached --quiet
+passed
+```

docs/release/m12-evidence.md

@@ -0,0 +1,43 @@
+# M12 evidence
+
+- Migration revision: `0008_notification_outbox`
+- Event schema version: `1`
+- Catalog: 22 live stable IDs only. Deferred/unimplemented operation types are not public notification contracts.
+- Delivery guarantee: durable at-least-once, stable event ID, leased worker retries; receiver deduplication is required.
+
+## Green focused evidence
+
+```text
+PYTHONPATH=.:backend/src .venv/bin/python -m pytest \
+  tests/contract/test_notification_contract.py \
+  tests/integration/test_migrations.py \
+  tests/integration/test_all_operational_events_deliver.py \
+  tests/integration/test_notifications.py \
+  tests/fault/test_notification_retries.py \
+  tests/security/test_webhook_ssrf.py -q
+28 passed (live-catalog contract, fair dispatch regression; no deferred event IDs).
+
+# Scheduler-role service-path regression
+PYTHONPATH=.:backend/src .venv/bin/python -m pytest \
+  tests/integration/test_scheduler_live_sync.py -q
+1 passed (scheduler role service path)
+```
+
+The suite uses temporary SQLite/repository roots, a fake SMTP implementation, fake resolver inputs, and dispatcher monkeypatches; it performs no real webhook DNS, HTTP, or SMTP delivery. It verifies receiver-visible versioned webhook headers/signatures, STARTTLS-before-AUTH SMTP behavior, transient/permanent SMTP classification, persisted SMTP attempt limits, lease-abandoned attempt closure, selected-only test sends, manual-retry idempotency, and the absence of a plaintext-secret idempotency verifier. Behavioral producer tests cover every live catalog family: execution, schedule, backup/verification, restore, and retention. They also prove fair dispatch under an execution backlog, scheduler-role delivery, and worker-owned retention/GC maintenance. The CLI scheduler role now runs `SchedulerService`; worker maintenance runs durable retention/GC on startup and at bounded intervals.
+
+## Quality evidence
+
+- `make test-fast`: 89 passed.
+- `make test-integration`: 61 passed in 27.00s (the execution wrapper nevertheless returned exit 124 at its fixed 30s wall limit).
+- `make test-fault`: focused fair-dispatch regression passed.
+- `make test-security`: 28 passed.
+- `make lint` and `make typecheck`: passed.
+- `make frontend-build`: passed.
+- `git diff --check`: passed.
+- staged-file check: no staged files.
+
+Final verification: `make check` passed — 90 unit/contract, 61 integration, 14 fault, and 28 security tests; Ruff, mypy, TypeScript, and the frontend build passed. `git diff --check` and the staged-file check also passed.
+
+## Rollback
+
+Disable or archive subscriptions and stop worker dispatch. Do not delete notification events, deliveries, or attempts: they remain audit history. A fresh host recovery intentionally starts with no notification settings or credentials and must be reconfigured.

docs/release/m13-foundation-evidence.md

@@ -0,0 +1,20 @@
+# M13 UI/OpenAPI evidence
+
+- OpenAPI is deterministically exported to `openapi/v2.json`; generated browser client drift is checked by `npm --prefix frontend run api:check`.
+- `text/event-stream` is declared in OpenAPI for execution events. The generated client intentionally emits `executionEventsUrl(...) -> URL`, not a misleading JSON `Promise`; UI opens that URL using browser `EventSource`.
+- Notification delivery retry requests send an `Idempotency-Key` and the browser CSRF header.
+- Recovery status is CLI/runbook-only. The browser has no export, import, bundle, key, or passphrase transfer control.
+- v2.1 PostgreSQL, MySQL, and TAR/download controls remain absent.
+
+## Green M13 checks
+
+```text
+npm --prefix frontend test -- --run  # 9 passed
+npm --prefix frontend run typecheck  # passed
+npm --prefix frontend run build      # passed
+npx --prefix frontend playwright test --config frontend/playwright.config.ts  # 1 passed
+.venv/bin/python tools/export_openapi.py --check openapi/v2.json  # current
+npm --prefix frontend run api:check  # current
+```
+
+Final verification: `make check` passed — 90 unit/contract, 61 integration, 14 fault, and 28 security tests; Ruff, mypy, TypeScript, and the frontend build passed. `git diff --check` and the staged-file check also passed.

docs/release/m14-evidence.md

@@ -0,0 +1,34 @@
+# M14 operations and packaging evidence
+
+## Delivered
+
+- Pinned non-root OCI application and proxy images; isolated web, scheduler, worker,
+  migrate, and admin roles; same-origin Unix-socket proxy; no reload or embedded roles.
+  The proxy port is loopback-bound by default (`127.0.0.1`), so the localhost public
+  URL cannot permit remote first-admin setup takeover. Operators exposing it through
+  an external reverse proxy must set a non-loopback public URL and bootstrap secret.
+- Role-aware readiness, JSON structured logs, safe worker claim shutdown, and
+  dependency-free Prometheus metrics for request volume/duration, active/stale/failed
+  execution state, schedule lag, corrupt/unavailable repositories, and free space.
+- SBOM generation at `tools/generate_sbom.py`, generated CycloneDX artifact
+  `m14-sbom.json`, and base-image/source provenance in `m14-provenance.md`.
+- Metadata, repository, key, upgrade, disaster-recovery, and observability runbooks.
+
+## Green verification
+
+```text
+ docker compose config --quiet                         # passed (loopback port binding)
+ docker compose build --pull                           # passed
+ make test-e2e                                         # 1 passed in 51.69s (loopback regression)
+ make check                                            # passed after loopback regression
+   91 unit/contract, 61 integration, 15 fault, 33 security
+   Ruff/format, mypy (40 files), TypeScript, frontend build all passed
+ python tools/generate_sbom.py                         # 256 components
+```
+
+`test_compose_v2.py` creates its source and host-bind key fixture in pytest temporary
+directories. The actual service-owned master key is generated only in an ephemeral
+Compose named volume, then `down --volumes --remove-orphans` removes it. No fixture
+secret or source directory is committed. The test runs migration, starts the stack,
+checks readiness and metrics, stops/restarts the worker cleanly, restarts runtime
+roles, verifies setup metadata persists, and tears down the project.

docs/release/m14-provenance.md

@@ -0,0 +1,13 @@
+# M14 build provenance
+
+- **Source revision:** `396219e776aa9a115900d2b7bfd9fb5c1cfde115`
+- **Application base:** `python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7`
+- **Frontend builder:** `node:22.17.1-alpine@sha256:5539840ce9d013fa13e3b9814c9353024be7ac75aca5db6d039504a56c04ea59`
+- **Proxy base:** `nginx:1.29.7-alpine@sha256:e7257f1ef28ba17cf7c248cb8ccf6f0c6e0228ab9c315c152f9c203cd34cf6d1`
+- **Build command:** `docker compose build --pull`
+- **SBOM:** `docs/release/m14-sbom.json`, generated deterministically with
+  `python tools/generate_sbom.py` from the pinned backend manifest and frontend lockfile.
+
+The build uses digest-pinned bases and a non-root runtime user. Provenance records
+inputs and generation instructions rather than embedding a mutable image tag or a
+secret-bearing build environment.

docs/release/m15-evidence.md

@@ -0,0 +1,23 @@
+# M15 v2.0 certification evidence
+
+## Reference host and method
+
+Current CI reference host: Linux 7.1.4, Python 3.14.6, 12 CPUs. Certification uses a deterministic synthetic metadata workload: 100 jobs, 1,000,000 declared entries, and 100,000 cataloged backups with 10 TiB **logical** bytes per backup. It does not claim a physical 10 TiB transfer.
+
+`m15-scale-report.json` records 100,000 backup inserts in 0.57 s, a 100-row deep pagination query in 0.004209 s, and a 5.32 MB SQLite fixture.
+
+## Capability certification
+
+```text
+python tools/assert_capabilities.py --release v2.0 \
+  --include local,ssh,restore,webhook,email \
+  --exclude postgresql,mysql,tar_download
+```
+
+Passed. The released contract exposes `local`, `ssh`, `restore`, `webhook`, and `email`; PostgreSQL, MySQL, and TAR download remain disabled.
+
+SSH is private-key-only and requires a dedicated forced-SFTP chroot account; no password, shell, or remote-command path is released.
+
+## Remaining certification gates
+
+Run the full project, container E2E, SSH live integration, fault/security/leakage suites, and review the synthetic workload boundaries before a release commit is created.

docs/release/m15-scale-report.json

@@ -0,0 +1,21 @@
+{
+  "method": "synthetic metadata certification; logical bytes are sparse and no physical 10 TiB payload is allocated",
+  "reference_host": {
+    "cpus": 12,
+    "platform": "Linux-7.1.4-arch1-1-x86_64-with-glibc2.44",
+    "python": "3.14.6"
+  },
+  "results": {
+    "backup_insert_seconds": 0.57,
+    "database_bytes": 5320704,
+    "pagination_rows": 100,
+    "pagination_seconds": 0.004209,
+    "total_seconds": 0.592
+  },
+  "workload": {
+    "backups": 100000,
+    "entries_declared": 1000000,
+    "jobs": 100,
+    "logical_bytes_per_backup": 10995116277760
+  }
+}

docs/release/m5-evidence.md

@@ -0,0 +1,12 @@
+# M5 Durable Execution Lifecycle Evidence
+
+- Coverage: state-transition contract and DB-enforced active-execution uniqueness
+  under concurrent enqueue; disabled/archived-job rejection; lease reclamation
+  and fencing; retry semantics; startup stale-worker reconciliation; durable
+  event revision counts; and idle worker shutdown.
+- Verification: the focused M5 suite passed: 29 tests across transition,
+  queue/lease, and worker-loss acceptance coverage.
+- Automation: `make test-fault` runs the worker-loss acceptance suite, and
+  `make check` now includes it.
+- Scope: validates M5 durable lifecycle behavior only. Snapshot publication,
+  verification, and restore remain M6 work.

docs/release/m6-evidence.md

@@ -0,0 +1,14 @@
+# M6 Backup, Verification, and Restore Evidence
+
+- Coverage: local full-snapshot staging; immutable SHA-256 blobs; canonical
+  Ed25519-signed manifests; manifest/blob verification; publication markers and
+  startup reconciliation; selected and dry-run restores; `fail`, `skip`, and
+  `replace` root policies; destination containment; corruption handling; and
+  restore recovery.
+- Focused acceptance: the M6 integration, publication-fault, and restore-path
+  suites pass.
+- Full verification: `make check` passed with 68 unit/contract, 37
+  integration, 7 fault, and 17 security tests, plus Ruff, mypy, frontend
+  typecheck, and frontend production build.
+- Scope: excludes M7 incremental baselines/exclusions and later scheduling,
+  remote source, encryption, retention, and UI milestones.

docs/release/m7-evidence.md

@@ -0,0 +1,12 @@
+# M7 Incrementals, Exclusions, and Empty-Source Safety Evidence
+
+- Coverage: ordered normalized exclusion rules, exclusion policy persisted in signed
+  manifests, incremental compatible-baseline selection, independently restorable
+  complete manifests, parent linkage, content-addressed blob reuse, and
+  empty-source opt-in.
+- Verification: M7 unit, incremental, and empty-source suites are included in
+  the passing project verification.
+- Full verification: `make check` passed with 78 unit/contract, 39
+  integration, 7 fault, and 17 security tests, plus Ruff, mypy, frontend
+  typecheck, and frontend production build.
+- Scope: scheduling remains M8 work.

docs/release/m8-evidence.md

@@ -0,0 +1,9 @@
+# M8 Scheduling Evidence
+
+- Coverage: five-field cron validation, IANA timezones, UTC nominal runs,
+  durable schedule CRUD, live next-run updates, misfire handling, and
+  no-overlap delivery through the execution enqueue guard.
+- Full verification: `make check` passed with 81 unit/contract, 40
+  integration, 10 fault, and 17 security tests, plus Ruff, mypy, frontend
+  typecheck, and frontend production build.
+- Scope: retention and garbage collection are M9 work.

docs/release/m9-evidence.md

@@ -0,0 +1,8 @@
+# M9 Retention and Garbage Collection Evidence
+
+- Coverage: union retention policies, newest-backup protection, pins, tombstones,
+  referenced-object preservation, garbage collection, and reconciliation safety.
+- Full verification: `make check` passed with 86 unit/contract, 42 integration,
+  11 fault, and 17 security tests, plus Ruff, mypy, frontend typecheck, and
+  frontend production build.
+- Scope: remote source support remains M10 work.

docs/runbooks/disaster-recovery.md

@@ -0,0 +1,11 @@
+# Disaster recovery runbook
+
+1. Isolate the failed host and preserve the metadata volume, repository roots, logs,
+   image digest, and master-key backup. Do not restart writers repeatedly.
+2. Provision a clean host with the same pinned image and non-root volume permissions.
+3. Restore the master key securely, restore repository roots read-only first, and
+   restore metadata from a verified backup or the passphrase-protected recovery bundle.
+4. Run `migrate current`, start only `web`, validate `/readyz` and repository
+   inspection, then start scheduler and worker one at a time.
+5. Perform and document a test restore before enabling scheduled work. Rotate secrets
+   if host compromise is possible.

docs/runbooks/keys.md

@@ -0,0 +1,13 @@
+# Master and repository key runbook
+
+- Store the Compose master key outside the checkout. It must be a regular file,
+  at least 32 bytes, mode `0600`, owned by the service UID (`10001` for Compose).
+  Loss of this key destroys access to encrypted metadata secrets.
+- Back up the master key independently from metadata and repositories; do not put it
+  in an image, Compose environment variable, log, ticket, or recovery bundle.
+- Use the `admin recovery export` command with a passphrase file descriptor to create
+  a separately protected recovery bundle. Validate it on an isolated host.
+- Rotate repository data keys only with `admin repository-key rotate`; retain prior
+  recovery material until a restore drill succeeds.
+- If compromise is suspected, stop the worker, preserve evidence, rotate credentials,
+  export a fresh recovery bundle, and run a restore drill before resuming writes.

docs/runbooks/metadata.md

@@ -0,0 +1,19 @@
+# Metadata backup and restore runbook
+
+## Backup
+
+1. Confirm `docker compose ps` shows exactly one scheduler and one worker.
+2. For an online SQLite backup, run a host-side SQLite `.backup` against the mounted
+   `metadata.db`; do **not** copy only the main file while WAL writers run.
+3. For a filesystem copy, stop `web`, `scheduler`, and `worker` first, then retain
+   `metadata.db`, `metadata.db-wal`, and `metadata.db-shm` together.
+4. Encrypt and test the backup outside the appliance. Never place a database dump in
+   the repository or OCI image.
+
+## Restore
+
+1. Stop all runtime roles and preserve the failed metadata volume unchanged.
+2. Restore the complete SQLite backup into the metadata volume with the service user
+   ownership (UID 10001 in the supplied Compose deployment).
+3. Run `docker compose run --rm migrate current`; only start the stack when it reports
+   the expected revision. Validate `/readyz` and a read-only API request after startup.

docs/runbooks/notifications.md

@@ -0,0 +1,15 @@
+# Notifications runbook
+
+## Configure
+
+Create a filtered email or webhook subscription through `/api/v2/notifications/subscriptions`. Webhooks require a write-only signing secret. Configure SMTP separately at `/api/v2/notifications/email-settings`; only authenticated STARTTLS SMTP is accepted. Verify a channel with `POST /subscriptions/{id}/test` and inspect delivery/attempt history before relying on it.
+
+Filters are a nonempty set of exact catalog IDs or family wildcards such as `execution.*`; they may be narrowed by job IDs, repository IDs, or severity. The public catalog is live-events-only: every listed type is emitted by a currently available operation. Deferred channels and source capabilities have no catalog entries.
+
+## Rotate and recover
+
+Rotate webhook keys using the signing-key rotate endpoint with an idempotency key and an explicit bounded overlap. Receivers must accept both signatures during overlap, then remove the old key after expiry. A recovery bundle deliberately excludes subscriptions, SMTP settings, signing secrets, event history, and delivery attempts. Reconfigure notifications after a fresh-host recovery.
+
+## Failure handling
+
+The worker claims due deliveries with a lease. Transient errors enter bounded exponential retry; interrupted leases recover as retryable work and may send an event again. Inspect response class and redacted diagnostics in history. A terminal failed delivery can be retried manually once the destination is corrected. To stop outbound traffic, disable/archive subscriptions or stop the worker; do not delete outbox history. Rollback consists of disabling subscriptions and worker dispatch while retaining audit/outbox records for investigation.

docs/runbooks/observability.md

@@ -0,0 +1,17 @@
+# Observability and alert response
+
+The proxy exposes `/livez`, `/readyz`, and Prometheus text at `/metrics`. Metrics use
+no source paths, IDs, credentials, tokens, or secret values. Runtime logs are JSON
+records with an event, timestamp, role, and request correlation ID where applicable.
+
+Alert when any of the following remains non-zero or grows:
+
+- `backup_tool_stale_execution_leases`
+- `backup_tool_failed_executions`
+- `backup_tool_corrupt_backups`
+- `backup_tool_unavailable_repositories`
+- `backup_tool_schedule_lag_seconds`
+
+Also alert on low `backup_tool_filesystem_free_bytes`. For any alert, preserve logs,
+validate `/readyz`, stop the worker before destructive repository investigation, and
+use the matching metadata, repository, key, upgrade, or disaster-recovery runbook.

docs/runbooks/recovery-bundle.md

@@ -0,0 +1,70 @@
+# Recovery bundle export and validation
+
+M11 recovery exports an offline, passphrase-encrypted catalog and key bundle.
+Import is a local CLI operation that reconstructs only the metadata required to
+restore existing encrypted backups; it does not reactivate backup scheduling.
+
+## Export
+
+Choose an absolute path in a trusted, non-symlinked directory. The destination
+must not already exist; export creates it with mode `0600` and never overwrites
+it.
+
+```sh
+read -r -s recovery_passphrase
+printf '\n'
+printf '%s\n' "$recovery_passphrase" | \
+  backup-tool admin recovery export \
+  --output /secure/offline/backup-tool-recovery.btrec \
+  --passphrase-fd 0
+unset recovery_passphrase
+```
+
+The passphrase is read from the inherited file descriptor. It is never a CLI
+argument. Store the resulting `BTREC` file away from the host and away from the
+live repository-key directories.
+
+## Validate
+
+Validation authenticates and decrypts the bundle, checks the versioned Argon2id
+and AES-GCM format, and verifies the included catalog/key relationships. It
+prints only a status and repository count.
+
+```sh
+read -r -s recovery_passphrase
+printf '\n'
+printf '%s\n' "$recovery_passphrase" | \
+  backup-tool admin recovery validate \
+  --input /secure/offline/backup-tool-recovery.btrec \
+  --passphrase-fd 0
+unset recovery_passphrase
+```
+
+Wrong passphrases, tampering, malformed headers, unsupported KDF parameters,
+and invalid encrypted payloads intentionally produce the same validation error.
+Do not use a failed validation result to diagnose which of those conditions
+occurred.
+
+## Fresh-host import
+
+Before importing, run migrations on the replacement host and configure its
+repository allowlist to include the surviving repository directory. The
+repository must pass normal metadata/path inspection. The replacement metadata
+database must be current and otherwise empty; import rejects a non-empty
+destination and any existing/conflicting key files.
+
+```sh
+backup-tool migrate upgrade
+read -r -s recovery_passphrase
+printf '\n'
+printf '%s\n' "$recovery_passphrase" | \
+  backup-tool admin recovery import \
+  --input /secure/offline/backup-tool-recovery.btrec \
+  --passphrase-fd 0
+unset recovery_passphrase
+```
+
+Import installs signing and data keys with restrictive modes, restores the
+repository/source/job/execution/backup catalog needed for restore, and marks
+sources unavailable plus jobs archived and disabled. Reconfigure sources and
+explicitly create or enable new jobs before taking another backup.

docs/runbooks/repositories.md

@@ -0,0 +1,12 @@
+# Repository recovery runbook
+
+1. Stop `worker` before inspecting or repairing a repository; never edit a live
+   repository behind an active lease.
+2. Preserve the repository directory and its metadata volume before remediation.
+3. Verify repository state through the operator API and verify individual backups
+   before any restore. Treat a corrupt verification result as an incident, not a
+   deletion request.
+4. Mount replacement repository roots at the same allowlisted path, restore metadata,
+   then start `migrate`, `web`, `scheduler`, and finally `worker`.
+5. Keep archived repositories mounted until retention and restore obligations expire.
+   Do not remove manifests or blobs manually.

docs/runbooks/ssh-sources.md

@@ -0,0 +1,19 @@
+# SSH sources
+
+SSH sources require a dedicated account confined by an OpenSSH `ChrootDirectory`
+and `ForceCommand internal-sftp`. The chroot directory is root-owned; writable
+content is below it. Disable passwords, keyboard-interactive authentication,
+shells, PTYs, TCP/X11/agent forwarding, and tunnelling. Configure the source
+root as `/` only.
+
+Generate a dedicated unencrypted Ed25519, ECDSA, or RSA-3072+ client key and
+store it through the write-only `ssh_private_key` secret endpoint. Do not put a
+key, passphrase, password, command, agent path, or key file path in source
+configuration. Pin the server's exact OpenSSH public host key (`algorithm
+base64`) before probing. On host-key rotation, obtain the replacement through
+an out-of-band administrative channel, update the source pin, then probe.
+
+The server administrator controls mutable content inside the chroot. The client
+rejects traversal names, symlinks, special files, changed files, and configured
+resource-limit overflows, but cannot claim atomic no-follow behavior against a
+maliciously changing filesystem inside that server-controlled boundary.

docs/runbooks/upgrade.md

@@ -0,0 +1,12 @@
+# Upgrade and rollback runbook
+
+1. Record the running image digest and take a tested metadata backup plus repository
+   recovery evidence before changing the image.
+2. Pull/build the pinned image, then run `docker compose run --rm migrate upgrade`.
+   Do not start web, scheduler, or worker against an unverified schema.
+3. Start the stack, wait for `/readyz`, and inspect `/metrics` for stale leases,
+   schedule lag, unavailable repositories, and corrupt backups.
+4. If migration fails, stop and restore the prior metadata backup and matching image;
+   do not attempt to downgrade an unknown partially migrated database in place.
+5. Preserve migration logs and verify a representative backup restore before closing
+   the change.

docs/security/notifications.md

@@ -0,0 +1,9 @@
+# Notification security policy (M12)
+
+Webhook callbacks accept absolute `http` and `https` URLs only. HTTP is an approved compatibility option, not a relaxation of egress controls. URLs with credentials, fragments, literal IP addresses, or ports outside 80/443/8080/8443 are rejected. Immediately before every request the worker resolves all A/AAAA answers; any non-global answer rejects the whole destination. The transport connects only to an approved answer, verifies the connected peer address, disables proxy environment use, preserves HTTPS SNI/certificate validation, and rejects redirects.
+
+Webhook bodies are canonical JSON event envelopes. `X-Backup-Event-ID` is stable over retries. Each attempt supplies the versioned HMAC-SHA-256 timestamp/body input and one `X-Backup-Signature` header per active or overlap key. Key IDs and monotonically increasing subscription-local versions identify keys; an old key is retained only until its configured overlap expiry. Key material is encrypted in `secrets` and is never returned, logged, exported in recovery bundles, or included in audit/event payloads.
+
+SMTP is configured as one write-only password-backed singleton. Delivery performs EHLO, verified STARTTLS, a second EHLO, and SMTP AUTH; any inability to do this fails closed. Message content is a compact event summary with no attachments, Bcc, paths, raw exception text, credentials, or full webhook payload.
+
+Delivery is at-least-once. A worker crash after a send but before recording success can lead to a duplicate; consumers must deduplicate using the event ID. Redirects, malformed destinations, and SSRF validation failures are terminal. Connect/read transport failures, SMTP transient failures, and HTTP 408/425/429/5xx use bounded retry. Deployment egress controls are defense in depth, not a substitute for this policy.

docs/security/repository-encryption.md

@@ -0,0 +1,35 @@
+# Repository encryption threat model
+
+## Status
+
+This document defines the M11 repository-encryption boundary. Encrypted repository
+creation remains disabled until the recovery host-loss acceptance test passes.
+
+## Confidential data
+
+For an `aes-256-gcm` repository, blob contents and signed manifest contents are
+AEAD-encrypted. Data keys and signing private keys are never stored in repository
+metadata, SQLite, logs, command arguments, environment output, or API responses.
+Recovery bundles are separately passphrase-encrypted.
+
+## Intentional leakage
+
+The v1 content-addressed layout retains plaintext SHA-256 blob names. Encryption
+therefore leaks blob equality, object count, repository layout, manifest/backup
+identifiers, and ciphertext sizes derived from plaintext sizes. It does not claim
+to hide those values, a compromised running host with unlocked keys, source data
+while it is read, or a recovery-bundle passphrase.
+
+## Key lifecycle
+
+Encryption uses distinct data and signing keys. Rotation creates a new data-key
+epoch for subsequent objects and retains historical epochs so existing backups
+remain verifiable and restorable. It neither changes immutable repository policy
+nor rewrites existing objects.
+
+## Recovery boundary
+
+Recovery export, validation, and import are local CLI operations only. A bundle
+contains the required key material and an authenticated catalog; passphrases are
+not accepted on command lines. Bundle parsing must reject malformed or unsupported
+parameters without revealing whether a passphrase, key, or ciphertext was wrong.

frontend/Dockerfile

@@ -1,20 +1,19 @@
-# Stage 1: Build
+# syntax=docker/dockerfile:1
-FROM node:20-alpine AS builder
+FROM node:22.17.1-alpine@sha256:5539840ce9d013fa13e3b9814c9353024be7ac75aca5db6d039504a56c04ea59 AS builder
 
 WORKDIR /app
-
 COPY package*.json ./
-RUN npm install
+RUN npm ci
-
 COPY . .
 RUN npm run build
 
-# Stage 2: Serve
+FROM nginx:1.29.7-alpine@sha256:e7257f1ef28ba17cf7c248cb8ccf6f0c6e0228ab9c315c152f9c203cd34cf6d1
-FROM nginx:alpine
 
 COPY --from=builder /app/dist /usr/share/nginx/html
-COPY nginx.conf /etc/nginx/conf.d/default.conf
+COPY nginx.conf /etc/nginx/nginx.conf
-
+RUN chown -R 10001:0 /usr/share/nginx/html /var/cache/nginx /var/run /etc/nginx && \
-EXPOSE 80
+    chmod -R g=u /usr/share/nginx/html /var/cache/nginx /var/run /etc/nginx
 
+USER 10001:0
+EXPOSE 8080
 CMD ["nginx", "-g", "daemon off;"]

frontend/e2e/operator.spec.ts

@@ -0,0 +1,13 @@
+import { expect, test } from "@playwright/test";
+
+test("renders an accessible authentication fallback at a narrow viewport", async ({ page }) => {
+	await page.setViewportSize({ width: 375, height: 667 });
+	await page.emulateMedia({ reducedMotion: "reduce" });
+	await page.goto("/");
+	await expect(page.getByRole("main")).toBeVisible();
+	await expect(page.getByRole("heading", { name: "Sign in" })).toBeVisible();
+	const username = page.getByLabel("Username");
+	await expect(username).toBeVisible();
+	await username.focus();
+	await expect(username).toBeFocused();
+});

frontend/nginx.conf

@@ -1,6 +1,25 @@
-server {
+pid /tmp/nginx.pid;
-    listen 80;
+
-    server_name localhost;
+worker_processes auto;
+error_log /dev/stderr warn;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    access_log /dev/stdout;
+    sendfile on;
+
+    upstream backup_tool_web {
+        server unix:/run/backup-tool/web.sock;
+    }
+
+    server {
+        listen 8080;
+        server_name _;
         root /usr/share/nginx/html;
         index index.html;
 
@@ -9,11 +28,23 @@ server {
         }
 
         location /api/ {
-        proxy_pass http://backend:8000/api/;
+            proxy_pass http://backup_tool_web;
             proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'upgrade';
             proxy_set_header Host $host;
-        proxy_cache_bypass $http_upgrade;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        location = /livez {
+            proxy_pass http://backup_tool_web;
+        }
+
+        location = /readyz {
+            proxy_pass http://backup_tool_web;
+        }
+
+        location = /metrics {
+            proxy_pass http://backup_tool_web;
+        }
     }
 }

frontend/package-lock.json

@@ -12,6 +12,7 @@
         "react-dom": "19.2.8"
       },
       "devDependencies": {
+        "@playwright/test": "^1.57.0",
         "@testing-library/jest-dom": "7.0.0",
         "@testing-library/react": "16.3.2",
         "@types/react": "19.2.17",
@@ -430,6 +431,22 @@
         "url": "https://github.com/sponsors/Boshen"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.57.0.tgz",
+      "integrity": "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.57.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@rolldown/binding-android-arm64": {
       "version": "1.1.5",
       "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
@@ -2600,6 +2617,53 @@
         "node": ">= 6"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz",
+      "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.57.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz",
+      "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.23",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",

frontend/package.json

@@ -9,6 +9,8 @@
   },
   "scripts": {
     "dev": "vite",
+    "api:generate": "python3 ../tools/generate_api_client.py --input ../openapi/v2.json --output src/api/generated/client.ts",
+    "api:check": "python3 ../tools/generate_api_client.py --input ../openapi/v2.json --output src/api/generated/client.ts --check",
     "typecheck": "tsc --noEmit",
     "test": "vitest run --passWithNoTests",
     "build": "tsc && vite build",
@@ -19,6 +21,7 @@
     "react-dom": "19.2.8"
   },
   "devDependencies": {
+    "@playwright/test": "^1.57.0",
     "@testing-library/jest-dom": "7.0.0",
     "@testing-library/react": "16.3.2",
     "@types/react": "19.2.17",

frontend/playwright.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from "@playwright/test";
+
+export default defineConfig({
+	testDir: "./e2e",
+	use: {
+		baseURL: "http://127.0.0.1:4173",
+		launchOptions: { executablePath: "/usr/sbin/chromium" },
+	},
+	webServer: {
+		command: "npm run dev -- --host 127.0.0.1 --port 4173",
+		port: 4173,
+		reuseExistingServer: false,
+	},
+});

frontend/src/api/generated/client.ts

@@ -0,0 +1,567 @@
+/* eslint-disable */
+/*
+ * Generated by tools/generate_api_client.py from openapi/v2.json.
+ * Do not edit this file directly. Run `npm --prefix frontend run api:generate`.
+ */
+
+export interface Components {
+  schemas: {
+    AuditList: { items : Array<Components['schemas']["AuditSummary"]>; next_cursor : string | null };
+    AuditSummary: { action : string; created_at : string; details : Record<string, unknown>; id : string; outcome : string; request_id : string; resource_id : string | null; resource_type : string };
+    AuthenticatedUser: { id : string; username : string };
+    BackupDeletePreview: { backup_id : string; destructive_action : string; eligible : boolean; reason : string | null };
+    BackupList: { items : Array<Components['schemas']["BackupSummary"]> };
+    BackupSummary: { created_at : string; execution_id : string; id : string; integrity : string; logical_bytes : number; manifest_id : string; pinned : boolean; stored_bytes : number; tombstoned_at : string | null };
+    EmailSettingsInput: { host : string; max_attempts?: number; password : string; port?: number; rate_limit_per_minute?: number; sender : string; username : string };
+    ExecutionList: { items : Array<Components['schemas']["ExecutionSummary"]> };
+    ExecutionSummary: { attempt : number; id : string; progress : Record<string, unknown>; reason_code : string | null; revision : number; state : string };
+    HTTPValidationError: { detail?: Array<Components['schemas']["ValidationError"]> };
+    JobInput: { allow_empty?: boolean; enabled?: boolean; exclusions?: Array<string>; name : string; repository_id : string; requested_mode?: string; retention?: Record<string, unknown>; source_id : string };
+    JobList: { items : Array<Components['schemas']["JobSummary"]> };
+    JobSummary: { enabled : boolean; id : string; name : string; repository_id : string; requested_mode : string; schedule : Components['schemas']["ScheduleSummary"] | null; source_id : string; state : string };
+    LocalSourceInput: { kind : string; name : string; public_config : Record<string, unknown> };
+    LoginInput: { password : string; username : string };
+    NotificationAttemptList: { items : Array<Components['schemas']["NotificationAttemptSummary"]> };
+    NotificationAttemptSummary: { completed_at : string | null; diagnostic : string | null; number : number; outcome : string; response_class : string | null; started_at : string };
+    NotificationDeliveryList: { items : Array<Components['schemas']["NotificationDeliverySummary"]> };
+    NotificationDeliverySummary: { attempt_count : number; due_at : string; event_id : string; id : string; response_class : string | null; response_summary : string | null; state : string; subscription_id : string; terminal_reason : string | null };
+    NotificationSubscriptionInput: { channel : "webhook" | "email"; destination : Record<string, unknown>; event_filters : Array<string>; rate_limit_per_minute?: number; signing_secret?: string | null };
+    NotificationSubscriptionList: { items : Array<Components['schemas']["NotificationSubscriptionSummary"]> };
+    NotificationSubscriptionPatch: { destination?: Record<string, unknown> | null; event_filters?: Array<string> | null; rate_limit_per_minute?: number | null; state?: "active" | "disabled" | "archived" | null };
+    NotificationSubscriptionSummary: { channel : string; created_at : string; destination : Record<string, unknown>; event_filters : Array<string>; id : string; rate_limit_per_minute : number; revision : number; state : string; updated_at : string };
+    RecoveryStatus: { encrypted_repository_count : number; recovery_mode : string; runbook : string };
+    RepositoryInput: { compression?: string; encryption?: string; name : string; relative_path : string };
+    RepositoryList: { items : Array<Components['schemas']["RepositorySummary"]> };
+    RepositoryPatch: { compression?: string | null; encryption?: string | null };
+    RepositorySummary: { compression : string; encryption : string; format_version : number; id : string; name : string; state : string };
+    RestoreInput: { destination : string; dry_run?: boolean; overwrite_policy?: string; selection?: Array<string> };
+    SSHSourceInput: { kind : string; name : string; private_key_secret_id : string; public_config : Components['schemas']["SSHSourcePublicConfig"] };
+    SSHSourcePublicConfig: { host_key : string; hostname : string; port : number; root : string; username : string };
+    ScheduleInput: { cron : string; enabled?: boolean; misfire_grace_seconds?: number; timezone : string };
+    ScheduleSummary: { cron : string; enabled : boolean; id : string; last_enqueue_outcome : string | null; next_nominal_at : string | null; timezone : string };
+    SecretInput: { purpose : string; value : string };
+    SessionUser: { id : string; state : string; username : string };
+    SetupInput: { bootstrap_secret?: string | null; password : string; username : string };
+    SigningKeyRotateInput: { overlap_seconds?: number; secret : string };
+    SourceList: { items : Array<Components['schemas']["SourceSummary"]> };
+    SourceSummary: { id : string; kind : string; name : string; public_config : Record<string, unknown>; state : string };
+    TokenInput: { expires_at?: string | null; scopes : Array<string> };
+    UserPatch: { state : string };
+    ValidationError: { ctx?: Record<string, unknown>; input?: unknown; loc : Array<string | number>; msg : string; type : string };
+  };
+}
+export type CreateSecretParams = { body : Components['schemas']["SecretInput"] };
+
+export type GetUserParams = { path: { user_id: string } };
+
+export type PatchUserParams = { path: { user_id: string }; body : Components['schemas']["UserPatch"] };
+
+export type ListAuditParams = { query?: { limit?: number; cursor?: string | null } };
+
+export type LoginParams = { body : Components['schemas']["LoginInput"] };
+
+export type CreateTokenParams = { body : Components['schemas']["TokenInput"] };
+
+export type RevokeTokenParams = { path: { token_id: string } };
+
+export type GetBackupParams = { path: { backup_id: string } };
+
+export type BackupDeletePreviewParams = { path: { backup_id: string } };
+
+export type CreateRestoreParams = { path: { backup_id: string }; body : Components['schemas']["RestoreInput"] };
+
+export type VerifyBackupParams = { path: { backup_id: string } };
+
+export type GetExecutionParams = { path: { execution_id: string } };
+
+export type CancelExecutionParams = { path: { execution_id: string } };
+
+export type ExecutionEventsParams = { path: { execution_id: string } };
+
+export type RetryExecutionParams = { path: { execution_id: string } };
+
+export type CreateJobParams = { body : Components['schemas']["JobInput"] };
+
+export type EnqueueExecutionParams = { path: { job_id: string } };
+
+export type DeleteScheduleParams = { path: { job_id: string } };
+
+export type GetScheduleParams = { path: { job_id: string } };
+
+export type PatchScheduleParams = { path: { job_id: string }; body : Components['schemas']["ScheduleInput"] };
+
+export type CreateScheduleParams = { path: { job_id: string }; body : Components['schemas']["ScheduleInput"] };
+
+export type ListNotificationDeliveriesParams = { query?: { limit?: number } };
+
+export type ListNotificationAttemptsParams = { path: { delivery_id: string } };
+
+export type RetryNotificationDeliveryParams = { path: { delivery_id: string } };
+
+export type PutNotificationEmailSettingsParams = { body : Components['schemas']["EmailSettingsInput"] };
+
+export type CreateNotificationSubscriptionParams = { body : Components['schemas']["NotificationSubscriptionInput"] };
+
+export type GetNotificationSubscriptionParams = { path: { subscription_id: string } };
+
+export type PatchNotificationSubscriptionParams = { path: { subscription_id: string }; body : Components['schemas']["NotificationSubscriptionPatch"] };
+
+export type RotateNotificationSigningKeyParams = { path: { subscription_id: string }; body : Components['schemas']["SigningKeyRotateInput"] };
+
+export type TestNotificationSubscriptionParams = { path: { subscription_id: string } };
+
+export type CreateRepositoryParams = { body : Components['schemas']["RepositoryInput"] };
+
+export type GetRepositoryParams = { path: { repository_id: string } };
+
+export type PatchRepositoryParams = { path: { repository_id: string }; body : Components['schemas']["RepositoryPatch"] };
+
+export type InspectRepositoryEndpointParams = { path: { repository_id: string } };
+
+export type GetRestoreParams = { path: { restore_id: string } };
+
+export type SetupParams = { body : Components['schemas']["SetupInput"] };
+
+export type CreateSourceParams = { body : Components['schemas']["LocalSourceInput"] | Components['schemas']["SSHSourceInput"] };
+
+export type ArchiveSourceParams = { path: { source_id: string } };
+
+export type ProbeSourceParams = { path: { source_id: string } };
+
+export type RequestOptions = Omit<RequestInit, "body">;
+
+export type Problem = {
+  type: string;
+  title: string;
+  status: number;
+  detail: string;
+  instance: string;
+  code: string;
+};
+
+export class ApiError extends Error {
+  readonly status: number;
+  readonly problem?: Problem;
+
+  constructor(status: number, problem?: Problem) {
+    super(problem?.detail ?? `Request failed with status ${status}.`);
+    this.name = "ApiError";
+    this.status = status;
+    this.problem = problem;
+  }
+}
+
+export function isApiError(error: unknown): error is ApiError {
+  return error instanceof ApiError;
+}
+
+function appendQuery(search: URLSearchParams, query: Record<string, unknown> | undefined): void {
+  if (!query) return;
+  for (const [key, value] of Object.entries(query)) {
+    if (value === undefined || value === null) continue;
+    for (const item of Array.isArray(value) ? value : [value]) search.append(key, String(item));
+  }
+}
+
+async function request<T>(
+  url: URL,
+  method: string,
+  options: RequestOptions,
+  body?: unknown,
+): Promise<T> {
+  const headers = new Headers(options.headers);
+  if (body !== undefined && !headers.has("Content-Type")) {
+    headers.set("Content-Type", "application/json");
+  }
+  const response = await fetch(url, {
+    ...options,
+    method,
+    headers,
+    credentials: options.credentials ?? "same-origin",
+    body: body === undefined ? undefined : JSON.stringify(body),
+  });
+  if (response.status === 204) return undefined as T;
+  const contentType = response.headers.get("content-type") ?? "";
+  const isJson = contentType.includes("application/json")
+    || contentType.includes("application/problem+json");
+  const payload: unknown = isJson ? await response.json() : undefined;
+  if (!response.ok) throw new ApiError(response.status, payload as Problem | undefined);
+  return payload as T;
+}
+
+export class BackupToolClient {
+  constructor(readonly baseUrl = window.location.origin) {}
+
+	async listSecrets(options: RequestOptions = {}): Promise<Array<Record<string, unknown>>> {
+		const url = new URL("/api/v2/admin/secrets", this.baseUrl);
+		return request<Array<Record<string, unknown>>>(
+			url, "GET", options
+		);
+	}
+
+	async createSecret(params: CreateSecretParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/admin/secrets", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async getUser(params: GetUserParams, options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/api/v2/admin/users/{user_id}".replace("{user_id}", encodeURIComponent(String(params.path.user_id))), this.baseUrl);
+		return request<Record<string, string>>(
+			url, "GET", options
+		);
+	}
+
+	async patchUser(params: PatchUserParams, options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/api/v2/admin/users/{user_id}".replace("{user_id}", encodeURIComponent(String(params.path.user_id))), this.baseUrl);
+		return request<Record<string, string>>(
+			url, "PATCH", options, params.body
+		);
+	}
+
+	async listAudit(params: ListAuditParams, options: RequestOptions = {}): Promise<Components['schemas']["AuditList"]> {
+		const url = new URL("/api/v2/audit", this.baseUrl);
+		appendQuery(url.searchParams, params.query);
+		return request<Components['schemas']["AuditList"]>(
+			url, "GET", options
+		);
+	}
+
+	async login(params: LoginParams, options: RequestOptions = {}): Promise<Components['schemas']["AuthenticatedUser"]> {
+		const url = new URL("/api/v2/auth/login", this.baseUrl);
+		return request<Components['schemas']["AuthenticatedUser"]>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async logout(options: RequestOptions = {}): Promise<void> {
+		const url = new URL("/api/v2/auth/logout", this.baseUrl);
+		return request<void>(
+			url, "POST", options
+		);
+	}
+
+	async getSession(options: RequestOptions = {}): Promise<Components['schemas']["SessionUser"]> {
+		const url = new URL("/api/v2/auth/session", this.baseUrl);
+		return request<Components['schemas']["SessionUser"]>(
+			url, "GET", options
+		);
+	}
+
+	async createToken(params: CreateTokenParams, options: RequestOptions = {}): Promise<unknown> {
+		const url = new URL("/api/v2/auth/tokens", this.baseUrl);
+		return request<unknown>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async revokeToken(params: RevokeTokenParams, options: RequestOptions = {}): Promise<void> {
+		const url = new URL("/api/v2/auth/tokens/{token_id}".replace("{token_id}", encodeURIComponent(String(params.path.token_id))), this.baseUrl);
+		return request<void>(
+			url, "DELETE", options
+		);
+	}
+
+	async listBackups(options: RequestOptions = {}): Promise<Components['schemas']["BackupList"]> {
+		const url = new URL("/api/v2/backups", this.baseUrl);
+		return request<Components['schemas']["BackupList"]>(
+			url, "GET", options
+		);
+	}
+
+	async getBackup(params: GetBackupParams, options: RequestOptions = {}): Promise<Components['schemas']["BackupSummary"]> {
+		const url = new URL("/api/v2/backups/{backup_id}".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl);
+		return request<Components['schemas']["BackupSummary"]>(
+			url, "GET", options
+		);
+	}
+
+	async backupDeletePreview(params: BackupDeletePreviewParams, options: RequestOptions = {}): Promise<Components['schemas']["BackupDeletePreview"]> {
+		const url = new URL("/api/v2/backups/{backup_id}/delete-preview".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl);
+		return request<Components['schemas']["BackupDeletePreview"]>(
+			url, "GET", options
+		);
+	}
+
+	async createRestore(params: CreateRestoreParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/backups/{backup_id}/restores".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async verifyBackup(params: VerifyBackupParams, options: RequestOptions = {}): Promise<Components['schemas']["BackupSummary"]> {
+		const url = new URL("/api/v2/backups/{backup_id}/verify".replace("{backup_id}", encodeURIComponent(String(params.path.backup_id))), this.baseUrl);
+		return request<Components['schemas']["BackupSummary"]>(
+			url, "POST", options
+		);
+	}
+
+	async listExecutions(options: RequestOptions = {}): Promise<Components['schemas']["ExecutionList"]> {
+		const url = new URL("/api/v2/executions", this.baseUrl);
+		return request<Components['schemas']["ExecutionList"]>(
+			url, "GET", options
+		);
+	}
+
+	async getExecution(params: GetExecutionParams, options: RequestOptions = {}): Promise<Components['schemas']["ExecutionSummary"]> {
+		const url = new URL("/api/v2/executions/{execution_id}".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl);
+		return request<Components['schemas']["ExecutionSummary"]>(
+			url, "GET", options
+		);
+	}
+
+	async cancelExecution(params: CancelExecutionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/executions/{execution_id}/cancel".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options
+		);
+	}
+
+	/** EventSource transport; intentionally not a JSON fetch Promise. */
+	executionEventsUrl(params: ExecutionEventsParams): URL {
+		const url = new URL("/api/v2/executions/{execution_id}/events".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl);
+		return url;
+	}
+
+	async retryExecution(params: RetryExecutionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/executions/{execution_id}/retry".replace("{execution_id}", encodeURIComponent(String(params.path.execution_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options
+		);
+	}
+
+	async listJobs(options: RequestOptions = {}): Promise<Components['schemas']["JobList"]> {
+		const url = new URL("/api/v2/jobs", this.baseUrl);
+		return request<Components['schemas']["JobList"]>(
+			url, "GET", options
+		);
+	}
+
+	async createJob(params: CreateJobParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/jobs", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async enqueueExecution(params: EnqueueExecutionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/jobs/{job_id}/executions".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options
+		);
+	}
+
+	async deleteSchedule(params: DeleteScheduleParams, options: RequestOptions = {}): Promise<void> {
+		const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl);
+		return request<void>(
+			url, "DELETE", options
+		);
+	}
+
+	async getSchedule(params: GetScheduleParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async patchSchedule(params: PatchScheduleParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "PATCH", options, params.body
+		);
+	}
+
+	async createSchedule(params: CreateScheduleParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/jobs/{job_id}/schedule".replace("{job_id}", encodeURIComponent(String(params.path.job_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async listNotificationDeliveries(params: ListNotificationDeliveriesParams, options: RequestOptions = {}): Promise<Components['schemas']["NotificationDeliveryList"]> {
+		const url = new URL("/api/v2/notifications/deliveries", this.baseUrl);
+		appendQuery(url.searchParams, params.query);
+		return request<Components['schemas']["NotificationDeliveryList"]>(
+			url, "GET", options
+		);
+	}
+
+	async listNotificationAttempts(params: ListNotificationAttemptsParams, options: RequestOptions = {}): Promise<Components['schemas']["NotificationAttemptList"]> {
+		const url = new URL("/api/v2/notifications/deliveries/{delivery_id}/attempts".replace("{delivery_id}", encodeURIComponent(String(params.path.delivery_id))), this.baseUrl);
+		return request<Components['schemas']["NotificationAttemptList"]>(
+			url, "GET", options
+		);
+	}
+
+	async retryNotificationDelivery(params: RetryNotificationDeliveryParams, options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/api/v2/notifications/deliveries/{delivery_id}/retry".replace("{delivery_id}", encodeURIComponent(String(params.path.delivery_id))), this.baseUrl);
+		return request<Record<string, string>>(
+			url, "POST", options
+		);
+	}
+
+	async getNotificationEmailSettings(options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/email-settings", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async putNotificationEmailSettings(params: PutNotificationEmailSettingsParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/email-settings", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "PUT", options, params.body
+		);
+	}
+
+	async notificationCatalog(options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/event-catalog", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async listNotificationSubscriptions(options: RequestOptions = {}): Promise<Components['schemas']["NotificationSubscriptionList"]> {
+		const url = new URL("/api/v2/notifications/subscriptions", this.baseUrl);
+		return request<Components['schemas']["NotificationSubscriptionList"]>(
+			url, "GET", options
+		);
+	}
+
+	async createNotificationSubscription(params: CreateNotificationSubscriptionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/subscriptions", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async getNotificationSubscription(params: GetNotificationSubscriptionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async patchNotificationSubscription(params: PatchNotificationSubscriptionParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "PATCH", options, params.body
+		);
+	}
+
+	async rotateNotificationSigningKey(params: RotateNotificationSigningKeyParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}/signing-keys/rotate".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async testNotificationSubscription(params: TestNotificationSubscriptionParams, options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/api/v2/notifications/subscriptions/{subscription_id}/test".replace("{subscription_id}", encodeURIComponent(String(params.path.subscription_id))), this.baseUrl);
+		return request<Record<string, string>>(
+			url, "POST", options
+		);
+	}
+
+	async listRepositories(options: RequestOptions = {}): Promise<Components['schemas']["RepositoryList"]> {
+		const url = new URL("/api/v2/repositories", this.baseUrl);
+		return request<Components['schemas']["RepositoryList"]>(
+			url, "GET", options
+		);
+	}
+
+	async createRepository(params: CreateRepositoryParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/repositories", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async getRepository(params: GetRepositoryParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/repositories/{repository_id}".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async patchRepository(params: PatchRepositoryParams, options: RequestOptions = {}): Promise<unknown> {
+		const url = new URL("/api/v2/repositories/{repository_id}".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl);
+		return request<unknown>(
+			url, "PATCH", options, params.body
+		);
+	}
+
+	async inspectRepositoryEndpoint(params: InspectRepositoryEndpointParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/repositories/{repository_id}/inspection".replace("{repository_id}", encodeURIComponent(String(params.path.repository_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async getRestore(params: GetRestoreParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/restores/{restore_id}".replace("{restore_id}", encodeURIComponent(String(params.path.restore_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "GET", options
+		);
+	}
+
+	async recoveryStatus(options: RequestOptions = {}): Promise<Components['schemas']["RecoveryStatus"]> {
+		const url = new URL("/api/v2/security/recovery/status", this.baseUrl);
+		return request<Components['schemas']["RecoveryStatus"]>(
+			url, "GET", options
+		);
+	}
+
+	async setup(params: SetupParams, options: RequestOptions = {}): Promise<Components['schemas']["AuthenticatedUser"]> {
+		const url = new URL("/api/v2/setup", this.baseUrl);
+		return request<Components['schemas']["AuthenticatedUser"]>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async listSources(options: RequestOptions = {}): Promise<Components['schemas']["SourceList"]> {
+		const url = new URL("/api/v2/sources", this.baseUrl);
+		return request<Components['schemas']["SourceList"]>(
+			url, "GET", options
+		);
+	}
+
+	async createSource(params: CreateSourceParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/sources", this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options, params.body
+		);
+	}
+
+	async archiveSource(params: ArchiveSourceParams, options: RequestOptions = {}): Promise<void> {
+		const url = new URL("/api/v2/sources/{source_id}".replace("{source_id}", encodeURIComponent(String(params.path.source_id))), this.baseUrl);
+		return request<void>(
+			url, "DELETE", options
+		);
+	}
+
+	async probeSource(params: ProbeSourceParams, options: RequestOptions = {}): Promise<Record<string, unknown>> {
+		const url = new URL("/api/v2/sources/{source_id}/probe".replace("{source_id}", encodeURIComponent(String(params.path.source_id))), this.baseUrl);
+		return request<Record<string, unknown>>(
+			url, "POST", options
+		);
+	}
+
+	async livez(options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/livez", this.baseUrl);
+		return request<Record<string, string>>(
+			url, "GET", options
+		);
+	}
+
+	async readyz(options: RequestOptions = {}): Promise<Record<string, string>> {
+		const url = new URL("/readyz", this.baseUrl);
+		return request<Record<string, string>>(
+			url, "GET", options
+		);
+	}
+
+}

frontend/src/app/App.test.tsx

@@ -0,0 +1,176 @@
+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { App } from "./App";
+
+function response(body: unknown, status = 200): Response {
+	return {
+		headers: new Headers({ "content-type": "application/json" }),
+		json: async () => body,
+		ok: status >= 200 && status < 300,
+		status,
+	} as Response;
+}
+
+function problem(status: number, code: string, detail = "Request failed."): Response {
+	return response({ type: `https://backup-tool.invalid/problems/${code}`, title: code, status, detail, instance: "/", code }, status);
+}
+
+function session() {
+	return response({ id: "user-1", username: "operator", state: "active" });
+}
+
+function repositories(items: unknown[] = []) {
+	return response({ items });
+}
+
+afterEach(() => {
+	cleanup();
+	vi.unstubAllGlobals();
+});
+
+describe("operator foundation", () => {
+	it("shows loading then the accessible empty dashboard", async () => {
+		const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(repositories());
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		expect(screen.getByRole("status")).toHaveTextContent("Checking your session");
+		expect(await screen.findByText("No repositories have been configured.")).toBeInTheDocument();
+		expect(fetch).toHaveBeenCalledTimes(2);
+	});
+
+	it("selects setup and creates a session for the first administrator", async () => {
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(problem(401, "authentication_required"))
+			.mockResolvedValueOnce(problem(503, "setup_required"))
+			.mockResolvedValueOnce(response({ id: "user-1", username: "operator" }, 201))
+			.mockResolvedValueOnce(repositories());
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		expect(await screen.findByRole("heading", { name: "Set up your administrator account" })).toBeInTheDocument();
+		fireEvent.change(screen.getByLabelText("Username"), { target: { value: "operator" } });
+		fireEvent.change(screen.getByLabelText("Password"), { target: { value: "correct horse battery staple" } });
+		fireEvent.click(screen.getByRole("button", { name: "Create administrator account" }));
+
+		expect(await screen.findByRole("heading", { name: "Dashboard" })).toBeInTheDocument();
+		const setupRequest = fetch.mock.calls[2]?.[0] as URL;
+		expect(setupRequest.pathname).toBe("/api/v2/setup");
+	});
+
+	it("signs in after setup is complete", async () => {
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(problem(401, "authentication_required"))
+			.mockResolvedValueOnce(response({ status: "ready" }))
+			.mockResolvedValueOnce(response({ id: "user-1", username: "operator" }))
+			.mockResolvedValueOnce(repositories());
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		expect(await screen.findByRole("heading", { name: "Sign in" })).toBeInTheDocument();
+		fireEvent.change(screen.getByLabelText("Username"), { target: { value: "operator" } });
+		fireEvent.change(screen.getByLabelText("Password"), { target: { value: "correct horse battery staple" } });
+		fireEvent.click(screen.getByRole("button", { name: "Sign in" }));
+
+		expect(await screen.findByRole("heading", { name: "Dashboard" })).toBeInTheDocument();
+		const loginRequest = fetch.mock.calls[2]?.[0] as URL;
+		expect(loginRequest.pathname).toBe("/api/v2/auth/login");
+	});
+
+	it("shows retryable dashboard errors", async () => {
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(session())
+			.mockResolvedValueOnce(problem(500, "service_unavailable", "Dashboard data is unavailable."))
+			.mockResolvedValueOnce(repositories());
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		expect(await screen.findByRole("alert")).toHaveTextContent("Dashboard data is unavailable.");
+		fireEvent.click(screen.getByRole("button", { name: "Try again" }));
+		await waitFor(() => expect(screen.getByText("No repositories have been configured.")).toBeInTheDocument());
+	});
+
+	it("loads source and job states from generated client methods", async () => {
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(session())
+			.mockResolvedValueOnce(repositories())
+			.mockResolvedValueOnce(response({ items: [] }))
+			.mockResolvedValueOnce(problem(500, "service_unavailable", "Jobs are unavailable."));
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		await screen.findByText("No repositories have been configured.");
+		fireEvent.click(screen.getByRole("button", { name: "Sources" }));
+		expect(await screen.findByText("No sources have been configured.")).toBeInTheDocument();
+		fireEvent.click(screen.getByRole("button", { name: "Jobs & schedules" }));
+		expect(await screen.findByRole("alert")).toHaveTextContent("Jobs are unavailable.");
+	});
+
+	it("loads execution detail after selecting an execution", async () => {
+		const execution = { id: "execution-1", state: "failed", attempt: 2, revision: 3, reason_code: "timeout", progress: {} };
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(session())
+			.mockResolvedValueOnce(repositories())
+			.mockResolvedValueOnce(response({ items: [execution] }))
+			.mockResolvedValueOnce(response(execution));
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		await screen.findByText("No repositories have been configured.");
+		fireEvent.click(screen.getByRole("button", { name: "Executions" }));
+		expect(await screen.findByRole("button", { name: "Execution execution-1" })).toBeInTheDocument();
+		fireEvent.click(screen.getByRole("button", { name: "Execution execution-1" }));
+		expect(await screen.findByRole("heading", { name: "Execution detail" })).toBeInTheDocument();
+		expect(screen.getByText("timeout")).toBeInTheDocument();
+	});
+
+	it("announces SSE reconnects and applies live execution updates", async () => {
+		class EventSourceMock {
+			static instances: EventSourceMock[] = [];
+			onmessage: ((event: MessageEvent<string>) => void) | null = null;
+			onerror: (() => void) | null = null;
+			constructor() { EventSourceMock.instances.push(this); }
+			close = vi.fn();
+		}
+		vi.stubGlobal("EventSource", EventSourceMock);
+		const execution = { id: "execution-1", state: "running", attempt: 1, revision: 1, reason_code: null, progress: {} };
+		const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(repositories()).mockResolvedValueOnce(response({ items: [execution] })).mockResolvedValueOnce(response(execution));
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+		await screen.findByText("No repositories have been configured.");
+		fireEvent.click(screen.getByRole("button", { name: "Executions" }));
+		fireEvent.click(await screen.findByRole("button", { name: "Execution execution-1" }));
+		await screen.findByRole("heading", { name: "Execution detail" });
+		EventSourceMock.instances[0]?.onerror?.();
+		expect(await screen.findByRole("alert")).toHaveTextContent("Live updates disconnected");
+		EventSourceMock.instances[0]?.onmessage?.({ data: JSON.stringify({ ...execution, state: "committed", revision: 2 }) } as MessageEvent<string>);
+		expect(await screen.findByText("committed")).toBeInTheDocument();
+	});
+
+	it("sends an idempotency key when retrying a failed notification delivery", async () => {
+		const fetch = vi.fn()
+			.mockResolvedValueOnce(session())
+			.mockResolvedValueOnce(repositories())
+			.mockResolvedValueOnce(response({ items: [] }))
+			.mockResolvedValueOnce(response({ items: [{ id: "delivery-1", event_id: "event-1", subscription_id: "subscription-1", state: "failed", attempt_count: 1, response_class: null, response_summary: null, terminal_reason: "timeout", due_at: "2026-01-01T00:00:00Z" }] }))
+			.mockResolvedValueOnce(response({ delivery_id: "delivery-1" }, 202));
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+		await screen.findByText("No repositories have been configured.");
+		fireEvent.click(screen.getByRole("button", { name: "Notifications" }));
+		fireEvent.click(await screen.findByRole("button", { name: "Retry delivery" }));
+		expect(await screen.findByRole("status")).toHaveTextContent("Delivery retry queued.");
+		const options = fetch.mock.calls[4]?.[1] as RequestInit;
+		expect(new Headers(options.headers).get("Idempotency-Key")).toBeTruthy();
+	});
+
+	it("returns to sign-in when the dashboard request finds an expired session", async () => {
+		const fetch = vi.fn().mockResolvedValueOnce(session()).mockResolvedValueOnce(problem(401, "authentication_required"));
+		vi.stubGlobal("fetch", fetch);
+		render(<App />);
+
+		expect(await screen.findByRole("heading", { name: "Sign in" })).toBeInTheDocument();
+		expect(screen.getByRole("status")).toHaveTextContent("Your session expired");
+	});
+});

frontend/src/app/App.tsx

@@ -1,16 +1,97 @@
-export function App() {
+import { type FormEvent, useEffect, useRef, useState } from "react";
-	return (
+
-		<main className="min-h-screen bg-slate-950 p-8 text-slate-100">
+import {
-			<section className="mx-auto max-w-3xl rounded-xl border border-slate-800 bg-slate-900 p-8">
+	BackupToolClient,
-				<p className="text-sm font-semibold uppercase tracking-widest text-emerald-400">
+	type Components,
-					Backup Tool v2
+	isApiError,
-				</p>
+} from "../api/generated/client";
-				<h1 className="mt-3 text-3xl font-bold">Protocol foundation ready</h1>
+import { OperatorViews } from "./OperatorViews";
-				<p className="mt-4 text-slate-300">
+
-					Operator workflows are added as their versioned API contracts become
+type SessionUser = Components["schemas"]["SessionUser"];
-					executable.
+type AuthMode = "setup" | "login";
-				</p>
+type Screen =
-			</section>
+	| { kind: "loading" }
-		</main>
+	| { kind: "auth"; mode: AuthMode; error?: string; sessionExpired?: boolean }
-	);
+	| { kind: "operator"; user: SessionUser };
+
+const defaultClient = new BackupToolClient();
+
+function errorMessage(error: unknown): string {
+	if (isApiError(error)) return error.problem?.detail ?? "The server could not complete that request.";
+	return "We could not reach the Backup Tool service. Check your connection and try again.";
+}
+
+function isSetupRequired(error: unknown): boolean {
+	return isApiError(error) && error.status === 503 && error.problem?.code === "setup_required";
+}
+
+function isSessionExpired(error: unknown): boolean {
+	return isApiError(error) && error.status === 401;
+}
+
+function AuthForm({
+	mode,
+	error,
+	sessionExpired,
+	onSubmit,
+}: {
+	mode: AuthMode;
+	error?: string;
+	sessionExpired?: boolean;
+	onSubmit: (username: string, password: string) => Promise<void>;
+}) {
+	const [username, setUsername] = useState("");
+	const [password, setPassword] = useState("");
+	const [submitting, setSubmitting] = useState(false);
+	const isSetup = mode === "setup";
+
+	async function submit(event: FormEvent<HTMLFormElement>) {
+		event.preventDefault();
+		if (isSetup && password.length < 12) return;
+		setSubmitting(true);
+		try {
+			await onSubmit(username, password);
+		} finally {
+			setSubmitting(false);
+		}
+	}
+
+	return <main className="flex min-h-screen items-center justify-center bg-slate-950 p-4 text-slate-100"><section aria-labelledby="auth-title" className="w-full max-w-md rounded-xl border border-slate-700 bg-slate-900 p-6 shadow-2xl"><p className="text-sm font-semibold uppercase tracking-widest text-emerald-400">Backup Tool</p><h1 className="mt-3 text-3xl font-bold" id="auth-title">{isSetup ? "Set up your administrator account" : "Sign in"}</h1><p className="mt-3 text-slate-300">{isSetup ? "Create the first administrator account to begin operating this backup service." : "Use an administrator account to continue."}</p>{sessionExpired ? <p role="status" className="mt-4 rounded-md border border-amber-400/50 bg-amber-950/40 p-3 text-amber-100">Your session expired. Sign in again to continue.</p> : null}{error ? <p role="alert" className="mt-4 rounded-md border border-rose-400/50 bg-rose-950/40 p-3 text-rose-100">{error}</p> : null}<form className="mt-6 space-y-4" onSubmit={submit}><div><label className="block text-sm font-medium" htmlFor="username">Username</label><input autoComplete="username" autoFocus className="mt-1 w-full rounded-md border border-slate-600 bg-slate-950 px-3 py-2" id="username" onChange={(event) => setUsername(event.target.value)} required value={username} /></div><div><label className="block text-sm font-medium" htmlFor="password">Password</label><input autoComplete={isSetup ? "new-password" : "current-password"} className="mt-1 w-full rounded-md border border-slate-600 bg-slate-950 px-3 py-2" id="password" minLength={isSetup ? 12 : undefined} onChange={(event) => setPassword(event.target.value)} required type="password" value={password} />{isSetup ? <p className="mt-1 text-sm text-slate-400">Use at least 12 characters.</p> : null}</div><button className="w-full rounded-md bg-emerald-500 px-4 py-2 font-semibold text-slate-950 hover:bg-emerald-400 disabled:cursor-not-allowed disabled:opacity-60" disabled={submitting} type="submit">{submitting ? "Working…" : isSetup ? "Create administrator account" : "Sign in"}</button></form></section></main>;
+}
+
+export function App({ client = defaultClient }: { client?: BackupToolClient }) {
+	const [screen, setScreen] = useState<Screen>({ kind: "loading" });
+	const clientRef = useRef(client);
+	clientRef.current = client;
+
+	async function discover() {
+		setScreen({ kind: "loading" });
+		try {
+			setScreen({ kind: "operator", user: await clientRef.current.getSession() });
+		} catch (error) {
+			if (!isSessionExpired(error)) {
+				setScreen({ kind: "auth", mode: "login", error: errorMessage(error) });
+				return;
+			}
+			try {
+				await clientRef.current.readyz();
+				setScreen({ kind: "auth", mode: "login" });
+			} catch (readinessError) {
+				setScreen({ kind: "auth", mode: isSetupRequired(readinessError) ? "setup" : "login", error: isSetupRequired(readinessError) ? undefined : errorMessage(readinessError) });
+			}
+		}
+	}
+
+	useEffect(() => { void discover(); }, []);
+	if (screen.kind === "loading") return <main aria-live="polite" className="flex min-h-screen items-center justify-center bg-slate-950 text-slate-100" role="status">Checking your session…</main>;
+	if (screen.kind === "auth") return <AuthForm {...screen} onSubmit={async (username, password) => {
+		try {
+			const user = screen.mode === "setup" ? await clientRef.current.setup({ body: { username, password } }) : await clientRef.current.login({ body: { username, password } });
+			setScreen({ kind: "operator", user: { ...user, state: "active" } });
+		} catch (error) {
+			if (screen.mode === "setup" && isApiError(error) && error.problem?.code === "setup_complete") setScreen({ kind: "auth", mode: "login", error: "Setup is already complete. Sign in with the administrator account." });
+			else setScreen({ ...screen, error: errorMessage(error) });
+		}
+	}} />;
+	return <OperatorViews client={clientRef.current} onSessionExpired={() => setScreen({ kind: "auth", mode: "login", sessionExpired: true })} user={screen.user} />;
 }

frontend/src/app/M13Workflows.tsx

@@ -0,0 +1,33 @@
+import { type ReactNode, useEffect, useState } from "react";
+
+import { type BackupToolClient, type Components, isApiError } from "../api/generated/client";
+
+type BackupSummary = Components["schemas"]["BackupSummary"];
+type Delivery = Components["schemas"]["NotificationDeliverySummary"];
+type Subscription = Components["schemas"]["NotificationSubscriptionSummary"];
+type Audit = Components["schemas"]["AuditSummary"];
+type State<T> = { loading: boolean; data?: T; error?: string };
+
+type Props = { client: BackupToolClient; onSessionExpired: () => void };
+
+function message(error: unknown) { return isApiError(error) ? error.problem?.detail ?? "The request failed." : "The service could not be reached."; }
+function expired(error: unknown) { return isApiError(error) && error.status === 401; }
+function Panel({ children, title }: { children: ReactNode; title: string }) { return <section aria-labelledby="page-title" className="mx-auto max-w-6xl p-4 sm:p-6"><h2 className="text-xl font-semibold" id="page-title">{title}</h2>{children}</section>; }
+function Retry({ error, load }: { error?: string; load: () => void }) { return error ? <div className="mt-4 rounded-lg border border-rose-400/50 bg-rose-950/40 p-4"><p role="alert">{error}</p><button className="mt-3 rounded border border-slate-500 px-3 py-2" onClick={load} type="button">Try again</button></div> : null; }
+function Loading({ label }: { label: string }) { return <p className="mt-4 text-slate-300" role="status">Loading {label}…</p>; }
+function csrfHeaders(): HeadersInit { const value = document.cookie.split("; ").find((entry) => entry.startsWith("backup_tool_csrf="))?.split("=", 2)[1]; return value ? { "X-CSRF-Token": value } : {}; }
+function idempotencyKey(): string { return typeof crypto.randomUUID === "function" ? crypto.randomUUID() : `${Date.now()}-${Math.random()}`; }
+
+export function BackupsPage({ client, onSessionExpired }: Props) {
+ const [state,setState]=useState<State<BackupSummary[]>>({loading:true}); const [selected,setSelected]=useState<BackupSummary>(); const [preview,setPreview]=useState<string>(); const [destination,setDestination]=useState(""); const [restoreStatus,setRestoreStatus]=useState<string>();
+ const load=async()=>{setState({loading:true});try{setState({loading:false,data:(await client.listBackups()).items});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}};
+ useEffect(()=>{void load();},[]);
+ const headers=csrfHeaders();
+ return <Panel title="Backups">{state.loading?<Loading label="backups"/>:null}<Retry error={state.error} load={()=>void load()}/>{state.data?.length===0?<p className="mt-4 text-slate-300">No backups have been committed.</p>:null}<ul className="mt-4 space-y-3" aria-label="Backups">{state.data?.map(item=><li className="rounded border border-slate-700 bg-slate-900 p-4" key={item.id}><button className="font-semibold hover:text-emerald-300" onClick={()=>setSelected(item)} type="button">Backup {item.id}</button><p className="text-sm text-slate-300">{item.integrity} · {item.logical_bytes} logical bytes</p></li>)}</ul>{selected?<section className="mt-6 rounded border border-slate-700 bg-slate-900 p-4" aria-labelledby="backup-detail"><h3 id="backup-detail" className="font-semibold">Backup detail</h3><p className="mt-2">Integrity: {selected.integrity}</p><div className="mt-3 flex flex-wrap gap-2"><button className="rounded border border-slate-500 px-3 py-2" onClick={async()=>{try{setSelected(await client.verifyBackup({path:{backup_id:selected.id}},{headers}));}catch(error){setPreview(message(error));}}} type="button">Verify</button><button className="rounded border border-slate-500 px-3 py-2" onClick={async()=>{try{const result=await client.backupDeletePreview({path:{backup_id:selected.id}},{headers});setPreview(result.reason??result.destructive_action);}catch(error){setPreview(message(error));}}} type="button">Preview deletion</button></div>{preview?<p className="mt-3" role="status">{preview}</p>:null}<form className="mt-4 space-y-2" onSubmit={async(event)=>{event.preventDefault();try{await client.createRestore({path:{backup_id:selected.id},body:{destination,dry_run:true,selection:[],overwrite_policy:"fail"}},{headers});setRestoreStatus("Restore dry run queued.");}catch(error){setRestoreStatus(message(error));}}}><label className="block text-sm font-medium" htmlFor="restore-destination">Restore destination</label><input className="w-full rounded border border-slate-500 bg-slate-950 p-2" id="restore-destination" onChange={(event)=>setDestination(event.target.value)} required value={destination}/><button className="rounded border border-slate-500 px-3 py-2" type="submit">Queue restore dry run</button>{restoreStatus?<p role="status">{restoreStatus}</p>:null}</form></section>:null}</Panel>;
+}
+
+export function SecurityPage({ client,onSessionExpired }: Props) { const [state,setState]=useState<State<Components["schemas"]["RecoveryStatus"]>>({loading:true}); const load=async()=>{setState({loading:true});try{setState({loading:false,data:await client.recoveryStatus()});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}};useEffect(()=>{void load();},[]);return <Panel title="Security & recovery">{state.loading?<Loading label="recovery status"/>:null}<Retry error={state.error} load={()=>void load()}/>{state.data?<article className="mt-4 rounded border border-slate-700 bg-slate-900 p-4"><p>Recovery is {state.data.recovery_mode.replace("_"," ")}.</p><p className="mt-2">Encrypted repositories: {state.data.encrypted_repository_count}</p><p className="mt-3 text-slate-300">Use the CLI and the recovery runbook: <code>{state.data.runbook}</code>. Passphrases and recovery bundles never enter the browser.</p></article>:null}</Panel>; }
+
+export function NotificationsPage({client,onSessionExpired}:Props){const [state,setState]=useState<State<{subscriptions:Subscription[];deliveries:Delivery[]}>>({loading:true});const [history,setHistory]=useState<string>();const load=async()=>{setState({loading:true});const results=await Promise.allSettled([client.listNotificationSubscriptions(),client.listNotificationDeliveries({query:{limit:20}})]);if(results.some((result)=>result.status==="rejected"&&expired(result.reason))){onSessionExpired();return;}const errors=results.filter((result)=>result.status==="rejected");setState({loading:false,data:{subscriptions:results[0].status==="fulfilled"?results[0].value.items:[],deliveries:results[1].status==="fulfilled"?results[1].value.items:[]},error:errors.length?"Some notification history could not be loaded.":undefined});};useEffect(()=>{void load();},[]);return <Panel title="Notifications & history">{state.loading?<Loading label="notifications"/>:null}<Retry error={state.error} load={()=>void load()}/>{state.data?<><h3 className="mt-4 font-semibold">Subscriptions</h3>{state.data.subscriptions.length?<ul className="mt-2">{state.data.subscriptions.map(item=><li key={item.id}>{item.channel} · {item.state}</li>)}</ul>:<p className="mt-2 text-slate-300">No notification subscriptions.</p>}<h3 className="mt-5 font-semibold">Delivery history</h3>{state.data.deliveries.length?<ul className="mt-2">{state.data.deliveries.map(item=><li key={item.id}>{item.state} · {item.attempt_count} attempts <button className="ml-2 underline" onClick={async()=>{try{const attempts=await client.listNotificationAttempts({path:{delivery_id:item.id}});setHistory(`${attempts.items.length} delivery attempts loaded.`);}catch(error){setHistory(message(error));}}} type="button">View attempts</button>{item.state==="failed"?<button className="ml-2 underline" onClick={async()=>{try{await client.retryNotificationDelivery({path:{delivery_id:item.id}},{headers:{...csrfHeaders(),"Idempotency-Key":idempotencyKey()}});setHistory("Delivery retry queued.");}catch(error){setHistory(message(error));}}} type="button">Retry delivery</button>:null}</li>)}</ul>:<p className="mt-2 text-slate-300">No notification deliveries.</p>}{history?<p className="mt-3" role="status">{history}</p>:null}</>:null}</Panel>}
+
+export function AuditPage({client,onSessionExpired}:Props){const [state,setState]=useState<State<Audit[]>>({loading:true});const load=async()=>{setState({loading:true});try{setState({loading:false,data:(await client.listAudit({query:{limit:50}})).items});}catch(error){if(expired(error))onSessionExpired();else setState({loading:false,error:message(error)});}};useEffect(()=>{void load();},[]);return <Panel title="Audit">{state.loading?<Loading label="audit events"/>:null}<Retry error={state.error} load={()=>void load()}/>{state.data?.length===0?<p className="mt-4 text-slate-300">No audit events.</p>:null}<ul className="mt-4 space-y-2">{state.data?.map(item=><li className="rounded border border-slate-700 p-3" key={item.id}>{item.action} {item.resource_type} · {item.outcome}</li>)}</ul></Panel>}

frontend/src/app/OperatorViews.tsx

@@ -0,0 +1,159 @@
+import { type ReactNode, useEffect, useState } from "react";
+
+import {
+	type BackupToolClient,
+	type Components,
+	isApiError,
+} from "../api/generated/client";
+import { AuditPage, BackupsPage, NotificationsPage, SecurityPage } from "./M13Workflows";
+
+type SessionUser = Components["schemas"]["SessionUser"];
+type SourceSummary = Components["schemas"]["SourceSummary"];
+type RepositorySummary = Components["schemas"]["RepositorySummary"];
+type JobSummary = Components["schemas"]["JobSummary"];
+type ExecutionSummary = Components["schemas"]["ExecutionSummary"];
+type Page = "dashboard" | "sources" | "repositories" | "jobs" | "executions" | "backups" | "security" | "notifications" | "audit";
+type LoadState<T> =
+	| { kind: "loading" }
+	| { kind: "ready"; items: T }
+	| { kind: "error"; message: string };
+
+const pages: Array<{ id: Page; label: string }> = [
+	{ id: "dashboard", label: "Dashboard" },
+	{ id: "sources", label: "Sources" },
+	{ id: "repositories", label: "Repositories" },
+	{ id: "jobs", label: "Jobs & schedules" },
+	{ id: "executions", label: "Executions" },
+	{ id: "backups", label: "Backups" },
+	{ id: "security", label: "Security & recovery" },
+	{ id: "notifications", label: "Notifications" },
+	{ id: "audit", label: "Audit" },
+];
+
+function errorMessage(error: unknown): string {
+	if (isApiError(error)) return error.problem?.detail ?? "The server could not complete that request.";
+	return "We could not reach the Backup Tool service. Check your connection and try again.";
+}
+
+function isSessionExpired(error: unknown): boolean {
+	return isApiError(error) && error.status === 401;
+}
+
+function ErrorPanel({ message, retry }: { message: string; retry: () => void }) {
+	return <div className="mt-4 rounded-lg border border-rose-400/50 bg-rose-950/40 p-4"><p role="alert">{message}</p><button className="mt-3 rounded-md border border-slate-500 px-3 py-2 font-semibold hover:bg-slate-800" onClick={retry} type="button">Try again</button></div>;
+}
+
+function Loading({ label }: { label: string }) {
+	return <p aria-live="polite" className="mt-4 text-slate-300" role="status">Loading {label}…</p>;
+}
+
+function ResourceSection({ children, title }: { children: ReactNode; title: string }) {
+	return <section aria-labelledby="page-title" className="mx-auto max-w-6xl p-4 sm:p-6"><h2 className="text-xl font-semibold" id="page-title">{title}</h2>{children}</section>;
+}
+
+function DashboardPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) {
+	const [state, setState] = useState<LoadState<RepositorySummary[]>>({ kind: "loading" });
+	async function load() {
+		setState({ kind: "loading" });
+		try { setState({ kind: "ready", items: (await client.listRepositories()).items }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); }
+	}
+	useEffect(() => { void load(); }, []);
+	return <ResourceSection title="Dashboard">
+		<p className="mt-2 text-slate-300">Repository availability at a glance.</p>
+		{state.kind === "loading" ? <Loading label="dashboard data" /> : null}
+		{state.kind === "error" ? <ErrorPanel message={state.message} retry={() => { void load(); }} /> : null}
+		{state.kind === "ready" && state.items.length === 0 ? <p className="mt-4 rounded-lg border border-slate-700 bg-slate-900 p-4 text-slate-300">No repositories have been configured.</p> : null}
+		{state.kind === "ready" && state.items.length > 0 ? <ul aria-label="Configured repositories" className="mt-4 grid gap-3 sm:grid-cols-2">{state.items.map((repository) => <li className="rounded-lg border border-slate-700 bg-slate-900 p-4" key={repository.id}><h3 className="font-semibold">{repository.name}</h3><p className="mt-2 text-sm text-slate-300">{repository.state} · {repository.encryption}</p></li>)}</ul> : null}
+	</ResourceSection>;
+}
+
+function SourcesPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) {
+	const [state, setState] = useState<LoadState<SourceSummary[]>>({ kind: "loading" });
+	async function load() {
+		setState({ kind: "loading" });
+		try { setState({ kind: "ready", items: (await client.listSources()).items }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); }
+	}
+	useEffect(() => { void load(); }, []);
+	return <ResourceSection title="Sources">
+		{state.kind === "loading" ? <Loading label="sources" /> : null}
+		{state.kind === "error" ? <ErrorPanel message={state.message} retry={() => { void load(); }} /> : null}
+		{state.kind === "ready" && state.items.length === 0 ? <p className="mt-4 rounded-lg border border-slate-700 bg-slate-900 p-4 text-slate-300">No sources have been configured.</p> : null}
+		{state.kind === "ready" && state.items.length > 0 ? <ul aria-label="Configured sources" className="mt-4 space-y-3">{state.items.map((source) => <li className="rounded-lg border border-slate-700 bg-slate-900 p-4" key={source.id}><h3 className="font-semibold">{source.name}</h3><p className="mt-1 text-sm text-slate-300">{source.kind} · {source.state}</p></li>)}</ul> : null}
+	</ResourceSection>;
+}
+
+function RepositoriesPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) {
+	const [state, setState] = useState<LoadState<RepositorySummary[]>>({ kind: "loading" });
+	async function load() {
+		setState({ kind: "loading" });
+		try { setState({ kind: "ready", items: (await client.listRepositories()).items }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); }
+	}
+	useEffect(() => { void load(); }, []);
+	return <ResourceSection title="Repositories">
+		{state.kind === "loading" ? <Loading label="repositories" /> : null}
+		{state.kind === "error" ? <ErrorPanel message={state.message} retry={() => { void load(); }} /> : null}
+		{state.kind === "ready" && state.items.length === 0 ? <p className="mt-4 rounded-lg border border-slate-700 bg-slate-900 p-4 text-slate-300">No repositories have been configured.</p> : null}
+		{state.kind === "ready" && state.items.length > 0 ? <ul aria-label="Configured repositories" className="mt-4 space-y-3">{state.items.map((repository) => <li className="rounded-lg border border-slate-700 bg-slate-900 p-4" key={repository.id}><h3 className="font-semibold">{repository.name}</h3><p className="mt-1 text-sm text-slate-300">Format {repository.format_version} · {repository.encryption} · {repository.state}</p></li>)}</ul> : null}
+	</ResourceSection>;
+}
+
+function JobsPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) {
+	const [state, setState] = useState<LoadState<JobSummary[]>>({ kind: "loading" });
+	async function load() {
+		setState({ kind: "loading" });
+		try { setState({ kind: "ready", items: (await client.listJobs()).items }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); }
+	}
+	useEffect(() => { void load(); }, []);
+	return <ResourceSection title="Jobs & schedules">
+		{state.kind === "loading" ? <Loading label="jobs" /> : null}
+		{state.kind === "error" ? <ErrorPanel message={state.message} retry={() => { void load(); }} /> : null}
+		{state.kind === "ready" && state.items.length === 0 ? <p className="mt-4 rounded-lg border border-slate-700 bg-slate-900 p-4 text-slate-300">No jobs have been configured.</p> : null}
+		{state.kind === "ready" && state.items.length > 0 ? <ul aria-label="Configured jobs" className="mt-4 space-y-3">{state.items.map((job) => <li className="rounded-lg border border-slate-700 bg-slate-900 p-4" key={job.id}><h3 className="font-semibold">{job.name}</h3><p className="mt-1 text-sm text-slate-300">{job.requested_mode} · {job.enabled ? "enabled" : "disabled"}</p><p className="mt-2 text-sm text-slate-300">{job.schedule ? `${job.schedule.cron} (${job.schedule.timezone})` : "No schedule configured."}</p></li>)}</ul> : null}
+	</ResourceSection>;
+}
+
+function ExecutionsPage({ client, onSessionExpired }: { client: BackupToolClient; onSessionExpired: () => void }) {
+	const [state, setState] = useState<LoadState<ExecutionSummary[]>>({ kind: "loading" });
+	const [selectedId, setSelectedId] = useState<string>();
+	const [detail, setDetail] = useState<LoadState<ExecutionSummary> | undefined>();
+	async function load() {
+		setState({ kind: "loading" });
+		try { setState({ kind: "ready", items: (await client.listExecutions()).items }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setState({ kind: "error", message: errorMessage(error) }); }
+	}
+	async function loadDetail(executionId: string) {
+		setSelectedId(executionId); setDetail({ kind: "loading" });
+		try { setDetail({ kind: "ready", items: await client.getExecution({ path: { execution_id: executionId } }) }); }
+		catch (error) { if (isSessionExpired(error)) onSessionExpired(); else setDetail({ kind: "error", message: errorMessage(error) }); }
+	}
+	useEffect(() => { void load(); }, []);
+	useEffect(() => {
+		if (!selectedId || typeof EventSource === "undefined") return;
+		const stream = new EventSource(client.executionEventsUrl({ path: { execution_id: selectedId } }));
+		stream.onmessage = (event) => {
+			try { setDetail({ kind: "ready", items: JSON.parse(event.data) as ExecutionSummary }); }
+			catch { setDetail({ kind: "error", message: "Live execution update was invalid. Reconnecting…" }); }
+		};
+		stream.onerror = () => { setDetail({ kind: "error", message: "Live updates disconnected. Reconnecting…" }); };
+		return () => stream.close();
+	}, [client, selectedId]);
+	return <ResourceSection title="Executions">
+		{state.kind === "loading" ? <Loading label="executions" /> : null}
+		{state.kind === "error" ? <ErrorPanel message={state.message} retry={() => { void load(); }} /> : null}
+		{state.kind === "ready" && state.items.length === 0 ? <p className="mt-4 rounded-lg border border-slate-700 bg-slate-900 p-4 text-slate-300">No executions have been queued.</p> : null}
+		{state.kind === "ready" && state.items.length > 0 ? <ul aria-label="Executions" className="mt-4 space-y-3">{state.items.map((execution) => <li className="rounded-lg border border-slate-700 bg-slate-900 p-4" key={execution.id}><button aria-current={selectedId === execution.id ? "true" : undefined} className="text-left font-semibold hover:text-emerald-300" onClick={() => { void loadDetail(execution.id); }} type="button">Execution {execution.id}</button><p className="mt-1 text-sm text-slate-300">{execution.state} · attempt {execution.attempt}</p></li>)}</ul> : null}
+		{detail?.kind === "loading" ? <Loading label="execution details" /> : null}
+		{detail?.kind === "error" && selectedId ? <ErrorPanel message={detail.message} retry={() => { void loadDetail(selectedId); }} /> : null}
+		{detail?.kind === "ready" ? <section aria-labelledby="execution-detail-heading" className="mt-6 rounded-lg border border-slate-700 bg-slate-900 p-4"><h3 id="execution-detail-heading" className="text-lg font-semibold">Execution detail</h3><dl className="mt-3 grid gap-2 text-sm sm:grid-cols-2"><div><dt className="text-slate-400">State</dt><dd>{detail.items.state}</dd></div><div><dt className="text-slate-400">Attempt</dt><dd>{detail.items.attempt}</dd></div><div><dt className="text-slate-400">Reason</dt><dd>{detail.items.reason_code ?? "None"}</dd></div></dl></section> : null}
+	</ResourceSection>;
+}
+
+export function OperatorViews({ client, onSessionExpired, user }: { client: BackupToolClient; onSessionExpired: () => void; user: SessionUser }) {
+	const [page, setPage] = useState<Page>("dashboard");
+	const common = { client, onSessionExpired };
+	return <main className="min-h-screen bg-slate-950 text-slate-100"><header className="border-b border-slate-800 bg-slate-900"><div className="mx-auto flex max-w-6xl flex-wrap items-center justify-between gap-4 px-4 py-5 sm:px-6"><div><p className="text-sm font-semibold uppercase tracking-widest text-emerald-400">Backup Tool</p><h1 className="mt-1 text-2xl font-bold">Operator console</h1></div><p className="text-sm text-slate-300"><span className="sr-only">Signed in as </span>{user.username}</p></div><nav aria-label="Primary" className="mx-auto flex max-w-6xl gap-1 overflow-x-auto px-4 pb-3 sm:px-6">{pages.map((item) => <button aria-current={page === item.id ? "page" : undefined} className="whitespace-nowrap rounded-md px-3 py-2 text-sm font-semibold hover:bg-slate-800" key={item.id} onClick={() => setPage(item.id)} type="button">{item.label}</button>)}</nav></header>{page === "dashboard" ? <DashboardPage {...common} /> : null}{page === "sources" ? <SourcesPage {...common} /> : null}{page === "repositories" ? <RepositoriesPage {...common} /> : null}{page === "jobs" ? <JobsPage {...common} /> : null}{page === "executions" ? <ExecutionsPage {...common} /> : null}{page === "backups" ? <BackupsPage {...common} /> : null}{page === "security" ? <SecurityPage {...common} /> : null}{page === "notifications" ? <NotificationsPage {...common} /> : null}{page === "audit" ? <AuditPage {...common} /> : null}</main>;
+}

frontend/src/index.css

@@ -1,3 +1,23 @@
 @tailwind base;
 @tailwind components;
 @tailwind utilities;
+
+:root {
+	color-scheme: dark;
+}
+
+:focus-visible {
+	outline: 3px solid #34d399;
+	outline-offset: 3px;
+}
+
+@media (prefers-reduced-motion: reduce) {
+	*,
+	*::before,
+	*::after {
+		animation-duration: 0.01ms !important;
+		animation-iteration-count: 1 !important;
+		scroll-behavior: auto !important;
+		transition-duration: 0.01ms !important;
+	}
+}

frontend/src/test/setup.ts

@@ -0,0 +1 @@
+import "@testing-library/jest-dom/vitest";

frontend/vite.config.ts

@@ -1,12 +1,12 @@
-import { defineConfig } from 'vite'
+import react from "@vitejs/plugin-react";
-import react from '@vitejs/plugin-react'
+import { defineConfig } from "vite";
 
 export default defineConfig({
   plugins: [react()],
   server: {
     port: 3000,
     proxy: {
-      '/api': 'http://localhost:8000'
+      "/api": "http://localhost:8000",
-    }
+    },
-  }
+  },
-})
+});

frontend/vitest.config.ts

@@ -0,0 +1,16 @@
+import { mergeConfig } from "vite";
+import { defineConfig } from "vitest/config";
+
+import viteConfig from "./vite.config";
+
+export default mergeConfig(
+	viteConfig,
+	defineConfig({
+		test: {
+			environment: "jsdom",
+			include: ["src/**/*.test.{ts,tsx}"],
+			exclude: ["e2e/**", "node_modules/**"],
+			setupFiles: ["./src/test/setup.ts"],
+		},
+	}),
+);

pyrightconfig.json

@@ -0,0 +1,3 @@
+{
+  "extraPaths": ["backend/src"]
+}

tests/compose.ssh.yaml

@@ -0,0 +1,32 @@
+name: backup-tool-ssh-test
+
+services:
+  sshd:
+    build:
+      context: ./ssh-fixture
+      dockerfile: Dockerfile
+    read_only: true
+    tmpfs:
+      - /run/sshd:uid=0,gid=0,mode=0755,size=8m
+      - /tmp:mode=1777,size=8m
+    cap_drop:
+      - ALL
+    # sshd needs only these capabilities to chroot then drop to the SFTP account.
+    cap_add:
+      - SYS_CHROOT
+      - SETUID
+      - SETGID
+      - KILL
+    security_opt:
+      - no-new-privileges:true
+    ports:
+      - "127.0.0.1:${SSH_FIXTURE_PORT}:2222"
+    volumes:
+      - type: bind
+        source: ${SSH_FIXTURE_DIR}
+        target: /fixture
+        read_only: true
+      - type: bind
+        source: ${SSH_FIXTURE_DIR}/source
+        target: /home/backup/data
+        read_only: true

tests/contract/test_api_conventions.py

@@ -35,16 +35,17 @@ async def test_readyz_rejects_unmigrated_database(tmp_path) -> None:
         response = await client.get("/readyz")
     await app.state.engine.dispose()
     assert response.status_code == 503
-    assert response.json()["code"] == "schema_not_current"
+    assert response.json()["code"] == "dependency_unavailable"
 
 
 @pytest.mark.asyncio
-async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None:
+async def test_readyz_reports_runtime_dependencies_before_and_after_setup(
+    app_client,
+) -> None:
     client, _ = app_client
     before = await client.get("/readyz")
-    assert before.status_code == 503
+    assert before.status_code == 200
-    assert before.headers["content-type"].startswith("application/problem+json")
+    assert before.json()["status"] == "ready"
-    assert before.json()["code"] == "setup_required"
 
     assert (await setup_admin(client)).status_code == 201
     after = await client.get("/readyz")
@@ -52,6 +53,22 @@ async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None:
     assert after.json()["status"] == "ready"
 
 
+@pytest.mark.asyncio
+async def test_metrics_are_prometheus_text_without_sensitive_request_data(
+    app_client,
+) -> None:
+    client, _ = app_client
+    assert (await client.get("/livez")).status_code == 200
+
+    response = await client.get("/metrics")
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/plain; version=0.0.4")
+    assert "backup_tool_http_requests_total" in response.text
+    assert "backup_tool_active_executions" in response.text
+    assert "backup_tool_filesystem_free_bytes" in response.text
+
+
 @pytest.mark.asyncio
 async def test_protected_endpoint_uses_rfc9457_problem(app_client) -> None:
     client, _ = app_client

tests/contract/test_notification_contract.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.db.models import IdempotencyRecord
+from sqlalchemy import select
+
+PASSWORD = "correct horse battery staple"
+
+
+@pytest.mark.asyncio
+async def test_catalog_subscription_and_write_only_webhook_secret(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    catalog = await client.get("/api/v2/notifications/event-catalog")
+    assert catalog.status_code == 200
+    assert catalog.json()["event_schema_version"] == 1
+    assert "execution.queued" in catalog.json()["events"]
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "webhook",
+            "event_filters": ["execution.*"],
+            "destination": {"url": "https://hooks.example.test/backup"},
+            "signing_secret": "not-returned-webhook-secret",
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    assert "signing_secret" not in created.text
+    assert "not-returned-webhook-secret" not in created.text
+    assert created.headers["ETag"]
+
+
+@pytest.mark.asyncio
+async def test_rotation_idempotency_never_persists_secret_verifier(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "webhook",
+            "event_filters": ["execution.*"],
+            "destination": {"url": "https://hooks.example.test/backup"},
+            "signing_secret": "first-signing-secret",
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    route = f"/api/v2/notifications/subscriptions/{created.json()['id']}/signing-keys/rotate"
+    first = await client.post(
+        route,
+        json={"secret": "rotation-secret-one", "overlap_seconds": 60},
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"},
+    )
+    replay = await client.post(
+        route,
+        json={"secret": "rotation-secret-two", "overlap_seconds": 60},
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"},
+    )
+    assert first.status_code == replay.status_code == 200
+    assert first.json() == replay.json()
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        record = await db.scalar(
+            select(IdempotencyRecord).where(
+                IdempotencyRecord.operation == "rotate_notification_signing_key"
+            )
+        )
+    assert record is not None
+    assert "rotation-secret" not in record.request_digest
+
+
+@pytest.mark.asyncio
+async def test_notification_rejects_empty_or_unknown_filters(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    response = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["unknown.event"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers={"X-CSRF-Token": client.cookies["backup_tool_csrf"]},
+    )
+    assert response.status_code == 422
+    assert response.json()["code"] == "validation_failed"

tests/contract/test_repository_format.py

@@ -144,6 +144,14 @@ def test_state_errors_capabilities_and_fault_points_are_frozen() -> None:
 
     capabilities = load(CONTRACT / "capabilities-v2.0.json")
     assert capabilities["sources"] == ["local", "ssh"]
+
+    manifest_schema = load(CONTRACT / "manifest.schema.json")
+    adapter_kinds = manifest_schema["properties"]["source_consistency"]["properties"]["adapter"][
+        "enum"
+    ]
+    assert adapter_kinds == ["local", "postgresql", "mysql"]
+    assert "ssh" not in adapter_kinds
+
     assert not capabilities["features"]["tar_download"]
     assert not capabilities["features"]["postgresql"]
     assert not capabilities["features"]["mysql"]

tests/e2e/test_compose_v2.py

@@ -0,0 +1,183 @@
+"""Opt-in production-like Compose checks; no fixture secret is written to the repository."""
+
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def _enabled() -> bool:
+    return os.environ.get("BACKUP_TOOL_COMPOSE_E2E") == "1"
+
+
+def _free_port() -> int:
+    try:
+        with socket.socket() as listener:
+            listener.bind(("127.0.0.1", 0))
+            address = listener.getsockname()
+    except OSError as error:
+        raise RuntimeError("could not allocate a Compose test port") from error
+    if not isinstance(address, tuple) or not isinstance(address[1], int):
+        raise RuntimeError("could not allocate a Compose test port")
+    return address[1]
+
+
+def _compose(environment: dict[str, str], *arguments: str) -> subprocess.CompletedProcess[str]:
+    command = [
+        "docker",
+        "compose",
+        "-f",
+        "docker-compose.yml",
+        "-f",
+        environment["COMPOSE_FILE"],
+        *arguments,
+    ]
+    return subprocess.run(
+        command,
+        cwd=ROOT,
+        env=environment,
+        check=True,
+        text=True,
+        capture_output=True,
+        timeout=120,
+    )
+
+
+def _eventually_get(url: str, expected_status: int) -> str:
+    deadline = time.monotonic() + 90
+    while time.monotonic() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=3) as response:
+                if response.status == expected_status:
+                    return response.read().decode()
+        except urllib.error.HTTPError as error:
+            if error.code == expected_status:
+                return error.read().decode()
+        except OSError:
+            pass
+        time.sleep(1)
+    raise AssertionError(f"{url} did not return {expected_status}")
+
+
+@pytest.mark.skipif(
+    not _enabled(), reason="set BACKUP_TOOL_COMPOSE_E2E=1 to run Docker Compose E2E"
+)
+def test_compose_persists_metadata_and_stops_workers_safely(tmp_path: Path) -> None:
+    source = tmp_path / "source"
+    source.mkdir(mode=0o755)
+    fixture_key = tmp_path / "unused-bind-master.key"
+    fixture_key.write_bytes(b"compose-test-host-fixture-key-material-32-bytes")
+    fixture_key.chmod(0o600)
+    port = _free_port()
+    project = f"backup-tool-e2e-{os.getpid()}"
+    override = tmp_path / "compose-e2e.yaml"
+    override.write_text(
+        "services:\n"
+        "  migrate:\n"
+        "    environment: &e2e-env\n"
+        "      BACKUP_TOOL_MASTER_KEY_FILE: /var/lib/backup-tool/master.key\n"
+        f"      BACKUP_TOOL_PUBLIC_BASE_URL: http://localhost:{port}\n"
+        "  web:\n"
+        "    environment: *e2e-env\n"
+        "  scheduler:\n"
+        "    environment: *e2e-env\n"
+        "  worker:\n"
+        "    environment: *e2e-env\n"
+        "  admin:\n"
+        "    environment: *e2e-env\n"
+    )
+    environment = os.environ | {
+        "BACKUP_TOOL_COMPOSE_E2E": "1",
+        "BACKUP_TOOL_MASTER_KEY_FILE": str(fixture_key),
+        "BACKUP_TOOL_PORT": str(port),
+        "BACKUP_TOOL_SOURCES_DIR": str(source),
+        "COMPOSE_FILE": str(override),
+        "COMPOSE_PROJECT_NAME": project,
+    }
+    try:
+        # The service user creates the actual key inside its private named volume;
+        # the host key exists only to satisfy the unused read-only Compose bind.
+        _compose(
+            environment,
+            "run",
+            "--rm",
+            "--no-deps",
+            "--entrypoint",
+            "/bin/sh",
+            "migrate",
+            "-c",
+            (
+                "umask 077; dd if=/dev/urandom of=/var/lib/backup-tool/master.key "
+                "bs=32 count=1 status=none"
+            ),
+        )
+        _compose(environment, "run", "--rm", "migrate")
+        _compose(environment, "up", "-d")
+        assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+        setup = urllib.request.Request(
+            f"http://127.0.0.1:{port}/api/v2/setup",
+            data=b'{"username":"operator","password":"correct horse battery staple"}',
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(setup, timeout=10) as response:
+            assert response.status == 201
+        assert "backup_tool_active_executions" in _eventually_get(
+            f"http://127.0.0.1:{port}/metrics", 200
+        )
+
+        _compose(environment, "stop", "--timeout", "15", "worker")
+        worker_id = _compose(environment, "ps", "-aq", "worker").stdout.strip()
+        assert worker_id
+        stopped = subprocess.run(
+            ["docker", "inspect", "--format", "{{.State.ExitCode}}", worker_id],
+            check=True,
+            text=True,
+            capture_output=True,
+            timeout=30,
+        )
+        assert stopped.stdout.strip() == "0"
+        _compose(environment, "up", "-d", "worker")
+        _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+
+        _compose(environment, "restart", "web", "scheduler", "worker", "proxy")
+        assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+        repeat_setup = urllib.request.Request(
+            f"http://127.0.0.1:{port}/api/v2/setup",
+            data=b'{"username":"operator","password":"correct horse battery staple"}',
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with pytest.raises(urllib.error.HTTPError) as repeated:
+            urllib.request.urlopen(repeat_setup, timeout=10)
+        assert repeated.value.code == 409
+    finally:
+        command = [
+            "docker",
+            "compose",
+            "-f",
+            "docker-compose.yml",
+            "-f",
+            str(override),
+            "down",
+            "--volumes",
+            "--remove-orphans",
+        ]
+        subprocess.run(
+            command,
+            cwd=ROOT,
+            env=environment,
+            check=False,
+            text=True,
+            capture_output=True,
+            timeout=120,
+        )

tests/fault/test_deletion_failure.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from backup_tool.gc import purge_repository
+
+
+def test_gc_keeps_manifest_when_unlink_fails(tmp_path: Path) -> None:
+    root = tmp_path / "repository"
+    manifest = root / "manifests" / "deleted.json"
+    manifest.parent.mkdir(parents=True)
+    manifest.write_text('{"entries": []}', encoding="utf-8")
+
+    with (
+        patch("pathlib.Path.unlink", side_effect=OSError("read-only")),
+        pytest.raises(OSError),
+    ):
+        purge_repository(root, {"deleted"}, grace=timedelta(0))
+
+    assert manifest.exists()

tests/fault/test_notification_retries.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.db.models import (
+    Execution,
+    Job,
+    NotificationDelivery,
+    NotificationDeliveryAttempt,
+    NotificationEmailSettings,
+    NotificationSubscription,
+    Repository,
+    Secret,
+    Source,
+)
+from backup_tool.ids import new_uuid7
+from backup_tool.notifications.dispatcher import (
+    dispatch_one,
+    recover_notification_leases,
+)
+from backup_tool.notifications.email import EmailResult, EmailTransportError
+from backup_tool.notifications.events import emit_event
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+
+@pytest.mark.asyncio
+async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None:
+    client, settings = app_client
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        ciphertext, key_id = app.state.cipher.encrypt(
+            "smtp-password", purpose="notification_smtp", version=1
+        )
+        secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp")
+        db.add(secret)
+        await db.flush()
+        db.add(
+            NotificationEmailSettings(
+                id=1,
+                host="smtp.example.test",
+                port=587,
+                username="operator",
+                password_secret_id=secret.id,
+                sender="sender@example.test",
+                max_attempts=2,
+                rate_limit_per_minute=60,
+            )
+        )
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["execution.queued"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add(subscription)
+        await db.flush()
+        event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="retry-test",
+        )
+        await db.commit()
+
+        async def transient(*_args, **_kwargs):
+            raise EmailTransportError("smtp_421", transient=True)
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == event.id)
+        )
+        assert delivery is not None
+        assert delivery.state == "retry"
+        assert delivery.attempt_count == 1
+        delivery.state = "leased"
+        delivery.attempt_count = 2
+        delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+        db.add(
+            NotificationDeliveryAttempt(
+                delivery_id=delivery.id,
+                number=2,
+                started_at=datetime.now(UTC),
+                outcome="started",
+            )
+        )
+        await db.commit()
+        assert await recover_notification_leases(db) == 1
+        await db.refresh(delivery)
+        abandoned = await db.scalar(
+            select(NotificationDeliveryAttempt).where(
+                NotificationDeliveryAttempt.delivery_id == delivery.id,
+                NotificationDeliveryAttempt.number == 2,
+            )
+        )
+        assert delivery.state == "retry"
+        assert abandoned is not None
+        assert abandoned.outcome == "retry"
+        assert abandoned.diagnostic == "abandoned_lease"
+
+        async def succeeded(*_args, **_kwargs):
+            return EmailResult(response_class="smtp_2xx")
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded)
+        delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        await db.refresh(delivery)
+        assert delivery.state == "delivered"
+        assert delivery.attempt_count == 3
+
+        max_event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="smtp-max-attempts",
+        )
+        await db.commit()
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        max_delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id)
+        )
+        assert max_delivery is not None and max_delivery.state == "retry"
+        max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        await db.refresh(max_delivery)
+        assert max_delivery.state == "failed"
+        assert max_delivery.attempt_count == 2
+
+        permanent_event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="smtp-permanent",
+        )
+        await db.commit()
+
+        async def permanent(*_args, **_kwargs):
+            raise EmailTransportError("smtp_550", transient=False)
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        permanent_delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id)
+        )
+        assert permanent_delivery is not None
+        assert permanent_delivery.state == "failed"
+        assert permanent_delivery.attempt_count == 1
+
+
+@pytest.mark.asyncio
+async def test_notification_dispatch_gets_a_turn_during_execution_backlog(
+    app_client, monkeypatch
+) -> None:
+    client, settings = app_client
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="fair-repository",
+            root="/fair-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="fair-source",
+            kind="local",
+            public_config={"root": "/fair-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        job = Job(
+            name="fair-job",
+            source_id=source.id,
+            repository_id=repository.id,
+            exclusions=[],
+            retention={},
+            requested_mode="full",
+        )
+        db.add(job)
+        await db.flush()
+        execution = Execution(job_id=job.id, trigger="manual", progress={})
+        db.add(execution)
+        await db.commit()
+        execution_id = execution.id
+
+    called = False
+
+    async def dispatched(*_args, **_kwargs) -> bool:
+        nonlocal called
+        called = True
+        return True
+
+    monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched)
+    worker = Worker(settings, owner="fair-worker")
+    worker._execution_turns = 1
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+    async with app.state.sessions() as db:
+        queued = await db.get(Execution, execution_id)
+    assert called
+    assert queued is not None and queued.state == "queued"

tests/fault/test_publication_crashes.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import importlib
+import os
+import stat
+from pathlib import Path
+
+import httpx
+import pytest
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+config = importlib.import_module("backup_tool.config")
+adapters = importlib.import_module("backup_tool.adapters")
+faults = importlib.import_module("backup_tool.faults")
+models = importlib.import_module("backup_tool.db.models")
+snapshot = importlib.import_module("backup_tool.snapshot")
+worker_module = importlib.import_module("backup_tool.worker")
+
+PASSWORD = "correct-horse-battery-staple"
+
+
+def settings_for(tmp_path: Path):
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m6-publication-fault-test-master-key-material")
+    key.chmod(0o600)
+    data = tmp_path / "data"
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (data, repositories, sources, restores):
+        directory.mkdir()
+    return config.Settings(
+        data_dir=data,
+        database_url=f"sqlite+aiosqlite:///{data / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+        min_free_bytes=1,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("point", ["blob.before_write", "blob.after_write", "blob.after_fsync"])
+async def test_blob_write_crash_points_leave_no_published_blob(tmp_path: Path, point: str) -> None:
+    settings = settings_for(tmp_path)
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("backup data", encoding="utf-8")
+    adapter = adapters.LocalAdapter(source_root, settings)
+    staged_blob = tmp_path / "staged.blob"
+
+    with pytest.raises(faults.InjectedCrash):
+        await snapshot._copy_file(adapter, "data.txt", staged_blob, faults.CrashAt(point))
+
+    assert not (settings.repository_roots[0] / "blobs" / "sha256").exists()
+
+
+@pytest.mark.asyncio
+async def test_staging_is_owner_only_with_a_permissive_umask(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("plaintext", encoding="utf-8")
+    adapter = adapters.LocalAdapter(source_root, settings)
+    staging = tmp_path / "staging"
+    staged_blobs = staging / "blobs"
+    staged_blob = staged_blobs / "0.blob"
+
+    old_umask = os.umask(0)
+    try:
+        snapshot._private_directory(staging)
+        snapshot._private_directory(staged_blobs)
+        with pytest.raises(faults.InjectedCrash):
+            await snapshot._copy_file(
+                adapter,
+                "data.txt",
+                staged_blob,
+                faults.CrashAt("blob.after_write"),
+            )
+    finally:
+        os.umask(old_umask)
+
+    assert stat.S_IMODE(staging.stat().st_mode) == 0o700
+    assert stat.S_IMODE(staged_blobs.stat().st_mode) == 0o700
+    assert stat.S_IMODE(staged_blob.stat().st_mode) == 0o600
+    assert staged_blob.read_text(encoding="utf-8") == "plaintext"
+
+
+def test_blob_install_crash_point_leaves_staged_blob_unpublished(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    staged_blob = tmp_path / "staged.blob"
+    staged_blob.write_bytes(b"backup data")
+    digest = "a" * 64
+    target = settings.repository_roots[0] / "blobs" / "sha256" / digest
+
+    with pytest.raises(faults.InjectedCrash):
+        snapshot._install_blob(staged_blob, target, digest, faults.CrashAt("blob.before_rename"))
+
+    assert staged_blob.exists()
+    assert not target.exists()
+
+
+async def _login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+@pytest.mark.asyncio
+async def test_metadata_crash_is_reconciled_without_republishing(
+    app_client: tuple[httpx.AsyncClient, Settings],
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("backup data", encoding="utf-8")
+    headers = await _login(client)
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={
+            "name": "repo",
+            "relative_path": "repo",
+            "compression": "none",
+            "encryption": "none",
+        },
+        headers=headers,
+    )
+    source = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "source",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "job",
+            "source_id": source.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
+    execution_id = execution.json()["id"]
+
+    crashing_worker = worker_module.Worker(
+        settings,
+        owner="crashing-worker",
+        fault_injector=faults.CrashAt("metadata.before_commit"),
+    )
+    try:
+        with pytest.raises(faults.InjectedCrash):
+            await crashing_worker.run_once()
+    finally:
+        await crashing_worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            assert (
+                await db.scalar(
+                    select(models.Backup).where(models.Backup.execution_id == execution_id)
+                )
+                is None
+            )
+    finally:
+        await engine.dispose()
+
+    recovery_worker = worker_module.Worker(settings, owner="recovery-worker")
+    try:
+        assert await recovery_worker.startup() == 1
+    finally:
+        await recovery_worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            recovered = await db.get(models.Execution, execution_id)
+            backups = list(
+                await db.scalars(
+                    select(models.Backup).where(models.Backup.execution_id == execution_id)
+                )
+            )
+    finally:
+        await engine.dispose()
+    assert recovered is not None and recovered.state == "committed"
+    assert len(backups) == 1

tests/fault/test_schedule_delivery.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.scheduler import next_nominal
+
+
+def test_misfire_window_is_deterministic_for_delivery() -> None:
+    nominal = datetime.now(UTC) - timedelta(seconds=901)
+    next_run = next_nominal("* * * * *", "UTC", nominal)
+
+    assert next_run > nominal
+    assert (datetime.now(UTC) - nominal).total_seconds() > 900
+
+
+@pytest.mark.parametrize("cron", ["* * * * * *", "invalid"])
+def test_delivery_rejects_invalid_cron_before_enqueue(cron: str) -> None:
+    with pytest.raises(ValueError):
+        next_nominal(cron, "UTC")

tests/fault/test_worker_loss.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+from alembic import command
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Execution, ExecutionEvent, Job, Repository, Source
+from backup_tool.execution import (
+    claim,
+    complete_cancellation,
+    heartbeat,
+    record_event,
+    request_cancellation,
+)
+from backup_tool.worker import Worker
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+cli = importlib.import_module("backup_tool.cli")
+
+
+@pytest_asyncio.fixture
+async def database(
+    tmp_path: Path,
+) -> AsyncIterator[tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine]]:
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m5-fault-test-master-key-material-32-bytes-minimum")
+    key.chmod(0o600)
+    data_dir = tmp_path / "data"
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (data_dir, repositories, sources, restores):
+        directory.mkdir()
+    settings = Settings(
+        data_dir=data_dir,
+        database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+    )
+    command.upgrade(cli.build_alembic_config(settings), "head")
+    engine = create_engine(settings)
+    yield settings, async_sessionmaker(engine, expire_on_commit=False), engine
+    await engine.dispose()
+
+
+async def create_stale_execution(db: AsyncSession, suffix: str, state: str) -> Execution:
+    repository = Repository(
+        name=f"repository-{suffix}",
+        root=f"/repositories/{suffix}",
+        format_version=1,
+        compression="none",
+        encryption="none",
+    )
+    source = Source(
+        name=f"source-{suffix}",
+        kind="local",
+        public_config={"root": f"/sources/{suffix}"},
+        secret_refs=[],
+    )
+    db.add_all([repository, source])
+    await db.flush()
+    job = Job(
+        name=f"job-{suffix}",
+        source_id=source.id,
+        repository_id=repository.id,
+        requested_mode="full",
+        exclusions=[],
+        retention={},
+    )
+    db.add(job)
+    await db.flush()
+    execution = Execution(
+        job_id=job.id,
+        trigger="manual",
+        state=state,
+        lease_owner="lost-worker",
+        lease_expires_at=datetime.now(UTC) - timedelta(seconds=1),
+        heartbeat_at=datetime.now(UTC) - timedelta(seconds=2),
+        progress={},
+        reason_code="cancellation_requested" if state == "cancelling" else None,
+    )
+    db.add(execution)
+    await db.flush()
+    await record_event(db, execution)
+    await db.commit()
+    return execution
+
+
+@pytest.mark.asyncio
+async def test_startup_recovers_each_stale_lease_once_and_fences_lost_owner(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    settings, sessions, _ = database
+    async with sessions() as db:
+        stale = {
+            state: await create_stale_execution(db, state, state)
+            for state in ("preparing", "running", "verifying", "cancelling")
+        }
+
+    worker = Worker(settings, owner="recovery-worker")
+    try:
+        assert await worker.startup() == len(stale)
+    finally:
+        await worker.engine.dispose()
+
+    async with sessions() as db:
+        recovered = {
+            state: await db.get(Execution, execution.id) for state, execution in stale.items()
+        }
+        for state in ("preparing", "running", "verifying"):
+            execution = recovered[state]
+            assert execution is not None
+            assert execution.state == "queued"
+            assert execution.reason_code == "worker_lost"
+            assert execution.lease_owner is None
+            assert execution.lease_expires_at is None
+            assert execution.heartbeat_at is None
+        cancelling = recovered["cancelling"]
+        assert cancelling is not None
+        assert cancelling.state == "cancelled"
+        assert cancelling.completed_at is not None
+        assert cancelling.reason_code == "cancellation_requested"
+        event_counts = {
+            execution.id: await db.scalar(
+                select(func.count()).where(ExecutionEvent.execution_id == execution.id)
+            )
+            for execution in recovered.values()
+            if execution is not None
+        }
+
+    assert event_counts == {execution.id: 2 for execution in stale.values()}
+
+    async with sessions() as db:
+        execution = recovered["running"]
+        assert execution is not None
+        assert await claim(db, execution.id, "replacement-worker") is not None
+        assert not await heartbeat(db, execution.id, "lost-worker")
+        assert await request_cancellation(db, execution.id) is not None
+        assert not await complete_cancellation(db, execution.id, "lost-worker")
+        assert await complete_cancellation(db, execution.id, "replacement-worker")
+
+
+@pytest.mark.asyncio
+async def test_stopping_worker_does_not_claim_new_work(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    settings, _, _ = database
+    worker = Worker(settings, owner="stopping-worker")
+    try:
+        worker.stop()
+        assert not await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_idle_worker_stops_promptly_and_disposes_its_engine(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    settings, _, _ = database
+    worker = Worker(settings, owner="idle-worker")
+    disposed = asyncio.Event()
+    dispose = AsyncEngine.dispose
+
+    async def track_dispose(engine: AsyncEngine, *, close: bool = True) -> None:
+        disposed.set()
+        await dispose(engine, close=close)
+
+    monkeypatch.setattr(AsyncEngine, "dispose", track_dispose)
+    task = asyncio.create_task(worker.run())
+    await asyncio.sleep(0)
+    worker.stop()
+    await asyncio.wait_for(task, timeout=1)
+    assert disposed.is_set()

tests/integration/test_all_operational_events_deliver.py

@@ -0,0 +1,389 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.db.models import (
+    Backup,
+    Execution,
+    Job,
+    NotificationDelivery,
+    NotificationEvent,
+    NotificationSubscription,
+    Repository,
+    Schedule,
+    Source,
+)
+from backup_tool.execution import record_event
+from backup_tool.notifications.events import EVENT_CATALOG
+from backup_tool.scheduler import deliver_due
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+
+@pytest.mark.asyncio
+async def test_execution_catalog_events_are_produced_and_delivered(app_client) -> None:
+    """Exercise the execution producer, not emit_event(), for each live execution type."""
+    client, _ = app_client
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    app = client._transport.app
+    expected = {
+        "execution.queued",
+        "execution.started",
+        "execution.committed",
+        "execution.failed",
+        "execution.cancelled",
+        "execution.retry_queued",
+        "execution.worker_recovered",
+    }
+    assert expected <= set(EVENT_CATALOG)
+    deferred_prefixes = ["source.", "gc.", "reconciliation."]
+    assert not any(item.startswith(tuple(deferred_prefixes)) for item in EVENT_CATALOG)
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="events-repository",
+            root="/events-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="events-source",
+            kind="local",
+            public_config={"root": "/events-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        job = Job(
+            name="events-job",
+            source_id=source.id,
+            repository_id=repository.id,
+            exclusions=[],
+            retention={},
+            requested_mode="full",
+        )
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["execution.*"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add_all([job, subscription])
+        await db.flush()
+        cases = (
+            ("queued", 1, None, "execution.queued"),
+            ("preparing", 1, None, "execution.started"),
+            ("committed", 1, None, "execution.committed"),
+            ("failed", 1, "transient_io", "execution.failed"),
+            ("cancelled", 1, "cancellation_requested", "execution.cancelled"),
+            ("queued", 2, None, "execution.retry_queued"),
+            ("queued", 1, "worker_lost", "execution.worker_recovered"),
+        )
+        for state, attempt, reason, _event_type in cases:
+            execution = Execution(
+                job_id=job.id,
+                trigger="manual",
+                state=state,
+                attempt=attempt,
+                reason_code=reason,
+                progress={},
+            )
+            db.add(execution)
+            await db.flush()
+            await record_event(db, execution)
+            if state in {"queued", "preparing"}:
+                execution.state = "failed"
+                execution.reason_code = "test_cleanup"
+            await db.flush()
+        await db.commit()
+        event_statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+        event_types = set((await db.scalars(event_statement)).all())
+        deliveries = await db.scalar(
+            select(NotificationDelivery.id)
+            .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id)
+            .where(NotificationEvent.type.in_(expected))
+            .limit(1)
+        )
+    assert event_types == expected
+    assert deliveries is not None
+
+
+@pytest.mark.asyncio
+async def test_schedule_catalog_events_are_produced_and_delivered(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    app = client._transport.app
+    expected = {
+        "schedule.created",
+        "schedule.updated",
+        "schedule.deleted",
+        "schedule.enabled",
+        "schedule.disabled",
+        "schedule.occurrence_enqueued",
+        "schedule.occurrence_misfired",
+        "schedule.occurrence_blocked",
+    }
+    assert expected <= set(EVENT_CATALOG)
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="schedule-repository",
+            root="/schedule-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="schedule-source",
+            kind="local",
+            public_config={"root": "/schedule-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        jobs = [
+            Job(
+                name=f"schedule-job-{number}",
+                source_id=source.id,
+                repository_id=repository.id,
+                exclusions=[],
+                retention={},
+                requested_mode="full",
+                enabled=number not in {2, 3},
+            )
+            for number in range(1, 5)
+        ]
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["schedule.*"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add_all([*jobs, subscription])
+        await db.commit()
+        job_ids = [job.id for job in jobs]
+
+    created = await client.post(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    disabled = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": False},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert disabled.status_code == 200
+    enabled = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "1 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert enabled.status_code == 200
+    updated = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "2 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert updated.status_code == 200
+    assert (
+        await client.delete(f"/api/v2/jobs/{job_ids[0]}/schedule", headers={"X-CSRF-Token": csrf})
+    ).status_code == 204
+
+    async with app.state.sessions() as db:
+        now = datetime.now(UTC)
+        db.add_all(
+            [
+                Schedule(
+                    job_id=job_ids[1],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=0,
+                    enabled=True,
+                    next_nominal_at=now - timedelta(hours=1),
+                ),
+                Schedule(
+                    job_id=job_ids[2],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=60,
+                    enabled=True,
+                    next_nominal_at=now,
+                ),
+                Schedule(
+                    job_id=job_ids[3],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=60,
+                    enabled=True,
+                    next_nominal_at=now,
+                ),
+            ]
+        )
+        await db.commit()
+        assert await deliver_due(db, now=now) == 1
+        statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+        event_types = set((await db.scalars(statement)).all())
+        delivery = await db.scalar(
+            select(NotificationDelivery.id)
+            .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id)
+            .where(NotificationEvent.type.in_(expected))
+            .limit(1)
+        )
+    assert event_types == expected
+    assert delivery is not None
+
+
+@pytest.mark.asyncio
+async def test_backup_restore_and_retention_events_are_produced_and_delivered(
+    app_client,
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "notification-project"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("notification data\n", encoding="utf-8")
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    headers = {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+    subscription = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["backup.*", "restore.*", "retention.*"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers=headers,
+    )
+    assert subscription.status_code == 201
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={"name": "notification-repo", "relative_path": "notification-repo"},
+        headers=headers,
+    )
+    source = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "notification-source",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    assert repository.status_code == source.status_code == 201
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "notification-job",
+            "source_id": source.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {"keep_last": 1},
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    assert job.status_code == 201
+    execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
+    assert execution.status_code == 202
+    worker = Worker(settings, owner="notification-backup-worker")
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        backup = await db.scalar(
+            select(Backup).where(Backup.execution_id == execution.json()["id"])
+        )
+        assert backup is not None
+        # A synthetic older catalog entry is valid business state; tombstoning is
+        # performed only through the real retention producer below.
+        older_execution = Execution(
+            job_id=job.json()["id"],
+            trigger="manual",
+            state="committed",
+            progress={},
+        )
+        db.add(older_execution)
+        await db.flush()
+        older = Backup(
+            execution_id=older_execution.id,
+            manifest_id="00000000-0000-7000-8000-000000000001",
+            manifest_digest="0" * 64,
+            logical_bytes=0,
+            stored_bytes=0,
+            integrity="verified",
+            created_at=datetime.now(UTC) - timedelta(days=1),
+        )
+        db.add(older)
+        await db.commit()
+
+    retention_worker = Worker(settings, owner="notification-retention-worker")
+    try:
+        # Retention/GC is executed by worker maintenance, not a direct helper call.
+        assert await retention_worker.run_once()
+    finally:
+        await retention_worker.engine.dispose()
+
+    restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(settings.restore_roots[0] / "notification-restore"),
+            "selection": [],
+            "dry_run": True,
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert restore.status_code == 202
+    restore_worker = Worker(settings, owner="notification-restore-worker")
+    try:
+        assert await restore_worker.run_once()
+    finally:
+        await restore_worker.engine.dispose()
+
+    expected = {
+        "backup.committed",
+        "backup.verification_succeeded",
+        "restore.queued",
+        "restore.committed",
+        "retention.tombstoned",
+    }
+    async with app.state.sessions() as db:
+        types = set(
+            (
+                await db.scalars(
+                    select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+                )
+            ).all()
+        )
+        deliveries = list(
+            (
+                await db.scalars(
+                    select(NotificationDelivery.id)
+                    .join(
+                        NotificationEvent,
+                        NotificationDelivery.event_id == NotificationEvent.id,
+                    )
+                    .where(NotificationEvent.type.in_(expected))
+                )
+            ).all()
+        )
+    assert types == expected
+    assert len(deliveries) >= len(expected)

tests/integration/test_empty_source.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import importlib
+
+import pytest
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+
+
+def test_empty_source_requires_explicit_opt_in() -> None:
+    with pytest.raises(snapshot.SnapshotError, match="source_empty"):
+        snapshot.require_nonempty([], False)
+    snapshot.require_nonempty([], True)