feat(v2): complete v2 reimplementation

tests/compose.ssh.yaml

@@ -0,0 +1,32 @@
+name: backup-tool-ssh-test
+
+services:
+  sshd:
+    build:
+      context: ./ssh-fixture
+      dockerfile: Dockerfile
+    read_only: true
+    tmpfs:
+      - /run/sshd:uid=0,gid=0,mode=0755,size=8m
+      - /tmp:mode=1777,size=8m
+    cap_drop:
+      - ALL
+    # sshd needs only these capabilities to chroot then drop to the SFTP account.
+    cap_add:
+      - SYS_CHROOT
+      - SETUID
+      - SETGID
+      - KILL
+    security_opt:
+      - no-new-privileges:true
+    ports:
+      - "127.0.0.1:${SSH_FIXTURE_PORT}:2222"
+    volumes:
+      - type: bind
+        source: ${SSH_FIXTURE_DIR}
+        target: /fixture
+        read_only: true
+      - type: bind
+        source: ${SSH_FIXTURE_DIR}/source
+        target: /home/backup/data
+        read_only: true

tests/contract/test_api_conventions.py

@@ -35,16 +35,17 @@ async def test_readyz_rejects_unmigrated_database(tmp_path) -> None:
         response = await client.get("/readyz")
     await app.state.engine.dispose()
     assert response.status_code == 503
-    assert response.json()["code"] == "schema_not_current"
+    assert response.json()["code"] == "dependency_unavailable"
 
 
 @pytest.mark.asyncio
-async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None:
+async def test_readyz_reports_runtime_dependencies_before_and_after_setup(
+    app_client,
+) -> None:
     client, _ = app_client
     before = await client.get("/readyz")
-    assert before.status_code == 503
-    assert before.headers["content-type"].startswith("application/problem+json")
-    assert before.json()["code"] == "setup_required"
+    assert before.status_code == 200
+    assert before.json()["status"] == "ready"
 
     assert (await setup_admin(client)).status_code == 201
     after = await client.get("/readyz")
@@ -52,6 +53,22 @@ async def test_readyz_is_truthful_before_and_after_setup(app_client) -> None:
     assert after.json()["status"] == "ready"
 
 
+@pytest.mark.asyncio
+async def test_metrics_are_prometheus_text_without_sensitive_request_data(
+    app_client,
+) -> None:
+    client, _ = app_client
+    assert (await client.get("/livez")).status_code == 200
+
+    response = await client.get("/metrics")
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/plain; version=0.0.4")
+    assert "backup_tool_http_requests_total" in response.text
+    assert "backup_tool_active_executions" in response.text
+    assert "backup_tool_filesystem_free_bytes" in response.text
+
+
 @pytest.mark.asyncio
 async def test_protected_endpoint_uses_rfc9457_problem(app_client) -> None:
     client, _ = app_client

tests/contract/test_notification_contract.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.db.models import IdempotencyRecord
+from sqlalchemy import select
+
+PASSWORD = "correct horse battery staple"
+
+
+@pytest.mark.asyncio
+async def test_catalog_subscription_and_write_only_webhook_secret(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    catalog = await client.get("/api/v2/notifications/event-catalog")
+    assert catalog.status_code == 200
+    assert catalog.json()["event_schema_version"] == 1
+    assert "execution.queued" in catalog.json()["events"]
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "webhook",
+            "event_filters": ["execution.*"],
+            "destination": {"url": "https://hooks.example.test/backup"},
+            "signing_secret": "not-returned-webhook-secret",
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    assert "signing_secret" not in created.text
+    assert "not-returned-webhook-secret" not in created.text
+    assert created.headers["ETag"]
+
+
+@pytest.mark.asyncio
+async def test_rotation_idempotency_never_persists_secret_verifier(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "webhook",
+            "event_filters": ["execution.*"],
+            "destination": {"url": "https://hooks.example.test/backup"},
+            "signing_secret": "first-signing-secret",
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    route = f"/api/v2/notifications/subscriptions/{created.json()['id']}/signing-keys/rotate"
+    first = await client.post(
+        route,
+        json={"secret": "rotation-secret-one", "overlap_seconds": 60},
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"},
+    )
+    replay = await client.post(
+        route,
+        json={"secret": "rotation-secret-two", "overlap_seconds": 60},
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "rotation-one"},
+    )
+    assert first.status_code == replay.status_code == 200
+    assert first.json() == replay.json()
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        record = await db.scalar(
+            select(IdempotencyRecord).where(
+                IdempotencyRecord.operation == "rotate_notification_signing_key"
+            )
+        )
+    assert record is not None
+    assert "rotation-secret" not in record.request_digest
+
+
+@pytest.mark.asyncio
+async def test_notification_rejects_empty_or_unknown_filters(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert setup.status_code == 201
+    response = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["unknown.event"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers={"X-CSRF-Token": client.cookies["backup_tool_csrf"]},
+    )
+    assert response.status_code == 422
+    assert response.json()["code"] == "validation_failed"

tests/contract/test_repository_format.py

@@ -144,6 +144,14 @@ def test_state_errors_capabilities_and_fault_points_are_frozen() -> None:
 
     capabilities = load(CONTRACT / "capabilities-v2.0.json")
     assert capabilities["sources"] == ["local", "ssh"]
+
+    manifest_schema = load(CONTRACT / "manifest.schema.json")
+    adapter_kinds = manifest_schema["properties"]["source_consistency"]["properties"]["adapter"][
+        "enum"
+    ]
+    assert adapter_kinds == ["local", "postgresql", "mysql"]
+    assert "ssh" not in adapter_kinds
+
     assert not capabilities["features"]["tar_download"]
     assert not capabilities["features"]["postgresql"]
     assert not capabilities["features"]["mysql"]

tests/e2e/test_compose_v2.py

@@ -0,0 +1,183 @@
+"""Opt-in production-like Compose checks; no fixture secret is written to the repository."""
+
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def _enabled() -> bool:
+    return os.environ.get("BACKUP_TOOL_COMPOSE_E2E") == "1"
+
+
+def _free_port() -> int:
+    try:
+        with socket.socket() as listener:
+            listener.bind(("127.0.0.1", 0))
+            address = listener.getsockname()
+    except OSError as error:
+        raise RuntimeError("could not allocate a Compose test port") from error
+    if not isinstance(address, tuple) or not isinstance(address[1], int):
+        raise RuntimeError("could not allocate a Compose test port")
+    return address[1]
+
+
+def _compose(environment: dict[str, str], *arguments: str) -> subprocess.CompletedProcess[str]:
+    command = [
+        "docker",
+        "compose",
+        "-f",
+        "docker-compose.yml",
+        "-f",
+        environment["COMPOSE_FILE"],
+        *arguments,
+    ]
+    return subprocess.run(
+        command,
+        cwd=ROOT,
+        env=environment,
+        check=True,
+        text=True,
+        capture_output=True,
+        timeout=120,
+    )
+
+
+def _eventually_get(url: str, expected_status: int) -> str:
+    deadline = time.monotonic() + 90
+    while time.monotonic() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=3) as response:
+                if response.status == expected_status:
+                    return response.read().decode()
+        except urllib.error.HTTPError as error:
+            if error.code == expected_status:
+                return error.read().decode()
+        except OSError:
+            pass
+        time.sleep(1)
+    raise AssertionError(f"{url} did not return {expected_status}")
+
+
+@pytest.mark.skipif(
+    not _enabled(), reason="set BACKUP_TOOL_COMPOSE_E2E=1 to run Docker Compose E2E"
+)
+def test_compose_persists_metadata_and_stops_workers_safely(tmp_path: Path) -> None:
+    source = tmp_path / "source"
+    source.mkdir(mode=0o755)
+    fixture_key = tmp_path / "unused-bind-master.key"
+    fixture_key.write_bytes(b"compose-test-host-fixture-key-material-32-bytes")
+    fixture_key.chmod(0o600)
+    port = _free_port()
+    project = f"backup-tool-e2e-{os.getpid()}"
+    override = tmp_path / "compose-e2e.yaml"
+    override.write_text(
+        "services:\n"
+        "  migrate:\n"
+        "    environment: &e2e-env\n"
+        "      BACKUP_TOOL_MASTER_KEY_FILE: /var/lib/backup-tool/master.key\n"
+        f"      BACKUP_TOOL_PUBLIC_BASE_URL: http://localhost:{port}\n"
+        "  web:\n"
+        "    environment: *e2e-env\n"
+        "  scheduler:\n"
+        "    environment: *e2e-env\n"
+        "  worker:\n"
+        "    environment: *e2e-env\n"
+        "  admin:\n"
+        "    environment: *e2e-env\n"
+    )
+    environment = os.environ | {
+        "BACKUP_TOOL_COMPOSE_E2E": "1",
+        "BACKUP_TOOL_MASTER_KEY_FILE": str(fixture_key),
+        "BACKUP_TOOL_PORT": str(port),
+        "BACKUP_TOOL_SOURCES_DIR": str(source),
+        "COMPOSE_FILE": str(override),
+        "COMPOSE_PROJECT_NAME": project,
+    }
+    try:
+        # The service user creates the actual key inside its private named volume;
+        # the host key exists only to satisfy the unused read-only Compose bind.
+        _compose(
+            environment,
+            "run",
+            "--rm",
+            "--no-deps",
+            "--entrypoint",
+            "/bin/sh",
+            "migrate",
+            "-c",
+            (
+                "umask 077; dd if=/dev/urandom of=/var/lib/backup-tool/master.key "
+                "bs=32 count=1 status=none"
+            ),
+        )
+        _compose(environment, "run", "--rm", "migrate")
+        _compose(environment, "up", "-d")
+        assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+        setup = urllib.request.Request(
+            f"http://127.0.0.1:{port}/api/v2/setup",
+            data=b'{"username":"operator","password":"correct horse battery staple"}',
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(setup, timeout=10) as response:
+            assert response.status == 201
+        assert "backup_tool_active_executions" in _eventually_get(
+            f"http://127.0.0.1:{port}/metrics", 200
+        )
+
+        _compose(environment, "stop", "--timeout", "15", "worker")
+        worker_id = _compose(environment, "ps", "-aq", "worker").stdout.strip()
+        assert worker_id
+        stopped = subprocess.run(
+            ["docker", "inspect", "--format", "{{.State.ExitCode}}", worker_id],
+            check=True,
+            text=True,
+            capture_output=True,
+            timeout=30,
+        )
+        assert stopped.stdout.strip() == "0"
+        _compose(environment, "up", "-d", "worker")
+        _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+
+        _compose(environment, "restart", "web", "scheduler", "worker", "proxy")
+        assert _eventually_get(f"http://127.0.0.1:{port}/readyz", 200)
+        repeat_setup = urllib.request.Request(
+            f"http://127.0.0.1:{port}/api/v2/setup",
+            data=b'{"username":"operator","password":"correct horse battery staple"}',
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with pytest.raises(urllib.error.HTTPError) as repeated:
+            urllib.request.urlopen(repeat_setup, timeout=10)
+        assert repeated.value.code == 409
+    finally:
+        command = [
+            "docker",
+            "compose",
+            "-f",
+            "docker-compose.yml",
+            "-f",
+            str(override),
+            "down",
+            "--volumes",
+            "--remove-orphans",
+        ]
+        subprocess.run(
+            command,
+            cwd=ROOT,
+            env=environment,
+            check=False,
+            text=True,
+            capture_output=True,
+            timeout=120,
+        )

tests/fault/test_deletion_failure.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from backup_tool.gc import purge_repository
+
+
+def test_gc_keeps_manifest_when_unlink_fails(tmp_path: Path) -> None:
+    root = tmp_path / "repository"
+    manifest = root / "manifests" / "deleted.json"
+    manifest.parent.mkdir(parents=True)
+    manifest.write_text('{"entries": []}', encoding="utf-8")
+
+    with (
+        patch("pathlib.Path.unlink", side_effect=OSError("read-only")),
+        pytest.raises(OSError),
+    ):
+        purge_repository(root, {"deleted"}, grace=timedelta(0))
+
+    assert manifest.exists()

tests/fault/test_notification_retries.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.db.models import (
+    Execution,
+    Job,
+    NotificationDelivery,
+    NotificationDeliveryAttempt,
+    NotificationEmailSettings,
+    NotificationSubscription,
+    Repository,
+    Secret,
+    Source,
+)
+from backup_tool.ids import new_uuid7
+from backup_tool.notifications.dispatcher import (
+    dispatch_one,
+    recover_notification_leases,
+)
+from backup_tool.notifications.email import EmailResult, EmailTransportError
+from backup_tool.notifications.events import emit_event
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+
+@pytest.mark.asyncio
+async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None:
+    client, settings = app_client
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        ciphertext, key_id = app.state.cipher.encrypt(
+            "smtp-password", purpose="notification_smtp", version=1
+        )
+        secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp")
+        db.add(secret)
+        await db.flush()
+        db.add(
+            NotificationEmailSettings(
+                id=1,
+                host="smtp.example.test",
+                port=587,
+                username="operator",
+                password_secret_id=secret.id,
+                sender="sender@example.test",
+                max_attempts=2,
+                rate_limit_per_minute=60,
+            )
+        )
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["execution.queued"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add(subscription)
+        await db.flush()
+        event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="retry-test",
+        )
+        await db.commit()
+
+        async def transient(*_args, **_kwargs):
+            raise EmailTransportError("smtp_421", transient=True)
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == event.id)
+        )
+        assert delivery is not None
+        assert delivery.state == "retry"
+        assert delivery.attempt_count == 1
+        delivery.state = "leased"
+        delivery.attempt_count = 2
+        delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+        db.add(
+            NotificationDeliveryAttempt(
+                delivery_id=delivery.id,
+                number=2,
+                started_at=datetime.now(UTC),
+                outcome="started",
+            )
+        )
+        await db.commit()
+        assert await recover_notification_leases(db) == 1
+        await db.refresh(delivery)
+        abandoned = await db.scalar(
+            select(NotificationDeliveryAttempt).where(
+                NotificationDeliveryAttempt.delivery_id == delivery.id,
+                NotificationDeliveryAttempt.number == 2,
+            )
+        )
+        assert delivery.state == "retry"
+        assert abandoned is not None
+        assert abandoned.outcome == "retry"
+        assert abandoned.diagnostic == "abandoned_lease"
+
+        async def succeeded(*_args, **_kwargs):
+            return EmailResult(response_class="smtp_2xx")
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded)
+        delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        await db.refresh(delivery)
+        assert delivery.state == "delivered"
+        assert delivery.attempt_count == 3
+
+        max_event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="smtp-max-attempts",
+        )
+        await db.commit()
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        max_delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id)
+        )
+        assert max_delivery is not None and max_delivery.state == "retry"
+        max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        await db.refresh(max_delivery)
+        assert max_delivery.state == "failed"
+        assert max_delivery.attempt_count == 2
+
+        permanent_event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="smtp-permanent",
+        )
+        await db.commit()
+
+        async def permanent(*_args, **_kwargs):
+            raise EmailTransportError("smtp_550", transient=False)
+
+        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent)
+        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
+        permanent_delivery = await db.scalar(
+            select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id)
+        )
+        assert permanent_delivery is not None
+        assert permanent_delivery.state == "failed"
+        assert permanent_delivery.attempt_count == 1
+
+
+@pytest.mark.asyncio
+async def test_notification_dispatch_gets_a_turn_during_execution_backlog(
+    app_client, monkeypatch
+) -> None:
+    client, settings = app_client
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="fair-repository",
+            root="/fair-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="fair-source",
+            kind="local",
+            public_config={"root": "/fair-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        job = Job(
+            name="fair-job",
+            source_id=source.id,
+            repository_id=repository.id,
+            exclusions=[],
+            retention={},
+            requested_mode="full",
+        )
+        db.add(job)
+        await db.flush()
+        execution = Execution(job_id=job.id, trigger="manual", progress={})
+        db.add(execution)
+        await db.commit()
+        execution_id = execution.id
+
+    called = False
+
+    async def dispatched(*_args, **_kwargs) -> bool:
+        nonlocal called
+        called = True
+        return True
+
+    monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched)
+    worker = Worker(settings, owner="fair-worker")
+    worker._execution_turns = 1
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+    async with app.state.sessions() as db:
+        queued = await db.get(Execution, execution_id)
+    assert called
+    assert queued is not None and queued.state == "queued"

tests/fault/test_publication_crashes.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import importlib
+import os
+import stat
+from pathlib import Path
+
+import httpx
+import pytest
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+config = importlib.import_module("backup_tool.config")
+adapters = importlib.import_module("backup_tool.adapters")
+faults = importlib.import_module("backup_tool.faults")
+models = importlib.import_module("backup_tool.db.models")
+snapshot = importlib.import_module("backup_tool.snapshot")
+worker_module = importlib.import_module("backup_tool.worker")
+
+PASSWORD = "correct-horse-battery-staple"
+
+
+def settings_for(tmp_path: Path):
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m6-publication-fault-test-master-key-material")
+    key.chmod(0o600)
+    data = tmp_path / "data"
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (data, repositories, sources, restores):
+        directory.mkdir()
+    return config.Settings(
+        data_dir=data,
+        database_url=f"sqlite+aiosqlite:///{data / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+        min_free_bytes=1,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("point", ["blob.before_write", "blob.after_write", "blob.after_fsync"])
+async def test_blob_write_crash_points_leave_no_published_blob(tmp_path: Path, point: str) -> None:
+    settings = settings_for(tmp_path)
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("backup data", encoding="utf-8")
+    adapter = adapters.LocalAdapter(source_root, settings)
+    staged_blob = tmp_path / "staged.blob"
+
+    with pytest.raises(faults.InjectedCrash):
+        await snapshot._copy_file(adapter, "data.txt", staged_blob, faults.CrashAt(point))
+
+    assert not (settings.repository_roots[0] / "blobs" / "sha256").exists()
+
+
+@pytest.mark.asyncio
+async def test_staging_is_owner_only_with_a_permissive_umask(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("plaintext", encoding="utf-8")
+    adapter = adapters.LocalAdapter(source_root, settings)
+    staging = tmp_path / "staging"
+    staged_blobs = staging / "blobs"
+    staged_blob = staged_blobs / "0.blob"
+
+    old_umask = os.umask(0)
+    try:
+        snapshot._private_directory(staging)
+        snapshot._private_directory(staged_blobs)
+        with pytest.raises(faults.InjectedCrash):
+            await snapshot._copy_file(
+                adapter,
+                "data.txt",
+                staged_blob,
+                faults.CrashAt("blob.after_write"),
+            )
+    finally:
+        os.umask(old_umask)
+
+    assert stat.S_IMODE(staging.stat().st_mode) == 0o700
+    assert stat.S_IMODE(staged_blobs.stat().st_mode) == 0o700
+    assert stat.S_IMODE(staged_blob.stat().st_mode) == 0o600
+    assert staged_blob.read_text(encoding="utf-8") == "plaintext"
+
+
+def test_blob_install_crash_point_leaves_staged_blob_unpublished(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    staged_blob = tmp_path / "staged.blob"
+    staged_blob.write_bytes(b"backup data")
+    digest = "a" * 64
+    target = settings.repository_roots[0] / "blobs" / "sha256" / digest
+
+    with pytest.raises(faults.InjectedCrash):
+        snapshot._install_blob(staged_blob, target, digest, faults.CrashAt("blob.before_rename"))
+
+    assert staged_blob.exists()
+    assert not target.exists()
+
+
+async def _login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+@pytest.mark.asyncio
+async def test_metadata_crash_is_reconciled_without_republishing(
+    app_client: tuple[httpx.AsyncClient, Settings],
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("backup data", encoding="utf-8")
+    headers = await _login(client)
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={
+            "name": "repo",
+            "relative_path": "repo",
+            "compression": "none",
+            "encryption": "none",
+        },
+        headers=headers,
+    )
+    source = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "source",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "job",
+            "source_id": source.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
+    execution_id = execution.json()["id"]
+
+    crashing_worker = worker_module.Worker(
+        settings,
+        owner="crashing-worker",
+        fault_injector=faults.CrashAt("metadata.before_commit"),
+    )
+    try:
+        with pytest.raises(faults.InjectedCrash):
+            await crashing_worker.run_once()
+    finally:
+        await crashing_worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            assert (
+                await db.scalar(
+                    select(models.Backup).where(models.Backup.execution_id == execution_id)
+                )
+                is None
+            )
+    finally:
+        await engine.dispose()
+
+    recovery_worker = worker_module.Worker(settings, owner="recovery-worker")
+    try:
+        assert await recovery_worker.startup() == 1
+    finally:
+        await recovery_worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            recovered = await db.get(models.Execution, execution_id)
+            backups = list(
+                await db.scalars(
+                    select(models.Backup).where(models.Backup.execution_id == execution_id)
+                )
+            )
+    finally:
+        await engine.dispose()
+    assert recovered is not None and recovered.state == "committed"
+    assert len(backups) == 1

tests/fault/test_schedule_delivery.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.scheduler import next_nominal
+
+
+def test_misfire_window_is_deterministic_for_delivery() -> None:
+    nominal = datetime.now(UTC) - timedelta(seconds=901)
+    next_run = next_nominal("* * * * *", "UTC", nominal)
+
+    assert next_run > nominal
+    assert (datetime.now(UTC) - nominal).total_seconds() > 900
+
+
+@pytest.mark.parametrize("cron", ["* * * * * *", "invalid"])
+def test_delivery_rejects_invalid_cron_before_enqueue(cron: str) -> None:
+    with pytest.raises(ValueError):
+        next_nominal(cron, "UTC")

tests/fault/test_worker_loss.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+from alembic import command
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Execution, ExecutionEvent, Job, Repository, Source
+from backup_tool.execution import (
+    claim,
+    complete_cancellation,
+    heartbeat,
+    record_event,
+    request_cancellation,
+)
+from backup_tool.worker import Worker
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+cli = importlib.import_module("backup_tool.cli")
+
+
+@pytest_asyncio.fixture
+async def database(
+    tmp_path: Path,
+) -> AsyncIterator[tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine]]:
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m5-fault-test-master-key-material-32-bytes-minimum")
+    key.chmod(0o600)
+    data_dir = tmp_path / "data"
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (data_dir, repositories, sources, restores):
+        directory.mkdir()
+    settings = Settings(
+        data_dir=data_dir,
+        database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+    )
+    command.upgrade(cli.build_alembic_config(settings), "head")
+    engine = create_engine(settings)
+    yield settings, async_sessionmaker(engine, expire_on_commit=False), engine
+    await engine.dispose()
+
+
+async def create_stale_execution(db: AsyncSession, suffix: str, state: str) -> Execution:
+    repository = Repository(
+        name=f"repository-{suffix}",
+        root=f"/repositories/{suffix}",
+        format_version=1,
+        compression="none",
+        encryption="none",
+    )
+    source = Source(
+        name=f"source-{suffix}",
+        kind="local",
+        public_config={"root": f"/sources/{suffix}"},
+        secret_refs=[],
+    )
+    db.add_all([repository, source])
+    await db.flush()
+    job = Job(
+        name=f"job-{suffix}",
+        source_id=source.id,
+        repository_id=repository.id,
+        requested_mode="full",
+        exclusions=[],
+        retention={},
+    )
+    db.add(job)
+    await db.flush()
+    execution = Execution(
+        job_id=job.id,
+        trigger="manual",
+        state=state,
+        lease_owner="lost-worker",
+        lease_expires_at=datetime.now(UTC) - timedelta(seconds=1),
+        heartbeat_at=datetime.now(UTC) - timedelta(seconds=2),
+        progress={},
+        reason_code="cancellation_requested" if state == "cancelling" else None,
+    )
+    db.add(execution)
+    await db.flush()
+    await record_event(db, execution)
+    await db.commit()
+    return execution
+
+
+@pytest.mark.asyncio
+async def test_startup_recovers_each_stale_lease_once_and_fences_lost_owner(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    settings, sessions, _ = database
+    async with sessions() as db:
+        stale = {
+            state: await create_stale_execution(db, state, state)
+            for state in ("preparing", "running", "verifying", "cancelling")
+        }
+
+    worker = Worker(settings, owner="recovery-worker")
+    try:
+        assert await worker.startup() == len(stale)
+    finally:
+        await worker.engine.dispose()
+
+    async with sessions() as db:
+        recovered = {
+            state: await db.get(Execution, execution.id) for state, execution in stale.items()
+        }
+        for state in ("preparing", "running", "verifying"):
+            execution = recovered[state]
+            assert execution is not None
+            assert execution.state == "queued"
+            assert execution.reason_code == "worker_lost"
+            assert execution.lease_owner is None
+            assert execution.lease_expires_at is None
+            assert execution.heartbeat_at is None
+        cancelling = recovered["cancelling"]
+        assert cancelling is not None
+        assert cancelling.state == "cancelled"
+        assert cancelling.completed_at is not None
+        assert cancelling.reason_code == "cancellation_requested"
+        event_counts = {
+            execution.id: await db.scalar(
+                select(func.count()).where(ExecutionEvent.execution_id == execution.id)
+            )
+            for execution in recovered.values()
+            if execution is not None
+        }
+
+    assert event_counts == {execution.id: 2 for execution in stale.values()}
+
+    async with sessions() as db:
+        execution = recovered["running"]
+        assert execution is not None
+        assert await claim(db, execution.id, "replacement-worker") is not None
+        assert not await heartbeat(db, execution.id, "lost-worker")
+        assert await request_cancellation(db, execution.id) is not None
+        assert not await complete_cancellation(db, execution.id, "lost-worker")
+        assert await complete_cancellation(db, execution.id, "replacement-worker")
+
+
+@pytest.mark.asyncio
+async def test_stopping_worker_does_not_claim_new_work(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    settings, _, _ = database
+    worker = Worker(settings, owner="stopping-worker")
+    try:
+        worker.stop()
+        assert not await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_idle_worker_stops_promptly_and_disposes_its_engine(
+    database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    settings, _, _ = database
+    worker = Worker(settings, owner="idle-worker")
+    disposed = asyncio.Event()
+    dispose = AsyncEngine.dispose
+
+    async def track_dispose(engine: AsyncEngine, *, close: bool = True) -> None:
+        disposed.set()
+        await dispose(engine, close=close)
+
+    monkeypatch.setattr(AsyncEngine, "dispose", track_dispose)
+    task = asyncio.create_task(worker.run())
+    await asyncio.sleep(0)
+    worker.stop()
+    await asyncio.wait_for(task, timeout=1)
+    assert disposed.is_set()

tests/integration/test_all_operational_events_deliver.py

@@ -0,0 +1,389 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.db.models import (
+    Backup,
+    Execution,
+    Job,
+    NotificationDelivery,
+    NotificationEvent,
+    NotificationSubscription,
+    Repository,
+    Schedule,
+    Source,
+)
+from backup_tool.execution import record_event
+from backup_tool.notifications.events import EVENT_CATALOG
+from backup_tool.scheduler import deliver_due
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+
+@pytest.mark.asyncio
+async def test_execution_catalog_events_are_produced_and_delivered(app_client) -> None:
+    """Exercise the execution producer, not emit_event(), for each live execution type."""
+    client, _ = app_client
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    app = client._transport.app
+    expected = {
+        "execution.queued",
+        "execution.started",
+        "execution.committed",
+        "execution.failed",
+        "execution.cancelled",
+        "execution.retry_queued",
+        "execution.worker_recovered",
+    }
+    assert expected <= set(EVENT_CATALOG)
+    deferred_prefixes = ["source.", "gc.", "reconciliation."]
+    assert not any(item.startswith(tuple(deferred_prefixes)) for item in EVENT_CATALOG)
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="events-repository",
+            root="/events-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="events-source",
+            kind="local",
+            public_config={"root": "/events-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        job = Job(
+            name="events-job",
+            source_id=source.id,
+            repository_id=repository.id,
+            exclusions=[],
+            retention={},
+            requested_mode="full",
+        )
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["execution.*"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add_all([job, subscription])
+        await db.flush()
+        cases = (
+            ("queued", 1, None, "execution.queued"),
+            ("preparing", 1, None, "execution.started"),
+            ("committed", 1, None, "execution.committed"),
+            ("failed", 1, "transient_io", "execution.failed"),
+            ("cancelled", 1, "cancellation_requested", "execution.cancelled"),
+            ("queued", 2, None, "execution.retry_queued"),
+            ("queued", 1, "worker_lost", "execution.worker_recovered"),
+        )
+        for state, attempt, reason, _event_type in cases:
+            execution = Execution(
+                job_id=job.id,
+                trigger="manual",
+                state=state,
+                attempt=attempt,
+                reason_code=reason,
+                progress={},
+            )
+            db.add(execution)
+            await db.flush()
+            await record_event(db, execution)
+            if state in {"queued", "preparing"}:
+                execution.state = "failed"
+                execution.reason_code = "test_cleanup"
+            await db.flush()
+        await db.commit()
+        event_statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+        event_types = set((await db.scalars(event_statement)).all())
+        deliveries = await db.scalar(
+            select(NotificationDelivery.id)
+            .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id)
+            .where(NotificationEvent.type.in_(expected))
+            .limit(1)
+        )
+    assert event_types == expected
+    assert deliveries is not None
+
+
+@pytest.mark.asyncio
+async def test_schedule_catalog_events_are_produced_and_delivered(app_client) -> None:
+    client, _ = app_client
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    csrf = client.cookies["backup_tool_csrf"]
+    app = client._transport.app
+    expected = {
+        "schedule.created",
+        "schedule.updated",
+        "schedule.deleted",
+        "schedule.enabled",
+        "schedule.disabled",
+        "schedule.occurrence_enqueued",
+        "schedule.occurrence_misfired",
+        "schedule.occurrence_blocked",
+    }
+    assert expected <= set(EVENT_CATALOG)
+    async with app.state.sessions() as db:
+        repository = Repository(
+            name="schedule-repository",
+            root="/schedule-repository",
+            format_version=1,
+            compression="none",
+            encryption="none",
+        )
+        source = Source(
+            name="schedule-source",
+            kind="local",
+            public_config={"root": "/schedule-source"},
+            secret_refs=[],
+        )
+        db.add_all([repository, source])
+        await db.flush()
+        jobs = [
+            Job(
+                name=f"schedule-job-{number}",
+                source_id=source.id,
+                repository_id=repository.id,
+                exclusions=[],
+                retention={},
+                requested_mode="full",
+                enabled=number not in {2, 3},
+            )
+            for number in range(1, 5)
+        ]
+        subscription = NotificationSubscription(
+            channel="email",
+            event_filters=["schedule.*"],
+            destination_config={"recipients": ["operator@example.test"]},
+            rate_limit_per_minute=60,
+            rate_tokens=60.0,
+        )
+        db.add_all([*jobs, subscription])
+        await db.commit()
+        job_ids = [job.id for job in jobs]
+
+    created = await client.post(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    disabled = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "0 0 * * *", "timezone": "UTC", "enabled": False},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert disabled.status_code == 200
+    enabled = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "1 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert enabled.status_code == 200
+    updated = await client.patch(
+        f"/api/v2/jobs/{job_ids[0]}/schedule",
+        json={"cron": "2 0 * * *", "timezone": "UTC", "enabled": True},
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert updated.status_code == 200
+    assert (
+        await client.delete(f"/api/v2/jobs/{job_ids[0]}/schedule", headers={"X-CSRF-Token": csrf})
+    ).status_code == 204
+
+    async with app.state.sessions() as db:
+        now = datetime.now(UTC)
+        db.add_all(
+            [
+                Schedule(
+                    job_id=job_ids[1],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=0,
+                    enabled=True,
+                    next_nominal_at=now - timedelta(hours=1),
+                ),
+                Schedule(
+                    job_id=job_ids[2],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=60,
+                    enabled=True,
+                    next_nominal_at=now,
+                ),
+                Schedule(
+                    job_id=job_ids[3],
+                    cron="* * * * *",
+                    timezone="UTC",
+                    misfire_grace_seconds=60,
+                    enabled=True,
+                    next_nominal_at=now,
+                ),
+            ]
+        )
+        await db.commit()
+        assert await deliver_due(db, now=now) == 1
+        statement = select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+        event_types = set((await db.scalars(statement)).all())
+        delivery = await db.scalar(
+            select(NotificationDelivery.id)
+            .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id)
+            .where(NotificationEvent.type.in_(expected))
+            .limit(1)
+        )
+    assert event_types == expected
+    assert delivery is not None
+
+
+@pytest.mark.asyncio
+async def test_backup_restore_and_retention_events_are_produced_and_delivered(
+    app_client,
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "notification-project"
+    source_root.mkdir()
+    (source_root / "data.txt").write_text("notification data\n", encoding="utf-8")
+    setup = await client.post(
+        "/api/v2/setup",
+        json={"username": "admin", "password": "correct horse battery staple"},
+    )
+    assert setup.status_code == 201
+    headers = {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+    subscription = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["backup.*", "restore.*", "retention.*"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers=headers,
+    )
+    assert subscription.status_code == 201
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={"name": "notification-repo", "relative_path": "notification-repo"},
+        headers=headers,
+    )
+    source = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "notification-source",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    assert repository.status_code == source.status_code == 201
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "notification-job",
+            "source_id": source.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {"keep_last": 1},
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    assert job.status_code == 201
+    execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
+    assert execution.status_code == 202
+    worker = Worker(settings, owner="notification-backup-worker")
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        backup = await db.scalar(
+            select(Backup).where(Backup.execution_id == execution.json()["id"])
+        )
+        assert backup is not None
+        # A synthetic older catalog entry is valid business state; tombstoning is
+        # performed only through the real retention producer below.
+        older_execution = Execution(
+            job_id=job.json()["id"],
+            trigger="manual",
+            state="committed",
+            progress={},
+        )
+        db.add(older_execution)
+        await db.flush()
+        older = Backup(
+            execution_id=older_execution.id,
+            manifest_id="00000000-0000-7000-8000-000000000001",
+            manifest_digest="0" * 64,
+            logical_bytes=0,
+            stored_bytes=0,
+            integrity="verified",
+            created_at=datetime.now(UTC) - timedelta(days=1),
+        )
+        db.add(older)
+        await db.commit()
+
+    retention_worker = Worker(settings, owner="notification-retention-worker")
+    try:
+        # Retention/GC is executed by worker maintenance, not a direct helper call.
+        assert await retention_worker.run_once()
+    finally:
+        await retention_worker.engine.dispose()
+
+    restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(settings.restore_roots[0] / "notification-restore"),
+            "selection": [],
+            "dry_run": True,
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert restore.status_code == 202
+    restore_worker = Worker(settings, owner="notification-restore-worker")
+    try:
+        assert await restore_worker.run_once()
+    finally:
+        await restore_worker.engine.dispose()
+
+    expected = {
+        "backup.committed",
+        "backup.verification_succeeded",
+        "restore.queued",
+        "restore.committed",
+        "retention.tombstoned",
+    }
+    async with app.state.sessions() as db:
+        types = set(
+            (
+                await db.scalars(
+                    select(NotificationEvent.type).where(NotificationEvent.type.in_(expected))
+                )
+            ).all()
+        )
+        deliveries = list(
+            (
+                await db.scalars(
+                    select(NotificationDelivery.id)
+                    .join(
+                        NotificationEvent,
+                        NotificationDelivery.event_id == NotificationEvent.id,
+                    )
+                    .where(NotificationEvent.type.in_(expected))
+                )
+            ).all()
+        )
+    assert types == expected
+    assert len(deliveries) >= len(expected)

tests/integration/test_empty_source.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import importlib
+
+import pytest
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+
+
+def test_empty_source_requires_explicit_opt_in() -> None:
+    with pytest.raises(snapshot.SnapshotError, match="source_empty"):
+        snapshot.require_nonempty([], False)
+    snapshot.require_nonempty([], True)

tests/integration/test_encrypted_repository.py

@@ -0,0 +1,496 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import stat
+from pathlib import Path
+
+import httpx
+import pytest
+from alembic import command
+from backup_tool.cli import (
+    build_alembic_config,
+    recovery_export_payload,
+    rotate_repository_key,
+)
+from backup_tool.cli import main as cli_main
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Backup, Execution, Repository, RepositoryDataKeyEpoch
+from backup_tool.repository import (
+    begin_key_rotation,
+    initialize,
+    inspect_repository,
+    replace_active_data_key,
+)
+from backup_tool.security.repository_crypto import create_data_key
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+from .test_repository_safety import settings_for
+
+PASSWORD = "correct-horse-battery-staple"
+
+
+async def login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+def test_encrypted_initialization_creates_private_data_key(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+    assert initialized.data_key_path is not None
+    assert stat.S_IMODE(initialized.data_key_path.stat().st_mode) == 0o600
+    try:
+        payload = json.loads((initialized.root / "repository.json").read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise AssertionError("encrypted repository metadata is unreadable") from error
+    assert payload["encryption"] == {
+        "mode": "aes-256-gcm",
+        "key_id": initialized.data_key_id,
+    }
+    inspected = inspect_repository(settings, initialized.root)
+    assert inspected.encryption == "aes-256-gcm"
+    assert inspected.data_key_id == initialized.data_key_id
+
+
+def test_rotation_rejects_metadata_database_epoch_mismatch(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    command.upgrade(build_alembic_config(settings), "head")
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+
+    async def create_repository() -> str:
+        engine = create_engine(settings)
+        try:
+            from sqlalchemy.ext.asyncio import async_sessionmaker
+
+            sessions = async_sessionmaker(engine, expire_on_commit=False)
+            async with sessions() as db:
+                repository = Repository(
+                    name="encrypted",
+                    root=str(initialized.root),
+                    format_version=initialized.format_version,
+                    compression=initialized.compression,
+                    encryption=initialized.encryption,
+                    signing_key_id=initialized.signing_key_id,
+                    signing_public_key=initialized.signing_public_key,
+                    active_data_key_id=initialized.data_key_id,
+                )
+                db.add(repository)
+                await db.flush()
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=initialized.data_key_id,
+                        state="active",
+                    )
+                )
+                await db.commit()
+                return repository.id
+        finally:
+            await engine.dispose()
+
+    repository_id = asyncio.run(create_repository())
+    replacement_id, replacement_path = create_data_key(settings, initialized.repository_id)
+    try:
+        replace_active_data_key(initialized.root, initialized.data_key_id, replacement_id)
+        with pytest.raises(ValueError, match="repository encryption metadata is invalid"):
+            asyncio.run(rotate_repository_key(settings, repository_id))
+    finally:
+        replacement_path.unlink(missing_ok=True)
+
+
+def test_rotation_reconciliation_clears_stale_rollback_journal_after_key_removal(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    command.upgrade(build_alembic_config(settings), "head")
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+
+    async def create_repository() -> str:
+        engine = create_engine(settings)
+        try:
+            from sqlalchemy.ext.asyncio import async_sessionmaker
+
+            sessions = async_sessionmaker(engine, expire_on_commit=False)
+            async with sessions() as db:
+                repository = Repository(
+                    name="encrypted",
+                    root=str(initialized.root),
+                    format_version=initialized.format_version,
+                    compression=initialized.compression,
+                    encryption=initialized.encryption,
+                    signing_key_id=initialized.signing_key_id,
+                    signing_public_key=initialized.signing_public_key,
+                    active_data_key_id=initialized.data_key_id,
+                )
+                db.add(repository)
+                await db.flush()
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=initialized.data_key_id,
+                        state="active",
+                    )
+                )
+                await db.commit()
+                return repository.id
+        finally:
+            await engine.dispose()
+
+    repository_id = asyncio.run(create_repository())
+    new_key_id, new_key_path = create_data_key(settings, initialized.repository_id)
+    begin_key_rotation(
+        initialized.root,
+        repository_id,
+        initialized.repository_id,
+        initialized.data_key_id,
+        new_key_id,
+    )
+    new_key_path.unlink()
+    assert (initialized.root / ".key-rotation.json").is_file()
+
+    async def reconcile_and_assert() -> None:
+        worker = Worker(settings, owner="stale-rollback-journal-worker")
+        try:
+            assert await worker.startup() == 1
+        finally:
+            await worker.engine.dispose()
+        engine = create_engine(settings)
+        try:
+            from sqlalchemy.ext.asyncio import async_sessionmaker
+
+            sessions = async_sessionmaker(engine, expire_on_commit=False)
+            async with sessions() as db:
+                repository = await db.get(Repository, repository_id)
+                assert repository is not None
+                assert repository.active_data_key_id == initialized.data_key_id
+                inspected = inspect_repository(settings, initialized.root)
+                assert inspected.data_key_id == initialized.data_key_id
+        finally:
+            await engine.dispose()
+
+    asyncio.run(reconcile_and_assert())
+    assert not (initialized.root / ".key-rotation.json").exists()
+
+
+def test_rotation_crash_after_db_commit_recovers_on_worker_startup(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    command.upgrade(build_alembic_config(settings), "head")
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+
+    async def create_repository() -> str:
+        engine = create_engine(settings)
+        try:
+            from sqlalchemy.ext.asyncio import async_sessionmaker
+
+            sessions = async_sessionmaker(engine, expire_on_commit=False)
+            async with sessions() as db:
+                repository = Repository(
+                    name="encrypted",
+                    root=str(initialized.root),
+                    format_version=initialized.format_version,
+                    compression=initialized.compression,
+                    encryption=initialized.encryption,
+                    signing_key_id=initialized.signing_key_id,
+                    signing_public_key=initialized.signing_public_key,
+                    active_data_key_id=initialized.data_key_id,
+                )
+                db.add(repository)
+                await db.flush()
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=initialized.data_key_id,
+                        state="active",
+                    )
+                )
+                await db.commit()
+                return repository.id
+        finally:
+            await engine.dispose()
+
+    repository_id = asyncio.run(create_repository())
+
+    def interrupted_after_database_commit() -> None:
+        raise OSError("simulated process loss after database commit")
+
+    with pytest.raises(OSError, match="simulated process loss"):
+        asyncio.run(
+            rotate_repository_key(
+                settings,
+                repository_id,
+                after_database_commit=interrupted_after_database_commit,
+            )
+        )
+    assert inspect_repository(settings, initialized.root).data_key_id == initialized.data_key_id
+    assert (initialized.root / ".key-rotation.json").is_file()
+
+    async def reconcile_and_assert() -> None:
+        worker = Worker(settings, owner="rotation-recovery-worker")
+        try:
+            assert await worker.startup() == 1
+        finally:
+            await worker.engine.dispose()
+        engine = create_engine(settings)
+        try:
+            from sqlalchemy.ext.asyncio import async_sessionmaker
+
+            sessions = async_sessionmaker(engine, expire_on_commit=False)
+            async with sessions() as db:
+                repository = await db.get(Repository, repository_id)
+                assert repository is not None
+                assert repository.active_data_key_id is not None
+                inspected = inspect_repository(settings, initialized.root)
+                assert inspected.data_key_id == repository.active_data_key_id
+                active_data_key_id = repository.active_data_key_id
+            exported = await recovery_export_payload(settings)
+            exported_repository = exported["catalog"]["repositories"][0]
+            assert exported_repository["active_data_key_id"] == active_data_key_id
+        finally:
+            await engine.dispose()
+
+    asyncio.run(reconcile_and_assert())
+    assert not (initialized.root / ".key-rotation.json").exists()
+
+
+@pytest.mark.asyncio
+async def test_encrypted_repository_worker_backup_and_restore(
+    app_client: tuple[httpx.AsyncClient, Settings],
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "project"
+    source_root.mkdir()
+    plaintext = b"encrypted backup content\n"
+    (source_root / "hello.txt").write_bytes(plaintext)
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            repository = Repository(
+                name="encrypted",
+                root=str(initialized.root),
+                format_version=initialized.format_version,
+                compression=initialized.compression,
+                encryption=initialized.encryption,
+                signing_key_id=initialized.signing_key_id,
+                signing_public_key=initialized.signing_public_key,
+                active_data_key_id=initialized.data_key_id,
+            )
+            db.add(repository)
+            await db.flush()
+            db.add(
+                RepositoryDataKeyEpoch(
+                    repository_id=repository.id,
+                    key_id=initialized.data_key_id,
+                    state="active",
+                )
+            )
+            await db.commit()
+            repository_id = repository.id
+    finally:
+        await engine.dispose()
+
+    headers = await login(client)
+    source_response = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "local",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    assert source_response.status_code == 201
+    job_response = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "encrypted-backup",
+            "source_id": source_response.json()["id"],
+            "repository_id": repository_id,
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    assert job_response.status_code == 201
+    execution_response = await client.post(
+        f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers
+    )
+    assert execution_response.status_code == 202
+    execution_id = execution_response.json()["id"]
+
+    worker = Worker(settings, owner="encrypted-backup-worker")
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            execution = await db.get(Execution, execution_id)
+            backup = await db.scalar(select(Backup).where(Backup.execution_id == execution_id))
+    finally:
+        await engine.dispose()
+    assert execution is not None
+    assert execution.state == "committed"
+    assert backup is not None
+
+    blob = next((initialized.root / "blobs" / "sha256").iterdir())
+    assert blob.read_bytes().startswith(b"BTENC\x01")
+    assert plaintext not in blob.read_bytes()
+    manifest_path = initialized.root / "manifests" / f"{backup.manifest_id}.json"
+    stored_manifest = manifest_path.read_bytes()
+    assert stored_manifest.startswith(b"BTENC\x01")
+    assert b'"entries"' not in stored_manifest
+    assert b"hello.txt" not in stored_manifest
+
+    assert (
+        await asyncio.to_thread(
+            cli_main,
+            ["admin", "repository-key", "rotate", "--repository-id", repository_id],
+            settings=settings,
+        )
+        == 0
+    )
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            repository = await db.get(Repository, repository_id)
+            epochs = list(
+                (
+                    await db.scalars(
+                        select(RepositoryDataKeyEpoch).where(
+                            RepositoryDataKeyEpoch.repository_id == repository_id
+                        )
+                    )
+                ).all()
+            )
+    finally:
+        await engine.dispose()
+    assert repository is not None
+    assert repository.active_data_key_id != initialized.data_key_id
+    epoch_states = {f"{epoch.key_id}:{epoch.state}" for epoch in epochs}
+    expected_epoch_states = {
+        f"{initialized.data_key_id}:retired",
+        f"{repository.active_data_key_id}:active",
+    }
+    assert epoch_states == expected_epoch_states
+    assert (
+        inspect_repository(settings, initialized.root).data_key_id == repository.active_data_key_id
+    )
+
+    plaintext_after_rotation = b"encrypted content after rotation\n"
+    (source_root / "hello.txt").write_bytes(plaintext_after_rotation)
+    second_execution_response = await client.post(
+        f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers
+    )
+    assert second_execution_response.status_code == 202
+    second_execution_id = second_execution_response.json()["id"]
+    second_worker = Worker(settings, owner="encrypted-rotated-backup-worker")
+    try:
+        assert await second_worker.run_once()
+    finally:
+        await second_worker.engine.dispose()
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            second_backup = await db.scalar(
+                select(Backup).where(Backup.execution_id == second_execution_id)
+            )
+    finally:
+        await engine.dispose()
+    assert second_backup is not None
+    assert second_backup.data_key_id == repository.active_data_key_id
+
+    destination = settings.restore_roots[0] / "restored"
+    restore_response = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(destination),
+            "selection": [],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert restore_response.status_code == 202
+    source_root.rename(settings.data_dir / "removed-source")
+    restore_worker = Worker(settings, owner="encrypted-restore-worker")
+    try:
+        assert await restore_worker.run_once()
+    finally:
+        await restore_worker.engine.dispose()
+
+    restored = await client.get(
+        f"/api/v2/restores/{restore_response.json()['id']}", headers=headers
+    )
+    assert restored.json()["state"] == "committed"
+    assert (destination / "hello.txt").read_bytes() == plaintext
+
+    rotated_destination = settings.restore_roots[0] / "rotated-restored"
+    rotated_restore_response = await client.post(
+        f"/api/v2/backups/{second_backup.id}/restores",
+        json={
+            "destination": str(rotated_destination),
+            "selection": [],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert rotated_restore_response.status_code == 202
+    rotated_restore_worker = Worker(settings, owner="encrypted-rotated-restore-worker")
+    try:
+        assert await rotated_restore_worker.run_once()
+    finally:
+        await rotated_restore_worker.engine.dispose()
+    assert (rotated_destination / "hello.txt").read_bytes() == plaintext_after_rotation
+
+    manifest_path.write_bytes(stored_manifest[:-1] + bytes([stored_manifest[-1] ^ 1]))
+    corrupt_destination = settings.restore_roots[0] / "corrupt-manifest"
+    corrupt_restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(corrupt_destination),
+            "selection": [],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert corrupt_restore.status_code == 202
+    corrupt_worker = Worker(settings, owner="encrypted-corrupt-manifest-worker")
+    try:
+        assert await corrupt_worker.run_once()
+    finally:
+        await corrupt_worker.engine.dispose()
+
+    corrupt_status = await client.get(
+        f"/api/v2/restores/{corrupt_restore.json()['id']}", headers=headers
+    )
+    assert corrupt_status.json()["state"] == "failed"
+    assert not corrupt_destination.exists()

tests/integration/test_full_backup_restore.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import httpx
+import pytest
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Backup, Execution, Repository, Restore
+from backup_tool.snapshot import verify_published_snapshot
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+PASSWORD = "correct-horse-battery-staple"
+
+
+async def login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+@pytest.mark.asyncio
+async def test_worker_publishes_a_verified_signed_full_backup_and_atomic_restore(
+    app_client: tuple[httpx.AsyncClient, Settings],
+) -> None:
+    client, settings = app_client
+    source_root = settings.local_source_roots[0] / "project"
+    source_root.mkdir()
+    (source_root / "nested").mkdir()
+    (source_root / "nested" / "hello.txt").write_text("hello backup\n", encoding="utf-8")
+    headers = await login(client)
+    repository_response = await client.post(
+        "/api/v2/repositories",
+        json={
+            "name": "primary",
+            "relative_path": "primary",
+            "compression": "none",
+            "encryption": "none",
+        },
+        headers=headers,
+    )
+    assert repository_response.status_code == 201
+    source_response = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "local",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    assert source_response.status_code == 201
+    job_response = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "full-backup",
+            "source_id": source_response.json()["id"],
+            "repository_id": repository_response.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    assert job_response.status_code == 201
+    execution_response = await client.post(
+        f"/api/v2/jobs/{job_response.json()['id']}/executions", headers=headers
+    )
+    assert execution_response.status_code == 202
+    execution_id = execution_response.json()["id"]
+
+    worker = Worker(settings, owner="snapshot-worker")
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            execution = await db.get(Execution, execution_id)
+            backup = await db.scalar(select(Backup).where(Backup.execution_id == execution_id))
+            repository = await db.get(Repository, repository_response.json()["id"])
+    finally:
+        await engine.dispose()
+
+    assert execution is not None
+    assert execution.state == "committed"
+    assert backup is not None
+    assert backup.integrity == "verified"
+    assert repository is not None
+    root = Path(repository.root)
+    manifest_path = root / "manifests" / f"{backup.manifest_id}.json"
+    manifest = verify_published_snapshot(root, manifest_path, repository.signing_public_key)
+    file_entry = next(entry for entry in manifest["entries"] if entry["type"] == "file")
+    assert file_entry["path"] == "nested/hello.txt"
+    assert (root / "blobs" / "sha256" / file_entry["blob_digest"]).read_text() == "hello backup\n"
+
+    dry_run_destination = settings.restore_roots[0] / "dry-run-backup"
+    dry_run_response = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(dry_run_destination),
+            "selection": ["nested"],
+            "dry_run": True,
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert dry_run_response.status_code == 202
+    dry_run_worker = Worker(settings, owner="dry-run-worker")
+    try:
+        assert await dry_run_worker.run_once()
+    finally:
+        await dry_run_worker.engine.dispose()
+    dry_run = await client.get(f"/api/v2/restores/{dry_run_response.json()['id']}", headers=headers)
+    assert dry_run.json()["state"] == "committed"
+    assert dry_run.json()["result"]["dry_run"]
+    assert dry_run.json()["result"]["entry_count"] == 2
+    assert not dry_run_destination.exists()
+
+    destination = settings.restore_roots[0] / "restored-backup"
+    restore_response = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(destination),
+            "selection": [],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert restore_response.status_code == 202
+    restore_id = restore_response.json()["id"]
+    assert restore_response.json()["state"] == "queued"
+    source_root.rename(settings.data_dir / "removed-source")
+
+    restore_worker = Worker(settings, owner="restore-worker")
+    try:
+        assert await restore_worker.run_once()
+    finally:
+        await restore_worker.engine.dispose()
+
+    restored = await client.get(f"/api/v2/restores/{restore_id}", headers=headers)
+    assert restored.status_code == 200
+    assert restored.json()["state"] == "committed"
+    assert restored.json()["result"]["manifest_digest"] == backup.manifest_digest
+    assert (destination / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello backup\n"
+
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            interrupted_restore = await db.get(Restore, restore_id)
+            assert interrupted_restore is not None
+            interrupted_restore.state = "running"
+            interrupted_restore.result = None
+            await db.commit()
+    finally:
+        await engine.dispose()
+    recovery_worker = Worker(settings, owner="recovery-worker")
+    try:
+        assert await recovery_worker.startup() == 1
+    finally:
+        await recovery_worker.engine.dispose()
+    recovered = await client.get(f"/api/v2/restores/{restore_id}", headers=headers)
+    assert recovered.json()["state"] == "committed"
+
+    skipped_restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(destination),
+            "selection": [],
+            "overwrite_policy": "skip",
+        },
+        headers=headers,
+    )
+    assert skipped_restore.status_code == 202
+    skip_worker = Worker(settings, owner="skip-worker")
+    try:
+        assert await skip_worker.run_once()
+    finally:
+        await skip_worker.engine.dispose()
+    skipped = await client.get(f"/api/v2/restores/{skipped_restore.json()['id']}", headers=headers)
+    assert skipped.json()["result"]["skipped"]
+
+    (destination / "nested" / "hello.txt").write_text("replaced", encoding="utf-8")
+    replaced_restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(destination),
+            "selection": [],
+            "overwrite_policy": "replace",
+        },
+        headers=headers,
+    )
+    assert replaced_restore.status_code == 202
+    replace_worker = Worker(settings, owner="replace-worker")
+    try:
+        assert await replace_worker.run_once()
+    finally:
+        await replace_worker.engine.dispose()
+    assert (destination / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello backup\n"
+
+    (root / "blobs" / "sha256" / file_entry["blob_digest"]).write_text("tampered")
+    corrupt_destination = settings.restore_roots[0] / "corrupt-restore"
+    corrupt_restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(corrupt_destination),
+            "selection": [],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert corrupt_restore.status_code == 202
+
+    corrupt_worker = Worker(settings, owner="corrupt-restore-worker")
+    try:
+        assert await corrupt_worker.run_once()
+    finally:
+        await corrupt_worker.engine.dispose()
+
+    corrupt_status = await client.get(
+        f"/api/v2/restores/{corrupt_restore.json()['id']}", headers=headers
+    )
+    assert corrupt_status.json()["state"] == "failed"
+    assert not corrupt_destination.exists()
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as db:
+            corrupted_backup = await db.get(Backup, backup.id)
+    finally:
+        await engine.dispose()
+    assert corrupted_backup is not None
+    assert corrupted_backup.integrity == "corrupt"

tests/integration/test_gc.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+import os
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from backup_tool.gc import purge_repository
+from backup_tool.security.repository_crypto import encrypt_object, object_aad
+
+
+def write_manifest(path: Path, digests: list[str]) -> None:
+    path.write_text(
+        json.dumps({"entries": [{"blob_digest": digest} for digest in digests]}),
+        encoding="utf-8",
+    )
+
+
+def age(path: Path, days: int = 8) -> None:
+    stamp = (datetime.now(UTC) - timedelta(days=days)).timestamp()
+    os.utime(path, (stamp, stamp))
+
+
+def test_gc_purges_tombstoned_manifest_and_only_unreferenced_old_blob(
+    tmp_path: Path,
+) -> None:
+    root = tmp_path / "repository"
+    manifests = root / "manifests"
+    blobs = root / "blobs" / "sha256"
+    manifests.mkdir(parents=True)
+    blobs.mkdir(parents=True)
+    kept = "a" * 64
+    removed = "b" * 64
+    write_manifest(manifests / "kept.json", [kept])
+    write_manifest(manifests / "deleted.json", [removed])
+    (blobs / kept).write_bytes(b"kept")
+    (blobs / removed).write_bytes(b"removed")
+    age(manifests / "deleted.json")
+    age(blobs / removed)
+
+    report = purge_repository(root, {"deleted"})
+
+    assert report.purged_manifests == 1
+    assert report.purged_blobs == 1
+    assert (manifests / "kept.json").exists()
+    assert (blobs / kept).exists()
+    assert not (blobs / removed).exists()
+
+
+@pytest.mark.parametrize("encrypted", [False, True], ids=["corrupt", "encrypted"])
+def test_gc_fails_closed_for_unreadable_manifest(tmp_path: Path, encrypted: bool) -> None:
+    root = tmp_path / "repository"
+    manifests = root / "manifests"
+    blobs = root / "blobs" / "sha256"
+    manifests.mkdir(parents=True)
+    blobs.mkdir(parents=True)
+    kept = "a" * 64
+    deleted = "b" * 64
+    write_manifest(manifests / "kept.json", [kept])
+    write_manifest(manifests / "deleted.json", [deleted])
+    unreadable = manifests / "unreadable.json"
+    unreadable.write_bytes(
+        encrypt_object(
+            b"k" * 32,
+            object_aad("repository", "key", "manifest", "unreadable"),
+            b'{"entries": []}',
+        )
+        if encrypted
+        else b"not json"
+    )
+    (blobs / kept).write_bytes(b"kept")
+    (blobs / deleted).write_bytes(b"deleted")
+    age(manifests / "deleted.json")
+    age(blobs / deleted)
+
+    report = purge_repository(root, {"deleted"})
+
+    assert report.purged_manifests == 0
+    assert report.purged_blobs == 0
+    assert (manifests / "deleted.json").exists()
+    assert (blobs / deleted).exists()
+
+
+def test_gc_purges_encrypted_manifests_with_known_epoch_keys(tmp_path: Path) -> None:
+    root = tmp_path / "repository"
+    manifests = root / "manifests"
+    blobs = root / "blobs" / "sha256"
+    manifests.mkdir(parents=True)
+    blobs.mkdir(parents=True)
+    key = b"k" * 32
+    repository_id = "repository"
+    key_id = "epoch"
+    kept = "a" * 64
+    deleted = "b" * 64
+    for manifest_id, digests in (("kept", [kept]), ("deleted", [deleted])):
+        plaintext = json.dumps(
+            {"entries": [{"blob_digest": digest} for digest in digests]}
+        ).encode()
+        (manifests / f"{manifest_id}.json").write_bytes(
+            encrypt_object(
+                key,
+                object_aad(repository_id, key_id, "manifest", manifest_id),
+                plaintext,
+            )
+        )
+    (blobs / kept).write_bytes(b"kept")
+    (blobs / deleted).write_bytes(b"deleted")
+    age(manifests / "deleted.json")
+    age(blobs / deleted)
+
+    report = purge_repository(
+        root,
+        {"deleted"},
+        repository_id=repository_id,
+        manifest_keys={"kept": (key_id, key), "deleted": (key_id, key)},
+    )
+
+    assert report.purged_manifests == 1
+    assert report.purged_blobs == 1
+    assert (blobs / kept).exists()
+    assert not (blobs / deleted).exists()
+
+
+def test_gc_quarantines_unknown_blob_name(tmp_path: Path) -> None:
+    root = tmp_path / "repository"
+    blobs = root / "blobs" / "sha256"
+    blobs.mkdir(parents=True)
+    unknown = blobs / "not-a-digest"
+    unknown.write_bytes(b"unknown")
+
+    report = purge_repository(root, set())
+
+    assert report.quarantined == 1
+    assert not unknown.exists()
+    assert (root / "quarantine" / "not-a-digest").exists()

tests/integration/test_incremental.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from backup_tool.snapshot import _unsigned_manifest
+
+
+class Object:
+    def __init__(self, **values: object) -> None:
+        self.__dict__.update(values)
+
+
+def test_incremental_request_without_baseline_emits_complete_full_manifest() -> None:
+    manifest = _unsigned_manifest(
+        "0198c57f-0000-7000-8000-000000000006",
+        "0198c57f-0000-7000-8000-000000000001",
+        Object(id="0198c57f-0000-7000-8000-000000000003", kind="local"),
+        Object(
+            id="0198c57f-0000-7000-8000-000000000004",
+            requested_mode="incremental",
+            exclusions=[],
+        ),
+        Object(id="0198c57f-0000-7000-8000-000000000005"),
+        [],
+        0,
+        0,
+    )
+
+    assert manifest["requested_mode"] == "incremental"
+    assert manifest["effective_mode"] == "full"

tests/integration/test_migrations.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import importlib
 import os
+import sqlite3
 import stat
 from datetime import UTC, datetime, timedelta, timezone
 from pathlib import Path
@@ -24,8 +25,13 @@ EXPECTED_TABLES = {
     "idempotency_records",
     "jobs",
     "notification_deliveries",
+    "notification_delivery_attempts",
+    "notification_email_settings",
+    "notification_events",
+    "notification_signing_keys",
     "notification_subscriptions",
     "repositories",
+    "repository_data_key_epochs",
     "restores",
     "schedules",
     "secrets",
@@ -103,6 +109,199 @@ async def test_startup_rejects_unmigrated_database(tmp_path: Path) -> None:
         await engine.dispose()
 
 
+def test_local_only_sources_migration_rejects_existing_remote_sources(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    migration = alembic_config(settings.database_url)
+    command.upgrade(migration, "0005_restore_dry_run")
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            ("remote", "sftp", "{}", "[]", "active", None, "source-remote"),
+        )
+
+    with pytest.raises(RuntimeError, match="found 1 non-local source row"):
+        command.upgrade(migration, "head")
+
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute("UPDATE sources SET kind = ? WHERE id = ?", ("local", "source-remote"))
+
+    command.upgrade(migration, "head")
+    with (
+        sqlite3.connect(settings.database_path) as connection,
+        pytest.raises(sqlite3.IntegrityError),
+    ):
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            ("remote-two", "sftp", "{}", "[]", "active", None, "source-remote-two"),
+        )
+
+    command.downgrade(migration, "0005_restore_dry_run")
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            ("remote-three", "sftp", "{}", "[]", "active", None, "source-remote-three"),
+        )
+
+
+def test_0009_allows_ssh_sources_and_refuses_populated_downgrade(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    migration = alembic_config(settings.database_url)
+    command.upgrade(migration, "0008_notification_outbox")
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            ("local", "local", '{"root":"/tmp"}', "[]", "active", None, "source-local"),
+        )
+
+    command.upgrade(migration, "head")
+    with sqlite3.connect(settings.database_path) as connection:
+        local_source = connection.execute("SELECT kind FROM sources").fetchone()
+        assert local_source is not None
+        assert local_source[0] == "local"
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                "ssh",
+                "ssh",
+                '{"hostname":"backup.example.test"}',
+                '["secret"]',
+                "active",
+                None,
+                "source-ssh",
+            ),
+        )
+        with pytest.raises(sqlite3.IntegrityError):
+            connection.execute(
+                """
+                INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                ("sftp", "sftp", "{}", "[]", "active", None, "source-sftp"),
+            )
+
+    with pytest.raises(RuntimeError, match="found 1 SSH source row"):
+        command.downgrade(migration, "0008_notification_outbox")
+
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute("DELETE FROM sources WHERE id = ?", ("source-ssh",))
+    command.downgrade(migration, "0008_notification_outbox")
+    with (
+        sqlite3.connect(settings.database_path) as connection,
+        pytest.raises(sqlite3.IntegrityError),
+    ):
+        connection.execute(
+            """
+            INSERT INTO sources (name, kind, public_config, secret_refs, state, last_probe, id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                "ssh-after-downgrade",
+                "ssh",
+                "{}",
+                "[]",
+                "active",
+                None,
+                "source-ssh-after",
+            ),
+        )
+
+
+def test_0008_migrates_unexpected_legacy_notification_delivery(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    migration = alembic_config(settings.database_url)
+    command.upgrade(migration, "0007_repository_data_key_epochs")
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO notification_subscriptions
+                (id, channel, event_filters, destination_config, state)
+            VALUES (?, ?, ?, ?, ?)
+            """,
+            (
+                "subscription",
+                "email",
+                '["execution.queued"]',
+                '{"recipients":["a@example.test"]}',
+                "active",
+            ),
+        )
+        connection.execute(
+            """
+            INSERT INTO notification_deliveries
+                (id, event_id, subscription_id, attempt, state, response_class)
+            VALUES (?, ?, ?, ?, ?, ?)
+            """,
+            ("delivery", "legacy-event", "subscription", 1, "delivered", "http_200"),
+        )
+    command.upgrade(migration, "head")
+    with sqlite3.connect(settings.database_path) as connection:
+        event = connection.execute("SELECT type, payload FROM notification_events").fetchone()
+        attempt = connection.execute(
+            "SELECT number, outcome, response_class FROM notification_delivery_attempts"
+        ).fetchone()
+    assert event is not None and event[0] == "notification.legacy"
+    assert "legacy-event" in event[1]
+    assert attempt is not None
+    assert attempt[0] == 1
+    assert attempt[1] == "delivered"
+    assert attempt[2] == "http_200"
+
+
+def test_0007_downgrade_refuses_populated_key_metadata(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    migration = alembic_config(settings.database_url)
+    command.upgrade(migration, "head")
+    with sqlite3.connect(settings.database_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO repositories
+                (id, name, root, format_version, compression, encryption, active_data_key_id, state)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                "repository",
+                "repository",
+                str(settings.repository_roots[0]),
+                1,
+                "none",
+                "none",
+                "epoch",
+                "active",
+            ),
+        )
+        connection.execute(
+            """
+            INSERT INTO repository_data_key_epochs (id, repository_id, key_id, state)
+            VALUES (?, ?, ?, ?)
+            """,
+            ("epoch-row", "repository", "epoch", "active"),
+        )
+
+    with pytest.raises(
+        RuntimeError, match="cannot downgrade while repository data key metadata exists"
+    ):
+        command.downgrade(migration, "0006_local_sources_only")
+
+
 def test_database_role_rejects_unmigrated_database(tmp_path: Path) -> None:
     cli = importlib.import_module("backup_tool.cli")
     db_module = importlib.import_module("backup_tool.db.engine")

tests/integration/test_notifications.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import cast
+
+import pytest
+from backup_tool.db.models import (
+    NotificationDelivery,
+    NotificationEmailSettings,
+    NotificationEvent,
+)
+from backup_tool.ids import new_uuid7
+from backup_tool.notifications.email import SMTPClient, deliver_email
+from backup_tool.notifications.events import emit_event
+from sqlalchemy import select
+
+PASSWORD = "correct horse battery staple"
+
+
+async def _setup(client) -> str:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return client.cookies["backup_tool_csrf"]
+
+
+@pytest.mark.asyncio
+async def test_filters_manual_test_retry_and_history(app_client) -> None:
+    client, _ = app_client
+    csrf = await _setup(client)
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["schedule.*", "notification.test_requested"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert created.status_code == 201
+    subscription_id = created.json()["id"]
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="filtered-out",
+        )
+        scheduled = await emit_event(
+            db,
+            "schedule.created",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            deduplication_key="filtered-in",
+        )
+        await db.commit()
+        statement = select(NotificationDelivery).where(
+            NotificationDelivery.event_id == scheduled.id
+        )
+        deliveries = list((await db.scalars(statement)).all())
+        assert len(deliveries) == 1
+        delivery = deliveries[0]
+        delivery.state = "failed"
+        delivery.terminal_reason = "http_permanent"
+        await db.commit()
+        delivery_id = delivery.id
+
+    tested = await client.post(
+        f"/api/v2/notifications/subscriptions/{subscription_id}/test",
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "test-one"},
+    )
+    assert tested.status_code == 202
+    retried = await client.post(
+        f"/api/v2/notifications/deliveries/{delivery_id}/retry",
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "retry-one"},
+    )
+    assert retried.status_code == 202
+    replayed = await client.post(
+        f"/api/v2/notifications/deliveries/{delivery_id}/retry",
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "retry-one"},
+    )
+    assert replayed.status_code == 202
+    assert replayed.json() == retried.json()
+    history = await client.get("/api/v2/notifications/deliveries")
+    assert history.status_code == 200
+    row = next(item for item in history.json()["items"] if item["id"] == delivery_id)
+    assert row["state"] == "retry"
+
+
+@pytest.mark.asyncio
+async def test_manual_test_bypasses_filters_and_targets_only_selected_subscription(
+    app_client,
+) -> None:
+    client, _ = app_client
+    csrf = await _setup(client)
+    selected = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["execution.failed"],
+            "destination": {"recipients": ["selected@example.test"]},
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    other = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["notification.test_requested"],
+            "destination": {"recipients": ["other@example.test"]},
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    assert selected.status_code == other.status_code == 201
+    response = await client.post(
+        f"/api/v2/notifications/subscriptions/{selected.json()['id']}/test",
+        headers={"X-CSRF-Token": csrf, "Idempotency-Key": "selected-test"},
+    )
+    assert response.status_code == 202
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        rows = list(
+            (
+                await db.scalars(
+                    select(NotificationDelivery.subscription_id).where(
+                        NotificationDelivery.event_id == response.json()["event_id"]
+                    )
+                )
+            ).all()
+        )
+    assert rows == [selected.json()["id"]]
+
+
+@pytest.mark.asyncio
+async def test_disabled_subscription_does_not_receive_future_events(app_client) -> None:
+    client, _ = app_client
+    csrf = await _setup(client)
+    created = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["execution.*"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers={"X-CSRF-Token": csrf},
+    )
+    subscription = created.json()
+    disabled = await client.patch(
+        f"/api/v2/notifications/subscriptions/{subscription['id']}",
+        json={"state": "disabled"},
+        headers={"X-CSRF-Token": csrf, "If-Match": created.headers["ETag"]},
+    )
+    assert disabled.status_code == 200
+    app = client._transport.app
+    async with app.state.sessions() as db:
+        event = await emit_event(
+            db,
+            "execution.queued",
+            correlation_id=str(new_uuid7()),
+            resource={},
+            occurred_at=datetime.now(UTC),
+        )
+        await db.commit()
+        assert (
+            await db.scalar(
+                select(NotificationDelivery.id).where(NotificationDelivery.event_id == event.id)
+            )
+            is None
+        )
+
+
+@pytest.mark.asyncio
+async def test_email_uses_ehlo_starttls_then_auth_with_hermetic_fake() -> None:
+    calls: list[str] = []
+
+    class FakeSMTP:
+        def __init__(self, *_args, **_kwargs) -> None:
+            calls.append("connect")
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args) -> None:
+            calls.append("close")
+
+        def ehlo(self) -> None:
+            calls.append("ehlo")
+
+        def starttls(self, *, context) -> None:
+            assert context.check_hostname
+            calls.append("starttls")
+
+        def login(self, username: str, password: str) -> None:
+            assert username == "operator"
+            assert password == "smtp-password"
+            calls.append("auth")
+
+        def send_message(self, message) -> None:
+            assert "smtp-password" not in message.as_string()
+            calls.append("send")
+
+    event = NotificationEvent(
+        id=str(new_uuid7()),
+        type="execution.queued",
+        schema_version=1,
+        occurred_at=datetime.now(UTC),
+        correlation_id=str(new_uuid7()),
+        severity="info",
+        resource_refs={},
+        payload={},
+        canonical_envelope="{}",
+    )
+    settings = NotificationEmailSettings(
+        id=1,
+        host="smtp.example.test",
+        port=587,
+        username="operator",
+        password_secret_id=str(new_uuid7()),
+        sender="sender@example.test",
+        max_attempts=5,
+        rate_limit_per_minute=60,
+    )
+    result = await deliver_email(
+        settings,
+        "smtp-password",
+        event,
+        ["operator@example.test"],
+        smtp_factory=cast(Callable[..., SMTPClient], FakeSMTP),
+    )
+    assert result.response_class == "smtp_2xx"
+    assert calls == ["connect", "ehlo", "starttls", "ehlo", "auth", "send", "close"]

tests/integration/test_queue_leases.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+from alembic import command
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Execution, Job, Repository, Source
+from backup_tool.execution import (
+    EnqueueError,
+    claim,
+    complete_cancellation,
+    enqueue,
+    heartbeat,
+    request_cancellation,
+    retry,
+)
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+cli = importlib.import_module("backup_tool.cli")
+
+
+@pytest_asyncio.fixture
+async def database(
+    tmp_path: Path,
+) -> AsyncIterator[tuple[async_sessionmaker[AsyncSession], AsyncEngine]]:
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m5-test-master-key-material-32-bytes-minimum")
+    key.chmod(0o600)
+    data_dir = tmp_path / "data"
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (data_dir, repositories, sources, restores):
+        directory.mkdir()
+    settings = Settings(
+        data_dir=data_dir,
+        database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+    )
+    command.upgrade(cli.build_alembic_config(settings), "head")
+    engine = create_engine(settings)
+    yield async_sessionmaker(engine, expire_on_commit=False), engine
+    await engine.dispose()
+
+
+async def create_job(
+    db: AsyncSession, suffix: str, *, enabled: bool = True, state: str = "active"
+) -> str:
+    repository = Repository(
+        name=f"repository-{suffix}",
+        root=f"/repositories/{suffix}",
+        format_version=1,
+        compression="none",
+        encryption="none",
+    )
+    source = Source(
+        name=f"source-{suffix}",
+        kind="local",
+        public_config={"root": f"/sources/{suffix}"},
+        secret_refs=[],
+    )
+    db.add_all([repository, source])
+    await db.flush()
+    job = Job(
+        name=f"job-{suffix}",
+        source_id=source.id,
+        repository_id=repository.id,
+        requested_mode="full",
+        exclusions=[],
+        retention={},
+        enabled=enabled,
+        state=state,
+    )
+    db.add(job)
+    await db.commit()
+    return job.id
+
+
+@pytest.mark.asyncio
+async def test_concurrent_enqueue_allows_exactly_one_active_execution(
+    database: tuple[async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    sessions, _ = database
+    async with sessions() as db:
+        job_id = await create_job(db, "concurrent")
+
+    async def start() -> Execution | EnqueueError:
+        async with sessions() as db:
+            try:
+                return await enqueue(db, job_id)
+            except EnqueueError as error:
+                return error
+
+    first, second = await asyncio.gather(start(), start())
+    results = [first, second]
+    successes = [result for result in results if isinstance(result, Execution)]
+    failures = [result for result in results if isinstance(result, EnqueueError)]
+
+    assert len(successes) == 1
+    assert len(failures) == 1
+    assert failures[0].code == "execution_active"
+    assert failures[0].active_execution_id == successes[0].id
+    async with sessions() as db:
+        executions = list(await db.scalars(select(Execution).where(Execution.job_id == job_id)))
+    assert [execution.id for execution in executions] == [successes[0].id]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("enabled", "state"), [(False, "active"), (True, "archived")])
+async def test_enqueue_rejects_disabled_or_archived_jobs(
+    database: tuple[async_sessionmaker[AsyncSession], AsyncEngine],
+    enabled: bool,
+    state: str,
+) -> None:
+    sessions, _ = database
+    async with sessions() as db:
+        job_id = await create_job(
+            db, f"unavailable-{enabled}-{state}", enabled=enabled, state=state
+        )
+        with pytest.raises(EnqueueError) as raised:
+            await enqueue(db, job_id)
+
+    assert raised.value.code == "job_disabled"
+
+
+@pytest.mark.asyncio
+async def test_reclaimed_lease_fences_the_previous_worker(
+    database: tuple[async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    sessions, _ = database
+    async with sessions() as db:
+        execution = await enqueue(db, await create_job(db, "leases"))
+        assert await claim(db, execution.id, "worker-a") is not None
+        assert await heartbeat(db, execution.id, "worker-a")
+        persisted = await db.get(Execution, execution.id)
+        assert persisted is not None
+        persisted.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+
+    async with sessions() as db:
+        assert await claim(db, execution.id, "worker-b") is not None
+        assert not await heartbeat(db, execution.id, "worker-a")
+        assert await request_cancellation(db, execution.id) is not None
+        assert not await complete_cancellation(db, execution.id, "worker-a")
+        assert await complete_cancellation(db, execution.id, "worker-b")
+
+    async with sessions() as db:
+        persisted = await db.get(Execution, execution.id)
+    assert persisted is not None
+    assert persisted.state == "cancelled"
+    assert persisted.lease_owner is None
+
+
+@pytest.mark.asyncio
+async def test_retry_reuses_execution_and_rejects_non_transient_failures(
+    database: tuple[async_sessionmaker[AsyncSession], AsyncEngine],
+) -> None:
+    sessions, _ = database
+    async with sessions() as db:
+        execution = await enqueue(db, await create_job(db, "retry"))
+        execution.state = "failed"
+        execution.reason_code = "transient_io"
+        execution.operator_message = "temporary failure"
+        execution.lease_owner = "worker-a"
+        execution.lease_expires_at = datetime.now(UTC) + timedelta(seconds=60)
+        await db.commit()
+
+        retried = await retry(db, execution.id)
+        assert retried is not None
+        assert retried.id == execution.id
+        assert retried.state == "queued"
+        assert retried.attempt == 2
+        assert retried.reason_code is None
+        assert retried.operator_message is None
+        assert retried.lease_owner is None
+        assert retried.lease_expires_at is None
+
+        retried.state = "failed"
+        retried.reason_code = "integrity_failure"
+        await db.commit()
+        with pytest.raises(EnqueueError) as raised:
+            await retry(db, execution.id)
+
+    assert raised.value.code == "retry_not_allowed"

tests/integration/test_recovery_bundle.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from alembic import command
+from backup_tool.cli import (
+    build_alembic_config,
+    import_recovery_payload,
+)
+from backup_tool.cli import (
+    main as cli_main,
+)
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import (
+    Backup,
+    Execution,
+    Job,
+    Repository,
+    RepositoryDataKeyEpoch,
+    Restore,
+    Source,
+)
+from backup_tool.ids import new_uuid7
+from backup_tool.repository import initialize
+from backup_tool.security.recovery_bundle import RecoveryBundleError, decrypt_bundle
+from backup_tool.snapshot import finalize_publication, publish_full_snapshot
+from backup_tool.worker import Worker
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from tests.conftest import make_settings
+
+PASSPHRASE = b"correct horse battery staple"
+
+
+def _passphrase_fd(value: bytes) -> int:
+    read_fd, write_fd = os.pipe()
+    os.write(write_fd, value + b"\n")
+    os.close(write_fd)
+    return read_fd
+
+
+def _recovered_settings(tmp_path: Path, original: Settings) -> Settings:
+    data_dir = tmp_path / "recovered-data"
+    source_root = tmp_path / "recovered-sources"
+    restore_root = tmp_path / "recovered-restores"
+    for path in (data_dir, source_root, restore_root):
+        path.mkdir()
+    master_key = tmp_path / "recovered-master.key"
+    master_key.write_bytes(b"recovered-host-master-key-material-32-bytes")
+    master_key.chmod(0o600)
+    return Settings(
+        data_dir=data_dir,
+        database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}",
+        repository_roots=original.repository_roots,
+        local_source_roots=(source_root,),
+        restore_roots=(restore_root,),
+        master_key_file=master_key,
+        min_free_bytes=1,
+    )
+
+
+def test_recovery_import_restores_encrypted_snapshot_after_host_loss(
+    tmp_path: Path,
+) -> None:
+    original_path = tmp_path / "original"
+    original_path.mkdir()
+    original = make_settings(original_path)
+    command.upgrade(build_alembic_config(original), "head")
+    source_root = original.local_source_roots[0] / "project"
+    source_root.mkdir()
+    plaintext = b"recovery host-loss content\n"
+    (source_root / "document.txt").write_bytes(plaintext)
+    initialized = initialize(original, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+
+    expected_created_at = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC)
+    expected_tombstoned_at = datetime(2024, 2, 3, 4, 5, 6, tzinfo=UTC)
+
+    async def create_snapshot() -> tuple[str, str]:
+        engine = create_engine(original)
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        try:
+            async with sessions() as db:
+                repository = Repository(
+                    name="encrypted",
+                    root=str(initialized.root),
+                    format_version=initialized.format_version,
+                    compression=initialized.compression,
+                    encryption=initialized.encryption,
+                    signing_key_id=initialized.signing_key_id,
+                    signing_public_key=initialized.signing_public_key,
+                    active_data_key_id=initialized.data_key_id,
+                )
+                source = Source(
+                    name="project",
+                    kind="local",
+                    public_config={"root": str(source_root)},
+                    secret_refs=[],
+                )
+                db.add_all([repository, source])
+                await db.flush()
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=initialized.data_key_id,
+                        state="active",
+                    )
+                )
+                job = Job(
+                    name="encrypted-job",
+                    source_id=source.id,
+                    repository_id=repository.id,
+                    requested_mode="full",
+                    exclusions=[],
+                    retention={},
+                    allow_empty=False,
+                )
+                db.add(job)
+                await db.flush()
+                execution = Execution(job_id=job.id, trigger="manual", progress={})
+                db.add(execution)
+                await db.flush()
+                backup = await publish_full_snapshot(
+                    original, db, execution, job, source, repository
+                )
+                backup.created_at = expected_created_at
+                execution.state = "committed"
+                tombstoned_execution = Execution(job_id=job.id, trigger="manual", progress={})
+                db.add(tombstoned_execution)
+                await db.flush()
+                tombstoned_backup = Backup(
+                    execution_id=tombstoned_execution.id,
+                    parent_backup_id=None,
+                    manifest_id=str(new_uuid7()),
+                    manifest_digest="0" * 64,
+                    logical_bytes=0,
+                    stored_bytes=0,
+                    integrity="verified",
+                    data_key_id=initialized.data_key_id,
+                    tombstoned_at=expected_tombstoned_at,
+                    created_at=expected_created_at,
+                )
+                db.add(tombstoned_backup)
+                await db.commit()
+                finalize_publication(initialized.root, execution.id)
+                return backup.id, tombstoned_backup.id
+        finally:
+            await engine.dispose()
+
+    original_backup_id, tombstoned_backup_id = asyncio.run(create_snapshot())
+    bundle = tmp_path / "offline.btrec"
+    export_fd = _passphrase_fd(PASSPHRASE)
+    try:
+        assert (
+            cli_main(
+                [
+                    "admin",
+                    "recovery",
+                    "export",
+                    "--output",
+                    str(bundle),
+                    "--passphrase-fd",
+                    str(export_fd),
+                ],
+                settings=original,
+            )
+            == 0
+        )
+    finally:
+        os.close(export_fd)
+
+    retry_path = tmp_path / "retry-recovery"
+    retry_path.mkdir()
+    retry = _recovered_settings(retry_path, original)
+    command.upgrade(build_alembic_config(retry), "head")
+    retry_payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE)
+
+    def interrupted_after_key_install() -> None:
+        raise OSError("simulated crash before metadata commit")
+
+    with pytest.raises(RecoveryBundleError, match="recovery import failed"):
+        asyncio.run(
+            import_recovery_payload(
+                retry,
+                retry_payload,
+                after_key_install=interrupted_after_key_install,
+            )
+        )
+    assert not list((retry.data_dir / "repository-keys").glob("*"))
+    assert not list((retry.data_dir / "repository-data-keys").glob("*"))
+    assert asyncio.run(import_recovery_payload(retry, retry_payload)) == 1
+
+    unsafe_path = tmp_path / "unsafe-recovery"
+    unsafe_path.mkdir()
+    unsafe = _recovered_settings(unsafe_path, original)
+    command.upgrade(build_alembic_config(unsafe), "head")
+    unsafe_payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE)
+    unsafe_payload["catalog"]["repositories"][0]["root"] = str(tmp_path)
+    with pytest.raises(RecoveryBundleError, match="recovery import failed"):
+        asyncio.run(import_recovery_payload(unsafe, unsafe_payload))
+    assert not (unsafe.data_dir / "repository-keys").exists()
+    assert not (unsafe.data_dir / "repository-data-keys").exists()
+
+    recovered = _recovered_settings(tmp_path, original)
+    command.upgrade(build_alembic_config(recovered), "head")
+    import_fd = _passphrase_fd(PASSPHRASE)
+    try:
+        assert (
+            cli_main(
+                [
+                    "admin",
+                    "recovery",
+                    "import",
+                    "--input",
+                    str(bundle),
+                    "--passphrase-fd",
+                    str(import_fd),
+                ],
+                settings=recovered,
+            )
+            == 0
+        )
+    finally:
+        os.close(import_fd)
+
+    async def restore_and_assert() -> None:
+        engine = create_engine(recovered)
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        try:
+            async with sessions() as db:
+                backup = await db.get(Backup, original_backup_id)
+                tombstoned_backup = await db.get(Backup, tombstoned_backup_id)
+                job = await db.scalar(select(Job))
+                recovered_source = await db.scalar(select(Source))
+                assert backup is not None
+                assert tombstoned_backup is not None
+                assert backup.created_at == expected_created_at
+                assert tombstoned_backup.created_at == expected_created_at
+                assert tombstoned_backup.tombstoned_at == expected_tombstoned_at
+                assert job is not None
+                assert recovered_source is not None
+                assert recovered_source.state == "unavailable"
+                assert job.state == "archived"
+                assert not job.enabled
+                restore = Restore(
+                    backup_id=backup.id,
+                    destination=str(recovered.restore_roots[0] / "restored"),
+                    selection=[],
+                    overwrite_policy="fail",
+                )
+                db.add(restore)
+                await db.commit()
+            worker = Worker(recovered, owner="host-loss-restore")
+            try:
+                assert await worker.run_once()
+            finally:
+                await worker.engine.dispose()
+        finally:
+            await engine.dispose()
+
+    asyncio.run(restore_and_assert())
+    assert (recovered.restore_roots[0] / "restored" / "document.txt").read_bytes() == plaintext
+
+    payload = decrypt_bundle(bundle.read_bytes(), PASSPHRASE)
+    with pytest.raises(RecoveryBundleError, match="destination is not empty"):
+        asyncio.run(import_recovery_payload(recovered, payload))

tests/integration/test_repositories.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import Any, cast
 
 import httpx
 import pytest
 from backup_tool.api.app import create_app
 from backup_tool.config import Settings
+from backup_tool.db.models import RepositoryDataKeyEpoch
+from sqlalchemy import select
 
 
 @pytest.mark.asyncio
@@ -30,7 +33,7 @@ async def test_admin_can_create_and_inspect_allowlisted_repository(
 
     async with app.state.engine.begin() as connection:
         await connection.run_sync(Base.metadata.create_all)
-    transport = httpx.ASGITransport(app=app)
+    transport = httpx.ASGITransport(app=cast(Any, app))
     async with httpx.AsyncClient(transport=transport, base_url="https://test") as client:
         setup = await client.post(
             "/api/v2/setup", json={"username": "admin", "password": "a secure password"}
@@ -52,6 +55,29 @@ async def test_admin_can_create_and_inspect_allowlisted_repository(
         assert body["name"] == "main"
         assert body["format_version"] == 1
         assert (root / "main" / "repository.json").is_file()
+        encrypted = await client.post(
+            "/api/v2/repositories",
+            json={
+                "name": "encrypted",
+                "relative_path": "encrypted",
+                "compression": "none",
+                "encryption": "aes-256-gcm",
+            },
+            headers={"X-CSRF-Token": csrf},
+        )
+        assert encrypted.status_code == 201, encrypted.text
+        async with app.state.sessions() as db:
+            epochs = list(
+                (
+                    await db.scalars(
+                        select(RepositoryDataKeyEpoch).where(
+                            RepositoryDataKeyEpoch.repository_id == encrypted.json()["id"]
+                        )
+                    )
+                ).all()
+            )
+        assert len(epochs) == 1
+        assert epochs[0].state == "active"
         got = await client.get(f"/api/v2/repositories/{body['id']}")
         assert got.status_code == 200
         changed = await client.patch(

tests/integration/test_repository_safety.py

@@ -42,6 +42,7 @@ def test_partial_initialization_is_removed_on_publish_failure(tmp_path: Path) ->
         initialize(settings, "main", "none", "none")
     assert not (settings.repository_roots[0] / "main").exists()
     assert not list(settings.repository_roots[0].glob(".main.staging-*"))
+    assert not list((settings.data_dir / "repository-keys").glob("*"))
 
 
 @pytest.mark.parametrize("relative_path", ["/absolute", "../escape"])

tests/integration/test_repository_signing.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol, cast
+
+import httpx
+import pytest
+from backup_tool.api.app import create_app
+from backup_tool.config import Settings
+from backup_tool.db.models import Repository
+from backup_tool.repository import initialize
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+PASSWORD = "a secure password"
+
+
+class SignedRepository(Protocol):
+    repository_id: str
+    signing_key_id: str
+    signing_public_key: str
+
+
+def settings_for(tmp_path: Path) -> Settings:
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m6-test-master-key-material-32-bytes-minimum")
+    key.chmod(0o600)
+    repositories = tmp_path / "repositories"
+    repositories.mkdir()
+    return Settings(
+        data_dir=tmp_path,
+        database_url=f"sqlite+aiosqlite:///{tmp_path / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(tmp_path,),
+        restore_roots=(tmp_path,),
+        master_key_file=key,
+        min_free_bytes=1,
+    )
+
+
+def test_repository_initialization_creates_a_bound_ed25519_keypair(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+
+    initialized = cast(SignedRepository, initialize(settings, "main", "none", "none"))
+
+    key_path = settings.data_dir / "repository-keys" / f"{initialized.repository_id}.ed25519"
+    assert key_path.read_bytes()
+    assert key_path.stat().st_mode & 0o777 == 0o600
+    private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes())
+    public_key = private_key.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    assert initialized.signing_public_key == public_key.hex()
+    assert initialized.signing_key_id.startswith("ed25519-")
+
+
+@pytest.mark.asyncio
+async def test_repository_api_persists_its_bound_public_signing_key(
+    tmp_path: Path,
+) -> None:
+    settings = settings_for(tmp_path)
+    app = create_app(settings)
+    from backup_tool.db.models import Base
+
+    async with app.state.engine.begin() as connection:
+        await connection.run_sync(Base.metadata.create_all)
+    async with httpx.AsyncClient(
+        transport=httpx.ASGITransport(app=app), base_url="https://test"
+    ) as client:
+        assert (
+            await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+        ).status_code == 201
+        created = await client.post(
+            "/api/v2/repositories",
+            json={
+                "name": "main",
+                "relative_path": "main",
+                "compression": "none",
+                "encryption": "none",
+            },
+            headers={"X-CSRF-Token": client.cookies["backup_tool_csrf"]},
+        )
+        assert created.status_code == 201
+        async with app.state.sessions() as db:
+            repository = await db.get(Repository, created.json()["id"])
+
+    await app.state.engine.dispose()
+    assert repository is not None
+    assert repository.signing_key_id.startswith("ed25519-")
+    assert len(repository.signing_public_key) == 64

tests/integration/test_scheduler_live_sync.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+
+import httpx
+import pytest
+from backup_tool.config import Settings
+from backup_tool.db.models import NotificationDelivery, NotificationEvent, Schedule
+from backup_tool.scheduler import SchedulerService
+from sqlalchemy import select
+
+PASSWORD = "correct-horse-battery-staple"
+
+
+async def setup_job(client: httpx.AsyncClient, settings: Settings) -> tuple[dict[str, str], str]:
+    source_root = settings.local_source_roots[0] / "source"
+    source_root.mkdir()
+    headers = await login(client)
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={
+            "name": "repo",
+            "relative_path": "repo",
+            "compression": "none",
+            "encryption": "none",
+        },
+        headers=headers,
+    )
+    source = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "source",
+            "kind": "local",
+            "public_config": {"root": str(source_root)},
+        },
+        headers=headers,
+    )
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "job",
+            "source_id": source.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": True,
+        },
+        headers=headers,
+    )
+    assert job.status_code == 201
+    return headers, job.json()["id"]
+
+
+async def login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+@pytest.mark.asyncio
+async def test_schedule_role_delivery_and_live_crud_sync(
+    app_client: tuple[httpx.AsyncClient, Settings],
+) -> None:
+    client, settings = app_client
+    headers, job_id = await setup_job(client, settings)
+    subscription = await client.post(
+        "/api/v2/notifications/subscriptions",
+        json={
+            "channel": "email",
+            "event_filters": ["schedule.occurrence_enqueued"],
+            "destination": {"recipients": ["operator@example.test"]},
+        },
+        headers=headers,
+    )
+    assert subscription.status_code == 201
+    created = await client.post(
+        f"/api/v2/jobs/{job_id}/schedule",
+        json={"cron": "* * * * *", "timezone": "UTC"},
+        headers=headers,
+    )
+    assert created.status_code == 201
+    app = cast(Any, client._transport).app
+    async with app.state.sessions() as db:
+        schedule = await db.scalar(select(Schedule).where(Schedule.job_id == job_id))
+        assert schedule is not None
+        schedule.next_nominal_at = datetime.now(UTC) - timedelta(seconds=1)
+        await db.commit()
+    service = SchedulerService(settings)
+    try:
+        assert await service.run_once() == 1
+    finally:
+        await service.engine.dispose()
+    async with app.state.sessions() as db:
+        delivery = await db.scalar(
+            select(NotificationDelivery.id)
+            .join(NotificationEvent, NotificationDelivery.event_id == NotificationEvent.id)
+            .where(NotificationEvent.type == "schedule.occurrence_enqueued")
+            .limit(1)
+        )
+    assert delivery is not None
+
+    updated = await client.patch(
+        f"/api/v2/jobs/{job_id}/schedule",
+        json={"cron": "0 10 * * *", "timezone": "UTC", "enabled": False},
+        headers=headers,
+    )
+    assert updated.status_code == 200
+    assert updated.json()["next_nominal_at"] is None
+    # Deletion with historical executions is deliberately restricted; schedule
+    # delete behavior is covered before occurrence enqueue in the catalog test.

tests/integration/test_sources_jobs.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
+from typing import Any, cast
 
 import httpx
 import pytest
@@ -18,7 +19,6 @@ from backup_tool.execution import (
     recover_stale,
     request_cancellation,
 )
-from backup_tool.worker import Worker
 
 PASSWORD = "correct-horse-battery-staple"
 
@@ -58,7 +58,7 @@ async def test_local_source_probe_archive_and_repository_targeted_job(
 
     async with app.state.engine.begin() as connection:
         await connection.run_sync(Base.metadata.create_all)
-    transport = httpx.ASGITransport(app=app)
+    transport = httpx.ASGITransport(app=cast(Any, app))
     async with httpx.AsyncClient(transport=transport, base_url="https://test") as client:
         headers = await login(client)
         repository = await client.post(
@@ -108,6 +108,24 @@ async def test_local_source_probe_archive_and_repository_targeted_job(
         assert duplicate.status_code == 409
         assert duplicate.json()["code"] == "execution_active"
         execution_id = execution.json()["id"]
+        listed_sources = await client.get("/api/v2/sources", headers=headers)
+        assert listed_sources.status_code == 200
+        assert listed_sources.json()["items"] == [
+            {
+                "id": source_id,
+                "name": "local",
+                "kind": "local",
+                "state": "active",
+                "public_config": {"root": str(source_root)},
+            }
+        ]
+        listed_jobs = await client.get("/api/v2/jobs", headers=headers)
+        assert listed_jobs.status_code == 200
+        assert listed_jobs.json()["items"][0]["id"] == job.json()["id"]
+        assert listed_jobs.json()["items"][0]["schedule"] is None
+        listed_executions = await client.get("/api/v2/executions", headers=headers)
+        assert listed_executions.status_code == 200
+        assert listed_executions.json()["items"][0]["id"] == execution_id
         scoped_token = await client.post(
             "/api/v2/auth/tokens",
             json={"scopes": ["audit:read"], "expires_at": None},
@@ -252,7 +270,9 @@ async def test_execution_sse_replays_later_redacted_revision(tmp_path: Path) ->
 
 
 @pytest.mark.asyncio
-async def test_execution_events_preserve_progress_replay_and_recovery_order(tmp_path: Path) -> None:
+async def test_execution_events_preserve_progress_replay_and_recovery_order(
+    tmp_path: Path,
+) -> None:
     source_root = tmp_path / "sources"
     source_root.mkdir()
     data_dir = tmp_path / "data"
@@ -306,8 +326,14 @@ async def test_execution_events_preserve_progress_replay_and_recovery_order(tmp_
         execution = await enqueue(db, job.id)
         execution_id = execution.id
 
-    worker = Worker(settings, owner="ordering-worker")
-    assert await worker.run_once()
+    async with app.state.sessions() as db:
+        assert await claim(db, execution_id, "ordering-worker") is not None
+        execution = await db.get(Execution, execution_id)
+        assert execution is not None and execution.state == "preparing"
+        execution.state = "running"
+        execution.started_at = datetime.now(UTC)
+        await record_event(db, execution)
+        await db.commit()
     async with app.state.sessions() as db:
         execution = await db.get(Execution, execution_id)
         assert execution is not None and execution.state == "running"
@@ -383,7 +409,7 @@ async def test_local_source_rejects_unallowlisted_root(tmp_path: Path) -> None:
     async with app.state.engine.begin() as connection:
         await connection.run_sync(Base.metadata.create_all)
     async with httpx.AsyncClient(
-        transport=httpx.ASGITransport(app=app), base_url="https://test"
+        transport=httpx.ASGITransport(app=cast(Any, app)), base_url="https://test"
     ) as client:
         headers = await login(client)
         response = await client.post(

tests/integration/test_ssh_backup_restore.py

@@ -0,0 +1,197 @@
+"""Opt-in live forced-SFTP chroot coverage; all SSH keys are generated under tmp_path."""
+
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+import time
+from pathlib import Path
+
+import httpx
+import pytest
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Backup, Execution, Repository
+from backup_tool.snapshot import verify_published_snapshot
+from backup_tool.worker import Worker
+from sqlalchemy import select
+
+ROOT = Path(__file__).resolve().parents[2]
+PASSWORD = "correct-horse-battery-staple"
+
+
+def _enabled() -> bool:
+    return os.environ.get("BACKUP_TOOL_SSH_INTEGRATION") == "1"
+
+
+def _port() -> int:
+    with socket.socket() as listener:
+        listener.bind(("127.0.0.1", 0))
+        address = listener.getsockname()
+    if not isinstance(address, tuple) or not isinstance(address[1], int):
+        raise RuntimeError("could not allocate SSH fixture port")
+    return address[1]
+
+
+def _wait(port: int) -> None:
+    deadline = time.monotonic() + 60
+    while time.monotonic() < deadline:
+        try:
+            with socket.create_connection(("127.0.0.1", port), timeout=1):
+                return
+        except OSError:
+            time.sleep(0.25)
+    raise AssertionError("SSHD fixture did not become reachable")
+
+
+@pytest.fixture
+def sshd_fixture(tmp_path: Path):
+    fixture = tmp_path / "fixture"
+    host = fixture / "host"
+    source = fixture / "source"
+    host.mkdir(parents=True)
+    source.mkdir()
+    fixture.chmod(0o755)
+    host.chmod(0o755)
+    source.chmod(0o755)
+    private = fixture / "client"
+    for target in (host / "ssh_host_ed25519_key", private):
+        subprocess.run(
+            ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(target)],
+            check=True,
+        )
+    (host / "ssh_host_ed25519_key").chmod(0o644)
+    (fixture / "authorized_keys").write_text(private.with_suffix(".pub").read_text())
+    # Public keys are copied into a backup-owned 0600 tmpfs file at startup.
+    (fixture / "authorized_keys").chmod(0o644)
+    port = _port()
+    environment = os.environ | {
+        "SSH_FIXTURE_DIR": str(fixture),
+        "SSH_FIXTURE_PORT": str(port),
+        "COMPOSE_PROJECT_NAME": f"backup-tool-ssh-{os.getpid()}-{port}",
+    }
+    command = ["docker", "compose", "-f", "tests/compose.ssh.yaml"]
+    try:
+        subprocess.run([*command, "up", "--build", "-d"], cwd=ROOT, env=environment, check=True)
+        _wait(port)
+        host_key = " ".join(host.joinpath("ssh_host_ed25519_key.pub").read_text().split()[:2])
+        yield {
+            "port": port,
+            "key": private,
+            "host_key": host_key,
+            "source": source,
+            "env": environment,
+        }
+    finally:
+        subprocess.run(
+            [*command, "down", "--volumes", "--remove-orphans"],
+            cwd=ROOT,
+            env=environment,
+            check=False,
+        )
+
+
+async def _login(client: httpx.AsyncClient) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+@pytest.mark.skipif(not _enabled(), reason="set BACKUP_TOOL_SSH_INTEGRATION=1")
+@pytest.mark.asyncio
+async def test_forced_sftp_chroot_probe_backup_and_restore(app_client, sshd_fixture) -> None:
+    client, settings = app_client
+    source = sshd_fixture["source"]
+    (source / "nested").mkdir()
+    (source / "nested" / "hello.txt").write_text("hello ssh\n")
+    headers = await _login(client)
+    secret = await client.post(
+        "/api/v2/admin/secrets",
+        json={"purpose": "ssh_private_key", "value": sshd_fixture["key"].read_text()},
+        headers=headers,
+    )
+    assert secret.status_code == 201
+    remote = await client.post(
+        "/api/v2/sources",
+        json={
+            "name": "ssh",
+            "kind": "ssh",
+            "private_key_secret_id": secret.json()["id"],
+            "public_config": {
+                "hostname": "127.0.0.1",
+                "port": sshd_fixture["port"],
+                "username": "backup",
+                "host_key": sshd_fixture["host_key"],
+                "root": "/",
+            },
+        },
+        headers=headers,
+    )
+    assert remote.status_code == 201, remote.text
+    assert (
+        await client.post(f"/api/v2/sources/{remote.json()['id']}/probe", headers=headers)
+    ).json() == {"entry_count": 1}
+    repository = await client.post(
+        "/api/v2/repositories",
+        json={"name": "repo", "relative_path": "ssh"},
+        headers=headers,
+    )
+    job = await client.post(
+        "/api/v2/jobs",
+        json={
+            "name": "ssh-job",
+            "source_id": remote.json()["id"],
+            "repository_id": repository.json()["id"],
+            "requested_mode": "full",
+            "exclusions": [],
+            "retention": {},
+            "enabled": True,
+            "allow_empty": False,
+        },
+        headers=headers,
+    )
+    execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
+    worker = Worker(settings, owner="ssh-live")
+    try:
+        assert await worker.run_once()
+    finally:
+        await worker.engine.dispose()
+    engine = create_engine(settings)
+    try:
+        from sqlalchemy.ext.asyncio import async_sessionmaker
+
+        async with async_sessionmaker(engine, expire_on_commit=False)() as db:
+            stored = await db.get(Execution, execution.json()["id"])
+            backup = await db.scalar(
+                select(Backup).where(Backup.execution_id == execution.json()["id"])
+            )
+            stored_repository = await db.get(Repository, repository.json()["id"])
+    finally:
+        await engine.dispose()
+    assert stored is not None and stored.state == "committed", (
+        stored.operator_message if stored else None
+    )
+    assert backup is not None and stored_repository is not None
+    manifest = verify_published_snapshot(
+        Path(stored_repository.root),
+        Path(stored_repository.root) / "manifests" / f"{backup.manifest_id}.json",
+        stored_repository.signing_public_key,
+    )
+    assert any(entry["path"] == "data/nested/hello.txt" for entry in manifest["entries"])
+    destination = settings.restore_roots[0] / "ssh-restored"
+    restore = await client.post(
+        f"/api/v2/backups/{backup.id}/restores",
+        json={
+            "destination": str(destination),
+            "selection": ["data/nested"],
+            "overwrite_policy": "fail",
+        },
+        headers=headers,
+    )
+    assert restore.status_code == 202
+    restore_worker = Worker(settings, owner="ssh-restore")
+    try:
+        assert await restore_worker.run_once()
+    finally:
+        await restore_worker.engine.dispose()
+    assert (destination / "data" / "nested" / "hello.txt").read_text() == "hello ssh\n"

tests/integration/test_ssh_source_api.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import pytest
+
+PASSWORD = "correct horse battery staple"
+HOST_KEY = "ssh-ed25519 AQID"
+
+
+async def _setup_headers(client) -> dict[str, str]:
+    response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
+    assert response.status_code == 201
+    return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
+
+
+async def _create_secret(client, headers: dict[str, str], purpose: str) -> str:
+    response = await client.post(
+        "/api/v2/admin/secrets",
+        json={"purpose": purpose, "value": "PRIVATE-KEY-CANARY"},
+        headers=headers,
+    )
+    assert response.status_code == 201
+    assert "PRIVATE-KEY-CANARY" not in response.text
+    return str(response.json()["id"])
+
+
+def _source(secret_id: str) -> dict[str, object]:
+    return {
+        "name": "remote",
+        "kind": "ssh",
+        "public_config": {
+            "hostname": "backup.example.test",
+            "port": 22,
+            "username": "backup",
+            "host_key": HOST_KEY,
+            "root": "/",
+        },
+        "private_key_secret_id": secret_id,
+    }
+
+
+@pytest.mark.asyncio
+async def test_ssh_source_persists_only_safe_public_config_and_one_key_reference(
+    app_client,
+) -> None:
+    client, _ = app_client
+    headers = await _setup_headers(client)
+    secret_id = await _create_secret(client, headers, "ssh_private_key")
+
+    created = await client.post("/api/v2/sources", json=_source(secret_id), headers=headers)
+
+    assert created.status_code == 201
+    body = created.json()
+    assert body["kind"] == "ssh"
+    assert body["public_config"] == _source(secret_id)["public_config"]
+    assert "secret" not in body
+    assert "PRIVATE-KEY-CANARY" not in created.text
+    listed = await client.get("/api/v2/sources", headers=headers)
+    assert listed.status_code == 200
+    assert listed.json()["items"] == [body]
+    assert "PRIVATE-KEY-CANARY" not in listed.text
+    probe = await client.post(f"/api/v2/sources/{body['id']}/probe", headers=headers)
+    assert probe.status_code == 409
+    assert probe.json()["code"] == "source_probe_failed"
+
+
+@pytest.mark.asyncio
+async def test_ssh_source_requires_exactly_one_existing_private_key_secret(
+    app_client,
+) -> None:
+    client, _ = app_client
+    headers = await _setup_headers(client)
+    wrong_purpose = await _create_secret(client, headers, "ssh")
+    request = _source(wrong_purpose)
+
+    rejected_purpose = await client.post("/api/v2/sources", json=request, headers=headers)
+    assert rejected_purpose.status_code == 422
+    assert rejected_purpose.json()["code"] == "validation_failed"
+    assert "PRIVATE-KEY-CANARY" not in rejected_purpose.text
+
+    missing_secret = await client.post(
+        "/api/v2/sources",
+        json=_source("00000000-0000-0000-0000-000000000000"),
+        headers=headers,
+    )
+    assert missing_secret.status_code == 422
+    assert missing_secret.json()["code"] == "validation_failed"
+
+    valid_secret = await _create_secret(client, headers, "ssh_private_key")
+    extra_secret_reference = {
+        **_source(valid_secret),
+        "secret_refs": [valid_secret, wrong_purpose],
+    }
+    rejected_extra = await client.post(
+        "/api/v2/sources", json=extra_secret_reference, headers=headers
+    )
+    assert rejected_extra.status_code == 422
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {"name": "unsupported", "kind": "sftp", "public_config": {}},
+        {"name": "unsupported", "kind": "postgresql", "public_config": {}},
+        {"name": "unsupported", "kind": "mysql", "public_config": {}},
+        {"name": "unsupported", "kind": "shell", "public_config": {}},
+    ],
+)
+async def test_source_api_rejects_all_non_local_ssh_kinds(
+    app_client, payload: dict[str, object]
+) -> None:
+    client, _ = app_client
+    headers = await _setup_headers(client)
+
+    response = await client.post("/api/v2/sources", json=payload, headers=headers)
+
+    assert response.status_code == 422
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "public_config",
+    [
+        {
+            "hostname": "backup.example.test",
+            "port": 22,
+            "username": "backup",
+            "root": "/",
+        },
+        {
+            "hostname": "backup.example.test",
+            "port": 0,
+            "username": "backup",
+            "host_key": HOST_KEY,
+            "root": "/",
+        },
+        {
+            "hostname": "backup.example.test",
+            "port": 22,
+            "username": "backup",
+            "host_key": HOST_KEY,
+            "root": "/not-the-chroot",
+        },
+        {
+            "hostname": "backup.example.test",
+            "port": 22,
+            "username": "backup",
+            "host_key": HOST_KEY,
+            "root": "/",
+            "password": "not-supported",
+        },
+        {
+            "hostname": "backup.example.test",
+            "port": 22,
+            "username": "backup",
+            "host_key": HOST_KEY,
+            "root": "/",
+            "remote_command": "not-supported",
+        },
+    ],
+)
+async def test_ssh_source_rejects_noncanonical_or_unsupported_config(
+    app_client, public_config: dict[str, object]
+) -> None:
+    client, _ = app_client
+    headers = await _setup_headers(client)
+    secret_id = await _create_secret(client, headers, "ssh_private_key")
+    payload = _source(secret_id)
+    payload["public_config"] = public_config
+
+    response = await client.post("/api/v2/sources", json=payload, headers=headers)
+
+    assert response.status_code == 422

tests/security/test_container.py

@@ -0,0 +1,82 @@
+"""Static checks for the production Compose packaging slice."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from backup_tool.cli import build_parser
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_health_command_selects_an_explicit_runtime_role() -> None:
+    parsed = build_parser().parse_args(["health", "worker"])
+
+    assert parsed.role == "health"
+    assert parsed.health_role == "worker"
+
+
+def test_runtime_images_are_pinned_non_root_and_exclude_database_clients() -> None:
+    runtime = (ROOT / "Dockerfile").read_text()
+    proxy = (ROOT / "frontend" / "Dockerfile").read_text()
+
+    for dockerfile in (runtime, proxy):
+        from_lines = [line for line in dockerfile.splitlines() if line.startswith("FROM ")]
+        assert from_lines
+        assert all("@sha256:" in line for line in from_lines)
+
+    assert "USER backup-tool:backup-tool" in runtime
+    assert 'ENTRYPOINT ["backup-tool"]' in runtime
+    assert 'CMD ["web"]' in runtime
+    assert "USER 10001:0" in proxy
+    assert not re.search(r"\b(pg_dump|mysqldump|postgresql-client|mysql-client)\b", runtime)
+
+
+def test_compose_runs_one_isolated_role_per_service_without_reload() -> None:
+    compose = (ROOT / "docker-compose.yml").read_text()
+
+    for role in ("web", "scheduler", "worker", "migrate", "admin"):
+        assert f"  {role}:" in compose
+    for command in (
+        '["web"]',
+        '["scheduler"]',
+        '["worker"]',
+        '["migrate", "upgrade"]',
+        '["admin", "--help"]',
+    ):
+        assert command in compose
+    assert "--reload" not in compose
+    assert "backup-tool-runtime:/run/backup-tool" in compose
+    assert '["CMD", "backup-tool", "health", "web"]' in compose
+    assert '["CMD", "backup-tool", "health", "scheduler"]' in compose
+    assert '["CMD", "backup-tool", "health", "worker"]' in compose
+
+
+def test_proxy_is_the_only_published_endpoint_and_uses_same_origin_socket() -> None:
+    compose = (ROOT / "docker-compose.yml").read_text()
+    nginx = (ROOT / "frontend" / "nginx.conf").read_text()
+
+    assert compose.count("    ports:") == 1
+    assert '"127.0.0.1:${BACKUP_TOOL_PORT:-8080}:8080"' in compose
+    assert '"${BACKUP_TOOL_PORT:-8080}:8080"' not in compose
+    assert "server unix:/run/backup-tool/web.sock;" in nginx
+    assert "location /api/" in nginx
+    assert "location = /readyz" in nginx
+    assert "location = /livez" in nginx
+    assert "location = /metrics" in nginx
+    assert "proxy_pass http://backup_tool_web;" in nginx
+
+
+def test_operational_artifacts_cover_sbom_provenance_and_recovery() -> None:
+    assert (ROOT / "docs/release/m14-sbom.json").is_file()
+    assert (ROOT / "docs/release/m14-provenance.md").is_file()
+    for runbook in (
+        "metadata.md",
+        "repositories.md",
+        "keys.md",
+        "upgrade.md",
+        "disaster-recovery.md",
+        "observability.md",
+    ):
+        assert (ROOT / "docs/runbooks" / runbook).is_file()

tests/security/test_crypto_leakage.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+from backup_tool.security.repository_crypto import (
+    RepositoryKeyError,
+    decrypt_object,
+    encrypt_object,
+    object_aad,
+)
+
+
+def test_encrypted_object_hides_plaintext_and_rejects_tampering() -> None:
+    key = os.urandom(32)
+    aad = object_aad("repository", "epoch", "blob", "identity")
+    stored = encrypt_object(key, aad, b"secret-content")
+    assert b"secret-content" not in stored
+    assert decrypt_object(key, aad, stored) == b"secret-content"
+    with pytest.raises(RepositoryKeyError):
+        decrypt_object(key, aad, stored[:-1] + bytes([stored[-1] ^ 1]))

tests/security/test_recovery_bundle.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+
+import pytest
+from alembic import command
+from backup_tool.cli import build_alembic_config, build_parser
+from backup_tool.cli import main as cli_main
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Repository, RepositoryDataKeyEpoch
+from backup_tool.repository import initialize
+from backup_tool.security.recovery_bundle import (
+    RecoveryBundleError,
+    RecoveryBundlePathError,
+    decrypt_bundle,
+    encrypt_bundle,
+    write_bundle_exclusive,
+)
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from tests.conftest import make_settings
+
+PASSPHRASE = b"correct horse battery staple"
+
+
+def _passphrase_fd(value: bytes) -> int:
+    read_fd, write_fd = os.pipe()
+    os.write(write_fd, value + b"\n")
+    os.close(write_fd)
+    return read_fd
+
+
+def test_recovery_bundle_rejects_wrong_passphrase_tampering_and_invalid_kdf() -> None:
+    bundle = encrypt_bundle({"catalog": {"version": 1}, "keys": []}, PASSPHRASE)
+    tampered = bundle[:-1] + bytes([bundle[-1] ^ 1])
+    unsupported_kdf = bundle[:6] + b"\x02" + bundle[7:]
+
+    for encoded, passphrase in (
+        (bundle, b"wrong passphrase"),
+        (tampered, PASSPHRASE),
+        (unsupported_kdf, PASSPHRASE),
+    ):
+        with pytest.raises(RecoveryBundleError, match="^recovery bundle is invalid$"):
+            decrypt_bundle(encoded, passphrase)
+
+
+def test_recovery_bundle_output_is_exclusive_and_does_not_follow_symlinks(
+    tmp_path: Path,
+) -> None:
+    bundle = encrypt_bundle({"catalog": {"version": 1}, "keys": []}, PASSPHRASE)
+    existing = tmp_path / "existing.btrec"
+    existing.write_bytes(b"keep")
+
+    with pytest.raises(RecoveryBundlePathError, match="^recovery bundle output is unsafe$"):
+        write_bundle_exclusive(existing, bundle)
+    assert existing.read_bytes() == b"keep"
+
+    target = tmp_path / "target.btrec"
+    target.write_bytes(b"keep")
+    symlink = tmp_path / "link.btrec"
+    symlink.symlink_to(target)
+    with pytest.raises(RecoveryBundlePathError, match="^recovery bundle output is unsafe$"):
+        write_bundle_exclusive(symlink, bundle)
+    assert target.read_bytes() == b"keep"
+
+
+def test_cli_recovery_export_validate_uses_fd_and_hides_plaintext_keys(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    settings = make_settings(tmp_path)
+    command.upgrade(build_alembic_config(settings), "head")
+    initialized = initialize(settings, "encrypted", "none", "aes-256-gcm")
+    assert initialized.data_key_id is not None
+    assert initialized.data_key_path is not None
+
+    async def create_repository() -> None:
+        engine = create_engine(settings)
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        try:
+            async with sessions() as db:
+                repository = Repository(
+                    name="encrypted",
+                    root=str(initialized.root),
+                    format_version=initialized.format_version,
+                    compression=initialized.compression,
+                    encryption=initialized.encryption,
+                    signing_key_id=initialized.signing_key_id,
+                    signing_public_key=initialized.signing_public_key,
+                    active_data_key_id=initialized.data_key_id,
+                )
+                db.add(repository)
+                await db.flush()
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=initialized.data_key_id,
+                        state="active",
+                    )
+                )
+                await db.commit()
+        finally:
+            await engine.dispose()
+
+    asyncio.run(create_repository())
+    output = tmp_path / "recovery.btrec"
+    export_fd = _passphrase_fd(PASSPHRASE)
+    try:
+        assert (
+            cli_main(
+                [
+                    "admin",
+                    "recovery",
+                    "export",
+                    "--output",
+                    str(output),
+                    "--passphrase-fd",
+                    str(export_fd),
+                ],
+                settings=settings,
+            )
+            == 0
+        )
+    finally:
+        os.close(export_fd)
+    export_output = capsys.readouterr().out
+    bundle = output.read_bytes()
+    assert bundle.startswith(b"BTREC\x01")
+    assert initialized.data_key_path.read_bytes() not in bundle
+    signing_key_path = (
+        settings.data_dir / "repository-keys" / f"{initialized.repository_id}.ed25519"
+    )
+    assert signing_key_path.read_bytes() not in bundle
+    assert "key" not in export_output.lower()
+
+    validate_fd = _passphrase_fd(PASSPHRASE)
+    try:
+        assert (
+            cli_main(
+                [
+                    "admin",
+                    "recovery",
+                    "validate",
+                    "--input",
+                    str(output),
+                    "--passphrase-fd",
+                    str(validate_fd),
+                ],
+                settings=settings,
+            )
+            == 0
+        )
+    finally:
+        os.close(validate_fd)
+    assert capsys.readouterr().out == '{"repositories": 1, "status": "valid"}\n'
+
+    parser = build_parser()
+    with pytest.raises(SystemExit):
+        parser.parse_args(["admin", "recovery", "export", "--passphrase", "not-allowed"])

tests/security/test_restore_manifest_entries.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import copy
+import importlib
+
+import pytest
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+
+
+def file_entry(path: str = "data.txt") -> dict[str, object]:
+    return {
+        "path": path,
+        "type": "file",
+        "size": 1,
+        "blob_digest": "a" * 64,
+        "mode": 0o600,
+        "mtime_ns": 0,
+        "link_target": None,
+        "metadata_support": ["mode", "mtime_ns"],
+    }
+
+
+@pytest.mark.parametrize(
+    "entries",
+    [
+        [file_entry("../escape")],
+        [file_entry("/absolute")],
+        [file_entry("windows\\escape")],
+        [file_entry(), file_entry()],
+        [file_entry("file"), file_entry("file/child")],
+        [
+            {
+                **file_entry("link"),
+                "type": "symlink",
+                "size": 0,
+                "blob_digest": None,
+                "link_target": "../escape",
+            }
+        ],
+    ],
+)
+def test_restore_rejects_semantically_unsafe_signed_manifest_entries(
+    entries: list[dict[str, object]],
+) -> None:
+    manifest = {"entries": copy.deepcopy(entries)}
+
+    with pytest.raises(snapshot.SnapshotIntegrityError):
+        snapshot._safe_restore_entries(manifest)

tests/security/test_restore_paths.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import importlib
+import stat
+from pathlib import Path
+
+import pytest
+from backup_tool.config import Settings
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+
+
+def settings_for(tmp_path: Path) -> Settings:
+    key = tmp_path / "master.key"
+    key.write_bytes(b"m6-restore-path-test-master-key-material")
+    key.chmod(0o600)
+    repositories = tmp_path / "repositories"
+    sources = tmp_path / "sources"
+    restores = tmp_path / "restores"
+    for directory in (repositories, sources, restores):
+        directory.mkdir()
+    return Settings(
+        data_dir=tmp_path,
+        database_url=f"sqlite+aiosqlite:///{tmp_path / 'metadata.db'}",
+        repository_roots=(repositories,),
+        local_source_roots=(sources,),
+        restore_roots=(restores,),
+        master_key_file=key,
+        min_free_bytes=1,
+    )
+
+
+@pytest.mark.parametrize("name", ["existing", "outside", "root"])
+def test_restore_destination_rejects_existing_or_outside_paths(tmp_path: Path, name: str) -> None:
+    settings = settings_for(tmp_path)
+    root = settings.restore_roots[0]
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    existing = root / "existing"
+    existing.mkdir()
+    destinations = {
+        "existing": existing,
+        "outside": outside / "restore",
+        "root": root,
+    }
+
+    with pytest.raises(snapshot.SnapshotError):
+        snapshot.validate_restore_destination(settings, str(destinations[name]))
+
+
+def test_restore_strips_special_permission_bits(tmp_path: Path) -> None:
+    target = tmp_path / "restored"
+    target.write_bytes(b"content")
+
+    snapshot._apply_metadata(
+        target,
+        {"metadata_support": ["mode"], "mode": 0o7777},
+    )
+
+    mode = stat.S_IMODE(target.stat().st_mode)
+    assert mode == 0o777
+    assert mode & (stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX) == 0
+
+
+def test_restore_destination_rejects_a_symlinked_parent(tmp_path: Path) -> None:
+    settings = settings_for(tmp_path)
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    (settings.restore_roots[0] / "linked").symlink_to(outside, target_is_directory=True)
+
+    with pytest.raises(snapshot.SnapshotError):
+        snapshot.validate_restore_destination(
+            settings, str(settings.restore_roots[0] / "linked" / "restore")
+        )

tests/security/test_webhook_ssrf.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.security.ssrf import (
+    SSRFError,
+    resolve_public_addresses,
+    validate_webhook_url,
+)
+
+
+@pytest.mark.asyncio
+async def test_private_or_mixed_answers_are_rejected() -> None:
+    async def private(_host: str, _port: int) -> tuple[str, ...]:
+        return ("8.8.8.8", "127.0.0.1")
+
+    with pytest.raises(SSRFError, match="non-public"):
+        await resolve_public_addresses("hooks.example.test", 443, private)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "https://127.0.0.1/callback",
+        "https://user:pass@hooks.example.test/callback",
+        "https://hooks.example.test/callback#fragment",
+        "ftp://hooks.example.test/callback",
+        "https://hooks.example.test:22/callback",
+    ],
+)
+def test_webhook_url_rejects_bypasses(value: str) -> None:
+    with pytest.raises(SSRFError):
+        validate_webhook_url(value)

tests/ssh-fixture/Dockerfile

@@ -0,0 +1,18 @@
+FROM debian:12-slim
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y openssh-server \
+    && rm -rf /var/lib/apt/lists/* \
+    && (getent group backup >/dev/null || groupadd --system backup) \
+    && (id backup >/dev/null 2>&1 || useradd --system --gid backup --home-dir /home/backup --shell /usr/sbin/nologin backup) \
+    && install -d -o root -g root -m 0755 /home/backup \
+    && install -d -o backup -g backup -m 0700 /home/backup/.ssh \
+    && install -d -o backup -g backup -m 0755 /home/backup/data \
+    && install -d -o root -g root -m 0755 /run/sshd
+
+COPY sshd_config /etc/ssh/sshd_config
+COPY entrypoint.sh /usr/local/bin/fixture-sshd
+RUN chmod 0755 /usr/local/bin/fixture-sshd
+
+EXPOSE 2222
+CMD ["/usr/local/bin/fixture-sshd"]

tests/ssh-fixture/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+# The ephemeral host key is bind-mounted from the test host and can therefore
+# have an untrusted numeric owner. Copy it into the root-owned tmpfs before
+# sshd checks host-key ownership and permissions.
+install -o root -g root -m 0600 /fixture/host/ssh_host_ed25519_key /run/sshd/ssh_host_ed25519_key
+# Authorized keys are public material; root ownership prevents fixture-user mutation.
+install -o root -g root -m 0644 /fixture/authorized_keys /run/sshd/authorized_keys
+exec /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config

tests/ssh-fixture/sshd_config

@@ -0,0 +1,28 @@
+Port 2222
+ListenAddress 0.0.0.0
+HostKey /run/sshd/ssh_host_ed25519_key
+PidFile /run/sshd/sshd.pid
+AuthorizedKeysFile /run/sshd/authorized_keys
+UsePAM no
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+ChallengeResponseAuthentication no
+PermitRootLogin no
+PermitEmptyPasswords no
+PubkeyAuthentication yes
+PermitUserEnvironment no
+AllowTcpForwarding no
+X11Forwarding no
+PermitTunnel no
+PermitTTY no
+GatewayPorts no
+AllowAgentForwarding no
+LogLevel VERBOSE
+Subsystem sftp internal-sftp
+
+Match User backup
+    ChrootDirectory /home/backup
+    ForceCommand internal-sftp
+    AllowTcpForwarding no
+    X11Forwarding no
+    PermitTTY no

tests/unit/test_cron_dst.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+from backup_tool.scheduler import ScheduleError, next_nominal
+
+
+def test_five_field_cron_returns_utc_nominal_time() -> None:
+    result = next_nominal("0 9 * * *", "America/New_York", datetime(2026, 1, 1, tzinfo=UTC))
+    assert result.tzinfo is UTC
+    assert result.hour == 14
+
+
+@pytest.mark.parametrize(("cron", "timezone"), [("* * * * * *", "UTC"), ("* * * * *", "Nope/Zone")])
+def test_invalid_cron_or_timezone_is_rejected(cron: str, timezone: str) -> None:
+    with pytest.raises(ScheduleError):
+        next_nominal(cron, timezone)

tests/unit/test_exclusions.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.exclusions import ExclusionError, matches
+
+
+@pytest.mark.parametrize(
+    ("path", "patterns", "expected"),
+    [
+        ("notes.tmp", ["*.tmp"], True),
+        ("nested/notes.tmp", ["*.tmp"], True),
+        ("cache/item.txt", ["cache/"], True),
+        ("cache/keep.txt", ["cache/", "!cache/keep.txt"], False),
+        ("data/keep.txt", ["data/**"], True),
+        ("data.txt", ["data/**"], False),
+    ],
+)
+def test_gitignore_like_exclusions(path: str, patterns: list[str], expected: bool) -> None:
+    assert matches(path, patterns) is expected
+
+
+@pytest.mark.parametrize("value", ["../escape", "/absolute", "windows\\path", ""])
+def test_exclusions_reject_unsafe_paths(value: str) -> None:
+    with pytest.raises(ExclusionError):
+        matches("safe.txt", [value])

tests/unit/test_execution_transitions.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.execution import TransitionError, transition
+
+
+@pytest.mark.parametrize(
+    ("current", "target"),
+    [
+        ("queued", "preparing"),
+        ("queued", "cancelled"),
+        ("queued", "failed"),
+        ("preparing", "running"),
+        ("preparing", "cancelling"),
+        ("preparing", "failed"),
+        ("running", "verifying"),
+        ("running", "cancelling"),
+        ("running", "failed"),
+        ("verifying", "committed"),
+        ("verifying", "failed"),
+        ("cancelling", "cancelled"),
+        ("cancelling", "failed"),
+    ],
+)
+def test_all_legal_execution_transitions_are_accepted(current: str, target: str) -> None:
+    assert transition(current, target) == target
+
+
+@pytest.mark.parametrize(
+    ("current", "target"),
+    [
+        ("queued", "running"),
+        ("preparing", "queued"),
+        ("running", "preparing"),
+        ("verifying", "cancelling"),
+        ("cancelling", "running"),
+        ("committed", "cancelled"),
+        ("cancelled", "queued"),
+        ("failed", "queued"),
+        ("unknown", "queued"),
+    ],
+)
+def test_invalid_or_backward_execution_transitions_are_rejected(current: str, target: str) -> None:
+    with pytest.raises(TransitionError) as raised:
+        transition(current, target)
+
+    assert raised.value.code == "invalid_transition"

tests/unit/test_restore_selection.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import importlib
+
+import pytest
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+
+
+def entry(path: str, kind: str) -> dict[str, object]:
+    return {"path": path, "type": kind}
+
+
+def test_restore_selection_includes_selected_descendants_and_ancestors() -> None:
+    entries = [
+        entry("top", "directory"),
+        entry("top/keep", "directory"),
+        entry("top/keep/file.txt", "file"),
+        entry("top/drop.txt", "file"),
+    ]
+
+    selected = snapshot._select_restore_entries(entries, ["top/keep"])
+
+    assert [item["path"] for item in selected] == [
+        "top",
+        "top/keep",
+        "top/keep/file.txt",
+    ]
+
+
+@pytest.mark.parametrize("selection", [["../escape"], ["/absolute"], ["a\\b"], [""]])
+def test_restore_selection_rejects_unsafe_paths(selection: list[str]) -> None:
+    with pytest.raises(snapshot.SnapshotError):
+        snapshot._select_restore_entries([entry("safe", "file")], selection)

tests/unit/test_retention.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from backup_tool.retention import RetentionError, RetentionPolicy, retained_ids
+
+
+@dataclass
+class Backup:
+    id: str
+    created_at: datetime
+    pinned: bool = False
+    tombstoned_at: datetime | None = None
+
+
+def test_retention_is_union_and_protects_newest() -> None:
+    now = datetime(2026, 7, 29, tzinfo=UTC)
+    backups = [Backup(str(index), now - timedelta(days=index)) for index in range(6)]
+    backups[-1].pinned = True
+
+    kept = retained_ids(backups, RetentionPolicy(keep_last=2, keep_daily=3), now)
+
+    assert {"0", "1", "2", "5"} <= kept
+
+
+def test_newest_is_kept_even_when_policy_is_zero() -> None:
+    now = datetime(2026, 7, 29, tzinfo=UTC)
+    assert retained_ids([Backup("new", now)], RetentionPolicy(keep_last=0), now) == {"new"}
+
+
+@pytest.mark.parametrize("policy", [{"keep_last": -1}, {"unknown": 1}, {"keep_days": True}])
+def test_invalid_retention_policy_is_rejected(policy: dict[str, object]) -> None:
+    with pytest.raises(RetentionError):
+        RetentionPolicy.from_dict(policy)

tests/unit/test_ssh_adapter.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import io
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+
+import paramiko
+import pytest
+from backup_tool.adapters import SourceError
+from backup_tool.ssh_adapter import SSHAdapter, load_private_key
+from backup_tool.ssh_source import SSHSourcePublicConfig
+
+from tests.conftest import make_settings
+
+
+@dataclass
+class Attributes:
+    filename: str
+    st_mode: int
+    st_size: int = 0
+    st_mtime: int = 1
+
+
+class ServerKey:
+    def __init__(self, algorithm: str = "ssh-ed25519", encoded: str = "AQID") -> None:
+        self.algorithm = algorithm
+        self.encoded = encoded
+
+    def get_name(self) -> str:
+        return self.algorithm
+
+    def get_base64(self) -> str:
+        return self.encoded
+
+
+class Transport:
+    def __init__(self, key: ServerKey) -> None:
+        self.key = key
+        self.events: list[str] = []
+        self.closed = False
+
+    def start_client(self, *, timeout: float) -> None:
+        self.events.append("start")
+
+    def get_remote_server_key(self) -> ServerKey:
+        self.events.append("host_key")
+        return self.key
+
+    def auth_publickey(self, username: str, private_key: object) -> None:
+        self.events.append("auth")
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class Channel:
+    def __init__(self, events: list[str]) -> None:
+        self.events = events
+
+    def settimeout(self, timeout: float) -> None:
+        self.events.append("timeout")
+
+
+class Handle:
+    def __init__(self, chunks: list[bytes]) -> None:
+        self.chunks = chunks
+        self.closed = False
+
+    def read(self, size: int) -> bytes:
+        assert size == 4096
+        return self.chunks.pop(0) if self.chunks else b""
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class SFTP:
+    def __init__(self, events: list[str], entries: list[Attributes]) -> None:
+        self.events = events
+        self.entries = entries
+        self.handle = Handle([b"one", b"two"])
+        self.closed = False
+
+    def get_channel(self) -> Channel:
+        return Channel(self.events)
+
+    def listdir_iter(self, path: str, *, read_aheads: int):
+        self.events.append(f"list:{path}:{read_aheads}")
+        return iter(self.entries)
+
+    def lstat(self, path: str) -> Attributes:
+        self.events.append(f"lstat:{path}")
+        return Attributes("file", stat.S_IFREG | 0o640, 6, 1)
+
+    def open(self, path: str, mode: str, bufsize: int) -> Handle:
+        self.events.append(f"open:{path}:{mode}:{bufsize}")
+        return self.handle
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def config() -> SSHSourcePublicConfig:
+    return SSHSourcePublicConfig(
+        hostname="backup.example.test",
+        port=22,
+        username="backup",
+        host_key="ssh-ed25519 AQID",
+        root="/",
+    )
+
+
+def adapter(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, transport: Transport, sftp: SFTP
+) -> SSHAdapter:
+    monkeypatch.setattr("backup_tool.ssh_adapter.load_private_key", lambda _: object())
+    settings = make_settings(tmp_path).model_copy(update={"ssh_read_chunk_bytes": 4096})
+    return SSHAdapter(
+        config(),
+        "private-key-is-never-sent-to-a-log",
+        settings,
+        transport_factory=lambda *_: transport,
+        sftp_factory=lambda _: sftp,
+    )
+
+
+@pytest.mark.asyncio
+async def test_host_pin_mismatch_never_authenticates_or_opens_sftp(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    transport = Transport(ServerKey(encoded="BAUG"))
+    sftp = SFTP(transport.events, [])
+    reader = adapter(tmp_path, monkeypatch, transport, sftp)
+
+    with pytest.raises(SourceError, match="host key") as error:
+        await reader.probe()
+
+    assert error.value.reason_code == "source_trust"
+    assert transport.events == ["start", "host_key"]
+    assert transport.closed
+    assert "timeout" not in sftp.events
+    assert not any(event.startswith("list:") for event in sftp.events)
+
+
+@pytest.mark.asyncio
+async def test_pinned_transport_authenticates_before_sftp_and_streams_bounded_reads(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    transport = Transport(ServerKey())
+    sftp = SFTP(transport.events, [Attributes("file", stat.S_IFREG | 0o640, 6, 1)])
+    reader = adapter(tmp_path, monkeypatch, transport, sftp)
+
+    entries = [entry async for entry in reader.enumerate_entries()]
+    content = b"".join([chunk async for chunk in reader.open_content("file")])
+    await reader.close()
+
+    assert entries[0].path == "file"
+    assert content == b"onetwo"
+    assert transport.events.index("host_key") < transport.events.index("auth")
+    assert transport.events.index("auth") < transport.events.index("timeout")
+    assert "list:/:32" in sftp.events
+    assert "open:/file:rb:4096" in sftp.events
+    assert sftp.handle.closed and sftp.closed and transport.closed
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("mode", [stat.S_IFLNK | 0o777, stat.S_IFIFO | 0o600])
+async def test_sftp_rejects_symlinks_and_special_entries(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: int
+) -> None:
+    transport = Transport(ServerKey())
+    sftp = SFTP(transport.events, [Attributes("unsafe", mode)])
+    reader = adapter(tmp_path, monkeypatch, transport, sftp)
+
+    with pytest.raises(SourceError, match="symlink|unsupported"):
+        await anext(reader.enumerate_entries())
+
+    assert "auth" in transport.events
+    await reader.close()
+
+
+def test_private_key_loader_rejects_short_rsa_and_accepts_strong_rsa() -> None:
+    short = paramiko.RSAKey.generate(2048)
+    strong = paramiko.RSAKey.generate(3072)
+    short_buffer = io.StringIO()
+    strong_buffer = io.StringIO()
+    short.write_private_key(short_buffer)
+    strong.write_private_key(strong_buffer)
+
+    with pytest.raises(SourceError, match="algorithm") as error:
+        load_private_key(short_buffer.getvalue())
+    assert error.value.reason_code == "source_auth"
+    loaded = load_private_key(strong_buffer.getvalue())
+    assert isinstance(loaded, paramiko.RSAKey)

tests/unit/test_ssh_source_config.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import pytest
+from backup_tool.ssh_source import SSHSourcePublicConfig
+from pydantic import ValidationError
+
+VALID_CONFIG = {
+    "hostname": "backup.example.test",
+    "port": 22,
+    "username": "backup",
+    "host_key": "ssh-ed25519 AQID",
+    "root": "/",
+}
+
+
+def test_ssh_source_config_is_closed_and_canonical() -> None:
+    config = SSHSourcePublicConfig.model_validate(VALID_CONFIG)
+
+    assert config.model_dump() == VALID_CONFIG
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("hostname", "backup user@example.test"),
+        ("hostname", "ssh://backup.example.test"),
+        ("port", 0),
+        ("port", 65536),
+        ("username", "backup user"),
+        ("username", "backup/root"),
+        ("host_key", "ssh-ed25519 not-base64!"),
+        ("host_key", "ssh-rsa AQID"),
+        ("root", "/data"),
+    ],
+)
+def test_ssh_source_config_rejects_invalid_public_values(field: str, value: object) -> None:
+    invalid = {**VALID_CONFIG, field: value}
+
+    with pytest.raises(ValidationError):
+        SSHSourcePublicConfig.model_validate(invalid)
+
+
+def test_ssh_source_config_rejects_non_public_connection_options() -> None:
+    with pytest.raises(ValidationError):
+        SSHSourcePublicConfig.model_validate({**VALID_CONFIG, "password": "not-allowed"})
+
+    with pytest.raises(ValidationError):
+        SSHSourcePublicConfig.model_validate({**VALID_CONFIG, "remote_command": "not-allowed"})

tests/unit/test_webhook_signing.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from backup_tool.notifications.webhook import (
+    SigningMaterial,
+    canonical_signing_input,
+    signatures,
+    webhook_headers,
+)
+
+
+def test_signature_is_stable_and_dual_key_versioned() -> None:
+    body = b'{"id":"018f"}'
+    timestamp = "2026-07-30T00:00:00+00:00"
+    keys = [
+        SigningMaterial("key-a", 1, "old-secret"),
+        SigningMaterial("key-b", 2, "new-secret"),
+    ]
+    values = signatures(timestamp, body, keys)
+    assert len(values) == 2
+    assert "key_id=key-a" in values[0] and "key_version=1" in values[0]
+    assert "key_id=key-b" in values[1] and "key_version=2" in values[1]
+    assert values == signatures(timestamp, body, keys)
+    assert canonical_signing_input(timestamp, body).endswith(body)
+    headers = webhook_headers("event-id", "execution.queued", timestamp, body, keys)
+    receiver_headers = {key: value for key, value in headers if key != "X-Backup-Signature"}
+    receiver_signatures = [value for key, value in headers if key == "X-Backup-Signature"]
+    assert receiver_headers["X-Backup-Event-ID"] == "event-id"
+    assert receiver_headers["X-Backup-Event-Type"] == "execution.queued"
+    assert receiver_headers["X-Backup-Timestamp"] == timestamp
+    assert receiver_signatures == values