feat(v2): complete v2 reimplementation

backend/alembic/versions/0004_repository_signing_keys.py

@@ -0,0 +1,29 @@
+"""bind repositories to manifest signing public keys
+
+Revision ID: 0004_repository_signing_keys
+Revises: 0003_execution_events
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0004_repository_signing_keys"
+down_revision = "0003_execution_events"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("repositories") as batch:
+        batch.add_column(
+            sa.Column("signing_key_id", sa.String(length=64), nullable=False, server_default="")
+        )
+        batch.add_column(
+            sa.Column("signing_public_key", sa.String(length=64), nullable=False, server_default="")
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("repositories") as batch:
+        batch.drop_column("signing_public_key")
+        batch.drop_column("signing_key_id")

backend/alembic/versions/0005_restore_dry_run.py

@@ -0,0 +1,23 @@
+"""persist restore dry-run intent
+
+Revision ID: 0005_restore_dry_run
+Revises: 0004_repository_signing_keys
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0005_restore_dry_run"
+down_revision = "0004_repository_signing_keys"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("restores") as batch:
+        batch.add_column(sa.Column("dry_run", sa.Boolean(), nullable=False, server_default="0"))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("restores") as batch:
+        batch.drop_column("dry_run")

backend/alembic/versions/0006_local_sources_only.py

@@ -0,0 +1,43 @@
+"""restrict persisted sources to local
+
+Revision ID: 0006_local_sources_only
+Revises: 0005_restore_dry_run
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0006_local_sources_only"
+down_revision = "0005_restore_dry_run"
+branch_labels = None
+depends_on = None
+
+
+def _reject_nonlocal_sources() -> None:
+    connection = op.get_bind()
+    sources = sa.table("sources", sa.column("kind"))
+    count = connection.scalar(
+        sa.select(sa.func.count()).select_from(sources).where(sources.c.kind != "local")
+    )
+    if count is None:
+        raise RuntimeError("Cannot inspect persisted source kinds before migration.")
+    if count:
+        raise RuntimeError(
+            "Cannot restrict sources to local: "
+            f"found {count} non-local source row(s). Remove or migrate them before upgrading."
+        )
+
+
+def upgrade() -> None:
+    _reject_nonlocal_sources()
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(
+            op.f("ck_sources_kind"), "kind IN ('local','sftp','postgresql','mysql')"
+        )

backend/alembic/versions/0007_repository_data_key_epochs.py

@@ -0,0 +1,87 @@
+"""add repository data key epochs
+
+Revision ID: 0007_repository_data_key_epochs
+Revises: 0006_local_sources_only
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0007_repository_data_key_epochs"
+down_revision = "0006_local_sources_only"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("repositories", sa.Column("active_data_key_id", sa.String(36), nullable=True))
+    op.add_column("backups", sa.Column("data_key_id", sa.String(36), nullable=True))
+    op.create_table(
+        "repository_data_key_epochs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "repository_id",
+            sa.String(36),
+            sa.ForeignKey("repositories.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("key_id", sa.String(36), nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("retired_at", sa.DateTime(timezone=True)),
+        sa.UniqueConstraint(
+            "repository_id", "key_id", name="uq_repository_data_key_epochs_repository_key_epoch"
+        ),
+        sa.CheckConstraint(
+            "state IN ('active','retired')",
+            name="ck_repository_data_key_epochs_repository_data_key_epoch_state",
+        ),
+    )
+    op.create_index(
+        "ix_repository_data_key_epochs_repository_id",
+        "repository_data_key_epochs",
+        ["repository_id"],
+    )
+    op.create_index(
+        "uq_repository_data_key_epochs_active",
+        "repository_data_key_epochs",
+        ["repository_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'active'"),
+    )
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    epochs = sa.table("repository_data_key_epochs")
+    repositories = sa.table("repositories", sa.column("active_data_key_id"))
+    backups = sa.table("backups", sa.column("data_key_id"))
+    epoch_count = connection.scalar(sa.select(sa.func.count()).select_from(epochs))
+    active_key_count = connection.scalar(
+        sa.select(sa.func.count())
+        .select_from(repositories)
+        .where(repositories.c.active_data_key_id.is_not(None))
+    )
+    backup_key_count = connection.scalar(
+        sa.select(sa.func.count()).select_from(backups).where(backups.c.data_key_id.is_not(None))
+    )
+    if epoch_count or active_key_count or backup_key_count:
+        raise RuntimeError("cannot downgrade while repository data key metadata exists")
+    op.drop_index("uq_repository_data_key_epochs_active", table_name="repository_data_key_epochs")
+    op.drop_index(
+        "ix_repository_data_key_epochs_repository_id", table_name="repository_data_key_epochs"
+    )
+    op.drop_table("repository_data_key_epochs")
+    op.drop_column("backups", "data_key_id")
+    op.drop_column("repositories", "active_data_key_id")

backend/alembic/versions/0008_notification_outbox.py

@@ -0,0 +1,445 @@
+"""replace notification attempt stub with a durable M12 outbox
+
+Revision ID: 0008_notification_outbox
+Revises: 0007_repository_data_key_epochs
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+import sqlalchemy as sa
+from alembic import op
+from backup_tool.ids import new_uuid7
+
+revision = "0008_notification_outbox"
+down_revision = "0007_repository_data_key_epochs"
+branch_labels = None
+depends_on = None
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _parameterized_execute(connection: sa.Connection, statement: object) -> Any:
+    """Execute only SQLAlchemy Core statements, never dynamic SQL strings."""
+    database = connection.execution_options()
+    return database.execute(statement)  # type: ignore[arg-type]
+
+
+def upgrade() -> None:
+    connection = op.get_bind()
+    # SQLite batch-rebuilds notification_subscriptions. Its old delivery table
+    # references this table, so enforcement must be suspended for this migration
+    # only while the legacy rows are copied into the replacement outbox shape.
+    if connection.dialect.name == "sqlite":
+        connection.exec_driver_sql("PRAGMA foreign_keys=OFF")
+    # Extend the existing subscription rows first: pre-M12 rows stay disabled until
+    # an operator explicitly configures a new credential/key.
+    with op.batch_alter_table("notification_subscriptions") as batch:
+        batch.add_column(
+            sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60")
+        )
+        batch.add_column(sa.Column("rate_tokens", sa.Float(), nullable=False, server_default="60"))
+        batch.add_column(sa.Column("rate_updated_at", sa.DateTime(timezone=True), nullable=True))
+        batch.add_column(sa.Column("revision", sa.Integer(), nullable=False, server_default="1"))
+        batch.create_check_constraint("rate_positive", "rate_limit_per_minute > 0")
+        batch.create_check_constraint("rate_tokens_nonnegative", "rate_tokens >= 0")
+        batch.create_check_constraint("revision_positive", "revision > 0")
+
+    op.create_table(
+        "notification_events",
+        sa.Column("type", sa.String(96), nullable=False),
+        sa.Column("schema_version", sa.Integer(), nullable=False, server_default="1"),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("correlation_id", sa.String(36), nullable=False),
+        sa.Column("severity", sa.String(16), nullable=False),
+        sa.Column("resource_refs", sa.JSON(), nullable=False),
+        sa.Column("payload", sa.JSON(), nullable=False),
+        sa.Column("canonical_envelope", sa.Text(), nullable=False),
+        sa.Column("deduplication_key", sa.String(255), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.CheckConstraint("schema_version = 1", name="ck_notification_events_schema_version"),
+        sa.CheckConstraint(
+            "severity IN ('info','warning','error','security')",
+            name="ck_notification_events_severity",
+        ),
+        sa.UniqueConstraint("deduplication_key", name="uq_notification_events_deduplication_key"),
+    )
+    op.create_index(
+        "ix_notification_events_type_occurred", "notification_events", ["type", "occurred_at"]
+    )
+
+    # Preserve unexpected rows from the unused baseline shape.  Renaming first
+    # keeps the original data intact if an upgrade is interrupted before copy.
+    op.rename_table("notification_deliveries", "notification_deliveries_legacy")
+    op.create_table(
+        "notification_deliveries",
+        sa.Column(
+            "event_id",
+            sa.String(36),
+            sa.ForeignKey("notification_events.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "subscription_id",
+            sa.String(36),
+            sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("state", sa.String(32), nullable=False, server_default="pending"),
+        sa.Column("due_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_owner", sa.String(255), nullable=True),
+        sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("terminal_reason", sa.String(96), nullable=True),
+        sa.Column("response_class", sa.String(64), nullable=True),
+        sa.Column("response_summary", sa.String(512), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint(
+            "attempt_count >= 0", name="ck_notification_deliveries_attempt_count_nonnegative"
+        ),
+        sa.CheckConstraint(
+            "state IN ('pending','leased','delivered','retry','failed')",
+            name="ck_notification_deliveries_state",
+        ),
+        sa.UniqueConstraint(
+            "event_id", "subscription_id", name="uq_notification_deliveries_event_subscription"
+        ),
+    )
+    op.create_index(
+        "ix_notification_deliveries_due", "notification_deliveries", ["state", "due_at"]
+    )
+    op.create_index(
+        "ix_notification_deliveries_lease", "notification_deliveries", ["state", "lease_expires_at"]
+    )
+    op.create_table(
+        "notification_delivery_attempts",
+        sa.Column(
+            "delivery_id",
+            sa.String(36),
+            sa.ForeignKey("notification_deliveries.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("number", sa.Integer(), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("outcome", sa.String(32), nullable=False, server_default="started"),
+        sa.Column("response_class", sa.String(64), nullable=True),
+        sa.Column("diagnostic", sa.String(512), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.CheckConstraint("number > 0", name="ck_notification_delivery_attempts_number_positive"),
+        sa.CheckConstraint(
+            "outcome IN ('started','delivered','retry','failed')",
+            name="ck_notification_delivery_attempts_outcome",
+        ),
+        sa.UniqueConstraint(
+            "delivery_id", "number", name="uq_notification_delivery_attempts_delivery_number"
+        ),
+    )
+    op.create_index(
+        "ix_notification_attempts_delivery",
+        "notification_delivery_attempts",
+        ["delivery_id", "number"],
+    )
+
+    legacy = sa.table(
+        "notification_deliveries_legacy",
+        sa.column("id"),
+        sa.column("event_id"),
+        sa.column("subscription_id"),
+        sa.column("attempt"),
+        sa.column("state"),
+        sa.column("response_class"),
+        sa.column("next_attempt_at"),
+        sa.column("created_at"),
+        sa.column("updated_at"),
+    )
+    rows = _parameterized_execute(connection, sa.select(legacy)).mappings().all()
+    events = sa.table(
+        "notification_events",
+        sa.column("id"),
+        sa.column("type"),
+        sa.column("schema_version"),
+        sa.column("occurred_at"),
+        sa.column("correlation_id"),
+        sa.column("severity"),
+        sa.column("resource_refs", sa.JSON()),
+        sa.column("payload", sa.JSON()),
+        sa.column("canonical_envelope"),
+        sa.column("deduplication_key"),
+    )
+    deliveries = sa.table(
+        "notification_deliveries",
+        *[
+            sa.column(name)
+            for name in (
+                "id",
+                "event_id",
+                "subscription_id",
+                "state",
+                "due_at",
+                "attempt_count",
+                "terminal_reason",
+                "response_class",
+                "response_summary",
+                "created_at",
+                "updated_at",
+            )
+        ],
+    )
+    attempts = sa.table(
+        "notification_delivery_attempts",
+        *[
+            sa.column(name)
+            for name in (
+                "id",
+                "delivery_id",
+                "number",
+                "started_at",
+                "completed_at",
+                "outcome",
+                "response_class",
+                "diagnostic",
+            )
+        ],
+    )
+    for row in rows:
+        occurred = row["created_at"] or _now()
+        event_id, delivery_id, attempt_id = (str(new_uuid7()), str(new_uuid7()), str(new_uuid7()))
+        payload = {"legacy_event_id": str(row["event_id"]), "legacy_delivery_id": str(row["id"])}
+        envelope = {
+            "event_schema_version": 1,
+            "id": event_id,
+            "type": "notification.legacy",
+            "occurred_at": occurred.isoformat()
+            if hasattr(occurred, "isoformat")
+            else str(occurred),
+            "correlation_id": event_id,
+            "severity": "warning",
+            "resource": {"subscription_id": str(row["subscription_id"])},
+            "payload": payload,
+        }
+        _parameterized_execute(
+            connection,
+            events.insert().values(
+                id=event_id,
+                type="notification.legacy",
+                schema_version=1,
+                occurred_at=occurred,
+                correlation_id=event_id,
+                severity="warning",
+                resource_refs=envelope["resource"],
+                payload=payload,
+                canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
+                deduplication_key=f"legacy:{row['id']}",
+            ),
+        )
+        old_state = str(row["state"])
+        new_state = "retry" if old_state == "pending" else old_state
+        try:
+            legacy_attempt = max(1, int(row["attempt"]))
+        except (TypeError, ValueError) as error:
+            raise RuntimeError("legacy notification delivery has an invalid attempt") from error
+        _parameterized_execute(
+            connection,
+            deliveries.insert().values(
+                id=delivery_id,
+                event_id=event_id,
+                subscription_id=row["subscription_id"],
+                state=new_state,
+                due_at=row["next_attempt_at"] or occurred,
+                attempt_count=legacy_attempt,
+                terminal_reason="legacy_migrated" if new_state == "failed" else None,
+                response_class=row["response_class"],
+                response_summary="legacy delivery migrated",
+                created_at=occurred,
+                updated_at=row["updated_at"] or occurred,
+            ),
+        )
+        _parameterized_execute(
+            connection,
+            attempts.insert().values(
+                id=attempt_id,
+                delivery_id=delivery_id,
+                number=legacy_attempt,
+                started_at=occurred,
+                completed_at=row["updated_at"] if new_state in {"delivered", "failed"} else None,
+                outcome="delivered"
+                if new_state == "delivered"
+                else ("failed" if new_state == "failed" else "retry"),
+                response_class=row["response_class"],
+                diagnostic="legacy delivery migrated",
+            ),
+        )
+    op.drop_table("notification_deliveries_legacy")
+
+    op.create_table(
+        "notification_signing_keys",
+        sa.Column(
+            "subscription_id",
+            sa.String(36),
+            sa.ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("version", sa.Integer(), nullable=False),
+        sa.Column(
+            "secret_id",
+            sa.String(36),
+            sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("state", sa.String(16), nullable=False, server_default="active"),
+        sa.Column("overlap_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint("version > 0", name="ck_notification_signing_keys_version_positive"),
+        sa.CheckConstraint(
+            "state IN ('active','overlap','retired')", name="ck_notification_signing_keys_state"
+        ),
+        sa.UniqueConstraint(
+            "subscription_id", "version", name="uq_notification_signing_keys_subscription_version"
+        ),
+    )
+    op.create_index(
+        "ix_notification_signing_keys_subscription",
+        "notification_signing_keys",
+        ["subscription_id", "state"],
+    )
+    op.create_index(
+        "uq_notification_signing_keys_active",
+        "notification_signing_keys",
+        ["subscription_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'active'"),
+    )
+    op.create_index(
+        "uq_notification_signing_keys_overlap",
+        "notification_signing_keys",
+        ["subscription_id"],
+        unique=True,
+        sqlite_where=sa.text("state = 'overlap'"),
+    )
+    op.create_table(
+        "notification_email_settings",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("host", sa.String(255), nullable=False),
+        sa.Column("port", sa.Integer(), nullable=False, server_default="587"),
+        sa.Column("username", sa.String(255), nullable=False),
+        sa.Column(
+            "password_secret_id",
+            sa.String(36),
+            sa.ForeignKey("secrets.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("sender", sa.String(320), nullable=False),
+        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="5"),
+        sa.Column("rate_limit_per_minute", sa.Integer(), nullable=False, server_default="60"),
+        sa.CheckConstraint("id = 1", name="ck_notification_email_settings_singleton"),
+        sa.CheckConstraint("port > 0 AND port < 65536", name="ck_notification_email_settings_port"),
+        sa.CheckConstraint("max_attempts > 0", name="ck_notification_email_settings_max_attempts"),
+        sa.CheckConstraint(
+            "rate_limit_per_minute > 0", name="ck_notification_email_settings_rate_positive"
+        ),
+    )
+    if connection.dialect.name == "sqlite":
+        connection.exec_driver_sql("PRAGMA foreign_keys=ON")
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    for table in (
+        "notification_events",
+        "notification_delivery_attempts",
+        "notification_signing_keys",
+        "notification_email_settings",
+    ):
+        if bind.scalar(sa.text(f"SELECT count(*) FROM {table}")):
+            raise RuntimeError(
+                "cannot downgrade while M12 notification history or configuration exists"
+            )
+    # A fresh M12 schema can safely return to the historical stub shape.
+    op.drop_table("notification_email_settings")
+    op.drop_index("uq_notification_signing_keys_overlap", table_name="notification_signing_keys")
+    op.drop_index("uq_notification_signing_keys_active", table_name="notification_signing_keys")
+    op.drop_index(
+        "ix_notification_signing_keys_subscription", table_name="notification_signing_keys"
+    )
+    op.drop_table("notification_signing_keys")
+    op.drop_index("ix_notification_attempts_delivery", table_name="notification_delivery_attempts")
+    op.drop_table("notification_delivery_attempts")
+    op.drop_index("ix_notification_deliveries_lease", table_name="notification_deliveries")
+    op.drop_index("ix_notification_deliveries_due", table_name="notification_deliveries")
+    op.drop_table("notification_deliveries")
+    op.drop_index("ix_notification_events_type_occurred", table_name="notification_events")
+    op.drop_table("notification_events")
+    op.create_table(
+        "notification_deliveries",
+        sa.Column("event_id", sa.String(36), nullable=False),
+        sa.Column("subscription_id", sa.String(36), nullable=False),
+        sa.Column("attempt", sa.Integer(), nullable=False),
+        sa.Column("state", sa.String(32), nullable=False),
+        sa.Column("response_class", sa.String(64)),
+        sa.Column("next_attempt_at", sa.DateTime(timezone=True)),
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.CheckConstraint("attempt > 0", name="ck_notification_deliveries_attempt_positive"),
+        sa.CheckConstraint(
+            "state IN ('pending','delivered','retry','failed')",
+            name="ck_notification_deliveries_state",
+        ),
+        sa.ForeignKeyConstraint(
+            ["subscription_id"], ["notification_subscriptions.id"], ondelete="RESTRICT"
+        ),
+        sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
+    )
+    op.create_index(
+        "ix_notification_deliveries_state_next",
+        "notification_deliveries",
+        ["state", "next_attempt_at"],
+    )
+    with op.batch_alter_table("notification_subscriptions") as batch:
+        batch.drop_constraint("revision_positive", type_="check")
+        batch.drop_constraint("rate_tokens_nonnegative", type_="check")
+        batch.drop_constraint("rate_positive", type_="check")
+        batch.drop_column("revision")
+        batch.drop_column("rate_updated_at")
+        batch.drop_column("rate_tokens")
+        batch.drop_column("rate_limit_per_minute")

backend/alembic/versions/0009_allow_ssh_sources.py

@@ -0,0 +1,43 @@
+"""allow staged SSH source definitions
+
+Revision ID: 0009_allow_ssh_sources
+Revises: 0008_notification_outbox
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0009_allow_ssh_sources"
+down_revision = "0008_notification_outbox"
+branch_labels = None
+depends_on = None
+
+
+def _reject_ssh_sources() -> None:
+    connection = op.get_bind()
+    sources = sa.table("sources", sa.column("kind"))
+    count = connection.scalar(
+        sa.select(sa.func.count()).select_from(sources).where(sources.c.kind == "ssh")
+    )
+    if count is None:
+        raise RuntimeError("Cannot inspect persisted SSH sources before migration.")
+    if count:
+        raise RuntimeError(
+            "Cannot restrict sources to local: "
+            f"found {count} SSH source row(s). Remove them before downgrading."
+        )
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind IN ('local','ssh')")
+
+
+def downgrade() -> None:
+    _reject_ssh_sources()
+    with op.batch_alter_table("sources") as batch:
+        batch.drop_constraint(op.f("ck_sources_kind"), type_="check")
+        batch.create_check_constraint(op.f("ck_sources_kind"), "kind = 'local'")

backend/pyproject.toml

@@ -18,9 +18,9 @@ dependencies = [
   "cryptography==49.0.0",
   "fastapi==0.136.1",
   "httpx==0.28.1",
-  "paramiko==5.0.0",
   "pydantic==2.13.4",
   "pydantic-settings==2.14.2",
+  "paramiko==5.0.0",
   "sqlalchemy[asyncio]==2.0.49",
   "uvicorn[standard]==0.51.0",
 ]

backend/src/backup_tool/adapters.py

@@ -1,21 +1,43 @@
 from __future__ import annotations
 
+import os
+import stat
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Protocol
 
 from backup_tool.config import Settings
 
 
 class SourceError(ValueError):
-    pass
+    """A redacted source failure with a stable execution reason."""
+
+    def __init__(self, message: str, *, reason_code: str = "source_invalid") -> None:
+        super().__init__(message)
+        self.reason_code = reason_code
 
 
 @dataclass(frozen=True)
 class Entry:
     path: str
     kind: str
-    size: int | None = None
+    size: int
+    mode: int
+    mtime_ns: int
+    link_target: str | None = None
+
+
+class SourceReader(Protocol):
+    def validate_config(self) -> None: ...
+
+    async def probe(self) -> dict[str, int]: ...
+
+    def enumerate_entries(self) -> AsyncIterator[Entry]: ...
+
+    def open_content(self, path: str) -> AsyncIterator[bytes]: ...
+
+    async def close(self) -> None: ...
 
 
 class LocalAdapter:
@@ -36,22 +58,59 @@ class LocalAdapter:
 
     async def enumerate_entries(self) -> AsyncIterator[Entry]:
         self.validate_config()
-        for item in self.root.rglob("*"):
+        for item in sorted(self.root.rglob("*"), key=lambda candidate: candidate.as_posix()):
             relative = item.relative_to(self.root).as_posix()
-            if item.is_symlink():
-                yield Entry(relative, "symlink")
-            elif item.is_file():
-                yield Entry(relative, "file", item.stat().st_size)
-            elif item.is_dir():
-                yield Entry(relative, "directory")
+            metadata = item.lstat()
+            if stat.S_ISLNK(metadata.st_mode):
+                target = os.readlink(item)
+                target_path = Path(target)
+                if (
+                    target_path.is_absolute()
+                    or "\\" in target
+                    or ".." in target_path.parts
+                    or not target
+                ):
+                    raise SourceError(f"symlink target for {relative!r} is unsafe")
+                yield Entry(
+                    relative,
+                    "symlink",
+                    0,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                    target,
+                )
+            elif stat.S_ISREG(metadata.st_mode):
+                yield Entry(
+                    relative,
+                    "file",
+                    metadata.st_size,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                )
+            elif stat.S_ISDIR(metadata.st_mode):
+                yield Entry(
+                    relative,
+                    "directory",
+                    0,
+                    stat.S_IMODE(metadata.st_mode),
+                    metadata.st_mtime_ns,
+                )
+            else:
+                raise SourceError(f"local source entry {relative!r} has an unsupported type")
+
+    async def close(self) -> None:
+        return None
 
     async def open_content(self, path: str) -> AsyncIterator[bytes]:
-        candidate = (self.root / path).resolve()
-        if (
-            not candidate.is_relative_to(self.root)
-            or not candidate.is_file()
-            or candidate.is_symlink()
-        ):
+        requested = Path(path)
+        if not path or requested.is_absolute() or "\\" in path or ".." in requested.parts:
+            raise SourceError("invalid local source entry")
+        candidate = self.root / requested
+        metadata = candidate.lstat()
+        if not stat.S_ISREG(metadata.st_mode) or candidate.is_symlink():
+            raise SourceError("invalid local source entry")
+        resolved = candidate.resolve()
+        if not resolved.is_relative_to(self.root):
             raise SourceError("invalid local source entry")
         with candidate.open("rb") as handle:
             while chunk := handle.read(1024 * 1024):

backend/src/backup_tool/cli.py

@@ -4,16 +4,62 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import base64
+import binascii
+import json
+import os
+import socket
 from collections.abc import Callable, Sequence
+from contextlib import suppress
+from datetime import UTC, datetime
 from pathlib import Path
+from typing import Any
 
 from alembic import command
 from alembic.config import Config
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from backup_tool import __version__
 from backup_tool.config import Settings
 
 from .db.engine import assert_schema_current, create_engine
+from .db.models import (
+    Backup,
+    Base,
+    Execution,
+    Job,
+    Repository,
+    RepositoryDataKeyEpoch,
+    Source,
+)
+from .repository import (
+    begin_key_rotation,
+    finish_key_rotation,
+    inspect_repository,
+    install_signing_key,
+    load_signing_key,
+    reconcile_key_rotations,
+    replace_active_data_key,
+)
+from .scheduler import run_scheduler
+from .security.recovery_bundle import (
+    RecoveryBundleError,
+    decrypt_bundle,
+    encrypt_bundle,
+    read_bundle_file,
+    read_passphrase_fd,
+    write_bundle_exclusive,
+)
+from .security.repository_crypto import (
+    RepositoryKeyError,
+    create_data_key,
+    install_data_key,
+    load_data_key,
+)
+from .web import run_web
 from .worker import run_worker
 
 RoleHandler = Callable[[Settings], int]
@@ -25,8 +71,8 @@ def _placeholder_role(_settings: Settings) -> int:
 
 
 ROLE_HANDLERS: dict[str, RoleHandler] = {
-    "web": _placeholder_role,
-    "scheduler": _placeholder_role,
+    "web": run_web,
+    "scheduler": run_scheduler,
     "worker": run_worker,
     "admin": _placeholder_role,
 }
@@ -37,14 +83,34 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--version", action="version", version=__version__)
     subparsers = parser.add_subparsers(dest="role", required=True)
     for role in DATABASE_ROLES:
-        subparsers.add_parser(role, help=f"run the {role} role")
+        role_parser = subparsers.add_parser(role, help=f"run the {role} role")
+        if role == "admin":
+            admin_parsers = role_parser.add_subparsers(dest="admin_command")
+            repository_key = admin_parsers.add_parser("repository-key")
+            repository_key_parsers = repository_key.add_subparsers(dest="repository_key_command")
+            rotate = repository_key_parsers.add_parser("rotate")
+            rotate.add_argument("--repository-id", required=True)
+            recovery = admin_parsers.add_parser("recovery")
+            recovery_parsers = recovery.add_subparsers(dest="recovery_command", required=True)
+            export = recovery_parsers.add_parser("export")
+            export.add_argument("--output", required=True, type=Path)
+            export.add_argument("--passphrase-fd", required=True, type=int)
+            validate = recovery_parsers.add_parser("validate")
+            validate.add_argument("--input", required=True, type=Path)
+            validate.add_argument("--passphrase-fd", required=True, type=int)
+            import_ = recovery_parsers.add_parser("import")
+            import_.add_argument("--input", required=True, type=Path)
+            import_.add_argument("--passphrase-fd", required=True, type=int)
     migrate = subparsers.add_parser("migrate", help="manage the metadata schema")
     migrate.add_argument("action", choices=("upgrade", "downgrade", "current"))
+    health = subparsers.add_parser("health", help="check runtime role readiness")
+    health.add_argument("health_role", choices=("web", "scheduler", "worker"))
     return parser
 
 
 def build_alembic_config(settings: Settings) -> Config:
-    backend_root = Path(__file__).resolve().parents[2]
+    default_root = Path(__file__).resolve().parents[2]
+    backend_root = Path(os.environ.get("BACKUP_TOOL_ALEMBIC_ROOT", default_root))
     migration = Config(backend_root / "alembic.ini")
     migration.set_main_option("sqlalchemy.url", settings.database_url)
     return migration
@@ -61,6 +127,24 @@ def require_current_schema(settings: Settings) -> None:
     asyncio.run(check())
 
 
+def run_role_healthcheck(settings: Settings, role: str) -> int:
+    """Verify the selected isolated runtime role is ready to serve work."""
+    if role != "web":
+        from backup_tool.observability.health import check_role_readiness
+
+        asyncio.run(check_role_readiness(settings, role))
+        return 0
+    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+        client.settimeout(2)
+        client.connect(str(settings.web_socket_path))
+        request = b"GET /readyz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
+        client.sendall(request)
+        status_line = client.recv(64).split(b"\r\n", 1)[0]
+    if status_line != b"HTTP/1.1 200 OK":
+        raise RuntimeError("web role is not ready")
+    return 0
+
+
 def run_migration(action: str, settings: Settings) -> int:
     migration = build_alembic_config(settings)
     if action == "upgrade":
@@ -76,12 +160,792 @@ def load_settings() -> Settings:
     return Settings()  # type: ignore[call-arg]
 
 
+async def rotate_repository_key(
+    settings: Settings,
+    repository_id: str,
+    *,
+    after_database_commit: Callable[[], None] | None = None,
+) -> dict[str, str]:
+    """Journal a key rotation so a crash always has a deterministic recovery path."""
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    new_key_path: Path | None = None
+    database_committed = False
+    journal_started = False
+    root: Path | None = None
+    old_key_id = ""
+    new_key_id = ""
+    try:
+        async with sessions() as db:
+            await reconcile_key_rotations(settings, db)
+            await db.rollback()
+            async with db.begin():
+                repository = await db.scalar(
+                    select(Repository).where(Repository.id == repository_id).with_for_update()
+                )
+                if repository is None:
+                    raise ValueError("repository was not found")
+                if repository.encryption != "aes-256-gcm" or repository.active_data_key_id is None:
+                    raise ValueError("repository encryption is not enabled")
+                inspected = inspect_repository(settings, Path(repository.root))
+                root = inspected.root
+                if (
+                    inspected.encryption != "aes-256-gcm"
+                    or inspected.data_key_id != repository.active_data_key_id
+                ):
+                    raise ValueError("repository encryption metadata is invalid")
+                epochs = list(
+                    (
+                        await db.scalars(
+                            select(RepositoryDataKeyEpoch).where(
+                                RepositoryDataKeyEpoch.repository_id == repository.id
+                            )
+                        )
+                    ).all()
+                )
+                active = [epoch for epoch in epochs if epoch.state == "active"]
+                if len(active) != 1 or active[0].key_id != repository.active_data_key_id:
+                    raise ValueError("repository key epochs are invalid")
+                old_key_id = active[0].key_id
+                new_key_id, new_key_path = create_data_key(settings, inspected.repository_id)
+                begin_key_rotation(
+                    inspected.root,
+                    repository.id,
+                    inspected.repository_id,
+                    old_key_id,
+                    new_key_id,
+                )
+                journal_started = True
+                active[0].state = "retired"
+                active[0].retired_at = datetime.now(UTC)
+                db.add(
+                    RepositoryDataKeyEpoch(
+                        repository_id=repository.id,
+                        key_id=new_key_id,
+                        state="active",
+                    )
+                )
+                repository.active_data_key_id = new_key_id
+            database_committed = True
+            if after_database_commit is not None:
+                after_database_commit()
+            if root is None:
+                raise ValueError("repository rotation metadata is invalid")
+            replace_active_data_key(root, old_key_id, new_key_id)
+            finish_key_rotation(root)
+            journal_started = False
+            return {
+                "repository_id": repository_id,
+                "retired_key_id": old_key_id,
+                "active_key_id": new_key_id,
+            }
+    finally:
+        if not database_committed:
+            if journal_started and root is not None:
+                with suppress(ValueError):
+                    finish_key_rotation(root)
+            if new_key_path is not None:
+                new_key_path.unlink(missing_ok=True)
+        await engine.dispose()
+
+
+async def recovery_export_payload(settings: Settings) -> dict[str, Any]:
+    """Collect the current repository trust catalog and offline key material."""
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    try:
+        async with sessions() as db:
+            await reconcile_key_rotations(settings, db)
+            await db.rollback()
+            repositories = list(
+                (await db.scalars(select(Repository).order_by(Repository.id))).all()
+            )
+            catalog_repositories: list[dict[str, Any]] = []
+            key_records: list[dict[str, Any]] = []
+            for repository in repositories:
+                inspected = inspect_repository(settings, Path(repository.root))
+                if inspected.encryption != repository.encryption or (
+                    repository.encryption == "aes-256-gcm"
+                    and repository.active_data_key_id != inspected.data_key_id
+                ):
+                    raise RecoveryBundleError("recovery bundle export is unavailable")
+                private_key = load_signing_key(
+                    settings,
+                    inspected.repository_id,
+                    repository.signing_key_id,
+                    repository.signing_public_key,
+                )
+                epochs = list(
+                    (
+                        await db.scalars(
+                            select(RepositoryDataKeyEpoch)
+                            .where(RepositoryDataKeyEpoch.repository_id == repository.id)
+                            .order_by(RepositoryDataKeyEpoch.key_id)
+                        )
+                    ).all()
+                )
+                active = [epoch for epoch in epochs if epoch.state == "active"]
+                if repository.encryption == "aes-256-gcm":
+                    if (
+                        repository.active_data_key_id is None
+                        or len(active) != 1
+                        or active[0].key_id != repository.active_data_key_id
+                    ):
+                        raise RecoveryBundleError("recovery bundle export is unavailable")
+                elif epochs:
+                    raise RecoveryBundleError("recovery bundle export is unavailable")
+                data_keys: list[dict[str, str]] = []
+                for epoch in epochs:
+                    data_keys.append(
+                        {
+                            "key_id": epoch.key_id,
+                            "key": base64.b64encode(
+                                load_data_key(settings, inspected.repository_id, epoch.key_id)
+                            ).decode("ascii"),
+                        }
+                    )
+                catalog_repositories.append(
+                    {
+                        "active_data_key_id": repository.active_data_key_id,
+                        "compression": repository.compression,
+                        "data_key_epochs": [
+                            {
+                                "key_id": epoch.key_id,
+                                "retired_at": (
+                                    epoch.retired_at.isoformat()
+                                    if epoch.retired_at is not None
+                                    else None
+                                ),
+                                "state": epoch.state,
+                            }
+                            for epoch in epochs
+                        ],
+                        "database_id": repository.id,
+                        "encryption": repository.encryption,
+                        "format_version": repository.format_version,
+                        "name": repository.name,
+                        "repository_id": inspected.repository_id,
+                        "root": repository.root,
+                        "signing_key_id": repository.signing_key_id,
+                        "signing_public_key": repository.signing_public_key,
+                        "state": repository.state,
+                    }
+                )
+                key_records.append(
+                    {
+                        "data_keys": data_keys,
+                        "repository_id": inspected.repository_id,
+                        "signing_private_key": base64.b64encode(
+                            private_key.private_bytes(
+                                serialization.Encoding.Raw,
+                                serialization.PrivateFormat.Raw,
+                                serialization.NoEncryption(),
+                            )
+                        ).decode("ascii"),
+                    }
+                )
+            sources = list((await db.scalars(select(Source).order_by(Source.id))).all())
+            jobs = list((await db.scalars(select(Job).order_by(Job.id))).all())
+            executions = list((await db.scalars(select(Execution).order_by(Execution.id))).all())
+            backups = list((await db.scalars(select(Backup).order_by(Backup.id))).all())
+            if any(source.secret_refs for source in sources):
+                raise RecoveryBundleError("recovery bundle export is unavailable")
+            return {
+                "catalog": {
+                    "backups": [
+                        {
+                            "created_at": backup.created_at.isoformat(),
+                            "data_key_id": backup.data_key_id,
+                            "execution_id": backup.execution_id,
+                            "id": backup.id,
+                            "integrity": backup.integrity,
+                            "logical_bytes": backup.logical_bytes,
+                            "manifest_digest": backup.manifest_digest,
+                            "manifest_id": backup.manifest_id,
+                            "parent_backup_id": backup.parent_backup_id,
+                            "pinned": backup.pinned,
+                            "stored_bytes": backup.stored_bytes,
+                            "tombstoned_at": (
+                                backup.tombstoned_at.isoformat()
+                                if backup.tombstoned_at is not None
+                                else None
+                            ),
+                        }
+                        for backup in backups
+                    ],
+                    "executions": [
+                        {
+                            "attempt": execution.attempt,
+                            "id": execution.id,
+                            "job_id": execution.job_id,
+                            "progress": execution.progress,
+                            "revision": execution.revision,
+                            "trigger": execution.trigger,
+                        }
+                        for execution in executions
+                    ],
+                    "format": "backup-tool-recovery-catalog",
+                    "jobs": [
+                        {
+                            "allow_empty": job.allow_empty,
+                            "exclusions": job.exclusions,
+                            "id": job.id,
+                            "name": job.name,
+                            "repository_id": job.repository_id,
+                            "requested_mode": job.requested_mode,
+                            "retention": job.retention,
+                            "source_id": job.source_id,
+                        }
+                        for job in jobs
+                    ],
+                    "repositories": catalog_repositories,
+                    "sources": [
+                        {
+                            "id": source.id,
+                            "kind": source.kind,
+                            "name": source.name,
+                            "public_config": source.public_config,
+                        }
+                        for source in sources
+                    ],
+                    "version": 2,
+                },
+                "keys": key_records,
+            }
+    except (OSError, RepositoryKeyError, ValueError) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise
+        raise RecoveryBundleError("recovery bundle export is unavailable") from error
+    finally:
+        await engine.dispose()
+
+
+def _validate_legacy_recovery_payload(payload: dict[str, Any]) -> int:
+    """Validate the authenticated export shape without exposing key material."""
+    try:
+        if set(payload) != {"catalog", "keys"}:
+            raise ValueError
+        catalog = payload["catalog"]
+        keys = payload["keys"]
+        if (
+            not isinstance(catalog, dict)
+            or set(catalog) != {"format", "repositories", "version"}
+            or catalog["format"] != "backup-tool-recovery-catalog"
+            or catalog["version"] != 1
+            or not isinstance(catalog["repositories"], list)
+            or not isinstance(keys, list)
+            or len(catalog["repositories"]) != len(keys)
+        ):
+            raise ValueError
+        seen: set[str] = set()
+        for repository, key_record in zip(catalog["repositories"], keys, strict=True):
+            if not isinstance(repository, dict) or not isinstance(key_record, dict):
+                raise ValueError
+            required_repository = {
+                "active_data_key_id",
+                "compression",
+                "data_key_epochs",
+                "database_id",
+                "encryption",
+                "format_version",
+                "name",
+                "repository_id",
+                "root",
+                "signing_key_id",
+                "signing_public_key",
+                "state",
+            }
+            if set(repository) != required_repository or set(key_record) != {
+                "data_keys",
+                "repository_id",
+                "signing_private_key",
+            }:
+                raise ValueError
+            repository_id = repository["repository_id"]
+            if (
+                not isinstance(repository_id, str)
+                or not repository_id
+                or repository_id in seen
+                or key_record["repository_id"] != repository_id
+                or not isinstance(repository["data_key_epochs"], list)
+                or not isinstance(key_record["data_keys"], list)
+            ):
+                raise ValueError
+            seen.add(repository_id)
+            private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+            public_key = bytes.fromhex(repository["signing_public_key"])
+            if len(private_key) != 32 or len(public_key) != 32:
+                raise ValueError
+            signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
+            derived_public = signing_key.public_key().public_bytes(
+                serialization.Encoding.Raw,
+                serialization.PublicFormat.Raw,
+            )
+            if derived_public != public_key:
+                raise ValueError
+            epochs = repository["data_key_epochs"]
+            data_keys = key_record["data_keys"]
+            if len(epochs) != len(data_keys):
+                raise ValueError
+            epoch_ids: set[str] = set()
+            for epoch, data_key in zip(epochs, data_keys, strict=True):
+                if (
+                    not isinstance(epoch, dict)
+                    or set(epoch) != {"key_id", "retired_at", "state"}
+                    or not isinstance(data_key, dict)
+                    or set(data_key) != {"key", "key_id"}
+                    or not isinstance(epoch["key_id"], str)
+                    or epoch["key_id"] in epoch_ids
+                    or data_key["key_id"] != epoch["key_id"]
+                    or len(base64.b64decode(data_key["key"], validate=True)) != 32
+                ):
+                    raise ValueError
+                epoch_ids.add(epoch["key_id"])
+            active = [epoch for epoch in epochs if epoch["state"] == "active"]
+            if repository["encryption"] == "aes-256-gcm":
+                if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
+                    raise ValueError
+            elif (
+                repository["encryption"] != "none"
+                or epochs
+                or repository["active_data_key_id"] is not None
+            ):
+                raise ValueError
+        return len(seen)
+    except (binascii.Error, KeyError, TypeError, ValueError) as error:
+        raise RecoveryBundleError("recovery bundle is invalid") from error
+
+
+def _validated_recovery_catalog(
+    payload: dict[str, Any],
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Validate a v2 catalog before any filesystem or metadata mutation."""
+    try:
+        if set(payload) != {"catalog", "keys"}:
+            raise ValueError
+        catalog = payload["catalog"]
+        keys = payload["keys"]
+        required_catalog = {
+            "backups",
+            "executions",
+            "format",
+            "jobs",
+            "repositories",
+            "sources",
+            "version",
+        }
+        if (
+            not isinstance(catalog, dict)
+            or set(catalog) != required_catalog
+            or catalog["format"] != "backup-tool-recovery-catalog"
+            or catalog["version"] != 2
+            or not isinstance(keys, list)
+            or any(
+                not isinstance(catalog[name], list)
+                for name in required_catalog - {"format", "version"}
+            )
+            or len(catalog["repositories"]) != len(keys)
+        ):
+            raise ValueError
+        repositories = catalog["repositories"]
+        key_by_repository: dict[str, dict[str, Any]] = {}
+        repository_by_id: dict[str, dict[str, Any]] = {}
+        canonical_ids: set[str] = set()
+        for repository, key_record in zip(repositories, keys, strict=True):
+            if not isinstance(repository, dict) or not isinstance(key_record, dict):
+                raise ValueError
+            required_repository = {
+                "active_data_key_id",
+                "compression",
+                "data_key_epochs",
+                "database_id",
+                "encryption",
+                "format_version",
+                "name",
+                "repository_id",
+                "root",
+                "signing_key_id",
+                "signing_public_key",
+                "state",
+            }
+            if set(repository) != required_repository or set(key_record) != {
+                "data_keys",
+                "repository_id",
+                "signing_private_key",
+            }:
+                raise ValueError
+            database_id = repository["database_id"]
+            canonical_id = repository["repository_id"]
+            if (
+                not isinstance(database_id, str)
+                or not isinstance(canonical_id, str)
+                or not database_id
+                or not canonical_id
+                or database_id in repository_by_id
+                or canonical_id in canonical_ids
+                or key_record["repository_id"] != canonical_id
+                or not isinstance(repository["data_key_epochs"], list)
+                or not isinstance(key_record["data_keys"], list)
+            ):
+                raise ValueError
+            private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+            public_key = bytes.fromhex(repository["signing_public_key"])
+            signing_key = Ed25519PrivateKey.from_private_bytes(private_key)
+            if (
+                len(public_key) != 32
+                or signing_key.public_key().public_bytes(
+                    serialization.Encoding.Raw, serialization.PublicFormat.Raw
+                )
+                != public_key
+            ):
+                raise ValueError
+            epochs = repository["data_key_epochs"]
+            data_keys = key_record["data_keys"]
+            if len(epochs) != len(data_keys):
+                raise ValueError
+            epoch_ids: set[str] = set()
+            for epoch, data_key in zip(epochs, data_keys, strict=True):
+                if (
+                    not isinstance(epoch, dict)
+                    or set(epoch) != {"key_id", "retired_at", "state"}
+                    or epoch.get("state") not in {"active", "retired"}
+                    or not isinstance(epoch.get("key_id"), str)
+                    or epoch["key_id"] in epoch_ids
+                    or not isinstance(data_key, dict)
+                    or set(data_key) != {"key", "key_id"}
+                    or data_key["key_id"] != epoch["key_id"]
+                    or len(base64.b64decode(data_key["key"], validate=True)) != 32
+                ):
+                    raise ValueError
+                epoch_ids.add(epoch["key_id"])
+            active = [epoch for epoch in epochs if epoch["state"] == "active"]
+            if repository["encryption"] == "aes-256-gcm":
+                if len(active) != 1 or active[0]["key_id"] != repository["active_data_key_id"]:
+                    raise ValueError
+            elif (
+                repository["encryption"] != "none"
+                or epochs
+                or repository["active_data_key_id"] is not None
+            ):
+                raise ValueError
+            repository_by_id[database_id] = repository
+            canonical_ids.add(canonical_id)
+            key_by_repository[canonical_id] = key_record
+        source_ids = _catalog_ids(catalog["sources"], {"id", "kind", "name", "public_config"})
+        job_ids = _catalog_ids(
+            catalog["jobs"],
+            {
+                "allow_empty",
+                "exclusions",
+                "id",
+                "name",
+                "repository_id",
+                "requested_mode",
+                "retention",
+                "source_id",
+            },
+        )
+        for job in catalog["jobs"]:
+            if job["source_id"] not in source_ids or job["repository_id"] not in repository_by_id:
+                raise ValueError
+        execution_ids = _catalog_ids(
+            catalog["executions"], {"attempt", "id", "job_id", "progress", "revision", "trigger"}
+        )
+        for execution in catalog["executions"]:
+            if execution["job_id"] not in job_ids:
+                raise ValueError
+        backup_ids = _catalog_ids(
+            catalog["backups"],
+            {
+                "created_at",
+                "data_key_id",
+                "execution_id",
+                "id",
+                "integrity",
+                "logical_bytes",
+                "manifest_digest",
+                "manifest_id",
+                "parent_backup_id",
+                "pinned",
+                "stored_bytes",
+                "tombstoned_at",
+            },
+        )
+        for backup in catalog["backups"]:
+            if backup["execution_id"] not in execution_ids:
+                raise ValueError
+            _catalog_datetime(backup["created_at"])
+            _catalog_datetime(backup["tombstoned_at"], allow_none=True)
+            if (
+                backup["parent_backup_id"] is not None
+                and backup["parent_backup_id"] not in backup_ids
+            ):
+                raise ValueError
+        return (
+            catalog,
+            [key_by_repository[repository["repository_id"]] for repository in repositories],
+        )
+    except (binascii.Error, KeyError, TypeError, ValueError) as error:
+        raise RecoveryBundleError("recovery bundle is invalid") from error
+
+
+def _catalog_ids(rows: list[Any], required: set[str]) -> set[str]:
+    identifiers: set[str] = set()
+    for row in rows:
+        if not isinstance(row, dict) or set(row) != required or not isinstance(row.get("id"), str):
+            raise ValueError
+        if not row["id"] or row["id"] in identifiers:
+            raise ValueError
+        identifiers.add(row["id"])
+    return identifiers
+
+
+def _catalog_datetime(value: object, *, allow_none: bool = False) -> datetime | None:
+    if value is None and allow_none:
+        return None
+    if not isinstance(value, str):
+        raise ValueError
+    parsed = datetime.fromisoformat(value.replace("Z", "+00"))
+    if parsed.tzinfo is None:
+        raise ValueError
+    return parsed.astimezone(UTC)
+
+
+def _validate_recovery_payload(payload: dict[str, Any]) -> int:
+    catalog = payload.get("catalog")
+    if isinstance(catalog, dict) and catalog.get("version") == 1:
+        return _validate_legacy_recovery_payload(payload)
+    validated_catalog, _keys = _validated_recovery_catalog(payload)
+    return len(validated_catalog["repositories"])
+
+
+def _install_or_verify_recovery_keys(
+    settings: Settings,
+    repository: dict[str, Any],
+    key_record: dict[str, Any],
+    installed_paths: list[Path],
+) -> None:
+    inspected = inspect_repository(settings, Path(repository["root"]))
+    if (
+        inspected.repository_id != repository["repository_id"]
+        or inspected.encryption != repository["encryption"]
+        or inspected.data_key_id != repository["active_data_key_id"]
+    ):
+        raise RecoveryBundleError("recovery bundle is invalid")
+    private_key = base64.b64decode(key_record["signing_private_key"], validate=True)
+    signing_path = settings.data_dir / "repository-keys" / f"{inspected.repository_id}.ed25519"
+    if signing_path.exists() or signing_path.is_symlink():
+        loaded = load_signing_key(
+            settings,
+            inspected.repository_id,
+            repository["signing_key_id"],
+            repository["signing_public_key"],
+        )
+        if (
+            loaded.private_bytes(
+                serialization.Encoding.Raw,
+                serialization.PrivateFormat.Raw,
+                serialization.NoEncryption(),
+            )
+            != private_key
+        ):
+            raise RecoveryBundleError("recovery destination has conflicting keys")
+    else:
+        installed_paths.append(
+            install_signing_key(
+                settings,
+                inspected.repository_id,
+                repository["signing_key_id"],
+                repository["signing_public_key"],
+                private_key,
+            )
+        )
+    for data_key in key_record["data_keys"]:
+        key_value = base64.b64decode(data_key["key"], validate=True)
+        key_path = (
+            settings.data_dir
+            / "repository-data-keys"
+            / f"{inspected.repository_id}.{data_key['key_id']}.key"
+        )
+        if key_path.exists() or key_path.is_symlink():
+            if load_data_key(settings, inspected.repository_id, data_key["key_id"]) != key_value:
+                raise RecoveryBundleError("recovery destination has conflicting keys")
+        else:
+            installed_paths.append(
+                install_data_key(settings, inspected.repository_id, data_key["key_id"], key_value)
+            )
+
+
+async def import_recovery_payload(
+    settings: Settings,
+    payload: dict[str, Any],
+    *,
+    after_key_install: Callable[[], None] | None = None,
+) -> int:
+    """Install a complete authenticated catalog into an empty, migrated database."""
+    catalog, key_records = _validated_recovery_catalog(payload)
+    engine = create_engine(settings)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    installed_paths: list[Path] = []
+    completed = False
+    try:
+        async with sessions() as db:
+            for table in Base.metadata.sorted_tables:
+                if await db.scalar(select(func.count()).select_from(table)):
+                    raise RecoveryBundleError("recovery destination is not empty")
+            await db.rollback()
+            for repository, key_record in zip(catalog["repositories"], key_records, strict=True):
+                _install_or_verify_recovery_keys(settings, repository, key_record, installed_paths)
+            if after_key_install is not None:
+                after_key_install()
+            async with db.begin():
+                for repository in catalog["repositories"]:
+                    db.add(
+                        Repository(
+                            id=repository["database_id"],
+                            name=repository["name"],
+                            root=repository["root"],
+                            format_version=repository["format_version"],
+                            compression=repository["compression"],
+                            encryption=repository["encryption"],
+                            signing_key_id=repository["signing_key_id"],
+                            signing_public_key=repository["signing_public_key"],
+                            active_data_key_id=repository["active_data_key_id"],
+                            state=repository["state"],
+                        )
+                    )
+                for repository in catalog["repositories"]:
+                    for epoch in repository["data_key_epochs"]:
+                        db.add(
+                            RepositoryDataKeyEpoch(
+                                repository_id=repository["database_id"],
+                                key_id=epoch["key_id"],
+                                state=epoch["state"],
+                            )
+                        )
+                await db.flush()
+                for source in catalog["sources"]:
+                    db.add(
+                        Source(
+                            id=source["id"],
+                            name=source["name"],
+                            kind=source["kind"],
+                            public_config=source["public_config"],
+                            secret_refs=[],
+                            state="unavailable",
+                        )
+                    )
+                await db.flush()
+                for job in catalog["jobs"]:
+                    db.add(
+                        Job(
+                            id=job["id"],
+                            name=job["name"],
+                            source_id=job["source_id"],
+                            repository_id=job["repository_id"],
+                            requested_mode=job["requested_mode"],
+                            exclusions=job["exclusions"],
+                            retention=job["retention"],
+                            enabled=False,
+                            allow_empty=job["allow_empty"],
+                            state="archived",
+                        )
+                    )
+                await db.flush()
+                for execution in catalog["executions"]:
+                    db.add(
+                        Execution(
+                            id=execution["id"],
+                            job_id=execution["job_id"],
+                            schedule_id=None,
+                            trigger=execution["trigger"],
+                            state="committed",
+                            attempt=execution["attempt"],
+                            progress=execution["progress"],
+                            revision=execution["revision"],
+                        )
+                    )
+                await db.flush()
+                for backup in catalog["backups"]:
+                    db.add(
+                        Backup(
+                            id=backup["id"],
+                            execution_id=backup["execution_id"],
+                            parent_backup_id=None,
+                            manifest_id=backup["manifest_id"],
+                            manifest_digest=backup["manifest_digest"],
+                            logical_bytes=backup["logical_bytes"],
+                            stored_bytes=backup["stored_bytes"],
+                            integrity=backup["integrity"],
+                            data_key_id=backup["data_key_id"],
+                            pinned=backup["pinned"],
+                            created_at=_catalog_datetime(backup["created_at"]),
+                            tombstoned_at=_catalog_datetime(
+                                backup["tombstoned_at"], allow_none=True
+                            ),
+                        )
+                    )
+                await db.flush()
+                for backup in catalog["backups"]:
+                    if backup["parent_backup_id"] is not None:
+                        imported = await db.get(Backup, backup["id"])
+                        if imported is None:
+                            raise RecoveryBundleError("recovery bundle is invalid")
+                        imported.parent_backup_id = backup["parent_backup_id"]
+        completed = True
+        return len(catalog["repositories"])
+    except (OSError, RepositoryKeyError, ValueError) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise
+        raise RecoveryBundleError("recovery import failed") from error
+    finally:
+        await engine.dispose()
+        if not completed:
+            for path in installed_paths:
+                path.unlink(missing_ok=True)
+
+
+def run_admin(settings: Settings, args: argparse.Namespace) -> int:
+    if (
+        getattr(args, "admin_command", None) == "repository-key"
+        and getattr(args, "repository_key_command", None) == "rotate"
+    ):
+        result = asyncio.run(rotate_repository_key(settings, args.repository_id))
+        print(json.dumps(result, sort_keys=True))
+        return 0
+    if getattr(args, "admin_command", None) == "recovery":
+        passphrase = read_passphrase_fd(args.passphrase_fd)
+        if args.recovery_command == "export":
+            payload = asyncio.run(recovery_export_payload(settings))
+            write_bundle_exclusive(args.output, encrypt_bundle(payload, passphrase))
+            print(json.dumps({"repositories": len(payload["keys"]), "status": "exported"}))
+            return 0
+        if args.recovery_command == "validate":
+            payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
+            validation_result: dict[str, object] = {
+                "repositories": _validate_recovery_payload(payload),
+                "status": "valid",
+            }
+            print(json.dumps(validation_result))
+            return 0
+        if args.recovery_command == "import":
+            payload = decrypt_bundle(read_bundle_file(args.input), passphrase)
+            imported = asyncio.run(import_recovery_payload(settings, payload))
+            print(json.dumps({"repositories": imported, "status": "imported"}))
+            return 0
+    raise ValueError("admin command is invalid")
+
+
 def main(argv: Sequence[str] | None = None, *, settings: Settings | None = None) -> int:
     args = build_parser().parse_args(argv)
     runtime_settings = settings or load_settings()
     if args.role == "migrate":
         return run_migration(args.action, runtime_settings)
+    if args.role == "health":
+        return run_role_healthcheck(runtime_settings, args.health_role)
     require_current_schema(runtime_settings)
+    if args.role == "admin":
+        return run_admin(runtime_settings, args)
     return ROLE_HANDLERS[args.role](runtime_settings)
 
 

backend/src/backup_tool/config.py

@@ -19,6 +19,7 @@ class Settings(BaseSettings):
     )
 
     data_dir: Path = Path("/var/lib/backup-tool")
+    web_socket_path: Path = Path("/tmp/backup-tool-web.sock")
     database_url: str = "sqlite+aiosqlite:////var/lib/backup-tool/metadata.db"
     repository_roots: tuple[Path, ...]
     local_source_roots: tuple[Path, ...]
@@ -29,6 +30,21 @@ class Settings(BaseSettings):
     session_ttl_seconds: int = Field(default=28_800, ge=60, le=2_592_000)
     cors_origins: tuple[str, ...] = ()
     worker_concurrency: Literal[1] = 1
+    notification_delivery_lease_seconds: int = Field(default=60, ge=5, le=3600)
+    notification_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
+    notification_read_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
+    notification_max_attempts: int = Field(default=5, ge=1, le=20)
+    notification_retry_cap_seconds: int = Field(default=3600, ge=1, le=86_400)
+    notification_global_rate_per_minute: int = Field(default=120, ge=1, le=10_000)
+    notification_default_rate_per_minute: int = Field(default=60, ge=1, le=10_000)
+    notification_max_webhook_body_bytes: int = Field(default=65_536, ge=256, le=1_048_576)
+    notification_max_response_bytes: int = Field(default=16_384, ge=256, le=1_048_576)
+    ssh_connect_timeout_seconds: float = Field(default=5.0, gt=0, le=60)
+    ssh_operation_timeout_seconds: float = Field(default=10.0, gt=0, le=120)
+    ssh_read_chunk_bytes: int = Field(default=1_048_576, ge=4096, le=8_388_608)
+    ssh_list_read_aheads: int = Field(default=32, ge=1, le=256)
+    ssh_max_entries: int = Field(default=1_000_000, ge=1, le=10_000_000)
+    ssh_max_traversal_depth: int = Field(default=128, ge=1, le=1024)
     min_free_bytes: int = Field(default=1_073_741_824, ge=0)
     min_free_percent: int = Field(default=5, gt=0, lt=100)
     sqlite_busy_timeout_ms: int = Field(default=5_000, ge=1_000, le=120_000)
@@ -37,6 +53,7 @@ class Settings(BaseSettings):
     @field_validator(
         "data_dir",
         "master_key_file",
+        "web_socket_path",
         mode="before",
     )
     @classmethod

backend/src/backup_tool/db/models.py

@@ -95,6 +95,9 @@ class Repository(IdentityMixin, TimestampMixin, Base):
     format_version: Mapped[int] = mapped_column(Integer, nullable=False)
     compression: Mapped[str] = mapped_column(String(32), nullable=False)
     encryption: Mapped[str] = mapped_column(String(32), nullable=False)
+    signing_key_id: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
+    signing_public_key: Mapped[str] = mapped_column(String(64), nullable=False, server_default="")
+    active_data_key_id: Mapped[str | None] = mapped_column(String(36))
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
     __table_args__ = (
         CheckConstraint("format_version > 0", name="format_version_positive"),
@@ -102,6 +105,25 @@ class Repository(IdentityMixin, TimestampMixin, Base):
     )
 
 
+class RepositoryDataKeyEpoch(IdentityMixin, TimestampMixin, Base):
+    __tablename__ = "repository_data_key_epochs"
+    repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
+    key_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    state: Mapped[str] = mapped_column(String(16), nullable=False)
+    retired_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    __table_args__ = (
+        UniqueConstraint("repository_id", "key_id", name="repository_key_epoch"),
+        CheckConstraint("state IN ('active','retired')", name="repository_data_key_epoch_state"),
+        Index("ix_repository_data_key_epochs_repository_id", "repository_id"),
+        Index(
+            "uq_repository_data_key_epochs_active",
+            "repository_id",
+            unique=True,
+            sqlite_where=text("state = 'active'"),
+        ),
+    )
+
+
 class Source(IdentityMixin, TimestampMixin, Base):
     __tablename__ = "sources"
     name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
@@ -111,7 +133,7 @@ class Source(IdentityMixin, TimestampMixin, Base):
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
     last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON)
     __table_args__ = (
-        CheckConstraint("kind IN ('local','sftp','postgresql','mysql')", name="kind"),
+        CheckConstraint("kind IN ('local','ssh')", name="kind"),
         CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
     )
 
@@ -216,6 +238,7 @@ class Backup(IdentityMixin, Base):
     logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified")
+    data_key_id: Mapped[str | None] = mapped_column(String(36))
     pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     tombstoned_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
     created_at: Mapped[datetime] = mapped_column(
@@ -236,6 +259,9 @@ class Restore(IdentityMixin, TimestampMixin, Base):
     backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT"))
     destination: Mapped[str] = mapped_column(Text, nullable=False)
     selection: Mapped[list[str]] = mapped_column(JSON, nullable=False)
+    dry_run: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=False, server_default="0"
+    )
     overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False)
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
     result: Mapped[dict[str, Any] | None] = mapped_column(JSON)
@@ -268,33 +294,143 @@ class AuditEvent(IdentityMixin, Base):
 
 
 class NotificationSubscription(IdentityMixin, TimestampMixin, Base):
+    """A typed, mutable outbound channel configuration; credential material is never here."""
+
     __tablename__ = "notification_subscriptions"
     channel: Mapped[str] = mapped_column(String(32), nullable=False)
     event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False)
     destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    # Retained only to make the migration from the unused v1-shaped table lossless.
+    # New webhook secrets live in NotificationSigningKey and SMTP secrets in settings.
     secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
     state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
+    rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
+    rate_tokens: Mapped[float] = mapped_column(nullable=False, default=60.0)
+    rate_updated_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
     __table_args__ = (
         CheckConstraint("channel IN ('webhook','email')", name="channel"),
         CheckConstraint("state IN ('active','disabled','archived')", name="state"),
+        CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
+        CheckConstraint("rate_tokens >= 0", name="rate_tokens_nonnegative"),
+        CheckConstraint("revision > 0", name="revision_positive"),
+    )
+
+
+class NotificationEvent(IdentityMixin, Base):
+    """Immutable canonical outbox envelope, created in the source mutation transaction."""
+
+    __tablename__ = "notification_events"
+    type: Mapped[str] = mapped_column(String(96), nullable=False)
+    schema_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    occurred_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    correlation_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    severity: Mapped[str] = mapped_column(String(16), nullable=False)
+    resource_refs: Mapped[dict[str, str]] = mapped_column(JSON, nullable=False)
+    payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    canonical_envelope: Mapped[str] = mapped_column(Text, nullable=False)
+    deduplication_key: Mapped[str | None] = mapped_column(String(255), unique=True)
+    __table_args__ = (
+        CheckConstraint("schema_version = 1", name="schema_version"),
+        CheckConstraint("severity IN ('info','warning','error','security')", name="severity"),
+        Index("ix_notification_events_type_occurred", "type", "occurred_at"),
     )
 
 
 class NotificationDelivery(IdentityMixin, TimestampMixin, Base):
+    """One durable delivery per matching subscription, with a lease-owned lifecycle."""
+
     __tablename__ = "notification_deliveries"
-    event_id: Mapped[str] = mapped_column(String(36), nullable=False)
-    subscription_id: Mapped[str] = mapped_column(
-        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT")
+    event_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_events.id", ondelete="RESTRICT"), nullable=False
     )
-    attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
-    state: Mapped[str] = mapped_column(String(32), nullable=False)
+    subscription_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
+    )
+    state: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
+    due_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    lease_owner: Mapped[str | None] = mapped_column(String(255))
+    lease_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    terminal_reason: Mapped[str | None] = mapped_column(String(96))
     response_class: Mapped[str | None] = mapped_column(String(64))
-    next_attempt_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    response_summary: Mapped[str | None] = mapped_column(String(512))
     __table_args__ = (
-        CheckConstraint("attempt > 0", name="attempt_positive"),
-        CheckConstraint("state IN ('pending','delivered','retry','failed')", name="state"),
-        UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
-        Index("ix_notification_deliveries_state_next", "state", "next_attempt_at"),
+        CheckConstraint("attempt_count >= 0", name="attempt_count_nonnegative"),
+        CheckConstraint("state IN ('pending','leased','delivered','retry','failed')", name="state"),
+        UniqueConstraint("event_id", "subscription_id", name="event_subscription"),
+        Index("ix_notification_deliveries_due", "state", "due_at"),
+        Index("ix_notification_deliveries_lease", "state", "lease_expires_at"),
+    )
+
+
+class NotificationDeliveryAttempt(IdentityMixin, Base):
+    """Append-only history.  Diagnostics are bounded/redacted before persistence."""
+
+    __tablename__ = "notification_delivery_attempts"
+    delivery_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_deliveries.id", ondelete="RESTRICT"), nullable=False
+    )
+    number: Mapped[int] = mapped_column(Integer, nullable=False)
+    started_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False)
+    completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    outcome: Mapped[str] = mapped_column(String(32), nullable=False, default="started")
+    response_class: Mapped[str | None] = mapped_column(String(64))
+    diagnostic: Mapped[str | None] = mapped_column(String(512))
+    __table_args__ = (
+        CheckConstraint("number > 0", name="number_positive"),
+        CheckConstraint("outcome IN ('started','delivered','retry','failed')", name="outcome"),
+        UniqueConstraint("delivery_id", "number", name="delivery_number"),
+        Index("ix_notification_attempts_delivery", "delivery_id", "number"),
+    )
+
+
+class NotificationSigningKey(IdentityMixin, TimestampMixin, Base):
+    __tablename__ = "notification_signing_keys"
+    subscription_id: Mapped[str] = mapped_column(
+        ForeignKey("notification_subscriptions.id", ondelete="RESTRICT"), nullable=False
+    )
+    version: Mapped[int] = mapped_column(Integer, nullable=False)
+    secret_id: Mapped[str] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
+    state: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
+    overlap_expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime())
+    __table_args__ = (
+        CheckConstraint("version > 0", name="version_positive"),
+        CheckConstraint("state IN ('active','overlap','retired')", name="state"),
+        UniqueConstraint("subscription_id", "version", name="subscription_version"),
+        Index("ix_notification_signing_keys_subscription", "subscription_id", "state"),
+        Index(
+            "uq_notification_signing_keys_active",
+            "subscription_id",
+            unique=True,
+            sqlite_where=text("state = 'active'"),
+        ),
+        Index(
+            "uq_notification_signing_keys_overlap",
+            "subscription_id",
+            unique=True,
+            sqlite_where=text("state = 'overlap'"),
+        ),
+    )
+
+
+class NotificationEmailSettings(Base):
+    __tablename__ = "notification_email_settings"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
+    host: Mapped[str] = mapped_column(String(255), nullable=False)
+    port: Mapped[int] = mapped_column(Integer, nullable=False, default=587)
+    username: Mapped[str] = mapped_column(String(255), nullable=False)
+    password_secret_id: Mapped[str] = mapped_column(
+        ForeignKey("secrets.id", ondelete="RESTRICT"), nullable=False
+    )
+    sender: Mapped[str] = mapped_column(String(320), nullable=False)
+    max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
+    rate_limit_per_minute: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
+    __table_args__ = (
+        CheckConstraint("id = 1", name="singleton"),
+        CheckConstraint("port > 0 AND port < 65536", name="port"),
+        CheckConstraint("max_attempts > 0", name="max_attempts"),
+        CheckConstraint("rate_limit_per_minute > 0", name="rate_positive"),
     )
 
 

backend/src/backup_tool/exclusions.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+
+
+class ExclusionError(ValueError):
+    pass
+
+
+def _validate(value: str) -> None:
+    path = PurePosixPath(value)
+    if not value or "\\" in value or path.is_absolute() or ".." in path.parts:
+        raise ExclusionError("exclusion paths must be normalized relative POSIX paths")
+
+
+def matches(path: str, patterns: list[str]) -> bool:
+    """Return whether normalized relative `path` is excluded by ordered gitignore-like rules."""
+    _validate(path)
+    excluded = False
+    candidate = PurePosixPath(path)
+    for pattern in patterns:
+        negated = pattern.startswith("!")
+        raw = pattern[1:] if negated else pattern
+        if not raw or raw.startswith("/") or "\\" in raw or ".." in PurePosixPath(raw).parts:
+            raise ExclusionError("exclusion pattern is invalid")
+        directory = raw.endswith("/")
+        raw = raw.rstrip("/")
+        if not raw:
+            raise ExclusionError("exclusion pattern is invalid")
+        matched = candidate.match(raw) or candidate.match(f"**/{raw}")
+        if directory:
+            matched = matched or any(
+                parent.match(raw) or parent.match(f"**/{raw}") for parent in candidate.parents
+            )
+        if matched:
+            excluded = not negated
+    return excluded

backend/src/backup_tool/execution.py

@@ -8,6 +8,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from backup_tool.db.models import Execution, ExecutionEvent, Job
+from backup_tool.notifications.events import emit_event
 from backup_tool.security.redaction import redact
 
 ACTIVE_STATES = frozenset({"queued", "preparing", "running", "verifying", "cancelling"})
@@ -44,7 +45,14 @@ def transition(current: str, target: str) -> str:
     return target
 
 
-async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Execution:
+async def enqueue(
+    db: AsyncSession,
+    job_id: str,
+    trigger: str = "manual",
+    *,
+    schedule_id: str | None = None,
+    nominal_run_at: datetime | None = None,
+) -> Execution:
     """Create exactly one active execution for an enabled active job."""
     job = await db.get(Job, job_id)
     if job is None:
@@ -52,7 +60,13 @@ async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Exe
     if job.state != "active" or not job.enabled:
         raise EnqueueError("job_disabled", "Job is disabled or archived.")
     job_identifier = job.id
-    execution = Execution(job_id=job_identifier, trigger=trigger, progress={})
+    execution = Execution(
+        job_id=job_identifier,
+        trigger=trigger,
+        schedule_id=schedule_id,
+        nominal_run_at=nominal_run_at,
+        progress={},
+    )
     db.add(execution)
     try:
         await db.flush()
@@ -221,6 +235,32 @@ async def record_event(db: AsyncSession, execution: Execution) -> ExecutionEvent
     )
     db.add(event)
     await db.flush()
+    notification_type = {
+        "preparing": "execution.started",
+        "committed": "execution.committed",
+        "failed": "execution.failed",
+        "cancelled": "execution.cancelled",
+    }.get(execution.state)
+    if execution.state == "queued":
+        if execution.reason_code == "worker_lost":
+            notification_type = "execution.worker_recovered"
+        elif execution.attempt > 1:
+            notification_type = "execution.retry_queued"
+        else:
+            notification_type = "execution.queued"
+    if notification_type is not None:
+        await emit_event(
+            db,
+            notification_type,
+            correlation_id=execution.id,
+            resource={"execution_id": execution.id, "job_id": execution.job_id},
+            payload={
+                "attempt": execution.attempt,
+                "state": execution.state,
+                "reason_code": execution.reason_code,
+            },
+            deduplication_key=f"execution:{execution.id}:revision:{execution.revision}",
+        )
     return event
 
 

backend/src/backup_tool/faults.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class FaultInjector(Protocol):
+    def hit(self, point: str) -> None: ...
+
+
+class NoFault:
+    def hit(self, point: str) -> None:
+        del point
+
+
+class InjectedCrash(BaseException):
+    """Test-only abrupt worker termination at a named durable fault point."""
+
+
+class CrashAt:
+    def __init__(self, point: str) -> None:
+        self.point = point
+
+    def hit(self, point: str) -> None:
+        if point == self.point:
+            raise InjectedCrash(point)

backend/src/backup_tool/gc.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import json
+import shutil
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import cast
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.db.models import Backup, Execution, Job, Repository
+from backup_tool.notifications.events import emit_event
+from backup_tool.retention import BackupLike, RetentionPolicy, retained_ids
+from backup_tool.security.repository_crypto import RepositoryKeyError, decrypt_object, object_aad
+
+
+@dataclass(frozen=True)
+class GcReport:
+    tombstoned: int
+    purged_manifests: int
+    purged_blobs: int
+    quarantined: int
+
+
+async def tombstone_expired(db: AsyncSession, now: datetime | None = None) -> int:
+    reference = now or datetime.now(UTC)
+    backups = list((await db.scalars(select(Backup))).all())
+    executions = {item.id: item for item in (await db.scalars(select(Execution))).all()}
+    jobs = {item.id: item for item in (await db.scalars(select(Job))).all()}
+    grouped: dict[str, list[Backup]] = {}
+    for backup in backups:
+        execution = executions.get(backup.execution_id)
+        if execution is None or execution.job_id not in jobs:
+            continue
+        grouped.setdefault(execution.job_id, []).append(backup)
+    tombstoned = 0
+    for job_id, items in grouped.items():
+        if not jobs[job_id].retention:
+            continue
+        policy = RetentionPolicy.from_dict(jobs[job_id].retention)
+        keep = retained_ids(cast(list[BackupLike], items), policy, reference)
+        for backup in items:
+            if backup.id not in keep and backup.tombstoned_at is None:
+                backup.tombstoned_at = reference
+                await emit_event(
+                    db,
+                    "retention.tombstoned",
+                    correlation_id=backup.id,
+                    resource={"backup_id": backup.id, "job_id": job_id},
+                    payload={"outcome": "tombstoned"},
+                    deduplication_key=f"backup:{backup.id}:tombstoned",
+                )
+                tombstoned += 1
+    await db.commit()
+    return tombstoned
+
+
+async def process_retention_gc(db: AsyncSession, now: datetime | None = None) -> GcReport:
+    """Run durable retention tombstoning and repository GC from the worker role."""
+    reference = now or datetime.now(UTC)
+    tombstoned = await tombstone_expired(db, reference)
+    reports: list[GcReport] = []
+    repositories = list((await db.scalars(select(Repository))).all())
+    for repository in repositories:
+        manifest_ids = set(
+            (
+                await db.scalars(
+                    select(Backup.manifest_id)
+                    .join(Execution, Backup.execution_id == Execution.id)
+                    .join(Job, Execution.job_id == Job.id)
+                    .where(
+                        Job.repository_id == repository.id,
+                        Backup.tombstoned_at.is_not(None),
+                    )
+                )
+            ).all()
+        )
+        if manifest_ids:
+            reports.append(purge_repository(Path(repository.root), manifest_ids, now=reference))
+    return GcReport(
+        tombstoned=tombstoned,
+        purged_manifests=sum(report.purged_manifests for report in reports),
+        purged_blobs=sum(report.purged_blobs for report in reports),
+        quarantined=sum(report.quarantined for report in reports),
+    )
+
+
+def _manifest_digests(
+    path: Path,
+    *,
+    repository_id: str | None = None,
+    manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
+) -> set[str] | None:
+    try:
+        if path.is_symlink() or not path.is_file():
+            return None
+        raw = path.read_bytes()
+        if raw.startswith(b"BTENC\x01"):
+            key_record = manifest_keys.get(path.stem) if manifest_keys is not None else None
+            if key_record is None or repository_id is None:
+                return None
+            key_id, key = key_record
+            raw = decrypt_object(key, object_aad(repository_id, key_id, "manifest", path.stem), raw)
+        payload = json.loads(raw.decode("utf-8"))
+    except (OSError, RepositoryKeyError, UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    if not isinstance(payload, dict) or not isinstance(entries := payload.get("entries"), list):
+        return None
+    digests: set[str] = set()
+    for entry in entries:
+        if not isinstance(entry, dict):
+            return None
+        digest = entry.get("blob_digest")
+        if digest is None:
+            continue
+        if (
+            not isinstance(digest, str)
+            or len(digest) != 64
+            or any(character not in "0123456789abcdef" for character in digest)
+        ):
+            return None
+        digests.add(digest)
+    return digests
+
+
+def purge_repository(
+    root: Path,
+    tombstoned_manifest_ids: set[str],
+    *,
+    repository_id: str | None = None,
+    manifest_keys: Mapping[str, tuple[str, bytes]] | None = None,
+    grace: timedelta = timedelta(days=7),
+    now: datetime | None = None,
+) -> GcReport:
+    reference = now or datetime.now(UTC)
+    manifests = root / "manifests"
+    blobs = root / "blobs" / "sha256"
+    quarantine = root / "quarantine"
+    quarantine.mkdir(exist_ok=True)
+    manifest_digests: dict[Path, set[str]] = {}
+    for manifest in manifests.glob("*.json"):
+        digests = _manifest_digests(
+            manifest,
+            repository_id=repository_id,
+            manifest_keys=manifest_keys,
+        )
+        if digests is None:
+            return GcReport(0, 0, 0, 0)
+        manifest_digests[manifest] = digests
+
+    purged_manifests = 0
+    for manifest_id in tombstoned_manifest_ids:
+        candidate = manifests / f"{manifest_id}.json"
+        if not candidate.is_file() or candidate.is_symlink():
+            continue
+        age = reference - datetime.fromtimestamp(candidate.stat().st_mtime, UTC)
+        if age >= grace:
+            candidate.unlink()
+            purged_manifests += 1
+    referenced: set[str] = set()
+    for manifest, digests in manifest_digests.items():
+        if manifest.exists():
+            referenced.update(digests)
+    purged_blobs = 0
+    quarantined = 0
+    if blobs.exists():
+        for blob in blobs.iterdir():
+            if not blob.is_file():
+                continue
+            if len(blob.name) != 64 or any(char not in "0123456789abcdef" for char in blob.name):
+                try:
+                    shutil.move(str(blob), quarantine / blob.name)
+                except OSError:
+                    continue
+                quarantined += 1
+            elif blob.name not in referenced and (
+                reference - datetime.fromtimestamp(blob.stat().st_mtime, UTC) >= grace
+            ):
+                blob.unlink()
+                purged_blobs += 1
+    return GcReport(0, purged_manifests, purged_blobs, quarantined)

backend/src/backup_tool/notifications/__init__.py

@@ -0,0 +1,5 @@
+"""Durable, worker-dispatched operational notifications."""
+
+from .events import EVENT_CATALOG, emit_event, validate_filters
+
+__all__ = ["EVENT_CATALOG", "emit_event", "validate_filters"]

backend/src/backup_tool/notifications/dispatcher.py

@@ -0,0 +1,335 @@
+"""Worker-owned leased outbox dispatcher; sends happen only after a committed lease."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.config import Settings
+from backup_tool.db.models import (
+    NotificationDelivery,
+    NotificationDeliveryAttempt,
+    NotificationEmailSettings,
+    NotificationEvent,
+    NotificationSigningKey,
+    NotificationSubscription,
+    Secret,
+)
+from backup_tool.notifications.email import EmailTransportError, deliver_email
+from backup_tool.notifications.retry import (
+    RetryDecision,
+    retry_delay,
+    transport_decision,
+    webhook_decision,
+)
+from backup_tool.notifications.webhook import (
+    SigningMaterial,
+    WebhookTransportError,
+    deliver_webhook,
+)
+from backup_tool.security.secrets import EnvelopeCipher
+from backup_tool.security.ssrf import Resolver, system_resolver
+
+
+async def recover_notification_leases(db: AsyncSession) -> int:
+    """An interrupted post-send lease becomes eligible again (at-least-once by design)."""
+    now = datetime.now(UTC)
+    deliveries = list(
+        (
+            await db.scalars(
+                select(NotificationDelivery).where(
+                    NotificationDelivery.state == "leased",
+                    NotificationDelivery.lease_expires_at < now,
+                )
+            )
+        ).all()
+    )
+    for delivery in deliveries:
+        attempt = await db.scalar(
+            select(NotificationDeliveryAttempt).where(
+                NotificationDeliveryAttempt.delivery_id == delivery.id,
+                NotificationDeliveryAttempt.number == delivery.attempt_count,
+                NotificationDeliveryAttempt.outcome == "started",
+            )
+        )
+        if attempt is not None:
+            attempt.completed_at = now
+            attempt.outcome = "retry"
+            attempt.response_class = "lease_expired"
+            attempt.diagnostic = "abandoned_lease"
+        delivery.state = "retry"
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        delivery.due_at = now
+    if deliveries:
+        await db.commit()
+    return len(deliveries)
+
+
+async def _claim_due(
+    db: AsyncSession, owner: str, lease_seconds: int
+) -> tuple[NotificationDelivery, NotificationSubscription, NotificationEvent] | None:
+    now = datetime.now(UTC)
+    delivery_id = await db.scalar(
+        select(NotificationDelivery.id)
+        .where(
+            NotificationDelivery.state.in_(("pending", "retry")),
+            NotificationDelivery.due_at <= now,
+        )
+        .order_by(NotificationDelivery.due_at, NotificationDelivery.created_at)
+        .limit(1)
+    )
+    if delivery_id is None:
+        return None
+    result = await db.execute(
+        update(NotificationDelivery)
+        .where(
+            NotificationDelivery.id == delivery_id,
+            NotificationDelivery.state.in_(("pending", "retry")),
+            NotificationDelivery.due_at <= now,
+        )
+        .values(
+            state="leased",
+            lease_owner=owner,
+            lease_expires_at=now + timedelta(seconds=lease_seconds),
+            attempt_count=NotificationDelivery.attempt_count + 1,
+        )
+    )
+    if getattr(result, "rowcount", 0) != 1:
+        await db.rollback()
+        return None
+    delivery = await db.get(NotificationDelivery, delivery_id)
+    if delivery is None:  # pragma: no cover - guarded by update
+        await db.rollback()
+        return None
+    subscription = await db.get(NotificationSubscription, delivery.subscription_id)
+    event = await db.get(NotificationEvent, delivery.event_id)
+    if subscription is None or event is None or subscription.state != "active":
+        delivery.state = "failed"
+        delivery.terminal_reason = "subscription_unavailable"
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        await db.commit()
+        return None
+    # Persist a token bucket before starting an attempt, so restarts cannot bypass
+    # the subscription rate limit.  Global process rate is intentionally a config
+    # ceiling; the durable subscription bucket protects cross-restart behavior.
+    last = subscription.rate_updated_at or now
+    elapsed = max(0.0, (now - last).total_seconds())
+    capacity = subscription.rate_limit_per_minute
+    try:
+        token_capacity = float(capacity)
+        tokens = min(token_capacity, subscription.rate_tokens + elapsed * capacity / 60)
+    except (TypeError, ValueError, ZeroDivisionError) as error:
+        raise RuntimeError("notification rate limit is invalid") from error
+    if tokens < 1:
+        try:
+            delay = max(1, int((1 - tokens) * 60 / capacity) + 1)
+        except (TypeError, ValueError, ZeroDivisionError) as error:
+            raise RuntimeError("notification rate limit is invalid") from error
+        delivery.state = "retry"
+        delivery.due_at = now + timedelta(seconds=delay)
+        delivery.lease_owner = None
+        delivery.lease_expires_at = None
+        subscription.rate_tokens = tokens
+        subscription.rate_updated_at = now
+        await db.commit()
+        return None
+    subscription.rate_tokens = tokens - 1
+    subscription.rate_updated_at = now
+    db.add(
+        NotificationDeliveryAttempt(
+            delivery_id=delivery.id,
+            number=delivery.attempt_count,
+            started_at=now,
+            outcome="started",
+        )
+    )
+    await db.commit()
+    return delivery, subscription, event
+
+
+async def _finish(
+    db: AsyncSession,
+    delivery_id: str,
+    owner: str,
+    *,
+    delivered: bool,
+    retryable: bool,
+    response_class: str,
+    reason: str,
+    retry_cap: int,
+    max_attempts: int,
+) -> None:
+    delivery = await db.get(NotificationDelivery, delivery_id)
+    if delivery is None or delivery.lease_owner != owner or delivery.state != "leased":
+        await db.rollback()
+        return
+    attempt = await db.scalar(
+        select(NotificationDeliveryAttempt).where(
+            NotificationDeliveryAttempt.delivery_id == delivery.id,
+            NotificationDeliveryAttempt.number == delivery.attempt_count,
+        )
+    )
+    if attempt is None:  # pragma: no cover - an invariant of _claim_due
+        await db.rollback()
+        return
+    now = datetime.now(UTC)
+    attempt.completed_at = now
+    attempt.response_class = response_class
+    attempt.diagnostic = reason[:512]
+    delivery.response_class = response_class
+    delivery.response_summary = reason[:512]
+    delivery.lease_owner = None
+    delivery.lease_expires_at = None
+    if delivered:
+        delivery.state = "delivered"
+        attempt.outcome = "delivered"
+    elif retryable and delivery.attempt_count < max_attempts:
+        delivery.state = "retry"
+        delivery.due_at = now + timedelta(seconds=retry_delay(delivery.attempt_count, retry_cap))
+        attempt.outcome = "retry"
+    else:
+        delivery.state = "failed"
+        delivery.terminal_reason = reason
+        attempt.outcome = "failed"
+    await db.commit()
+
+
+async def dispatch_one(
+    db: AsyncSession,
+    settings: Settings,
+    cipher: EnvelopeCipher,
+    owner: str,
+    *,
+    resolver: Resolver = system_resolver,
+) -> bool:
+    claimed = await _claim_due(db, owner, settings.notification_delivery_lease_seconds)
+    if claimed is None:
+        return False
+    delivery, subscription, event = claimed
+    max_attempts = settings.notification_max_attempts
+    try:
+        if subscription.channel == "webhook":
+            key_rows = list(
+                (
+                    await db.scalars(
+                        select(NotificationSigningKey).where(
+                            NotificationSigningKey.subscription_id == subscription.id,
+                            NotificationSigningKey.state.in_(("active", "overlap")),
+                        )
+                    )
+                ).all()
+            )
+            keys: list[SigningMaterial] = []
+            now = datetime.now(UTC)
+            for key in key_rows:
+                if (
+                    key.state == "overlap"
+                    and key.overlap_expires_at is not None
+                    and key.overlap_expires_at <= now
+                ):
+                    key.state = "retired"
+                    continue
+                secret = await db.get(Secret, key.secret_id)
+                if secret is None:
+                    raise WebhookTransportError("webhook signing secret is unavailable")
+                keys.append(
+                    SigningMaterial(
+                        key_id=key.id,
+                        version=key.version,
+                        secret=cipher.decrypt(
+                            secret.ciphertext,
+                            purpose=secret.purpose,
+                            version=secret.version,
+                        ),
+                    )
+                )
+            await db.commit()
+            webhook_result = await deliver_webhook(
+                str(subscription.destination_config["url"]),
+                event.canonical_envelope.encode(),
+                event_id=event.id,
+                event_type=event.type,
+                timestamp=event.occurred_at.isoformat(),
+                keys=keys,
+                resolver=resolver,
+                connect_timeout=settings.notification_connect_timeout_seconds,
+                read_timeout=settings.notification_read_timeout_seconds,
+                max_response_bytes=settings.notification_max_response_bytes,
+            )
+            decision = webhook_decision(webhook_result.status_code)
+        elif subscription.channel == "email":
+            email_settings = await db.get(NotificationEmailSettings, 1)
+            if email_settings is None:
+                raise EmailTransportError("SMTP settings are unavailable")
+            max_attempts = email_settings.max_attempts
+            password_secret = await db.get(Secret, email_settings.password_secret_id)
+            if password_secret is None:
+                raise EmailTransportError("SMTP password is unavailable")
+            email_result = await deliver_email(
+                email_settings,
+                cipher.decrypt(
+                    password_secret.ciphertext,
+                    purpose=password_secret.purpose,
+                    version=password_secret.version,
+                ),
+                event,
+                subscription.destination_config["recipients"],
+            )
+            decision = RetryDecision(False, email_result.response_class, "delivered")
+        else:  # guarded by DB constraint
+            raise WebhookTransportError("notification channel is unavailable")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=decision.reason == "delivered",
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=decision.reason,
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except EmailTransportError as error:
+        decision = transport_decision(error.transient, "smtp_transport")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=str(error),
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except WebhookTransportError as error:
+        decision = transport_decision(error.transient, "webhook_transport")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=decision.retry,
+            response_class=decision.response_class,
+            reason=str(error),
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    except (KeyError, ValueError):
+        decision = transport_decision(False, "webhook_validation")
+        await _finish(
+            db,
+            delivery.id,
+            owner,
+            delivered=False,
+            retryable=False,
+            response_class=decision.response_class,
+            reason=decision.reason,
+            retry_cap=settings.notification_retry_cap_seconds,
+            max_attempts=max_attempts,
+        )
+    return True

backend/src/backup_tool/notifications/email.py

@@ -0,0 +1,120 @@
+"""Authenticated, certificate-verified STARTTLS email notification transport."""
+
+from __future__ import annotations
+
+import asyncio
+import smtplib
+import ssl
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.utils import formataddr
+from typing import Protocol, Self, cast
+
+from backup_tool.db.models import NotificationEmailSettings, NotificationEvent
+
+
+class SMTPClient(Protocol):
+    def __enter__(self) -> Self: ...
+
+    def __exit__(self, *args: object) -> None: ...
+
+    def ehlo(self) -> object: ...
+
+    def starttls(self, *, context: ssl.SSLContext) -> object: ...
+
+    def login(self, user: str, password: str) -> object: ...
+
+    def send_message(self, msg: EmailMessage) -> object: ...
+
+
+class EmailTransportError(RuntimeError):
+    def __init__(self, message: str, *, transient: bool = False) -> None:
+        super().__init__(message)
+        self.transient = transient
+
+
+@dataclass(frozen=True)
+class EmailResult:
+    response_class: str
+
+
+def validate_address(value: str) -> str:
+    if not value or len(value) > 320 or any(character in value for character in "\r\n"):
+        raise EmailTransportError("email address is invalid")
+    local, separator, domain = value.rpartition("@")
+    if not separator or not local or not domain or any(character.isspace() for character in value):
+        raise EmailTransportError("email address is invalid")
+    return value
+
+
+def validate_recipients(values: Sequence[str]) -> list[str]:
+    if not values or len(values) > 20:
+        raise EmailTransportError("one to 20 email recipients are required")
+    recipients: list[str] = []
+    for value in values:
+        address = validate_address(value)
+        if address not in recipients:
+            recipients.append(address)
+    return recipients
+
+
+def _message(
+    settings: NotificationEmailSettings,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+) -> EmailMessage:
+    sender = validate_address(settings.sender)
+    safe_recipients = validate_recipients(recipients)
+    message = EmailMessage()
+    message["From"] = formataddr(("Backup Tool", sender))
+    message["To"] = ", ".join(safe_recipients)
+    message["Subject"] = f"Backup Tool: {event.type} ({event.severity})"
+    message["X-Backup-Event-ID"] = event.id
+    # Do not put the full envelope, paths, raw errors, or credentials into mail.
+    message.set_content(
+        "Backup Tool operational event\n"
+        f"Event ID: {event.id}\n"
+        f"Type: {event.type}\n"
+        f"Severity: {event.severity}\n"
+        f"Occurred: {event.occurred_at.isoformat()}\n"
+    )
+    return message
+
+
+def _deliver_sync(
+    settings: NotificationEmailSettings,
+    password: str,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+    smtp_factory: Callable[..., SMTPClient],
+) -> EmailResult:
+    message = _message(settings, event, recipients)
+    try:
+        with smtp_factory(settings.host, settings.port, timeout=10) as client:
+            client.ehlo()
+            context = ssl.create_default_context()
+            client.starttls(context=context)
+            client.ehlo()
+            client.login(settings.username, password)
+            client.send_message(message)
+    except smtplib.SMTPResponseException as error:
+        raise EmailTransportError(
+            f"smtp_{error.smtp_code}", transient=400 <= error.smtp_code < 500
+        ) from error
+    except (smtplib.SMTPException, OSError) as error:
+        raise EmailTransportError("smtp_transport_failed", transient=True) from error
+    return EmailResult(response_class="smtp_2xx")
+
+
+async def deliver_email(
+    settings: NotificationEmailSettings,
+    password: str,
+    event: NotificationEvent,
+    recipients: Sequence[str],
+    *,
+    smtp_factory: Callable[..., SMTPClient] | None = None,
+) -> EmailResult:
+    """Run blocking SMTP only in the worker thread, never in the web process."""
+    factory = smtp_factory or cast(Callable[..., SMTPClient], smtplib.SMTP)
+    return await asyncio.to_thread(_deliver_sync, settings, password, event, recipients, factory)

backend/src/backup_tool/notifications/events.py

@@ -0,0 +1,279 @@
+"""Versioned notification event catalog and transactional outbox fan-out."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from typing import Any
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.db.models import NotificationDelivery, NotificationEvent, NotificationSubscription
+from backup_tool.ids import new_uuid7
+from backup_tool.security.redaction import redact
+from backup_tool.security.ssrf import SSRFError, validate_webhook_url
+
+EVENT_SCHEMA_VERSION = 1
+# Live-events-only policy: every public type below has a current production emitter.
+_EVENT_TYPES = (
+    "execution.queued",
+    "execution.started",
+    "execution.committed",
+    "execution.failed",
+    "execution.cancelled",
+    "execution.retry_queued",
+    "execution.worker_recovered",
+    "schedule.created",
+    "schedule.updated",
+    "schedule.deleted",
+    "schedule.enabled",
+    "schedule.disabled",
+    "schedule.occurrence_enqueued",
+    "schedule.occurrence_misfired",
+    "schedule.occurrence_blocked",
+    "backup.committed",
+    "backup.verification_succeeded",
+    "restore.queued",
+    "restore.committed",
+    "restore.failed",
+    "retention.tombstoned",
+    "notification.test_requested",
+)
+EVENT_CATALOG: dict[str, dict[str, Any]] = {
+    event_type: {
+        "event_schema_version": EVENT_SCHEMA_VERSION,
+        "severity": "error" if event_type.endswith(("failed", "blocked", "rejected")) else "info",
+        "payload_keys": (
+            "attempt",
+            "count",
+            "dry_run",
+            "integrity",
+            "message",
+            "outcome",
+            "reason_code",
+            "requested_mode",
+            "effective_mode",
+            "state",
+        ),
+        "reserved": False,
+    }
+    for event_type in _EVENT_TYPES
+}
+_SAFE_RESOURCE_KEYS = frozenset(
+    {
+        "execution_id",
+        "job_id",
+        "schedule_id",
+        "backup_id",
+        "restore_id",
+        "repository_id",
+        "subscription_id",
+    }
+)
+_SAFE_PAYLOAD_KEYS = frozenset().union(
+    *(set(spec["payload_keys"]) for spec in EVENT_CATALOG.values())
+)
+
+
+class NotificationEventError(ValueError):
+    """A caller attempted to produce data outside the stable public catalog."""
+
+
+def _as_uuid(value: str, field: str) -> str:
+    try:
+        parsed = UUID(value)
+    except (TypeError, ValueError, AttributeError) as error:
+        raise NotificationEventError(f"{field} must be a UUID") from error
+    return str(parsed)
+
+
+def _safe_value(value: Any) -> Any:
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    if isinstance(value, str):
+        return redact(value)[:512]
+    if isinstance(value, list):
+        if len(value) > 32:
+            raise NotificationEventError("payload arrays are limited to 32 values")
+        return [_safe_value(item) for item in value]
+    if isinstance(value, Mapping):
+        if len(value) > 32:
+            raise NotificationEventError("payload objects are limited to 32 fields")
+        return {str(key)[:64]: _safe_value(item) for key, item in value.items()}
+    raise NotificationEventError("payload contains an unsupported value")
+
+
+def _is_filter_match(filter_value: str, event_type: str) -> bool:
+    if filter_value == event_type:
+        return True
+    family, wildcard = filter_value.rsplit(".", 1) if "." in filter_value else ("", "")
+    return wildcard == "*" and event_type.startswith(f"{family}.")
+
+
+def validate_filters(filters: Sequence[str]) -> list[str]:
+    if not filters:
+        raise NotificationEventError("at least one event filter is required")
+    if len(filters) > len(EVENT_CATALOG):
+        raise NotificationEventError("too many event filters")
+    output: list[str] = []
+    for filter_value in filters:
+        if not isinstance(filter_value, str) or len(filter_value) > 96:
+            raise NotificationEventError("invalid event filter")
+        if filter_value.endswith(".*"):
+            family = filter_value[:-2]
+            if not family or not any(item.startswith(f"{family}.") for item in EVENT_CATALOG):
+                raise NotificationEventError("unknown event filter")
+        elif filter_value not in EVENT_CATALOG:
+            raise NotificationEventError("unknown event filter")
+        if filter_value not in output:
+            output.append(filter_value)
+    return output
+
+
+def validate_destination(channel: str, destination: Mapping[str, Any]) -> dict[str, Any]:
+    resource_filters = destination.get("resource_filters", {})
+    if not isinstance(resource_filters, Mapping):
+        raise NotificationEventError("resource filters are invalid")
+    unknown = set(resource_filters) - {"job_ids", "repository_ids", "severities"}
+    if unknown:
+        raise NotificationEventError("resource filter is unknown")
+    normalized_filters: dict[str, list[str]] = {}
+    for key in ("job_ids", "repository_ids"):
+        values = resource_filters.get(key)
+        if values is None:
+            continue
+        if not isinstance(values, list) or not values:
+            raise NotificationEventError("resource filter is invalid")
+        normalized_filters[key] = [_as_uuid(value, key) for value in values]
+    severities = resource_filters.get("severities")
+    if severities is not None:
+        if not isinstance(severities, list) or not severities:
+            raise NotificationEventError("severity filter is invalid")
+        if any(value not in {"info", "warning", "error", "security"} for value in severities):
+            raise NotificationEventError("severity filter is invalid")
+        normalized_filters["severities"] = list(severities)
+    if channel == "webhook":
+        url = destination.get("url")
+        if not isinstance(url, str) or len(url) > 2048:
+            raise NotificationEventError("webhook URL is required")
+        try:
+            validate_webhook_url(url)
+        except SSRFError as error:
+            raise NotificationEventError("webhook URL is invalid") from error
+        return {"url": url, "resource_filters": normalized_filters}
+    if channel == "email":
+        recipients = destination.get("recipients")
+        if not isinstance(recipients, list) or not recipients or len(recipients) > 20:
+            raise NotificationEventError("one to 20 email recipients are required")
+        safe_recipients: list[str] = []
+        for recipient in recipients:
+            if not isinstance(recipient, str) or any(char in recipient for char in "\r\n"):
+                raise NotificationEventError("invalid email recipient")
+            if "@" not in recipient or len(recipient) > 320:
+                raise NotificationEventError("invalid email recipient")
+            if recipient not in safe_recipients:
+                safe_recipients.append(recipient)
+        return {"recipients": safe_recipients, "resource_filters": normalized_filters}
+    raise NotificationEventError("unsupported notification channel")
+
+
+def _matches(subscription: NotificationSubscription, event: dict[str, Any]) -> bool:
+    if subscription.state != "active":
+        return False
+    if not any(_is_filter_match(item, str(event["type"])) for item in subscription.event_filters):
+        return False
+    filters = subscription.destination_config.get("resource_filters", {})
+    if not isinstance(filters, Mapping):
+        return False
+    resources = event["resource"]
+    for key in ("job_ids", "repository_ids"):
+        selected = filters.get(key)
+        resource_key = key[:-1]
+        if selected is not None and resources.get(resource_key) not in selected:
+            return False
+    severities = filters.get("severities")
+    return severities is None or event["severity"] in severities
+
+
+async def emit_event(
+    db: AsyncSession,
+    event_type: str,
+    *,
+    correlation_id: str,
+    resource: Mapping[str, str] | None = None,
+    payload: Mapping[str, Any] | None = None,
+    severity: str | None = None,
+    deduplication_key: str | None = None,
+    occurred_at: datetime | None = None,
+    only_subscription_id: str | None = None,
+) -> NotificationEvent:
+    """Append an immutable event and matching deliveries; intentionally never commits."""
+    if event_type not in EVENT_CATALOG:
+        raise NotificationEventError("unknown operational event type")
+    correlation_id = _as_uuid(correlation_id, "correlation_id")
+    resource = resource or {}
+    if set(resource) - _SAFE_RESOURCE_KEYS:
+        raise NotificationEventError("unknown resource reference")
+    safe_resource = {key: _as_uuid(value, key) for key, value in resource.items()}
+    payload = payload or {}
+    if set(payload) - _SAFE_PAYLOAD_KEYS:
+        raise NotificationEventError("payload key is not allowlisted")
+    safe_payload = {key: _safe_value(value) for key, value in payload.items()}
+    event_severity = severity or str(EVENT_CATALOG[event_type]["severity"])
+    if event_severity not in {"info", "warning", "error", "security"}:
+        raise NotificationEventError("invalid severity")
+    if deduplication_key is not None and (not deduplication_key or len(deduplication_key) > 255):
+        raise NotificationEventError("invalid deduplication key")
+    if deduplication_key is not None:
+        existing = await db.scalar(
+            select(NotificationEvent).where(
+                NotificationEvent.deduplication_key == deduplication_key
+            )
+        )
+        if existing is not None:
+            return existing
+    event_id = str(new_uuid7())
+    timestamp = (occurred_at or datetime.now(UTC)).astimezone(UTC)
+    envelope = {
+        "event_schema_version": EVENT_SCHEMA_VERSION,
+        "id": event_id,
+        "type": event_type,
+        "occurred_at": timestamp.isoformat(),
+        "correlation_id": correlation_id,
+        "severity": event_severity,
+        "resource": safe_resource,
+        "payload": safe_payload,
+    }
+    event = NotificationEvent(
+        id=event_id,
+        type=event_type,
+        schema_version=EVENT_SCHEMA_VERSION,
+        occurred_at=timestamp,
+        correlation_id=correlation_id,
+        severity=event_severity,
+        resource_refs=safe_resource,
+        payload=safe_payload,
+        canonical_envelope=json.dumps(envelope, sort_keys=True, separators=(",", ":")),
+        deduplication_key=deduplication_key,
+    )
+    db.add(event)
+    await db.flush()
+    active_subscriptions = select(NotificationSubscription).where(
+        NotificationSubscription.state == "active"
+    )
+    subscriptions = list((await db.scalars(active_subscriptions)).all())
+    for subscription in subscriptions:
+        selected_for_test = only_subscription_id == subscription.id
+        if selected_for_test or (only_subscription_id is None and _matches(subscription, envelope)):
+            db.add(
+                NotificationDelivery(
+                    event_id=event.id,
+                    subscription_id=subscription.id,
+                    due_at=timestamp,
+                )
+            )
+    await db.flush()
+    return event

backend/src/backup_tool/notifications/retry.py

@@ -0,0 +1,33 @@
+"""Deterministic bounded retry classification for notification delivery."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RetryDecision:
+    retry: bool
+    response_class: str
+    reason: str
+
+
+def retry_delay(attempt: int, cap_seconds: int) -> int:
+    """Bounded exponential delay; no jitter keeps durable tests/restarts deterministic."""
+    exponent = max(0, attempt - 1)
+    return min(cap_seconds, 1 << exponent)
+
+
+def webhook_decision(status_code: int) -> RetryDecision:
+    if 200 <= status_code < 300:
+        return RetryDecision(False, "http_2xx", "delivered")
+    if status_code in {408, 425, 429} or status_code >= 500:
+        return RetryDecision(True, f"http_{status_code}", "http_transient")
+    if 300 <= status_code < 400:
+        return RetryDecision(False, f"http_{status_code}", "redirect_rejected")
+    return RetryDecision(False, f"http_{status_code}", "http_permanent")
+
+
+def transport_decision(transient: bool, response_class: str) -> RetryDecision:
+    reason = "transport_transient" if transient else "transport_failed"
+    return RetryDecision(transient, response_class, reason)

backend/src/backup_tool/notifications/webhook.py

@@ -0,0 +1,210 @@
+"""Canonical, dual-key HMAC webhook requests on a DNS-pinned HTTPX transport."""
+
+from __future__ import annotations
+
+import asyncio
+import hmac
+import ssl
+from collections.abc import Sequence
+from dataclasses import dataclass
+from hashlib import sha256
+
+import httpx
+
+from backup_tool.security.ssrf import (
+    ResolvedWebhookTarget,
+    Resolver,
+    SSRFError,
+    resolve_webhook_target,
+    verify_connected_peer,
+)
+
+SIGNATURE_VERSION = "v1"
+
+
+class WebhookTransportError(RuntimeError):
+    def __init__(self, message: str, *, transient: bool = False) -> None:
+        super().__init__(message)
+        self.transient = transient
+
+
+@dataclass(frozen=True)
+class SigningMaterial:
+    key_id: str
+    version: int
+    secret: str
+
+
+@dataclass(frozen=True)
+class WebhookResult:
+    status_code: int
+    response_bytes: int
+
+
+def canonical_signing_input(timestamp: str, body: bytes) -> bytes:
+    return SIGNATURE_VERSION.encode() + b"." + timestamp.encode("ascii") + b"." + body
+
+
+def signatures(timestamp: str, body: bytes, keys: Sequence[SigningMaterial]) -> list[str]:
+    signing_input = canonical_signing_input(timestamp, body)
+    return [
+        f"{SIGNATURE_VERSION};key_id={key.key_id};key_version={key.version};sha256="
+        f"{hmac.new(key.secret.encode(), signing_input, sha256).hexdigest()}"
+        for key in keys
+    ]
+
+
+class PinnedWebhookTransport(httpx.AsyncBaseTransport):
+    """HTTPX transport that never lets a post-validation DNS lookup choose a peer."""
+
+    def __init__(
+        self,
+        *,
+        target: ResolvedWebhookTarget,
+        connect_timeout: float,
+        read_timeout: float,
+        max_response_bytes: int,
+    ) -> None:
+        self._target = target
+        self._connect_timeout = connect_timeout
+        self._read_timeout = read_timeout
+        self._max_response_bytes = max_response_bytes
+
+    async def _connect(
+        self, target: ResolvedWebhookTarget
+    ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]:
+        hostname = target.url.hostname
+        if hostname is None:  # guarded by resolve_webhook_target
+            raise WebhookTransportError("webhook hostname is unavailable")
+        context = ssl.create_default_context() if target.url.scheme == "https" else None
+        last_error: OSError | None = None
+        for address in target.addresses:
+            try:
+                reader, writer = await asyncio.wait_for(
+                    asyncio.open_connection(
+                        address,
+                        target.port,
+                        ssl=context,
+                        server_hostname=hostname if context is not None else None,
+                    ),
+                    timeout=self._connect_timeout,
+                )
+                verify_connected_peer(writer.get_extra_info("peername"), target.addresses)
+                return reader, writer
+            except (TimeoutError, OSError, ssl.SSLError, SSRFError) as error:
+                last_error = error if isinstance(error, OSError) else None
+        raise WebhookTransportError("webhook connection failed", transient=True) from last_error
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        target = self._target
+        if str(request.url) != target.url.geturl():
+            raise WebhookTransportError("webhook target changed")
+        body = await request.aread()
+        if len(body) > 65_536:
+            raise WebhookTransportError("webhook body is too large")
+        reader, writer = await self._connect(target)
+        try:
+            raw_path = target.url.path or "/"
+            if target.url.query:
+                raw_path += "?" + target.url.query
+            headers = [(key, value) for key, value in request.headers.multi_items()]
+            header_names = {key.lower() for key, _ in headers}
+            if "host" not in header_names:
+                host = target.url.hostname or ""
+                headers.append(("Host", host))
+            if "content-length" not in header_names:
+                headers.append(("Content-Length", str(len(body))))
+            headers.append(("Connection", "close"))
+            serialized = [f"{request.method} {raw_path} HTTP/1.1\r\n".encode()]
+            serialized.extend(f"{key}: {value}\r\n".encode("ascii") for key, value in headers)
+            writer.write(b"".join(serialized) + b"\r\n" + body)
+            await asyncio.wait_for(writer.drain(), timeout=self._read_timeout)
+            head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=self._read_timeout)
+            if len(head) > 16_384:
+                raise WebhookTransportError("webhook response headers are too large")
+            lines = head.decode("iso-8859-1").split("\r\n")
+            try:
+                _protocol, code, _reason = lines[0].split(" ", 2)
+                status_code = int(code)
+            except (IndexError, ValueError) as error:
+                raise WebhookTransportError("webhook response is malformed") from error
+            response_headers: list[tuple[str, str]] = []
+            for line in lines[1:]:
+                if not line:
+                    continue
+                key, separator, value = line.partition(":")
+                if not separator:
+                    raise WebhookTransportError("webhook response headers are malformed")
+                response_headers.append((key.strip(), value.strip()))
+            content = await asyncio.wait_for(
+                reader.read(self._max_response_bytes + 1), timeout=self._read_timeout
+            )
+            if len(content) > self._max_response_bytes:
+                raise WebhookTransportError("webhook response is too large")
+            return httpx.Response(
+                status_code,
+                headers=response_headers,
+                content=content,
+                request=request,
+            )
+        except (TimeoutError, OSError, asyncio.IncompleteReadError) as error:
+            raise WebhookTransportError("webhook request failed", transient=True) from error
+        finally:
+            writer.close()
+            with __import__("contextlib").suppress(OSError):
+                await writer.wait_closed()
+
+
+def webhook_headers(
+    event_id: str,
+    event_type: str,
+    timestamp: str,
+    body: bytes,
+    keys: Sequence[SigningMaterial],
+) -> list[tuple[str, str]]:
+    headers: list[tuple[str, str]] = [
+        ("Content-Type", "application/json"),
+        ("X-Backup-Event-ID", event_id),
+        ("X-Backup-Event-Type", event_type),
+        ("X-Backup-Signature-Version", SIGNATURE_VERSION),
+        ("X-Backup-Timestamp", timestamp),
+    ]
+    headers.extend(("X-Backup-Signature", value) for value in signatures(timestamp, body, keys))
+    return headers
+
+
+async def deliver_webhook(
+    url: str,
+    body: bytes,
+    *,
+    event_id: str,
+    event_type: str,
+    timestamp: str,
+    keys: Sequence[SigningMaterial],
+    resolver: Resolver,
+    connect_timeout: float,
+    read_timeout: float,
+    max_response_bytes: int,
+) -> WebhookResult:
+    if not keys:
+        raise WebhookTransportError("webhook subscription has no active signing key")
+    try:
+        # Resolve immediately before this individual attempt.  The resulting
+        # addresses are passed to the transport, so it cannot rebind at connect.
+        target = await resolve_webhook_target(url, resolver)
+    except SSRFError as error:
+        raise WebhookTransportError("webhook_target_rejected") from error
+    transport = PinnedWebhookTransport(
+        target=target,
+        connect_timeout=connect_timeout,
+        read_timeout=read_timeout,
+        max_response_bytes=max_response_bytes,
+    )
+    headers = webhook_headers(event_id, event_type, timestamp, body, keys)
+    async with httpx.AsyncClient(
+        transport=transport, follow_redirects=False, trust_env=False
+    ) as client:
+        response = await client.post(url, content=body, headers=headers)
+    if 300 <= response.status_code < 400:
+        raise WebhookTransportError("redirect_rejected")
+    return WebhookResult(status_code=response.status_code, response_bytes=len(response.content))

backend/src/backup_tool/observability/__init__.py

@@ -0,0 +1 @@
+"""Operational logging, metrics, and readiness primitives."""

backend/src/backup_tool/observability/health.py

@@ -0,0 +1,39 @@
+"""Readiness checks shared by HTTP and background process roles."""
+
+from __future__ import annotations
+
+import os
+
+from backup_tool.cli import build_alembic_config
+from backup_tool.config import Settings
+from backup_tool.db.engine import assert_schema_current, create_engine
+
+
+class ReadinessError(RuntimeError):
+    """A dependency needed by the selected role is unavailable."""
+
+
+def _require_access(path: object, mode: int) -> None:
+    try:
+        candidate = path if isinstance(path, str) else str(path)
+        if not os.path.isdir(candidate) or not os.access(candidate, mode):
+            raise ReadinessError("required storage is unavailable")
+    except OSError as error:
+        raise ReadinessError("required storage is unavailable") from error
+
+
+async def check_role_readiness(settings: Settings, role: str) -> None:
+    engine = create_engine(settings)
+    try:
+        await assert_schema_current(engine, build_alembic_config(settings))
+    except Exception as error:
+        raise ReadinessError("metadata is unavailable") from error
+    finally:
+        await engine.dispose()
+    if role == "scheduler":
+        return
+    _require_access(settings.data_dir, os.R_OK | os.W_OK | os.X_OK)
+    for root in settings.repository_roots + settings.restore_roots:
+        _require_access(root, os.R_OK | os.W_OK | os.X_OK)
+    for root in settings.local_source_roots:
+        _require_access(root, os.R_OK | os.X_OK)

backend/src/backup_tool/observability/logging.py

@@ -0,0 +1,39 @@
+"""JSON logging that keeps operational context machine-readable and secret-free."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import UTC, datetime
+from typing import Any
+
+_STANDARD_RECORD_KEYS = frozenset(logging.makeLogRecord({}).__dict__)
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "event": record.getMessage(),
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "timestamp": datetime.now(UTC).isoformat(),
+        }
+        for key, value in record.__dict__.items():
+            if key not in _STANDARD_RECORD_KEYS and key not in {"message", "asctime"}:
+                payload[key] = value
+        return json.dumps(payload, default=str, separators=(",", ":"), sort_keys=True)
+
+
+def configure_logging(role: str, level: str) -> None:
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(JsonFormatter())
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.addHandler(handler)
+    root.setLevel(level)
+    logging.getLogger("backup_tool").info("role_started", extra={"role": role})
+
+
+def log_event(name: str, **fields: object) -> None:
+    logging.getLogger("backup_tool").info(name, extra=fields)

backend/src/backup_tool/observability/metrics.py

@@ -0,0 +1,102 @@
+"""Small dependency-free Prometheus exposition for the single-node appliance."""
+
+from __future__ import annotations
+
+import os
+import threading
+from collections import defaultdict
+from collections.abc import Iterable
+from datetime import UTC, datetime
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backup_tool.config import Settings
+from backup_tool.db.models import Backup, Execution, Repository, Schedule
+from backup_tool.execution import ACTIVE_STATES
+
+
+class Metrics:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
+        self._durations: dict[tuple[str, str], tuple[int, float]] = {}
+
+    def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None:
+        route = path if path in {"/livez", "/readyz", "/metrics"} else "/api"
+        with self._lock:
+            self._requests[(method, route, status)] += 1
+            count, total = self._durations.get((method, route), (0, 0.0))
+            self._durations[(method, route)] = (count + 1, total + duration_seconds)
+
+    def render(self, operational: Iterable[tuple[str, float]]) -> str:
+        lines = [
+            "# HELP backup_tool_http_requests_total HTTP requests handled by the web role.",
+            "# TYPE backup_tool_http_requests_total counter",
+        ]
+        with self._lock:
+            for (method, path, status), value in sorted(self._requests.items()):
+                labels = f'method="{method}",path="{path}",status="{status}"'
+                lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}")
+            lines.extend(
+                [
+                    "# HELP backup_tool_http_request_duration_seconds HTTP request duration.",
+                    "# TYPE backup_tool_http_request_duration_seconds summary",
+                ]
+            )
+            for (method, path), (count, total) in sorted(self._durations.items()):
+                labels = f'method="{method}",path="{path}"'
+                lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}")
+                lines.append(
+                    f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}"
+                )
+        lines.extend(f"{name} {value}" for name, value in operational)
+        return "\n".join(lines) + "\n"
+
+
+async def collect_operational_metrics(
+    settings: Settings, db: AsyncSession
+) -> list[tuple[str, float]]:
+    now = datetime.now(UTC)
+    active = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES))
+    )
+    stale = await db.scalar(
+        select(func.count())
+        .select_from(Execution)
+        .where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now)
+    )
+    failed = await db.scalar(
+        select(func.count()).select_from(Execution).where(Execution.state == "failed")
+    )
+    corrupt = await db.scalar(
+        select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt")
+    )
+    schedule_lag = await db.scalar(
+        select(func.min(Schedule.next_nominal_at)).where(
+            Schedule.enabled, Schedule.next_nominal_at.is_not(None)
+        )
+    )
+    values = [
+        ("backup_tool_active_executions", active or 0),
+        ("backup_tool_stale_execution_leases", stale or 0),
+        ("backup_tool_failed_executions", failed or 0),
+        ("backup_tool_corrupt_backups", corrupt or 0),
+        (
+            "backup_tool_schedule_lag_seconds",
+            max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0,
+        ),
+    ]
+    roots = list(settings.repository_roots) + list(settings.restore_roots)
+    for index, root in enumerate(roots):
+        try:
+            stats = os.statvfs(root)
+        except OSError:
+            continue
+        name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}'
+        values.append((name, stats.f_bavail * stats.f_frsize))
+    unavailable = await db.scalar(
+        select(func.count()).select_from(Repository).where(Repository.state == "unavailable")
+    )
+    values.append(("backup_tool_unavailable_repositories", unavailable or 0))
+    return values

backend/src/backup_tool/repository.py

@@ -4,12 +4,19 @@ import hashlib
 import json
 import os
 import shutil
+import stat
+from contextlib import suppress
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from sqlalchemy.ext.asyncio import AsyncSession
+
 from backup_tool.config import Settings
 from backup_tool.ids import new_uuid7
+from backup_tool.security.repository_crypto import RepositoryKeyError, create_data_key
 
 
 class RepositoryError(ValueError):
@@ -19,9 +26,15 @@ class RepositoryError(ValueError):
 @dataclass(frozen=True)
 class InitializedRepository:
     root: Path
+    repository_id: str
     format_version: int = 1
     compression: str = "none"
     encryption: str = "none"
+    signing_key_id: str = ""
+    signing_public_key: str = ""
+    signing_key_path: Path | None = None
+    data_key_id: str | None = None
+    data_key_path: Path | None = None
 
 
 def blob_digest(content: bytes) -> str:
@@ -47,14 +60,14 @@ def _contained(root: Path, relative_path: str) -> Path:
 
 
 def _canonical_payload(compression: str, encryption: str) -> dict[str, object]:
-    if compression != "none" or encryption != "none":
+    if compression != "none" or encryption not in {"none", "aes-256-gcm"}:
         raise RepositoryError("requested repository policy is unavailable")
     return {
         "repository_id": str(new_uuid7()),
         "format_version": 1,
         "digest_algorithm": "sha256",
         "compression": compression,
-        "encryption": {"mode": "none", "key_id": None},
+        "encryption": {"mode": encryption, "key_id": None},
         "created_at": datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z"),
     }
 
@@ -63,7 +76,205 @@ def _canonical_json(payload: dict[str, object]) -> str:
     return json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n"
 
 
-def _assert_capacity(settings: Settings, root: Path) -> None:
+def replace_active_data_key(root: Path, expected_key_id: str, new_key_id: str) -> None:
+    """Durably replace the active, non-secret epoch reference in repository metadata."""
+    metadata = root / "repository.json"
+    try:
+        payload = json.loads(metadata.read_text(encoding="utf-8"))
+        encryption = payload.get("encryption") if isinstance(payload, dict) else None
+        if (
+            not isinstance(encryption, dict)
+            or encryption.get("mode") != "aes-256-gcm"
+            or encryption.get("key_id") != expected_key_id
+        ):
+            raise ValueError
+    except (OSError, ValueError, json.JSONDecodeError) as error:
+        raise RepositoryError("repository metadata is invalid") from error
+    encryption["key_id"] = new_key_id
+    staging = root / f".repository.json.{os.urandom(8).hex()}.tmp"
+    try:
+        descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(_canonical_json(payload))
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(staging, metadata)
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository metadata cannot be updated") from error
+    finally:
+        staging.unlink(missing_ok=True)
+
+
+_ROTATION_JOURNAL = ".key-rotation.json"
+
+
+def begin_key_rotation(
+    root: Path,
+    repository_database_id: str,
+    repository_id: str,
+    old_key_id: str,
+    new_key_id: str,
+) -> None:
+    """Persist an intent record before changing durable key-epoch state."""
+    if not all((repository_database_id, repository_id, old_key_id, new_key_id)):
+        raise RepositoryError("repository rotation journal is invalid")
+    journal = root / _ROTATION_JOURNAL
+    payload: dict[str, object] = {
+        "new_key_id": new_key_id,
+        "old_key_id": old_key_id,
+        "repository_database_id": repository_database_id,
+        "repository_id": repository_id,
+        "version": 1,
+    }
+    try:
+        descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(_canonical_json(payload))
+            handle.flush()
+            os.fsync(handle.fileno())
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository rotation is already in progress") from error
+
+
+def read_key_rotation(root: Path) -> dict[str, str] | None:
+    journal = root / _ROTATION_JOURNAL
+    if not journal.exists():
+        return None
+    try:
+        if journal.is_symlink() or stat.S_IMODE(journal.stat().st_mode) != 0o600:
+            raise ValueError
+        payload = json.loads(journal.read_text(encoding="utf-8"))
+        expected = {
+            "new_key_id",
+            "old_key_id",
+            "repository_database_id",
+            "repository_id",
+            "version",
+        }
+        if (
+            not isinstance(payload, dict)
+            or set(payload) != expected
+            or payload.get("version") != 1
+            or not all(
+                isinstance(payload[key], str) and payload[key] for key in expected - {"version"}
+            )
+        ):
+            raise ValueError
+        return {key: payload[key] for key in expected - {"version"}}
+    except (OSError, ValueError, json.JSONDecodeError) as error:
+        raise RepositoryError("repository rotation journal is invalid") from error
+
+
+def finish_key_rotation(root: Path) -> None:
+    journal = root / _ROTATION_JOURNAL
+    try:
+        journal.unlink()
+        descriptor = os.open(root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    except OSError as error:
+        raise RepositoryError("repository rotation journal cannot be cleared") from error
+
+
+async def reconcile_key_rotations(settings: Settings, db: AsyncSession) -> int:
+    """Converge a journaled key rotation after a process crash.
+
+    A rotation whose DB transaction did not commit remains on the old active
+    epoch. A committed transaction deterministically advances repository.json.
+    """
+    from sqlalchemy import select
+
+    from backup_tool.db.models import Repository, RepositoryDataKeyEpoch
+    from backup_tool.security.repository_crypto import load_data_key
+
+    repositories = list((await db.scalars(select(Repository))).all())
+    reconciled = 0
+    for repository in repositories:
+        if repository.encryption != "aes-256-gcm":
+            continue
+        root = Path(repository.root)
+        journal = read_key_rotation(root)
+        if journal is None:
+            continue
+        inspected = inspect_repository(settings, root)
+        if (
+            journal["repository_database_id"] != repository.id
+            or journal["repository_id"] != inspected.repository_id
+            or repository.active_data_key_id is None
+        ):
+            raise RepositoryError("repository rotation journal does not match metadata")
+        epochs = list(
+            (
+                await db.scalars(
+                    select(RepositoryDataKeyEpoch).where(
+                        RepositoryDataKeyEpoch.repository_id == repository.id
+                    )
+                )
+            ).all()
+        )
+        active = [epoch for epoch in epochs if epoch.state == "active"]
+        old_key_id = journal["old_key_id"]
+        new_key_id = journal["new_key_id"]
+        if len(active) != 1:
+            raise RepositoryError("repository rotation epochs are invalid")
+        if repository.active_data_key_id == old_key_id and active[0].key_id == old_key_id:
+            if inspected.data_key_id != old_key_id:
+                raise RepositoryError("repository rotation metadata is invalid")
+            new_path = (
+                settings.data_dir
+                / "repository-data-keys"
+                / (f"{inspected.repository_id}.{new_key_id}.key")
+            )
+            # Clear the durable intent before deleting an unreferenced key. A
+            # crash afterward leaves only an orphaned key, not a journal whose
+            # retry depends on a key that no longer exists.
+            if new_path.exists() or new_path.is_symlink():
+                try:
+                    load_data_key(settings, inspected.repository_id, new_key_id)
+                except RepositoryKeyError as error:
+                    raise RepositoryError("repository rotation key is unavailable") from error
+            finish_key_rotation(inspected.root)
+            if new_path.exists() or new_path.is_symlink():
+                # The journal is already gone, so a cleanup failure only leaves
+                # an unreferenced key and must not disable the old-active epoch.
+                with suppress(OSError):
+                    new_path.unlink()
+                    descriptor = os.open(new_path.parent, os.O_RDONLY)
+                    try:
+                        os.fsync(descriptor)
+                    finally:
+                        os.close(descriptor)
+            reconciled += 1
+            continue
+        if repository.active_data_key_id == new_key_id and active[0].key_id == new_key_id:
+            try:
+                load_data_key(settings, inspected.repository_id, new_key_id)
+            except RepositoryKeyError as error:
+                raise RepositoryError("repository rotation key is unavailable") from error
+            if inspected.data_key_id == old_key_id:
+                replace_active_data_key(inspected.root, old_key_id, new_key_id)
+            elif inspected.data_key_id != new_key_id:
+                raise RepositoryError("repository rotation metadata is invalid")
+            finish_key_rotation(inspected.root)
+            reconciled += 1
+            continue
+        raise RepositoryError("repository rotation state is invalid")
+    return reconciled
+
+
+def assert_capacity(settings: Settings, root: Path) -> None:
     capacity_root = root
     while not capacity_root.exists():
         parent = capacity_root.parent
@@ -76,13 +287,57 @@ def _assert_capacity(settings: Settings, root: Path) -> None:
         raise RepositoryError("repository root does not meet minimum free capacity")
 
 
+def _signing_key_directory(settings: Settings) -> Path:
+    directory = settings.data_dir / "repository-keys"
+    directory.mkdir(mode=0o700, exist_ok=True)
+    if directory.is_symlink() or not directory.is_dir():
+        raise RepositoryError("repository signing key directory is unsafe")
+    if stat.S_IMODE(directory.stat().st_mode) != 0o700:
+        raise RepositoryError("repository signing key directory permissions must be 0700")
+    return directory
+
+
+def _write_private_key_staging(directory: Path, repository_id: str, private_key: bytes) -> Path:
+    staging = directory / f".{repository_id}.{os.urandom(8).hex()}.tmp"
+    descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(descriptor, "wb") as handle:
+        handle.write(private_key)
+        handle.flush()
+        os.fsync(handle.fileno())
+    return staging
+
+
 def initialize(
     settings: Settings, relative_path: str, compression: str, encryption: str
 ) -> InitializedRepository:
     payload = _canonical_payload(compression, encryption)
+    repository_id = str(payload["repository_id"])
+    data_key_id: str | None = None
+    data_key_path: Path | None = None
+    if encryption == "aes-256-gcm":
+        try:
+            data_key_id, data_key_path = create_data_key(settings, repository_id)
+        except RepositoryKeyError as error:
+            raise RepositoryError("repository data key is unavailable") from error
+        payload["encryption"] = {"mode": encryption, "key_id": data_key_id}
+    private_key = Ed25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        serialization.Encoding.Raw,
+        serialization.PrivateFormat.Raw,
+        serialization.NoEncryption(),
+    )
+    public_bytes = private_key.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    signing_key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
+    key_directory = _signing_key_directory(settings)
+    key_path = key_directory / f"{repository_id}.ed25519"
+    key_staging = _write_private_key_staging(key_directory, repository_id, private_bytes)
     root = _contained(settings.repository_roots[0], relative_path)
-    _assert_capacity(settings, root.parent)
+    assert_capacity(settings, root.parent)
     if root.exists():
+        key_staging.unlink(missing_ok=True)
         raise RepositoryError("repository path already exists")
     staging = root.with_name(f".{root.name}.staging-{os.urandom(8).hex()}")
     try:
@@ -94,17 +349,106 @@ def initialize(
         with metadata.open("rb") as handle:
             os.fsync(handle.fileno())
         os.replace(staging, root)
+        os.replace(key_staging, key_path)
     except Exception:
         shutil.rmtree(staging, ignore_errors=True)
+        shutil.rmtree(root, ignore_errors=True)
+        key_staging.unlink(missing_ok=True)
+        key_path.unlink(missing_ok=True)
+        if data_key_path is not None:
+            data_key_path.unlink(missing_ok=True)
         raise
-    return InitializedRepository(root=root, compression=compression, encryption=encryption)
+    return InitializedRepository(
+        root=root,
+        repository_id=repository_id,
+        compression=compression,
+        encryption=encryption,
+        signing_key_id=signing_key_id,
+        signing_public_key=public_bytes.hex(),
+        signing_key_path=key_path,
+        data_key_id=data_key_id,
+        data_key_path=data_key_path,
+    )
 
 
-def remove_repository(root: Path) -> None:
-    try:
+def remove_repository(
+    root: Path,
+    signing_key_path: Path | None = None,
+    data_key_path: Path | None = None,
+) -> None:
+    try:  # noqa: SIM105 - cleanup must not race a concurrent remover
         shutil.rmtree(root)
     except FileNotFoundError:
-        return
+        pass
+    for key_path in (signing_key_path, data_key_path):
+        if key_path is None:
+            continue
+        try:  # noqa: SIM105 - cleanup must not race a concurrent remover
+            key_path.unlink()
+        except FileNotFoundError:
+            pass
+
+
+def install_signing_key(
+    settings: Settings,
+    repository_id: str,
+    expected_key_id: str,
+    expected_public_key: str,
+    private_bytes: bytes,
+) -> Path:
+    """Install a recovered signing key once after checking its public trust anchor."""
+    if len(private_bytes) != 32 or "/" in repository_id:
+        raise RepositoryError("repository signing key is invalid")
+    try:
+        private_key = Ed25519PrivateKey.from_private_bytes(private_bytes)
+        public_bytes = private_key.public_key().public_bytes(
+            serialization.Encoding.Raw,
+            serialization.PublicFormat.Raw,
+        )
+    except ValueError as error:
+        raise RepositoryError("repository signing key is invalid") from error
+    key_id = f"ed25519-{hashlib.sha256(public_bytes).hexdigest()[:32]}"
+    if key_id != expected_key_id or public_bytes.hex() != expected_public_key:
+        raise RepositoryError("repository signing key is invalid")
+    path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(private_bytes)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            path.unlink(missing_ok=True)
+            raise RepositoryError("repository signing key is unsafe")
+    except OSError as error:
+        raise RepositoryError("repository signing key cannot be installed") from error
+    return path
+
+
+def load_signing_key(
+    settings: Settings,
+    repository_id: str,
+    expected_key_id: str,
+    expected_public_key: str,
+) -> Ed25519PrivateKey:
+    key_path = _signing_key_directory(settings) / f"{repository_id}.ed25519"
+    try:
+        if key_path.is_symlink() or stat.S_IMODE(key_path.stat().st_mode) != 0o600:
+            raise RepositoryError("repository signing key is unsafe")
+        private_key = Ed25519PrivateKey.from_private_bytes(key_path.read_bytes())
+    except (OSError, ValueError) as error:
+        raise RepositoryError("repository signing key is unreadable") from error
+    public_key = private_key.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    key_id = f"ed25519-{hashlib.sha256(public_key).hexdigest()[:32]}"
+    if key_id != expected_key_id or public_key.hex() != expected_public_key:
+        raise RepositoryError("repository signing key does not match its trust anchor")
+    return private_key
 
 
 def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
@@ -140,7 +484,14 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
     ):
         raise RepositoryError("repository metadata is not canonical")
     encryption = payload.get("encryption")
-    if payload.get("compression") != "none" or encryption != {"mode": "none", "key_id": None}:
+    if payload.get("compression") != "none" or not isinstance(encryption, dict):
+        raise RepositoryError("repository metadata is invalid")
+    mode = encryption.get("mode")
+    key_id = encryption.get("key_id")
+    valid_policy = (mode == "none" and key_id is None) or (
+        mode == "aes-256-gcm" and isinstance(key_id, str) and bool(key_id)
+    )
+    if not valid_policy:
         raise RepositoryError("repository metadata is invalid")
     try:
         from uuid import UUID
@@ -152,4 +503,10 @@ def inspect_repository(settings: Settings, root: Path) -> InitializedRepository:
         datetime.fromisoformat(created_at[:-1] + "+00:00")
     except (ValueError, TypeError) as error:
         raise RepositoryError("repository metadata is invalid") from error
-    return InitializedRepository(root=resolved_root, compression="none", encryption="none")
+    return InitializedRepository(
+        root=resolved_root,
+        repository_id=str(payload["repository_id"]),
+        compression="none",
+        encryption=str(encryption["mode"]),
+        data_key_id=key_id if isinstance(key_id, str) else None,
+    )

backend/src/backup_tool/retention.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Protocol
+
+
+class RetentionError(ValueError):
+    pass
+
+
+class BackupLike(Protocol):
+    id: str
+    created_at: datetime
+    pinned: bool
+    tombstoned_at: datetime | None
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+    keep_last: int = 1
+    keep_days: int = 0
+    keep_daily: int = 0
+    keep_weekly: int = 0
+    keep_monthly: int = 0
+
+    @classmethod
+    def from_dict(cls, value: dict[str, object]) -> RetentionPolicy:
+        allowed = {"keep_last", "keep_days", "keep_daily", "keep_weekly", "keep_monthly"}
+        if set(value) - allowed:
+            raise RetentionError("retention policy contains unknown keys")
+        kwargs: dict[str, int] = {}
+        for key in allowed:
+            raw = value.get(key, 0 if key != "keep_last" else 1)
+            if not isinstance(raw, int) or isinstance(raw, bool) or raw < 0:
+                raise RetentionError("retention values must be non-negative integers")
+            kwargs[key] = raw
+        return cls(**kwargs)
+
+
+def retained_ids(
+    backups: Iterable[BackupLike], policy: RetentionPolicy, now: datetime | None = None
+) -> set[str]:
+    """Return union retention set; the newest non-tombstoned backup is always protected."""
+    reference = (now or datetime.now(UTC)).astimezone(UTC)
+    items = sorted(
+        (item for item in backups if item.tombstoned_at is None),
+        key=lambda item: item.created_at.astimezone(UTC),
+        reverse=True,
+    )
+    if not items:
+        return set()
+    kept = {items[0].id}
+    kept.update(item.id for item in items[: policy.keep_last])
+    kept.update(item.id for item in items if item.pinned)
+    if policy.keep_days:
+        cutoff = reference - timedelta(days=policy.keep_days)
+        kept.update(item.id for item in items if item.created_at.astimezone(UTC) >= cutoff)
+    for count, key in (
+        (policy.keep_daily, lambda stamp: stamp.date()),
+        (policy.keep_weekly, lambda stamp: stamp.isocalendar()[:2]),
+        (policy.keep_monthly, lambda stamp: (stamp.year, stamp.month)),
+    ):
+        buckets: set[object] = set()
+        for item in items:
+            stamp = item.created_at.astimezone(UTC)
+            bucket = key(stamp)
+            if len(buckets) >= count and bucket not in buckets:
+                continue
+            buckets.add(bucket)
+            if bucket in buckets:
+                kept.add(item.id)
+    return kept

backend/src/backup_tool/scheduler.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+from datetime import UTC, datetime
+from typing import cast
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from apscheduler.triggers.cron import CronTrigger  # type: ignore[import-untyped]
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from backup_tool.config import Settings
+from backup_tool.db.engine import create_engine
+from backup_tool.db.models import Schedule
+from backup_tool.execution import EnqueueError, enqueue
+from backup_tool.notifications.events import emit_event
+from backup_tool.observability.logging import configure_logging, log_event
+
+
+class ScheduleError(ValueError):
+    pass
+
+
+def next_nominal(cron: str, timezone: str, after: datetime | None = None) -> datetime:
+    if len(cron.split()) != 5:
+        raise ScheduleError("cron must contain exactly five fields")
+    try:
+        zone = ZoneInfo(timezone)
+        trigger = CronTrigger.from_crontab(cron, timezone=zone)
+    except (ValueError, ZoneInfoNotFoundError) as error:
+        raise ScheduleError("cron or timezone is invalid") from error
+    reference = (after or datetime.now(UTC)).astimezone(zone)
+    next_run = trigger.get_next_fire_time(None, reference)
+    if next_run is None:
+        raise ScheduleError("cron has no future occurrence")
+    return cast(datetime, next_run.astimezone(UTC))
+
+
+class SchedulerService:
+    """Dedicated scheduler role using the same transactional enqueue service."""
+
+    def __init__(self, settings: Settings) -> None:
+        self.engine = create_engine(settings)
+        self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
+        self._stopping = asyncio.Event()
+
+    async def run_once(self) -> int:
+        async with self.sessions() as db:
+            return await deliver_due(db)
+
+    async def run(self) -> None:
+        while not self._stopping.is_set():
+            await self.run_once()
+            with contextlib.suppress(TimeoutError):
+                await asyncio.wait_for(self._stopping.wait(), timeout=0.25)
+        await self.engine.dispose()
+
+    def stop(self) -> None:
+        log_event("role_stopping", role="scheduler")
+        self._stopping.set()
+
+
+def run_scheduler(settings: Settings) -> int:
+    configure_logging("scheduler", settings.log_level)
+    service = SchedulerService(settings)
+    loop = asyncio.new_event_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        with contextlib.suppress(NotImplementedError):
+            loop.add_signal_handler(sig, service.stop)
+    try:
+        loop.run_until_complete(service.run())
+    finally:
+        loop.close()
+        log_event("role_stopped", role="scheduler")
+    return 0
+
+
+async def deliver_due(db: AsyncSession, now: datetime | None = None) -> int:
+    current = now or datetime.now(UTC)
+    schedules = list(
+        (
+            await db.scalars(
+                select(Schedule).where(
+                    Schedule.enabled,
+                    Schedule.next_nominal_at.is_not(None),
+                    Schedule.next_nominal_at <= current,
+                )
+            )
+        ).all()
+    )
+    delivered = 0
+    for schedule in schedules:
+        nominal = schedule.next_nominal_at
+        if nominal is None:
+            continue
+        schedule.next_nominal_at = next_nominal(schedule.cron, schedule.timezone, nominal)
+        if (current - nominal).total_seconds() > schedule.misfire_grace_seconds:
+            schedule.last_enqueue_outcome = "misfire"
+            await emit_event(
+                db,
+                "schedule.occurrence_misfired",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"outcome": "misfire"},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:misfire",
+            )
+            continue
+        try:
+            await enqueue(
+                db,
+                schedule.job_id,
+                "schedule",
+                schedule_id=schedule.id,
+                nominal_run_at=nominal,
+            )
+        except EnqueueError as error:
+            schedule.last_enqueue_outcome = error.code
+            await emit_event(
+                db,
+                "schedule.occurrence_blocked",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"reason_code": error.code},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:blocked",
+            )
+        else:
+            schedule.last_enqueue_outcome = "enqueued"
+            await emit_event(
+                db,
+                "schedule.occurrence_enqueued",
+                correlation_id=schedule.id,
+                resource={"schedule_id": schedule.id, "job_id": schedule.job_id},
+                payload={"outcome": "enqueued"},
+                deduplication_key=f"schedule:{schedule.id}:nominal:{nominal.isoformat()}:enqueued",
+            )
+            delivered += 1
+    await db.commit()
+    return delivered

backend/src/backup_tool/security/recovery_bundle.py

@@ -0,0 +1,249 @@
+"""Offline, passphrase-protected recovery bundle codec.
+
+The binary format is deliberately small and versioned so validation can reject
+unsupported inputs before attempting expensive password derivation.  Every
+failure while parsing or authenticating a bundle is reported as the same error
+so callers cannot distinguish a malformed bundle from a wrong passphrase.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import struct
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from argon2.low_level import Type, hash_secret_raw
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+class RecoveryBundleError(ValueError):
+    """A non-disclosing recovery bundle validation failure."""
+
+
+class RecoveryBundlePathError(ValueError):
+    """A requested recovery bundle path cannot be used safely."""
+
+
+_MAGIC = b"BTREC"
+_VERSION = 1
+_KDF_ARGON2ID = 1
+_SALT_BYTES = 16
+_NONCE_BYTES = 12
+_KEY_BYTES = 32
+_TAG_BYTES = 16
+_TIME_COST = 3
+_MEMORY_COST_KIB = 65_536
+_PARALLELISM = 1
+_MAX_PASSPHRASE_BYTES = 4_096
+_MAX_PLAINTEXT_BYTES = 8 * 1024 * 1024
+# magic, version, KDF id, Argon2 time/memory/parallelism, salt/nonce lengths,
+# and the AES-GCM ciphertext (including tag) length.
+_HEADER = struct.Struct(">5sBBIIHBBQ")
+_MAX_BUNDLE_BYTES = _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _MAX_PLAINTEXT_BYTES + _TAG_BYTES
+_ERROR = "recovery bundle is invalid"
+
+
+def _invalid() -> RecoveryBundleError:
+    return RecoveryBundleError(_ERROR)
+
+
+def _canonical_json(payload: Mapping[str, Any]) -> bytes:
+    try:
+        encoded = json.dumps(
+            payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True
+        ).encode("utf-8")
+    except (TypeError, ValueError) as error:
+        raise _invalid() from error
+    if not encoded or len(encoded) > _MAX_PLAINTEXT_BYTES:
+        raise _invalid()
+    return encoded
+
+
+def _passphrase(value: bytes) -> bytes:
+    if not isinstance(value, bytes) or not value or len(value) > _MAX_PASSPHRASE_BYTES:
+        raise _invalid()
+    return value
+
+
+def _derive_key(passphrase: bytes, salt: bytes) -> bytes:
+    return hash_secret_raw(
+        secret=passphrase,
+        salt=salt,
+        time_cost=_TIME_COST,
+        memory_cost=_MEMORY_COST_KIB,
+        parallelism=_PARALLELISM,
+        hash_len=_KEY_BYTES,
+        type=Type.ID,
+    )
+
+
+def encrypt_bundle(payload: Mapping[str, Any], passphrase: bytes) -> bytes:
+    """Serialize and encrypt a canonical recovery payload as a BTREC v1 bundle."""
+    plaintext = _canonical_json(payload)
+    secret = _passphrase(passphrase)
+    salt = os.urandom(_SALT_BYTES)
+    nonce = os.urandom(_NONCE_BYTES)
+    ciphertext_length = len(plaintext) + _TAG_BYTES
+    header = _HEADER.pack(
+        _MAGIC,
+        _VERSION,
+        _KDF_ARGON2ID,
+        _TIME_COST,
+        _MEMORY_COST_KIB,
+        _PARALLELISM,
+        _SALT_BYTES,
+        _NONCE_BYTES,
+        ciphertext_length,
+    )
+    ciphertext = AESGCM(_derive_key(secret, salt)).encrypt(nonce, plaintext, header)
+    return header + salt + nonce + ciphertext
+
+
+def decrypt_bundle(encoded: bytes, passphrase: bytes) -> dict[str, Any]:
+    """Authenticate and decode a BTREC v1 bundle without disclosing failure cause."""
+    try:
+        if not isinstance(encoded, bytes) or len(encoded) > _MAX_BUNDLE_BYTES:
+            raise _invalid()
+        if len(encoded) < _HEADER.size + _SALT_BYTES + _NONCE_BYTES + _TAG_BYTES:
+            raise _invalid()
+        (
+            magic,
+            version,
+            kdf_id,
+            time_cost,
+            memory_cost,
+            parallelism,
+            salt_length,
+            nonce_length,
+            ciphertext_length,
+        ) = _HEADER.unpack(encoded[: _HEADER.size])
+        if (
+            magic != _MAGIC
+            or version != _VERSION
+            or kdf_id != _KDF_ARGON2ID
+            or time_cost != _TIME_COST
+            or memory_cost != _MEMORY_COST_KIB
+            or parallelism != _PARALLELISM
+            or salt_length != _SALT_BYTES
+            or nonce_length != _NONCE_BYTES
+            or ciphertext_length < _TAG_BYTES
+            or ciphertext_length > _MAX_PLAINTEXT_BYTES + _TAG_BYTES
+            or len(encoded) != _HEADER.size + salt_length + nonce_length + ciphertext_length
+        ):
+            raise _invalid()
+        secret = _passphrase(passphrase)
+        salt_start = _HEADER.size
+        nonce_start = salt_start + salt_length
+        ciphertext_start = nonce_start + nonce_length
+        plaintext = AESGCM(_derive_key(secret, encoded[salt_start:nonce_start])).decrypt(
+            encoded[nonce_start:ciphertext_start],
+            encoded[ciphertext_start:],
+            encoded[: _HEADER.size],
+        )
+        if not plaintext or len(plaintext) > _MAX_PLAINTEXT_BYTES:
+            raise _invalid()
+        payload = json.loads(plaintext.decode("utf-8"))
+        if not isinstance(payload, dict):
+            raise _invalid()
+        # Reject non-canonical encodings to make catalog serialization deterministic.
+        if _canonical_json(payload) != plaintext:
+            raise _invalid()
+        return payload
+    except (
+        InvalidTag,
+        UnicodeDecodeError,
+        json.JSONDecodeError,
+        struct.error,
+        ValueError,
+    ) as error:
+        if isinstance(error, RecoveryBundleError):
+            raise error
+        raise _invalid() from error
+
+
+def _check_path_components(path: Path) -> None:
+    if not path.is_absolute() or path.name in {"", ".", ".."}:
+        raise RecoveryBundlePathError("recovery bundle path is unsafe")
+    current = Path(path.anchor)
+    for component in path.parts[1:-1]:
+        current /= component
+        try:
+            info = current.lstat()
+        except OSError as error:
+            raise RecoveryBundlePathError("recovery bundle path is unsafe") from error
+        if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
+            raise RecoveryBundlePathError("recovery bundle path is unsafe")
+
+
+def write_bundle_exclusive(path: Path, encoded: bytes) -> None:
+    """Write a bundle once with restrictive permissions and no symlink following."""
+    if not isinstance(encoded, bytes) or not encoded or len(encoded) > _MAX_BUNDLE_BYTES:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe")
+    _check_path_components(path)
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(encoded)
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
+    try:
+        info = path.lstat()
+        if (
+            stat.S_ISLNK(info.st_mode)
+            or not stat.S_ISREG(info.st_mode)
+            or stat.S_IMODE(info.st_mode) != 0o600
+        ):
+            path.unlink(missing_ok=True)
+            raise RecoveryBundlePathError("recovery bundle output is unsafe")
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle output is unsafe") from error
+
+
+def read_bundle_file(path: Path) -> bytes:
+    """Read a regular, non-symlink bundle with a bounded size."""
+    _check_path_components(path)
+    flags = os.O_RDONLY
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags)
+        with os.fdopen(descriptor, "rb") as handle:
+            info = os.fstat(handle.fileno())
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_size <= 0
+                or info.st_size > _MAX_BUNDLE_BYTES
+            ):
+                raise RecoveryBundlePathError("recovery bundle input is unsafe")
+            return handle.read()
+    except RecoveryBundlePathError:
+        raise
+    except OSError as error:
+        raise RecoveryBundlePathError("recovery bundle input is unsafe") from error
+
+
+def read_passphrase_fd(fd: int) -> bytes:
+    """Read one newline-terminated passphrase from an inherited file descriptor."""
+    if not isinstance(fd, int) or fd < 0:
+        raise RecoveryBundleError("recovery passphrase is unavailable")
+    try:
+        value = os.read(fd, _MAX_PASSPHRASE_BYTES + 2)
+    except OSError as error:
+        raise RecoveryBundleError("recovery passphrase is unavailable") from error
+    if value.endswith(b"\r\n"):
+        value = value[:-2]
+    elif value.endswith(b"\n"):
+        value = value[:-1]
+    if not value or len(value) > _MAX_PASSPHRASE_BYTES:
+        raise RecoveryBundleError("recovery passphrase is unavailable")
+    return value

backend/src/backup_tool/security/repository_crypto.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from backup_tool.config import Settings
+from backup_tool.ids import new_uuid7
+
+
+class RepositoryKeyError(ValueError):
+    pass
+
+
+def _directory(settings: Settings) -> Path:
+    directory = settings.data_dir / "repository-data-keys"
+    directory.mkdir(mode=0o700, exist_ok=True)
+    if (
+        directory.is_symlink()
+        or not directory.is_dir()
+        or stat.S_IMODE(directory.stat().st_mode) != 0o700
+    ):
+        raise RepositoryKeyError("repository data key directory is unsafe")
+    return directory
+
+
+def create_data_key(settings: Settings, repository_id: str) -> tuple[str, Path]:
+    key_id = str(new_uuid7())
+    directory = _directory(settings)
+    path = directory / f"{repository_id}.{key_id}.key"
+    try:
+        descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(os.urandom(32))
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as error:
+        raise RepositoryKeyError("repository data key cannot be created") from error
+    if stat.S_IMODE(path.stat().st_mode) != 0o600 or path.is_symlink():
+        path.unlink(missing_ok=True)
+        raise RepositoryKeyError("repository data key is unsafe")
+    return key_id, path
+
+
+def install_data_key(settings: Settings, repository_id: str, key_id: str, key: bytes) -> Path:
+    """Install recovered key material once; never replace an existing key file."""
+    if len(key) != 32 or "/" in repository_id or "/" in key_id:
+        raise RepositoryKeyError("repository data key is invalid")
+    path = _directory(settings) / f"{repository_id}.{key_id}.key"
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(key)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            path.unlink(missing_ok=True)
+            raise RepositoryKeyError("repository data key is unsafe")
+    except OSError as error:
+        raise RepositoryKeyError("repository data key cannot be installed") from error
+    return path
+
+
+def object_aad(repository_id: str, key_id: str, kind: str, identity: str) -> bytes:
+    if kind not in {"blob", "manifest"} or not all((repository_id, key_id, identity)):
+        raise RepositoryKeyError("encrypted object metadata is invalid")
+    return f"BTENC:1:{repository_id}:{key_id}:{kind}:{identity}".encode()
+
+
+def encrypt_object(key: bytes, aad: bytes, plaintext: bytes) -> bytes:
+    if len(key) != 32:
+        raise RepositoryKeyError("repository data key is unavailable")
+    nonce = os.urandom(12)
+    return b"BTENC\x01" + nonce + AESGCM(key).encrypt(nonce, plaintext, aad)
+
+
+def decrypt_object(key: bytes, aad: bytes, stored: bytes) -> bytes:
+    if len(key) != 32 or not stored.startswith(b"BTENC\x01") or len(stored) < 35:
+        raise RepositoryKeyError("encrypted object is invalid")
+    try:
+        return AESGCM(key).decrypt(stored[6:18], stored[18:], aad)
+    except InvalidTag as error:
+        raise RepositoryKeyError("encrypted object is invalid") from error
+
+
+def load_data_key(settings: Settings, repository_id: str, key_id: str) -> bytes:
+    path = _directory(settings) / f"{repository_id}.{key_id}.key"
+    try:
+        if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o600:
+            raise RepositoryKeyError("repository data key is unsafe")
+        key = path.read_bytes()
+    except OSError as error:
+        raise RepositoryKeyError("repository data key is unavailable") from error
+    if len(key) != 32:
+        raise RepositoryKeyError("repository data key is unavailable")
+    return key

backend/src/backup_tool/security/ssrf.py

@@ -0,0 +1,121 @@
+"""Fail-closed, DNS-rebinding-resistant webhook egress validation."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import socket
+from collections.abc import Awaitable, Callable, Sequence
+from dataclasses import dataclass
+from urllib.parse import SplitResult, urlsplit
+
+
+class SSRFError(ValueError):
+    """The callback target is not safe for the notification egress boundary."""
+
+
+Resolver = Callable[[str, int], Awaitable[Sequence[str]]]
+_ALLOWED_PORTS = frozenset({80, 443, 8080, 8443})
+
+
+def _is_global(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """ipaddress.is_global misses some policy-important mapped/special ranges."""
+    if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
+        return _is_global(address.ipv4_mapped)
+    return bool(address.is_global) and not any(
+        (
+            address.is_loopback,
+            address.is_private,
+            address.is_link_local,
+            address.is_multicast,
+            address.is_unspecified,
+            address.is_reserved,
+        )
+    )
+
+
+def validate_webhook_url(value: str) -> SplitResult:
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except ValueError as error:
+        raise SSRFError("webhook URL is malformed") from error
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        raise SSRFError("webhook URL must be absolute HTTP(S)")
+    if parsed.username is not None or parsed.password is not None or parsed.fragment:
+        raise SSRFError("webhook URL credentials and fragments are forbidden")
+    if len(value) > 2048 or any(character.isspace() for character in value):
+        raise SSRFError("webhook URL is malformed")
+    if port is not None and port not in _ALLOWED_PORTS:
+        raise SSRFError("webhook URL port is not permitted")
+    try:
+        ipaddress.ip_address(parsed.hostname)
+    except ValueError:
+        pass
+    else:
+        # Callback literals are never accepted: names are resolved immediately
+        # before every request and connected addresses are pinned/rechecked.
+        raise SSRFError("literal IP webhook targets are forbidden")
+    return parsed
+
+
+async def system_resolver(hostname: str, port: int) -> Sequence[str]:
+    records = await asyncio.get_running_loop().getaddrinfo(
+        hostname, port, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP
+    )
+    addresses: set[str] = set()
+    for record in records:
+        socket_address = record[4]
+        if socket_address and isinstance(socket_address[0], str):
+            addresses.add(socket_address[0])
+    return tuple(sorted(addresses))
+
+
+async def resolve_public_addresses(
+    hostname: str, port: int, resolver: Resolver = system_resolver
+) -> tuple[str, ...]:
+    try:
+        candidates = tuple(await resolver(hostname, port))
+    except (TimeoutError, OSError) as error:
+        raise SSRFError("webhook DNS resolution failed") from error
+    if not candidates:
+        raise SSRFError("webhook hostname has no addresses")
+    approved: list[str] = []
+    for candidate in candidates:
+        try:
+            address = ipaddress.ip_address(candidate)
+        except ValueError as error:
+            raise SSRFError("webhook resolver returned an invalid address") from error
+        if not _is_global(address):
+            # One unsafe answer poisons the hostname, including mixed public/private.
+            raise SSRFError("webhook hostname resolves to a non-public address")
+        approved.append(str(address))
+    return tuple(approved)
+
+
+@dataclass(frozen=True)
+class ResolvedWebhookTarget:
+    url: SplitResult
+    port: int
+    addresses: tuple[str, ...]
+
+
+async def resolve_webhook_target(
+    value: str, resolver: Resolver = system_resolver
+) -> ResolvedWebhookTarget:
+    parsed = validate_webhook_url(value)
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    addresses = await resolve_public_addresses(parsed.hostname or "", port, resolver)
+    return ResolvedWebhookTarget(url=parsed, port=port, addresses=addresses)
+
+
+def verify_connected_peer(peername: object, approved: Sequence[str]) -> str:
+    if not isinstance(peername, tuple) or not peername or not isinstance(peername[0], str):
+        raise SSRFError("webhook peer address is unavailable")
+    try:
+        peer = str(ipaddress.ip_address(peername[0]))
+    except ValueError as error:
+        raise SSRFError("webhook peer address is invalid") from error
+    if peer not in approved:
+        raise SSRFError("webhook peer changed after DNS resolution")
+    return peer

backend/src/backup_tool/ssh_adapter.py

@@ -0,0 +1,288 @@
+"""Pinned-host-key, forced-SFTP-only source reader.
+
+The adapter has no command-channel API.  It authenticates with the caller's
+already decrypted private key only after verifying the configured host key.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hmac
+import io
+import socket
+import stat
+from collections.abc import AsyncIterator, Callable, Iterator
+from contextlib import suppress
+from typing import Any
+
+import paramiko  # type: ignore[import-untyped]
+
+from backup_tool.adapters import Entry, SourceError
+from backup_tool.config import Settings
+from backup_tool.ssh_source import SSHSourcePublicConfig
+
+TransportFactory = Callable[[str, int, float], Any]
+SFTPFactory = Callable[[Any], Any]
+
+
+def _transport_for(hostname: str, port: int, timeout: float) -> Any:
+    connection = socket.create_connection((hostname, port), timeout=timeout)
+    connection.settimeout(timeout)
+    return paramiko.Transport(connection)
+
+
+def _sftp_for(transport: Any) -> Any:
+    return paramiko.SFTPClient.from_transport(transport)
+
+
+def _next_or_none(iterator: Iterator[Any]) -> Any | None:
+    try:
+        return next(iterator)
+    except StopIteration:
+        return None
+
+
+def load_private_key(value: str) -> Any:
+    """Load only unencrypted Ed25519, ECDSA, or sufficiently strong RSA keys."""
+    for key_type in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey):
+        try:
+            key = key_type.from_private_key(io.StringIO(value), password=None)
+        except (paramiko.SSHException, ValueError):
+            continue
+        if isinstance(key, paramiko.RSAKey) and key.get_bits() < 3072:
+            raise SourceError(
+                "SSH private key algorithm is not permitted", reason_code="source_auth"
+            )
+        return key
+    raise SourceError("SSH private key algorithm is not permitted", reason_code="source_auth")
+
+
+class SSHAdapter:
+    """A stateful SFTP reader rooted at the forced-SFTP account chroot only."""
+
+    def __init__(
+        self,
+        config: SSHSourcePublicConfig,
+        private_key: str,
+        settings: Settings,
+        *,
+        transport_factory: TransportFactory = _transport_for,
+        sftp_factory: SFTPFactory = _sftp_for,
+    ) -> None:
+        self.config = config
+        self._private_key = load_private_key(private_key)
+        self.settings = settings
+        self._transport_factory = transport_factory
+        self._sftp_factory = sftp_factory
+        self._transport: Any | None = None
+        self._sftp: Any | None = None
+        self._files: dict[str, tuple[int, int]] = {}
+
+    def validate_config(self) -> None:
+        if self.config.root != "/":  # defensive: persisted JSON can bypass API validation
+            raise SourceError(
+                "SSH source root must be the forced-SFTP chroot", reason_code="source_invalid"
+            )
+
+    def _connect(self) -> None:
+        if self._sftp is not None:
+            return
+        self.validate_config()
+        transport = self._transport_factory(
+            self.config.hostname, self.config.port, self.settings.ssh_connect_timeout_seconds
+        )
+        self._transport = transport
+        try:
+            transport.start_client(timeout=self.settings.ssh_connect_timeout_seconds)
+            algorithm, encoded_key = self.config.host_key.split(" ", 1)
+            server_key = transport.get_remote_server_key()
+            if server_key.get_name() != algorithm or not hmac.compare_digest(
+                server_key.get_base64(), encoded_key
+            ):
+                raise SourceError(
+                    "SSH host key does not match configured pin", reason_code="source_trust"
+                )
+            # Authentication deliberately occurs only after the exact pin comparison.
+            transport.auth_publickey(self.config.username, self._private_key)
+            sftp = self._sftp_factory(transport)
+            channel = sftp.get_channel()
+            channel.settimeout(self.settings.ssh_operation_timeout_seconds)
+            self._sftp = sftp
+        except SourceError:
+            self._close_sync()
+            raise
+        except paramiko.AuthenticationException as error:
+            self._close_sync()
+            raise SourceError("SSH authentication failed", reason_code="source_auth") from error
+        except (TimeoutError, OSError) as error:
+            self._close_sync()
+            raise SourceError(
+                "SSH source is unavailable", reason_code="source_unavailable"
+            ) from error
+        except (paramiko.SSHException, ValueError) as error:
+            self._close_sync()
+            raise SourceError(
+                "SSH source connection failed", reason_code="source_unavailable"
+            ) from error
+
+    def _sftp_client(self) -> Any:
+        if self._sftp is None:
+            raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
+        return self._sftp
+
+    def _read_client(self) -> Any:
+        if self._transport is None:
+            raise SourceError("SSH source is unavailable", reason_code="source_unavailable")
+        client = self._sftp_factory(self._transport)
+        client.get_channel().settimeout(self.settings.ssh_operation_timeout_seconds)
+        return client
+
+    @staticmethod
+    def _relative(parent: str, name: object) -> str:
+        if not isinstance(name, str) or not name or name in {".", ".."}:
+            raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
+        if "/" in name or "\\" in name or "\x00" in name:
+            raise SourceError("SSH source returned an invalid entry", reason_code="source_invalid")
+        return name if not parent else f"{parent}/{name}"
+
+    @staticmethod
+    def _remote_path(relative: str) -> str:
+        if (
+            not relative
+            or relative.startswith("/")
+            or "\\" in relative
+            or ".." in relative.split("/")
+        ):
+            raise SourceError("SSH source entry is invalid", reason_code="source_invalid")
+        return f"/{relative}"
+
+    @staticmethod
+    def _entry_from_attributes(path: str, attributes: Any) -> Entry:
+        mode = getattr(attributes, "st_mode", None)
+        if not isinstance(mode, int):
+            raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
+        if stat.S_ISLNK(mode):
+            raise SourceError("SSH source symlinks are not supported", reason_code="source_invalid")
+        size = getattr(attributes, "st_size", 0)
+        mtime = getattr(attributes, "st_mtime", 0)
+        if not isinstance(size, int) or size < 0 or not isinstance(mtime, int):
+            raise SourceError("SSH source entry metadata is invalid", reason_code="source_invalid")
+        if stat.S_ISDIR(mode):
+            return Entry(path, "directory", 0, stat.S_IMODE(mode), mtime * 1_000_000_000)
+        if stat.S_ISREG(mode):
+            return Entry(path, "file", size, stat.S_IMODE(mode), mtime * 1_000_000_000)
+        raise SourceError("SSH source contains an unsupported entry", reason_code="source_invalid")
+
+    async def probe(self) -> dict[str, int]:
+        count = 0
+        try:
+            async for entry in self.enumerate_entries():
+                if entry.kind == "file":
+                    count += 1
+            return {"entry_count": count}
+        finally:
+            await self.close()
+
+    async def enumerate_entries(self) -> AsyncIterator[Entry]:
+        await asyncio.to_thread(self._connect)
+        pending = [""]
+        entry_count = 0
+        try:
+            while pending:
+                parent = pending.pop()
+                remote_parent = "/" if not parent else self._remote_path(parent)
+                try:
+                    iterator = await asyncio.to_thread(
+                        self._sftp_client().listdir_iter,
+                        remote_parent,
+                        read_aheads=self.settings.ssh_list_read_aheads,
+                    )
+                    while (
+                        attributes := await asyncio.to_thread(_next_or_none, iterator)
+                    ) is not None:
+                        path = self._relative(parent, getattr(attributes, "filename", None))
+                        entry_count += 1
+                        if entry_count > self.settings.ssh_max_entries:
+                            raise SourceError(
+                                "SSH source entry limit exceeded", reason_code="source_limit"
+                            )
+                        entry = self._entry_from_attributes(path, attributes)
+                        if entry.kind == "directory":
+                            if path.count("/") + 1 > self.settings.ssh_max_traversal_depth:
+                                raise SourceError(
+                                    "SSH source traversal depth exceeded",
+                                    reason_code="source_limit",
+                                )
+                            pending.append(path)
+                        else:
+                            self._files[path] = (entry.size, entry.mtime_ns)
+                        yield entry
+                except SourceError:
+                    raise
+                except (TimeoutError, OSError, paramiko.SSHException) as error:
+                    raise SourceError(
+                        "SSH source enumeration failed", reason_code="source_unavailable"
+                    ) from error
+        except BaseException:
+            await self.close()
+            raise
+
+    async def open_content(self, path: str) -> AsyncIterator[bytes]:
+        expected = self._files.get(path)
+        if expected is None:
+            raise SourceError("SSH source entry was not enumerated", reason_code="source_invalid")
+        remote_path = self._remote_path(path)
+        handle: Any | None = None
+        reader: Any | None = None
+        try:
+            await asyncio.to_thread(self._connect)
+            reader = await asyncio.to_thread(self._read_client)
+            assert reader is not None
+            attributes = await asyncio.to_thread(reader.lstat, remote_path)
+            entry = self._entry_from_attributes(path, attributes)
+            if entry.kind != "file" or (entry.size, entry.mtime_ns) != expected:
+                raise SourceError(
+                    "SSH source file changed during backup", reason_code="source_changed"
+                )
+            handle = await asyncio.to_thread(
+                reader.open,
+                remote_path,
+                "rb",
+                self.settings.ssh_read_chunk_bytes,
+            )
+            assert handle is not None
+            while chunk := await asyncio.to_thread(handle.read, self.settings.ssh_read_chunk_bytes):
+                if not isinstance(chunk, bytes) or len(chunk) > self.settings.ssh_read_chunk_bytes:
+                    raise SourceError(
+                        "SSH source returned an invalid read", reason_code="source_invalid"
+                    )
+                yield chunk
+            after = await asyncio.to_thread(reader.lstat, remote_path)
+            verified = self._entry_from_attributes(path, after)
+            if verified.kind != "file" or (verified.size, verified.mtime_ns) != expected:
+                raise SourceError(
+                    "SSH source file changed during backup", reason_code="source_changed"
+                )
+        except SourceError:
+            raise
+        except (TimeoutError, OSError, paramiko.SSHException) as error:
+            raise SourceError("SSH source read failed", reason_code="source_unavailable") from error
+        finally:
+            if handle is not None:
+                await asyncio.to_thread(handle.close)
+            if reader is not None and reader is not self._sftp:
+                await asyncio.to_thread(reader.close)
+
+    def _close_sync(self) -> None:
+        sftp, transport = self._sftp, self._transport
+        self._sftp = None
+        self._transport = None
+        if sftp is not None:
+            with suppress(OSError, paramiko.SSHException):
+                sftp.close()
+        if transport is not None:
+            with suppress(OSError, paramiko.SSHException):
+                transport.close()
+
+    async def close(self) -> None:
+        await asyncio.to_thread(self._close_sync)

backend/src/backup_tool/ssh_source.py

@@ -0,0 +1,72 @@
+"""Closed public configuration for the staged SSH source capability.
+
+This module deliberately describes configuration only.  It must not create a
+network transport or load a private key; those capabilities are outside this
+slice.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+SSH_PRIVATE_KEY_PURPOSE = "ssh_private_key"
+_ALLOWED_HOST_KEY_ALGORITHMS = frozenset(
+    {
+        "ssh-ed25519",
+        "ecdsa-sha2-nistp256",
+        "ecdsa-sha2-nistp384",
+        "ecdsa-sha2-nistp521",
+        "rsa-sha2-256",
+        "rsa-sha2-512",
+    }
+)
+
+
+def _has_control_or_space(value: str) -> bool:
+    return any(character.isspace() or ord(character) < 32 for character in value)
+
+
+class SSHSourcePublicConfig(BaseModel):
+    """The public, SFTP-chroot-only portion of an SSH source definition."""
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    hostname: str = Field(min_length=1, max_length=253)
+    port: int = Field(ge=1, le=65535)
+    username: str = Field(min_length=1, max_length=255)
+    host_key: str = Field(min_length=1, max_length=16384)
+    # The server-side forced-SFTP account's chroot is the only permitted root.
+    root: Literal["/"]
+
+    @field_validator("hostname")
+    @classmethod
+    def validate_hostname(cls, value: str) -> str:
+        if _has_control_or_space(value) or any(character in value for character in "/\\@?#"):
+            raise ValueError("SSH hostname is invalid")
+        return value
+
+    @field_validator("username")
+    @classmethod
+    def validate_username(cls, value: str) -> str:
+        if _has_control_or_space(value) or any(character in value for character in "/\\:@"):
+            raise ValueError("SSH username is invalid")
+        return value
+
+    @field_validator("host_key")
+    @classmethod
+    def validate_host_key(cls, value: str) -> str:
+        algorithm, separator, encoded_key = value.partition(" ")
+        if not separator or not algorithm or not encoded_key or " " in encoded_key:
+            raise ValueError("SSH host key must be an algorithm and base64 key")
+        if algorithm not in _ALLOWED_HOST_KEY_ALGORITHMS:
+            raise ValueError("SSH host key algorithm is not supported")
+        try:
+            decoded = base64.b64decode(encoded_key, validate=True)
+        except (ValueError, UnicodeEncodeError) as error:
+            raise ValueError("SSH host key is not valid base64") from error
+        if not decoded:
+            raise ValueError("SSH host key is empty")
+        return value

backend/src/backup_tool/web.py

@@ -0,0 +1,43 @@
+"""Dedicated HTTP runtime role."""
+
+from __future__ import annotations
+
+import stat
+
+import uvicorn
+
+from backup_tool.config import Settings
+from backup_tool.observability.logging import configure_logging, log_event
+
+
+def _prepare_socket(settings: Settings) -> str:
+    path = settings.web_socket_path
+    path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
+    try:
+        existing = path.lstat()
+    except FileNotFoundError:
+        return str(path)
+    if not stat.S_ISSOCK(existing.st_mode):
+        raise RuntimeError("web socket path is not a socket")
+    path.unlink()
+    return str(path)
+
+
+def run_web(settings: Settings) -> int:
+    """Run exactly one ASGI server without reload or embedded background roles."""
+    from backup_tool.api.app import create_app
+
+    configure_logging("web", settings.log_level)
+    server = uvicorn.Server(
+        uvicorn.Config(
+            create_app(settings),
+            uds=_prepare_socket(settings),
+            log_level=settings.log_level.lower(),
+            reload=False,
+            workers=1,
+            log_config=None,
+        )
+    )
+    server.run()
+    log_event("role_stopped", role="web")
+    return 0

backend/src/backup_tool/worker.py

@@ -1,22 +1,25 @@
 """Single-node durable worker role.
 
-The worker owns leases; backup publishing is deliberately supplied by later M6 work.
+The worker owns leases and performs repository I/O outside the API process.
 """
 
 from __future__ import annotations
 
 import asyncio
 import contextlib
+import importlib
 import signal
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import cast
 from uuid import uuid4
 
 from sqlalchemy import select, update
-from sqlalchemy.ext.asyncio import async_sessionmaker
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from backup_tool.config import Settings
 from backup_tool.db.engine import create_engine
-from backup_tool.db.models import Execution
+from backup_tool.db.models import Backup, Execution, Job, Repository, Restore, Source
 from backup_tool.execution import (
     claim,
     complete_cancellation,
@@ -25,22 +28,158 @@ from backup_tool.execution import (
     recover_stale,
     transition,
 )
+from backup_tool.faults import FaultInjector, NoFault
+from backup_tool.gc import process_retention_gc
+from backup_tool.notifications.dispatcher import dispatch_one, recover_notification_leases
+from backup_tool.notifications.events import emit_event
+from backup_tool.observability.logging import configure_logging, log_event
+from backup_tool.repository import reconcile_key_rotations
+from backup_tool.security.secrets import EnvelopeCipher
+
+snapshot = importlib.import_module("backup_tool.snapshot")
+SnapshotError = snapshot.SnapshotError
+SnapshotIntegrityError = snapshot.SnapshotIntegrityError
 
 
 class Worker:
-    def __init__(self, settings: Settings, *, owner: str | None = None) -> None:
+    def __init__(
+        self,
+        settings: Settings,
+        *,
+        owner: str | None = None,
+        fault_injector: FaultInjector | None = None,
+    ) -> None:
         self.settings = settings
         self.owner = owner or f"worker-{uuid4()}"
+        self.fault_injector = fault_injector or NoFault()
         self._stopping = asyncio.Event()
         self.engine = create_engine(settings)
         self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
+        self.cipher = EnvelopeCipher.from_file(settings.master_key_file)
+        self._execution_turns = 0
+        self._next_maintenance_at: datetime | None = None
 
     async def startup(self) -> int:
         async with self.sessions() as db:
-            return await recover_stale(db)
+            rotations = await reconcile_key_rotations(self.settings, db)
+            publications = cast(int, await snapshot.reconcile_publications(self.settings, db))
+            restored = cast(int, await snapshot.reconcile_restores(self.settings, db))
+            recovered_executions = await recover_stale(db)
+            recovered_deliveries = await recover_notification_leases(db)
+            maintenance = await process_retention_gc(db)
+            self._next_maintenance_at = datetime.now(UTC) + timedelta(seconds=60)
+            return (
+                rotations
+                + publications
+                + restored
+                + recovered_executions
+                + recovered_deliveries
+                + maintenance.tombstoned
+            )
+
+    async def _run_restore(self, db: AsyncSession) -> bool:
+        restore_id = await db.scalar(
+            select(Restore.id)
+            .where(Restore.state == "queued")
+            .order_by(Restore.created_at)
+            .limit(1)
+        )
+        if restore_id is None:
+            return False
+        result = await db.execute(
+            update(Restore)
+            .where(Restore.id == restore_id, Restore.state == "queued")
+            .values(state="running")
+        )
+        if getattr(result, "rowcount", 0) != 1:
+            await db.rollback()
+            return False
+        await db.commit()
+        restore = await db.get(Restore, restore_id)
+        if restore is None:
+            return False
+        try:
+            backup = await db.get(Backup, restore.backup_id)
+            if backup is None or backup.integrity != "verified" or backup.tombstoned_at is not None:
+                raise SnapshotError("backup is unavailable")
+            execution = await db.get(Execution, backup.execution_id)
+            if execution is None:
+                raise SnapshotError("backup execution is unavailable")
+            job = await db.get(Job, execution.job_id)
+            if job is None:
+                raise SnapshotError("backup job is unavailable")
+            repository = await db.get(Repository, job.repository_id)
+            if repository is None:
+                raise SnapshotError("backup repository is unavailable")
+            restore.result = await snapshot.restore_full_snapshot(
+                self.settings, db, restore, backup, repository
+            )
+            restore.state = "committed"
+            await emit_event(
+                db,
+                "restore.committed",
+                correlation_id=restore.id,
+                resource={"restore_id": restore.id, "backup_id": restore.backup_id},
+                payload={"dry_run": restore.dry_run, "outcome": "committed"},
+                deduplication_key=f"restore:{restore.id}:committed",
+            )
+            await db.commit()
+        except SnapshotIntegrityError:
+            await db.rollback()
+            failed = await db.get(Restore, restore_id)
+            if failed is not None:
+                corrupted_backup = await db.get(Backup, failed.backup_id)
+                if corrupted_backup is not None:
+                    corrupted_backup.integrity = "corrupt"
+                failed.state = "failed"
+                failed.result = {"reason": "restore_failed"}
+                await emit_event(
+                    db,
+                    "restore.failed",
+                    correlation_id=failed.id,
+                    resource={"restore_id": failed.id, "backup_id": failed.backup_id},
+                    payload={"reason_code": "restore_failed"},
+                    deduplication_key=f"restore:{failed.id}:failed",
+                )
+                await db.commit()
+        except SnapshotError:
+            await db.rollback()
+            failed = await db.get(Restore, restore_id)
+            if failed is not None:
+                failed.state = "failed"
+                failed.result = {"reason": "restore_failed"}
+                await emit_event(
+                    db,
+                    "restore.failed",
+                    correlation_id=failed.id,
+                    resource={"restore_id": failed.id, "backup_id": failed.backup_id},
+                    payload={"reason_code": "restore_failed"},
+                    deduplication_key=f"restore:{failed.id}:failed",
+                )
+                await db.commit()
+        return True
+
+    async def _run_maintenance(self, db: AsyncSession) -> None:
+        now = datetime.now(UTC)
+        if self._next_maintenance_at is None or now >= self._next_maintenance_at:
+            await process_retention_gc(db, now)
+            self._next_maintenance_at = now + timedelta(seconds=60)
 
     async def run_once(self) -> bool:
+        if self._stopping.is_set():
+            return False
         async with self.sessions() as db:
+            # Retention/GC is a worker responsibility, but is rate-limited so it
+            # cannot turn a sustained backup queue into a metadata polling loop.
+            await self._run_maintenance(db)
+            # Never let an always-nonempty execution queue starve due notifications.
+            if self._execution_turns >= 1 and await dispatch_one(
+                db, self.settings, self.cipher, self.owner
+            ):
+                self._execution_turns = 0
+                return True
+            if self._stopping.is_set():
+                return False
             execution_id = await db.scalar(
                 select(Execution.id)
                 .where(Execution.state == "queued")
@@ -48,10 +187,13 @@ class Worker:
                 .limit(1)
             )
             if execution_id is None:
-                return False
+                if await self._run_restore(db):
+                    return True
+                return await dispatch_one(db, self.settings, self.cipher, self.owner)
             execution = await claim(db, execution_id, self.owner)
             if execution is None:
                 return False
+            self._execution_turns += 1
             # Reload after the claim: a control request can race the lease acquisition.
             await db.refresh(execution)
             if execution.state == "cancelling":
@@ -73,6 +215,76 @@ class Worker:
                 await record_event(db, execution)
             await db.commit()
             await heartbeat(db, execution.id, self.owner)
+            failure_detail: str | None = None
+            failure_reason = "transient_io"
+            try:
+                job = await db.get(Job, execution.job_id)
+                if job is None:
+                    raise SnapshotError("execution job is unavailable")
+                source = await db.get(Source, job.source_id)
+                repository = await db.get(Repository, job.repository_id)
+                if source is None or repository is None:
+                    raise SnapshotError("execution source or repository is unavailable")
+                backup = await snapshot.publish_full_snapshot(
+                    self.settings,
+                    db,
+                    execution,
+                    job,
+                    source,
+                    repository,
+                    self.fault_injector,
+                    cipher=self.cipher,
+                )
+                await db.flush()
+                await emit_event(
+                    db,
+                    "backup.committed",
+                    correlation_id=execution.id,
+                    resource={
+                        "execution_id": execution.id,
+                        "job_id": job.id,
+                        "repository_id": repository.id,
+                        "backup_id": backup.id,
+                    },
+                    payload={"integrity": backup.integrity, "effective_mode": job.requested_mode},
+                    deduplication_key=f"backup:{backup.id}:committed",
+                )
+                await emit_event(
+                    db,
+                    "backup.verification_succeeded",
+                    correlation_id=execution.id,
+                    resource={"execution_id": execution.id, "backup_id": backup.id},
+                    payload={"integrity": backup.integrity},
+                    deduplication_key=f"backup:{backup.id}:verified",
+                )
+                execution.state = transition("running", "verifying")
+                await record_event(db, execution)
+                self.fault_injector.hit("metadata.before_commit")
+                await db.commit()
+                self.fault_injector.hit("metadata.after_commit")
+                execution.state = transition("verifying", "committed")
+                execution.completed_at = datetime.now(UTC)
+                execution.lease_owner = None
+                execution.lease_expires_at = None
+                await record_event(db, execution)
+                await db.commit()
+                snapshot.finalize_publication(Path(repository.root), execution.id)
+            except SnapshotError as error:
+                await db.rollback()
+                failure_detail = str(error)
+                failure_reason = error.reason_code
+            if failure_detail is not None:
+                failed = await db.get(Execution, execution_id)
+                if failed is None or failed.state not in {"preparing", "running", "verifying"}:
+                    return True
+                failed.state = "failed"
+                failed.reason_code = failure_reason
+                failed.operator_message = failure_detail
+                failed.completed_at = datetime.now(UTC)
+                failed.lease_owner = None
+                failed.lease_expires_at = None
+                await record_event(db, failed)
+                await db.commit()
             return True
 
     async def run(self) -> None:
@@ -84,10 +296,12 @@ class Worker:
         await self.engine.dispose()
 
     def stop(self) -> None:
+        log_event("role_stopping", role="worker")
         self._stopping.set()
 
 
 def run_worker(settings: Settings) -> int:
+    configure_logging("worker", settings.log_level)
     worker = Worker(settings)
     loop = asyncio.new_event_loop()
     for sig in (signal.SIGINT, signal.SIGTERM):
@@ -97,4 +311,5 @@ def run_worker(settings: Settings) -> int:
         loop.run_until_complete(worker.run())
     finally:
         loop.close()
+        log_event("role_stopped", role="worker")
     return 0