feat(v2): complete v2 reimplementation
This commit is contained in:
@@ -1,22 +1,25 @@
|
||||
"""Single-node durable worker role.
|
||||
|
||||
The worker owns leases; backup publishing is deliberately supplied by later M6 work.
|
||||
The worker owns leases and performs repository I/O outside the API process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import importlib
|
||||
import signal
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import create_engine
|
||||
from backup_tool.db.models import Execution
|
||||
from backup_tool.db.models import Backup, Execution, Job, Repository, Restore, Source
|
||||
from backup_tool.execution import (
|
||||
claim,
|
||||
complete_cancellation,
|
||||
@@ -25,22 +28,158 @@ from backup_tool.execution import (
|
||||
recover_stale,
|
||||
transition,
|
||||
)
|
||||
from backup_tool.faults import FaultInjector, NoFault
|
||||
from backup_tool.gc import process_retention_gc
|
||||
from backup_tool.notifications.dispatcher import dispatch_one, recover_notification_leases
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.observability.logging import configure_logging, log_event
|
||||
from backup_tool.repository import reconcile_key_rotations
|
||||
from backup_tool.security.secrets import EnvelopeCipher
|
||||
|
||||
snapshot = importlib.import_module("backup_tool.snapshot")
|
||||
SnapshotError = snapshot.SnapshotError
|
||||
SnapshotIntegrityError = snapshot.SnapshotIntegrityError
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, settings: Settings, *, owner: str | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
*,
|
||||
owner: str | None = None,
|
||||
fault_injector: FaultInjector | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.owner = owner or f"worker-{uuid4()}"
|
||||
self.fault_injector = fault_injector or NoFault()
|
||||
self._stopping = asyncio.Event()
|
||||
self.engine = create_engine(settings)
|
||||
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
|
||||
self.cipher = EnvelopeCipher.from_file(settings.master_key_file)
|
||||
self._execution_turns = 0
|
||||
self._next_maintenance_at: datetime | None = None
|
||||
|
||||
async def startup(self) -> int:
|
||||
async with self.sessions() as db:
|
||||
return await recover_stale(db)
|
||||
rotations = await reconcile_key_rotations(self.settings, db)
|
||||
publications = cast(int, await snapshot.reconcile_publications(self.settings, db))
|
||||
restored = cast(int, await snapshot.reconcile_restores(self.settings, db))
|
||||
recovered_executions = await recover_stale(db)
|
||||
recovered_deliveries = await recover_notification_leases(db)
|
||||
maintenance = await process_retention_gc(db)
|
||||
self._next_maintenance_at = datetime.now(UTC) + timedelta(seconds=60)
|
||||
return (
|
||||
rotations
|
||||
+ publications
|
||||
+ restored
|
||||
+ recovered_executions
|
||||
+ recovered_deliveries
|
||||
+ maintenance.tombstoned
|
||||
)
|
||||
|
||||
async def _run_restore(self, db: AsyncSession) -> bool:
|
||||
restore_id = await db.scalar(
|
||||
select(Restore.id)
|
||||
.where(Restore.state == "queued")
|
||||
.order_by(Restore.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if restore_id is None:
|
||||
return False
|
||||
result = await db.execute(
|
||||
update(Restore)
|
||||
.where(Restore.id == restore_id, Restore.state == "queued")
|
||||
.values(state="running")
|
||||
)
|
||||
if getattr(result, "rowcount", 0) != 1:
|
||||
await db.rollback()
|
||||
return False
|
||||
await db.commit()
|
||||
restore = await db.get(Restore, restore_id)
|
||||
if restore is None:
|
||||
return False
|
||||
try:
|
||||
backup = await db.get(Backup, restore.backup_id)
|
||||
if backup is None or backup.integrity != "verified" or backup.tombstoned_at is not None:
|
||||
raise SnapshotError("backup is unavailable")
|
||||
execution = await db.get(Execution, backup.execution_id)
|
||||
if execution is None:
|
||||
raise SnapshotError("backup execution is unavailable")
|
||||
job = await db.get(Job, execution.job_id)
|
||||
if job is None:
|
||||
raise SnapshotError("backup job is unavailable")
|
||||
repository = await db.get(Repository, job.repository_id)
|
||||
if repository is None:
|
||||
raise SnapshotError("backup repository is unavailable")
|
||||
restore.result = await snapshot.restore_full_snapshot(
|
||||
self.settings, db, restore, backup, repository
|
||||
)
|
||||
restore.state = "committed"
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.committed",
|
||||
correlation_id=restore.id,
|
||||
resource={"restore_id": restore.id, "backup_id": restore.backup_id},
|
||||
payload={"dry_run": restore.dry_run, "outcome": "committed"},
|
||||
deduplication_key=f"restore:{restore.id}:committed",
|
||||
)
|
||||
await db.commit()
|
||||
except SnapshotIntegrityError:
|
||||
await db.rollback()
|
||||
failed = await db.get(Restore, restore_id)
|
||||
if failed is not None:
|
||||
corrupted_backup = await db.get(Backup, failed.backup_id)
|
||||
if corrupted_backup is not None:
|
||||
corrupted_backup.integrity = "corrupt"
|
||||
failed.state = "failed"
|
||||
failed.result = {"reason": "restore_failed"}
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.failed",
|
||||
correlation_id=failed.id,
|
||||
resource={"restore_id": failed.id, "backup_id": failed.backup_id},
|
||||
payload={"reason_code": "restore_failed"},
|
||||
deduplication_key=f"restore:{failed.id}:failed",
|
||||
)
|
||||
await db.commit()
|
||||
except SnapshotError:
|
||||
await db.rollback()
|
||||
failed = await db.get(Restore, restore_id)
|
||||
if failed is not None:
|
||||
failed.state = "failed"
|
||||
failed.result = {"reason": "restore_failed"}
|
||||
await emit_event(
|
||||
db,
|
||||
"restore.failed",
|
||||
correlation_id=failed.id,
|
||||
resource={"restore_id": failed.id, "backup_id": failed.backup_id},
|
||||
payload={"reason_code": "restore_failed"},
|
||||
deduplication_key=f"restore:{failed.id}:failed",
|
||||
)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def _run_maintenance(self, db: AsyncSession) -> None:
|
||||
now = datetime.now(UTC)
|
||||
if self._next_maintenance_at is None or now >= self._next_maintenance_at:
|
||||
await process_retention_gc(db, now)
|
||||
self._next_maintenance_at = now + timedelta(seconds=60)
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
if self._stopping.is_set():
|
||||
return False
|
||||
async with self.sessions() as db:
|
||||
# Retention/GC is a worker responsibility, but is rate-limited so it
|
||||
# cannot turn a sustained backup queue into a metadata polling loop.
|
||||
await self._run_maintenance(db)
|
||||
# Never let an always-nonempty execution queue starve due notifications.
|
||||
if self._execution_turns >= 1 and await dispatch_one(
|
||||
db, self.settings, self.cipher, self.owner
|
||||
):
|
||||
self._execution_turns = 0
|
||||
return True
|
||||
if self._stopping.is_set():
|
||||
return False
|
||||
execution_id = await db.scalar(
|
||||
select(Execution.id)
|
||||
.where(Execution.state == "queued")
|
||||
@@ -48,10 +187,13 @@ class Worker:
|
||||
.limit(1)
|
||||
)
|
||||
if execution_id is None:
|
||||
return False
|
||||
if await self._run_restore(db):
|
||||
return True
|
||||
return await dispatch_one(db, self.settings, self.cipher, self.owner)
|
||||
execution = await claim(db, execution_id, self.owner)
|
||||
if execution is None:
|
||||
return False
|
||||
self._execution_turns += 1
|
||||
# Reload after the claim: a control request can race the lease acquisition.
|
||||
await db.refresh(execution)
|
||||
if execution.state == "cancelling":
|
||||
@@ -73,6 +215,76 @@ class Worker:
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
await heartbeat(db, execution.id, self.owner)
|
||||
failure_detail: str | None = None
|
||||
failure_reason = "transient_io"
|
||||
try:
|
||||
job = await db.get(Job, execution.job_id)
|
||||
if job is None:
|
||||
raise SnapshotError("execution job is unavailable")
|
||||
source = await db.get(Source, job.source_id)
|
||||
repository = await db.get(Repository, job.repository_id)
|
||||
if source is None or repository is None:
|
||||
raise SnapshotError("execution source or repository is unavailable")
|
||||
backup = await snapshot.publish_full_snapshot(
|
||||
self.settings,
|
||||
db,
|
||||
execution,
|
||||
job,
|
||||
source,
|
||||
repository,
|
||||
self.fault_injector,
|
||||
cipher=self.cipher,
|
||||
)
|
||||
await db.flush()
|
||||
await emit_event(
|
||||
db,
|
||||
"backup.committed",
|
||||
correlation_id=execution.id,
|
||||
resource={
|
||||
"execution_id": execution.id,
|
||||
"job_id": job.id,
|
||||
"repository_id": repository.id,
|
||||
"backup_id": backup.id,
|
||||
},
|
||||
payload={"integrity": backup.integrity, "effective_mode": job.requested_mode},
|
||||
deduplication_key=f"backup:{backup.id}:committed",
|
||||
)
|
||||
await emit_event(
|
||||
db,
|
||||
"backup.verification_succeeded",
|
||||
correlation_id=execution.id,
|
||||
resource={"execution_id": execution.id, "backup_id": backup.id},
|
||||
payload={"integrity": backup.integrity},
|
||||
deduplication_key=f"backup:{backup.id}:verified",
|
||||
)
|
||||
execution.state = transition("running", "verifying")
|
||||
await record_event(db, execution)
|
||||
self.fault_injector.hit("metadata.before_commit")
|
||||
await db.commit()
|
||||
self.fault_injector.hit("metadata.after_commit")
|
||||
execution.state = transition("verifying", "committed")
|
||||
execution.completed_at = datetime.now(UTC)
|
||||
execution.lease_owner = None
|
||||
execution.lease_expires_at = None
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
snapshot.finalize_publication(Path(repository.root), execution.id)
|
||||
except SnapshotError as error:
|
||||
await db.rollback()
|
||||
failure_detail = str(error)
|
||||
failure_reason = error.reason_code
|
||||
if failure_detail is not None:
|
||||
failed = await db.get(Execution, execution_id)
|
||||
if failed is None or failed.state not in {"preparing", "running", "verifying"}:
|
||||
return True
|
||||
failed.state = "failed"
|
||||
failed.reason_code = failure_reason
|
||||
failed.operator_message = failure_detail
|
||||
failed.completed_at = datetime.now(UTC)
|
||||
failed.lease_owner = None
|
||||
failed.lease_expires_at = None
|
||||
await record_event(db, failed)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def run(self) -> None:
|
||||
@@ -84,10 +296,12 @@ class Worker:
|
||||
await self.engine.dispose()
|
||||
|
||||
def stop(self) -> None:
|
||||
log_event("role_stopping", role="worker")
|
||||
self._stopping.set()
|
||||
|
||||
|
||||
def run_worker(settings: Settings) -> int:
|
||||
configure_logging("worker", settings.log_level)
|
||||
worker = Worker(settings)
|
||||
loop = asyncio.new_event_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
@@ -97,4 +311,5 @@ def run_worker(settings: Settings) -> int:
|
||||
loop.run_until_complete(worker.run())
|
||||
finally:
|
||||
loop.close()
|
||||
log_event("role_stopped", role="worker")
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user