125 lines
4.2 KiB
Python
125 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from backup_tool.db.models import Execution, Job
|
|
|
|
ACTIVE_STATES = frozenset({"queued", "preparing", "running", "verifying", "cancelling"})
|
|
TERMINAL_STATES = frozenset({"committed", "cancelled", "failed"})
|
|
|
|
_ALLOWED: Mapping[str, frozenset[str]] = {
|
|
"queued": frozenset({"preparing", "cancelled", "failed"}),
|
|
"preparing": frozenset({"running", "cancelling", "failed"}),
|
|
"running": frozenset({"verifying", "cancelling", "failed"}),
|
|
"verifying": frozenset({"committed", "failed"}),
|
|
"cancelling": frozenset({"cancelled", "failed"}),
|
|
"committed": frozenset(),
|
|
"cancelled": frozenset(),
|
|
"failed": frozenset(),
|
|
}
|
|
|
|
|
|
class TransitionError(ValueError):
|
|
code = "invalid_transition"
|
|
|
|
|
|
class EnqueueError(ValueError):
|
|
def __init__(self, code: str, detail: str, active_execution_id: str | None = None):
|
|
super().__init__(detail)
|
|
self.code = code
|
|
self.active_execution_id = active_execution_id
|
|
|
|
|
|
def transition(current: str, target: str) -> str:
|
|
"""Validate and return one durable execution-state transition."""
|
|
if target not in _ALLOWED.get(current, frozenset()):
|
|
raise TransitionError(f"Cannot transition execution from {current!r} to {target!r}.")
|
|
return target
|
|
|
|
|
|
async def enqueue(db: AsyncSession, job_id: str, trigger: str = "manual") -> Execution:
|
|
"""Create exactly one active execution for an enabled active job."""
|
|
job = await db.get(Job, job_id)
|
|
if job is None:
|
|
raise EnqueueError("resource_not_found", "Job was not found.")
|
|
if job.state != "active" or not job.enabled:
|
|
raise EnqueueError("job_disabled", "Job is disabled or archived.")
|
|
job_identifier = job.id
|
|
execution = Execution(job_id=job_identifier, trigger=trigger, progress={})
|
|
db.add(execution)
|
|
try:
|
|
await db.commit()
|
|
except IntegrityError as error:
|
|
await db.rollback()
|
|
existing = await db.scalar(
|
|
select(Execution.id).where(
|
|
Execution.job_id == job_identifier, Execution.state.in_(ACTIVE_STATES)
|
|
)
|
|
)
|
|
raise EnqueueError(
|
|
"execution_active", "Job already has an active execution.", existing
|
|
) from error
|
|
await db.refresh(execution)
|
|
return execution
|
|
|
|
|
|
async def claim(
|
|
db: AsyncSession, execution_id: str, owner: str, lease_seconds: int = 60
|
|
) -> Execution | None:
|
|
"""Atomically lease a queued execution; a stale lease may be recovered."""
|
|
now = datetime.now(UTC)
|
|
expires = now + timedelta(seconds=lease_seconds)
|
|
result = await db.execute(
|
|
update(Execution)
|
|
.where(
|
|
Execution.id == execution_id,
|
|
Execution.state.in_({"queued", "preparing"}),
|
|
(Execution.lease_expires_at.is_(None)) | (Execution.lease_expires_at < now),
|
|
)
|
|
.values(state="preparing", lease_owner=owner, lease_expires_at=expires, heartbeat_at=now)
|
|
)
|
|
if getattr(result, "rowcount", 0) != 1:
|
|
await db.rollback()
|
|
return None
|
|
await db.commit()
|
|
return await db.get(Execution, execution_id)
|
|
|
|
|
|
async def heartbeat(
|
|
db: AsyncSession, execution_id: str, owner: str, lease_seconds: int = 60
|
|
) -> bool:
|
|
now = datetime.now(UTC)
|
|
result = await db.execute(
|
|
update(Execution)
|
|
.where(
|
|
Execution.id == execution_id,
|
|
Execution.lease_owner == owner,
|
|
Execution.state.in_(ACTIVE_STATES),
|
|
)
|
|
.values(heartbeat_at=now, lease_expires_at=now + timedelta(seconds=lease_seconds))
|
|
)
|
|
await db.commit()
|
|
return getattr(result, "rowcount", 0) == 1
|
|
|
|
|
|
async def recover_stale(db: AsyncSession) -> int:
|
|
now = datetime.now(UTC)
|
|
result = await db.execute(
|
|
update(Execution)
|
|
.where(Execution.state.in_({"preparing", "running"}), Execution.lease_expires_at < now)
|
|
.values(
|
|
state="queued",
|
|
lease_owner=None,
|
|
lease_expires_at=None,
|
|
heartbeat_at=None,
|
|
reason_code="worker_lost",
|
|
)
|
|
)
|
|
await db.commit()
|
|
return getattr(result, "rowcount", 0) or 0
|