fix(v2): preserve execution cancellation and retry safety

This commit is contained in:
2026-07-27 22:48:08 +02:00
parent 3b7ef2e874
commit bb87b04ca8
2 changed files with 29 additions and 5 deletions
+27 -4
View File
@@ -164,7 +164,18 @@ async def retry(db: AsyncSession, execution_id: str) -> Execution | None:
execution.heartbeat_at = None execution.heartbeat_at = None
execution.reason_code = None execution.reason_code = None
execution.operator_message = None execution.operator_message = None
await db.commit() try:
await db.commit()
except IntegrityError as error:
await db.rollback()
active_id = await db.scalar(
select(Execution.id).where(
Execution.job_id == execution.job_id, Execution.state.in_(ACTIVE_STATES)
)
)
raise EnqueueError(
"execution_active", "Job already has an active execution.", active_id
) from error
await db.refresh(execution) await db.refresh(execution)
return execution return execution
@@ -199,10 +210,22 @@ def public_event(execution: Execution) -> dict[str, object]:
async def recover_stale(db: AsyncSession) -> int: async def recover_stale(db: AsyncSession) -> int:
now = datetime.now(UTC) now = datetime.now(UTC)
result = await db.execute( cancelled = await db.execute(
update(Execution)
.where(Execution.state == "cancelling", Execution.lease_expires_at < now)
.values(
state="cancelled",
completed_at=now,
lease_owner=None,
lease_expires_at=None,
heartbeat_at=None,
reason_code="cancellation_requested",
)
)
recovered = await db.execute(
update(Execution) update(Execution)
.where( .where(
Execution.state.in_({"preparing", "running", "verifying", "cancelling"}), Execution.state.in_({"preparing", "running", "verifying"}),
Execution.lease_expires_at < now, Execution.lease_expires_at < now,
) )
.values( .values(
@@ -214,4 +237,4 @@ async def recover_stale(db: AsyncSession) -> int:
) )
) )
await db.commit() await db.commit()
return getattr(result, "rowcount", 0) or 0 return (getattr(cancelled, "rowcount", 0) or 0) + (getattr(recovered, "rowcount", 0) or 0)
+2 -1
View File
@@ -45,7 +45,8 @@ class Worker:
execution = await claim(db, execution_id, self.owner) execution = await claim(db, execution_id, self.owner)
if execution is None: if execution is None:
return False return False
# M6 supplies source/repository work; this slice proves durable ownership. # Reload after the claim: a control request can race the lease acquisition.
await db.refresh(execution)
if execution.state == "cancelling": if execution.state == "cancelling":
await complete_cancellation(db, execution.id, self.owner) await complete_cancellation(db, execution.id, self.owner)
return True return True