feat(v2): complete v2 reimplementation
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from backup_tool.gc import purge_repository
|
||||
|
||||
|
||||
def test_gc_keeps_manifest_when_unlink_fails(tmp_path: Path) -> None:
|
||||
root = tmp_path / "repository"
|
||||
manifest = root / "manifests" / "deleted.json"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_text('{"entries": []}', encoding="utf-8")
|
||||
|
||||
with (
|
||||
patch("pathlib.Path.unlink", side_effect=OSError("read-only")),
|
||||
pytest.raises(OSError),
|
||||
):
|
||||
purge_repository(root, {"deleted"}, grace=timedelta(0))
|
||||
|
||||
assert manifest.exists()
|
||||
@@ -0,0 +1,213 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from backup_tool.db.models import (
|
||||
Execution,
|
||||
Job,
|
||||
NotificationDelivery,
|
||||
NotificationDeliveryAttempt,
|
||||
NotificationEmailSettings,
|
||||
NotificationSubscription,
|
||||
Repository,
|
||||
Secret,
|
||||
Source,
|
||||
)
|
||||
from backup_tool.ids import new_uuid7
|
||||
from backup_tool.notifications.dispatcher import (
|
||||
dispatch_one,
|
||||
recover_notification_leases,
|
||||
)
|
||||
from backup_tool.notifications.email import EmailResult, EmailTransportError
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.worker import Worker
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None:
|
||||
client, settings = app_client
|
||||
app = client._transport.app
|
||||
async with app.state.sessions() as db:
|
||||
ciphertext, key_id = app.state.cipher.encrypt(
|
||||
"smtp-password", purpose="notification_smtp", version=1
|
||||
)
|
||||
secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp")
|
||||
db.add(secret)
|
||||
await db.flush()
|
||||
db.add(
|
||||
NotificationEmailSettings(
|
||||
id=1,
|
||||
host="smtp.example.test",
|
||||
port=587,
|
||||
username="operator",
|
||||
password_secret_id=secret.id,
|
||||
sender="sender@example.test",
|
||||
max_attempts=2,
|
||||
rate_limit_per_minute=60,
|
||||
)
|
||||
)
|
||||
subscription = NotificationSubscription(
|
||||
channel="email",
|
||||
event_filters=["execution.queued"],
|
||||
destination_config={"recipients": ["operator@example.test"]},
|
||||
rate_limit_per_minute=60,
|
||||
rate_tokens=60.0,
|
||||
)
|
||||
db.add(subscription)
|
||||
await db.flush()
|
||||
event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="retry-test",
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def transient(*_args, **_kwargs):
|
||||
raise EmailTransportError("smtp_421", transient=True)
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == event.id)
|
||||
)
|
||||
assert delivery is not None
|
||||
assert delivery.state == "retry"
|
||||
assert delivery.attempt_count == 1
|
||||
delivery.state = "leased"
|
||||
delivery.attempt_count = 2
|
||||
delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
db.add(
|
||||
NotificationDeliveryAttempt(
|
||||
delivery_id=delivery.id,
|
||||
number=2,
|
||||
started_at=datetime.now(UTC),
|
||||
outcome="started",
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
assert await recover_notification_leases(db) == 1
|
||||
await db.refresh(delivery)
|
||||
abandoned = await db.scalar(
|
||||
select(NotificationDeliveryAttempt).where(
|
||||
NotificationDeliveryAttempt.delivery_id == delivery.id,
|
||||
NotificationDeliveryAttempt.number == 2,
|
||||
)
|
||||
)
|
||||
assert delivery.state == "retry"
|
||||
assert abandoned is not None
|
||||
assert abandoned.outcome == "retry"
|
||||
assert abandoned.diagnostic == "abandoned_lease"
|
||||
|
||||
async def succeeded(*_args, **_kwargs):
|
||||
return EmailResult(response_class="smtp_2xx")
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded)
|
||||
delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await db.commit()
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
await db.refresh(delivery)
|
||||
assert delivery.state == "delivered"
|
||||
assert delivery.attempt_count == 3
|
||||
|
||||
max_event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="smtp-max-attempts",
|
||||
)
|
||||
await db.commit()
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
max_delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id)
|
||||
)
|
||||
assert max_delivery is not None and max_delivery.state == "retry"
|
||||
max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await db.commit()
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
await db.refresh(max_delivery)
|
||||
assert max_delivery.state == "failed"
|
||||
assert max_delivery.attempt_count == 2
|
||||
|
||||
permanent_event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="smtp-permanent",
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def permanent(*_args, **_kwargs):
|
||||
raise EmailTransportError("smtp_550", transient=False)
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
permanent_delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id)
|
||||
)
|
||||
assert permanent_delivery is not None
|
||||
assert permanent_delivery.state == "failed"
|
||||
assert permanent_delivery.attempt_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notification_dispatch_gets_a_turn_during_execution_backlog(
|
||||
app_client, monkeypatch
|
||||
) -> None:
|
||||
client, settings = app_client
|
||||
app = client._transport.app
|
||||
async with app.state.sessions() as db:
|
||||
repository = Repository(
|
||||
name="fair-repository",
|
||||
root="/fair-repository",
|
||||
format_version=1,
|
||||
compression="none",
|
||||
encryption="none",
|
||||
)
|
||||
source = Source(
|
||||
name="fair-source",
|
||||
kind="local",
|
||||
public_config={"root": "/fair-source"},
|
||||
secret_refs=[],
|
||||
)
|
||||
db.add_all([repository, source])
|
||||
await db.flush()
|
||||
job = Job(
|
||||
name="fair-job",
|
||||
source_id=source.id,
|
||||
repository_id=repository.id,
|
||||
exclusions=[],
|
||||
retention={},
|
||||
requested_mode="full",
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
execution = Execution(job_id=job.id, trigger="manual", progress={})
|
||||
db.add(execution)
|
||||
await db.commit()
|
||||
execution_id = execution.id
|
||||
|
||||
called = False
|
||||
|
||||
async def dispatched(*_args, **_kwargs) -> bool:
|
||||
nonlocal called
|
||||
called = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched)
|
||||
worker = Worker(settings, owner="fair-worker")
|
||||
worker._execution_turns = 1
|
||||
try:
|
||||
assert await worker.run_once()
|
||||
finally:
|
||||
await worker.engine.dispose()
|
||||
async with app.state.sessions() as db:
|
||||
queued = await db.get(Execution, execution_id)
|
||||
assert called
|
||||
assert queued is not None and queued.state == "queued"
|
||||
@@ -0,0 +1,203 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import create_engine
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
config = importlib.import_module("backup_tool.config")
|
||||
adapters = importlib.import_module("backup_tool.adapters")
|
||||
faults = importlib.import_module("backup_tool.faults")
|
||||
models = importlib.import_module("backup_tool.db.models")
|
||||
snapshot = importlib.import_module("backup_tool.snapshot")
|
||||
worker_module = importlib.import_module("backup_tool.worker")
|
||||
|
||||
PASSWORD = "correct-horse-battery-staple"
|
||||
|
||||
|
||||
def settings_for(tmp_path: Path):
|
||||
key = tmp_path / "master.key"
|
||||
key.write_bytes(b"m6-publication-fault-test-master-key-material")
|
||||
key.chmod(0o600)
|
||||
data = tmp_path / "data"
|
||||
repositories = tmp_path / "repositories"
|
||||
sources = tmp_path / "sources"
|
||||
restores = tmp_path / "restores"
|
||||
for directory in (data, repositories, sources, restores):
|
||||
directory.mkdir()
|
||||
return config.Settings(
|
||||
data_dir=data,
|
||||
database_url=f"sqlite+aiosqlite:///{data / 'metadata.db'}",
|
||||
repository_roots=(repositories,),
|
||||
local_source_roots=(sources,),
|
||||
restore_roots=(restores,),
|
||||
master_key_file=key,
|
||||
min_free_bytes=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("point", ["blob.before_write", "blob.after_write", "blob.after_fsync"])
|
||||
async def test_blob_write_crash_points_leave_no_published_blob(tmp_path: Path, point: str) -> None:
|
||||
settings = settings_for(tmp_path)
|
||||
source_root = settings.local_source_roots[0] / "source"
|
||||
source_root.mkdir()
|
||||
(source_root / "data.txt").write_text("backup data", encoding="utf-8")
|
||||
adapter = adapters.LocalAdapter(source_root, settings)
|
||||
staged_blob = tmp_path / "staged.blob"
|
||||
|
||||
with pytest.raises(faults.InjectedCrash):
|
||||
await snapshot._copy_file(adapter, "data.txt", staged_blob, faults.CrashAt(point))
|
||||
|
||||
assert not (settings.repository_roots[0] / "blobs" / "sha256").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_staging_is_owner_only_with_a_permissive_umask(tmp_path: Path) -> None:
|
||||
settings = settings_for(tmp_path)
|
||||
source_root = settings.local_source_roots[0] / "source"
|
||||
source_root.mkdir()
|
||||
(source_root / "data.txt").write_text("plaintext", encoding="utf-8")
|
||||
adapter = adapters.LocalAdapter(source_root, settings)
|
||||
staging = tmp_path / "staging"
|
||||
staged_blobs = staging / "blobs"
|
||||
staged_blob = staged_blobs / "0.blob"
|
||||
|
||||
old_umask = os.umask(0)
|
||||
try:
|
||||
snapshot._private_directory(staging)
|
||||
snapshot._private_directory(staged_blobs)
|
||||
with pytest.raises(faults.InjectedCrash):
|
||||
await snapshot._copy_file(
|
||||
adapter,
|
||||
"data.txt",
|
||||
staged_blob,
|
||||
faults.CrashAt("blob.after_write"),
|
||||
)
|
||||
finally:
|
||||
os.umask(old_umask)
|
||||
|
||||
assert stat.S_IMODE(staging.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(staged_blobs.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(staged_blob.stat().st_mode) == 0o600
|
||||
assert staged_blob.read_text(encoding="utf-8") == "plaintext"
|
||||
|
||||
|
||||
def test_blob_install_crash_point_leaves_staged_blob_unpublished(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
settings = settings_for(tmp_path)
|
||||
staged_blob = tmp_path / "staged.blob"
|
||||
staged_blob.write_bytes(b"backup data")
|
||||
digest = "a" * 64
|
||||
target = settings.repository_roots[0] / "blobs" / "sha256" / digest
|
||||
|
||||
with pytest.raises(faults.InjectedCrash):
|
||||
snapshot._install_blob(staged_blob, target, digest, faults.CrashAt("blob.before_rename"))
|
||||
|
||||
assert staged_blob.exists()
|
||||
assert not target.exists()
|
||||
|
||||
|
||||
async def _login(client: httpx.AsyncClient) -> dict[str, str]:
|
||||
response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
|
||||
assert response.status_code == 201
|
||||
return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_crash_is_reconciled_without_republishing(
|
||||
app_client: tuple[httpx.AsyncClient, Settings],
|
||||
) -> None:
|
||||
client, settings = app_client
|
||||
source_root = settings.local_source_roots[0] / "source"
|
||||
source_root.mkdir()
|
||||
(source_root / "data.txt").write_text("backup data", encoding="utf-8")
|
||||
headers = await _login(client)
|
||||
repository = await client.post(
|
||||
"/api/v2/repositories",
|
||||
json={
|
||||
"name": "repo",
|
||||
"relative_path": "repo",
|
||||
"compression": "none",
|
||||
"encryption": "none",
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
source = await client.post(
|
||||
"/api/v2/sources",
|
||||
json={
|
||||
"name": "source",
|
||||
"kind": "local",
|
||||
"public_config": {"root": str(source_root)},
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
job = await client.post(
|
||||
"/api/v2/jobs",
|
||||
json={
|
||||
"name": "job",
|
||||
"source_id": source.json()["id"],
|
||||
"repository_id": repository.json()["id"],
|
||||
"requested_mode": "full",
|
||||
"exclusions": [],
|
||||
"retention": {},
|
||||
"enabled": True,
|
||||
"allow_empty": False,
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
execution = await client.post(f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers)
|
||||
execution_id = execution.json()["id"]
|
||||
|
||||
crashing_worker = worker_module.Worker(
|
||||
settings,
|
||||
owner="crashing-worker",
|
||||
fault_injector=faults.CrashAt("metadata.before_commit"),
|
||||
)
|
||||
try:
|
||||
with pytest.raises(faults.InjectedCrash):
|
||||
await crashing_worker.run_once()
|
||||
finally:
|
||||
await crashing_worker.engine.dispose()
|
||||
|
||||
engine = create_engine(settings)
|
||||
try:
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with sessions() as db:
|
||||
assert (
|
||||
await db.scalar(
|
||||
select(models.Backup).where(models.Backup.execution_id == execution_id)
|
||||
)
|
||||
is None
|
||||
)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
recovery_worker = worker_module.Worker(settings, owner="recovery-worker")
|
||||
try:
|
||||
assert await recovery_worker.startup() == 1
|
||||
finally:
|
||||
await recovery_worker.engine.dispose()
|
||||
|
||||
engine = create_engine(settings)
|
||||
try:
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with sessions() as db:
|
||||
recovered = await db.get(models.Execution, execution_id)
|
||||
backups = list(
|
||||
await db.scalars(
|
||||
select(models.Backup).where(models.Backup.execution_id == execution_id)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
assert recovered is not None and recovered.state == "committed"
|
||||
assert len(backups) == 1
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from backup_tool.scheduler import next_nominal
|
||||
|
||||
|
||||
def test_misfire_window_is_deterministic_for_delivery() -> None:
|
||||
nominal = datetime.now(UTC) - timedelta(seconds=901)
|
||||
next_run = next_nominal("* * * * *", "UTC", nominal)
|
||||
|
||||
assert next_run > nominal
|
||||
assert (datetime.now(UTC) - nominal).total_seconds() > 900
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cron", ["* * * * * *", "invalid"])
|
||||
def test_delivery_rejects_invalid_cron_before_enqueue(cron: str) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
next_nominal(cron, "UTC")
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from alembic import command
|
||||
from backup_tool.config import Settings
|
||||
from backup_tool.db.engine import create_engine
|
||||
from backup_tool.db.models import Execution, ExecutionEvent, Job, Repository, Source
|
||||
from backup_tool.execution import (
|
||||
claim,
|
||||
complete_cancellation,
|
||||
heartbeat,
|
||||
record_event,
|
||||
request_cancellation,
|
||||
)
|
||||
from backup_tool.worker import Worker
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
cli = importlib.import_module("backup_tool.cli")
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def database(
|
||||
tmp_path: Path,
|
||||
) -> AsyncIterator[tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine]]:
|
||||
key = tmp_path / "master.key"
|
||||
key.write_bytes(b"m5-fault-test-master-key-material-32-bytes-minimum")
|
||||
key.chmod(0o600)
|
||||
data_dir = tmp_path / "data"
|
||||
repositories = tmp_path / "repositories"
|
||||
sources = tmp_path / "sources"
|
||||
restores = tmp_path / "restores"
|
||||
for directory in (data_dir, repositories, sources, restores):
|
||||
directory.mkdir()
|
||||
settings = Settings(
|
||||
data_dir=data_dir,
|
||||
database_url=f"sqlite+aiosqlite:///{data_dir / 'metadata.db'}",
|
||||
repository_roots=(repositories,),
|
||||
local_source_roots=(sources,),
|
||||
restore_roots=(restores,),
|
||||
master_key_file=key,
|
||||
)
|
||||
command.upgrade(cli.build_alembic_config(settings), "head")
|
||||
engine = create_engine(settings)
|
||||
yield settings, async_sessionmaker(engine, expire_on_commit=False), engine
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def create_stale_execution(db: AsyncSession, suffix: str, state: str) -> Execution:
|
||||
repository = Repository(
|
||||
name=f"repository-{suffix}",
|
||||
root=f"/repositories/{suffix}",
|
||||
format_version=1,
|
||||
compression="none",
|
||||
encryption="none",
|
||||
)
|
||||
source = Source(
|
||||
name=f"source-{suffix}",
|
||||
kind="local",
|
||||
public_config={"root": f"/sources/{suffix}"},
|
||||
secret_refs=[],
|
||||
)
|
||||
db.add_all([repository, source])
|
||||
await db.flush()
|
||||
job = Job(
|
||||
name=f"job-{suffix}",
|
||||
source_id=source.id,
|
||||
repository_id=repository.id,
|
||||
requested_mode="full",
|
||||
exclusions=[],
|
||||
retention={},
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
execution = Execution(
|
||||
job_id=job.id,
|
||||
trigger="manual",
|
||||
state=state,
|
||||
lease_owner="lost-worker",
|
||||
lease_expires_at=datetime.now(UTC) - timedelta(seconds=1),
|
||||
heartbeat_at=datetime.now(UTC) - timedelta(seconds=2),
|
||||
progress={},
|
||||
reason_code="cancellation_requested" if state == "cancelling" else None,
|
||||
)
|
||||
db.add(execution)
|
||||
await db.flush()
|
||||
await record_event(db, execution)
|
||||
await db.commit()
|
||||
return execution
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_recovers_each_stale_lease_once_and_fences_lost_owner(
|
||||
database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
|
||||
) -> None:
|
||||
settings, sessions, _ = database
|
||||
async with sessions() as db:
|
||||
stale = {
|
||||
state: await create_stale_execution(db, state, state)
|
||||
for state in ("preparing", "running", "verifying", "cancelling")
|
||||
}
|
||||
|
||||
worker = Worker(settings, owner="recovery-worker")
|
||||
try:
|
||||
assert await worker.startup() == len(stale)
|
||||
finally:
|
||||
await worker.engine.dispose()
|
||||
|
||||
async with sessions() as db:
|
||||
recovered = {
|
||||
state: await db.get(Execution, execution.id) for state, execution in stale.items()
|
||||
}
|
||||
for state in ("preparing", "running", "verifying"):
|
||||
execution = recovered[state]
|
||||
assert execution is not None
|
||||
assert execution.state == "queued"
|
||||
assert execution.reason_code == "worker_lost"
|
||||
assert execution.lease_owner is None
|
||||
assert execution.lease_expires_at is None
|
||||
assert execution.heartbeat_at is None
|
||||
cancelling = recovered["cancelling"]
|
||||
assert cancelling is not None
|
||||
assert cancelling.state == "cancelled"
|
||||
assert cancelling.completed_at is not None
|
||||
assert cancelling.reason_code == "cancellation_requested"
|
||||
event_counts = {
|
||||
execution.id: await db.scalar(
|
||||
select(func.count()).where(ExecutionEvent.execution_id == execution.id)
|
||||
)
|
||||
for execution in recovered.values()
|
||||
if execution is not None
|
||||
}
|
||||
|
||||
assert event_counts == {execution.id: 2 for execution in stale.values()}
|
||||
|
||||
async with sessions() as db:
|
||||
execution = recovered["running"]
|
||||
assert execution is not None
|
||||
assert await claim(db, execution.id, "replacement-worker") is not None
|
||||
assert not await heartbeat(db, execution.id, "lost-worker")
|
||||
assert await request_cancellation(db, execution.id) is not None
|
||||
assert not await complete_cancellation(db, execution.id, "lost-worker")
|
||||
assert await complete_cancellation(db, execution.id, "replacement-worker")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stopping_worker_does_not_claim_new_work(
|
||||
database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
|
||||
) -> None:
|
||||
settings, _, _ = database
|
||||
worker = Worker(settings, owner="stopping-worker")
|
||||
try:
|
||||
worker.stop()
|
||||
assert not await worker.run_once()
|
||||
finally:
|
||||
await worker.engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_worker_stops_promptly_and_disposes_its_engine(
|
||||
database: tuple[Settings, async_sessionmaker[AsyncSession], AsyncEngine],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
settings, _, _ = database
|
||||
worker = Worker(settings, owner="idle-worker")
|
||||
disposed = asyncio.Event()
|
||||
dispose = AsyncEngine.dispose
|
||||
|
||||
async def track_dispose(engine: AsyncEngine, *, close: bool = True) -> None:
|
||||
disposed.set()
|
||||
await dispose(engine, close=close)
|
||||
|
||||
monkeypatch.setattr(AsyncEngine, "dispose", track_dispose)
|
||||
task = asyncio.create_task(worker.run())
|
||||
await asyncio.sleep(0)
|
||||
worker.stop()
|
||||
await asyncio.wait_for(task, timeout=1)
|
||||
assert disposed.is_set()
|
||||
Reference in New Issue
Block a user