feat(v2): complete v2 reimplementation
This commit is contained in:
@@ -0,0 +1,213 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from backup_tool.db.models import (
|
||||
Execution,
|
||||
Job,
|
||||
NotificationDelivery,
|
||||
NotificationDeliveryAttempt,
|
||||
NotificationEmailSettings,
|
||||
NotificationSubscription,
|
||||
Repository,
|
||||
Secret,
|
||||
Source,
|
||||
)
|
||||
from backup_tool.ids import new_uuid7
|
||||
from backup_tool.notifications.dispatcher import (
|
||||
dispatch_one,
|
||||
recover_notification_leases,
|
||||
)
|
||||
from backup_tool.notifications.email import EmailResult, EmailTransportError
|
||||
from backup_tool.notifications.events import emit_event
|
||||
from backup_tool.worker import Worker
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None:
|
||||
client, settings = app_client
|
||||
app = client._transport.app
|
||||
async with app.state.sessions() as db:
|
||||
ciphertext, key_id = app.state.cipher.encrypt(
|
||||
"smtp-password", purpose="notification_smtp", version=1
|
||||
)
|
||||
secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp")
|
||||
db.add(secret)
|
||||
await db.flush()
|
||||
db.add(
|
||||
NotificationEmailSettings(
|
||||
id=1,
|
||||
host="smtp.example.test",
|
||||
port=587,
|
||||
username="operator",
|
||||
password_secret_id=secret.id,
|
||||
sender="sender@example.test",
|
||||
max_attempts=2,
|
||||
rate_limit_per_minute=60,
|
||||
)
|
||||
)
|
||||
subscription = NotificationSubscription(
|
||||
channel="email",
|
||||
event_filters=["execution.queued"],
|
||||
destination_config={"recipients": ["operator@example.test"]},
|
||||
rate_limit_per_minute=60,
|
||||
rate_tokens=60.0,
|
||||
)
|
||||
db.add(subscription)
|
||||
await db.flush()
|
||||
event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="retry-test",
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def transient(*_args, **_kwargs):
|
||||
raise EmailTransportError("smtp_421", transient=True)
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == event.id)
|
||||
)
|
||||
assert delivery is not None
|
||||
assert delivery.state == "retry"
|
||||
assert delivery.attempt_count == 1
|
||||
delivery.state = "leased"
|
||||
delivery.attempt_count = 2
|
||||
delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
db.add(
|
||||
NotificationDeliveryAttempt(
|
||||
delivery_id=delivery.id,
|
||||
number=2,
|
||||
started_at=datetime.now(UTC),
|
||||
outcome="started",
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
assert await recover_notification_leases(db) == 1
|
||||
await db.refresh(delivery)
|
||||
abandoned = await db.scalar(
|
||||
select(NotificationDeliveryAttempt).where(
|
||||
NotificationDeliveryAttempt.delivery_id == delivery.id,
|
||||
NotificationDeliveryAttempt.number == 2,
|
||||
)
|
||||
)
|
||||
assert delivery.state == "retry"
|
||||
assert abandoned is not None
|
||||
assert abandoned.outcome == "retry"
|
||||
assert abandoned.diagnostic == "abandoned_lease"
|
||||
|
||||
async def succeeded(*_args, **_kwargs):
|
||||
return EmailResult(response_class="smtp_2xx")
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded)
|
||||
delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await db.commit()
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
await db.refresh(delivery)
|
||||
assert delivery.state == "delivered"
|
||||
assert delivery.attempt_count == 3
|
||||
|
||||
max_event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="smtp-max-attempts",
|
||||
)
|
||||
await db.commit()
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
max_delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id)
|
||||
)
|
||||
assert max_delivery is not None and max_delivery.state == "retry"
|
||||
max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await db.commit()
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
await db.refresh(max_delivery)
|
||||
assert max_delivery.state == "failed"
|
||||
assert max_delivery.attempt_count == 2
|
||||
|
||||
permanent_event = await emit_event(
|
||||
db,
|
||||
"execution.queued",
|
||||
correlation_id=str(new_uuid7()),
|
||||
resource={},
|
||||
deduplication_key="smtp-permanent",
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def permanent(*_args, **_kwargs):
|
||||
raise EmailTransportError("smtp_550", transient=False)
|
||||
|
||||
monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent)
|
||||
assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
|
||||
permanent_delivery = await db.scalar(
|
||||
select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id)
|
||||
)
|
||||
assert permanent_delivery is not None
|
||||
assert permanent_delivery.state == "failed"
|
||||
assert permanent_delivery.attempt_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notification_dispatch_gets_a_turn_during_execution_backlog(
|
||||
app_client, monkeypatch
|
||||
) -> None:
|
||||
client, settings = app_client
|
||||
app = client._transport.app
|
||||
async with app.state.sessions() as db:
|
||||
repository = Repository(
|
||||
name="fair-repository",
|
||||
root="/fair-repository",
|
||||
format_version=1,
|
||||
compression="none",
|
||||
encryption="none",
|
||||
)
|
||||
source = Source(
|
||||
name="fair-source",
|
||||
kind="local",
|
||||
public_config={"root": "/fair-source"},
|
||||
secret_refs=[],
|
||||
)
|
||||
db.add_all([repository, source])
|
||||
await db.flush()
|
||||
job = Job(
|
||||
name="fair-job",
|
||||
source_id=source.id,
|
||||
repository_id=repository.id,
|
||||
exclusions=[],
|
||||
retention={},
|
||||
requested_mode="full",
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
execution = Execution(job_id=job.id, trigger="manual", progress={})
|
||||
db.add(execution)
|
||||
await db.commit()
|
||||
execution_id = execution.id
|
||||
|
||||
called = False
|
||||
|
||||
async def dispatched(*_args, **_kwargs) -> bool:
|
||||
nonlocal called
|
||||
called = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched)
|
||||
worker = Worker(settings, owner="fair-worker")
|
||||
worker._execution_turns = 1
|
||||
try:
|
||||
assert await worker.run_once()
|
||||
finally:
|
||||
await worker.engine.dispose()
|
||||
async with app.state.sessions() as db:
|
||||
queued = await db.get(Execution, execution_id)
|
||||
assert called
|
||||
assert queued is not None and queued.state == "queued"
|
||||
Reference in New Issue
Block a user