backup-tool/tests/fault/test_notification_retries.py

from __future__ import annotations

from datetime import UTC, datetime, timedelta

import pytest
from backup_tool.db.models import (
    Execution,
    Job,
    NotificationDelivery,
    NotificationDeliveryAttempt,
    NotificationEmailSettings,
    NotificationSubscription,
    Repository,
    Secret,
    Source,
)
from backup_tool.ids import new_uuid7
from backup_tool.notifications.dispatcher import (
    dispatch_one,
    recover_notification_leases,
)
from backup_tool.notifications.email import EmailResult, EmailTransportError
from backup_tool.notifications.events import emit_event
from backup_tool.worker import Worker
from sqlalchemy import select


@pytest.mark.asyncio
async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None:
    client, settings = app_client
    app = client._transport.app
    async with app.state.sessions() as db:
        ciphertext, key_id = app.state.cipher.encrypt(
            "smtp-password", purpose="notification_smtp", version=1
        )
        secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp")
        db.add(secret)
        await db.flush()
        db.add(
            NotificationEmailSettings(
                id=1,
                host="smtp.example.test",
                port=587,
                username="operator",
                password_secret_id=secret.id,
                sender="sender@example.test",
                max_attempts=2,
                rate_limit_per_minute=60,
            )
        )
        subscription = NotificationSubscription(
            channel="email",
            event_filters=["execution.queued"],
            destination_config={"recipients": ["operator@example.test"]},
            rate_limit_per_minute=60,
            rate_tokens=60.0,
        )
        db.add(subscription)
        await db.flush()
        event = await emit_event(
            db,
            "execution.queued",
            correlation_id=str(new_uuid7()),
            resource={},
            deduplication_key="retry-test",
        )
        await db.commit()

        async def transient(*_args, **_kwargs):
            raise EmailTransportError("smtp_421", transient=True)

        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
        delivery = await db.scalar(
            select(NotificationDelivery).where(NotificationDelivery.event_id == event.id)
        )
        assert delivery is not None
        assert delivery.state == "retry"
        assert delivery.attempt_count == 1
        delivery.state = "leased"
        delivery.attempt_count = 2
        delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
        db.add(
            NotificationDeliveryAttempt(
                delivery_id=delivery.id,
                number=2,
                started_at=datetime.now(UTC),
                outcome="started",
            )
        )
        await db.commit()
        assert await recover_notification_leases(db) == 1
        await db.refresh(delivery)
        abandoned = await db.scalar(
            select(NotificationDeliveryAttempt).where(
                NotificationDeliveryAttempt.delivery_id == delivery.id,
                NotificationDeliveryAttempt.number == 2,
            )
        )
        assert delivery.state == "retry"
        assert abandoned is not None
        assert abandoned.outcome == "retry"
        assert abandoned.diagnostic == "abandoned_lease"

        async def succeeded(*_args, **_kwargs):
            return EmailResult(response_class="smtp_2xx")

        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded)
        delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
        await db.commit()
        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
        await db.refresh(delivery)
        assert delivery.state == "delivered"
        assert delivery.attempt_count == 3

        max_event = await emit_event(
            db,
            "execution.queued",
            correlation_id=str(new_uuid7()),
            resource={},
            deduplication_key="smtp-max-attempts",
        )
        await db.commit()
        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient)
        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
        max_delivery = await db.scalar(
            select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id)
        )
        assert max_delivery is not None and max_delivery.state == "retry"
        max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1)
        await db.commit()
        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
        await db.refresh(max_delivery)
        assert max_delivery.state == "failed"
        assert max_delivery.attempt_count == 2

        permanent_event = await emit_event(
            db,
            "execution.queued",
            correlation_id=str(new_uuid7()),
            resource={},
            deduplication_key="smtp-permanent",
        )
        await db.commit()

        async def permanent(*_args, **_kwargs):
            raise EmailTransportError("smtp_550", transient=False)

        monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent)
        assert await dispatch_one(db, settings, app.state.cipher, "worker-test")
        permanent_delivery = await db.scalar(
            select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id)
        )
        assert permanent_delivery is not None
        assert permanent_delivery.state == "failed"
        assert permanent_delivery.attempt_count == 1


@pytest.mark.asyncio
async def test_notification_dispatch_gets_a_turn_during_execution_backlog(
    app_client, monkeypatch
) -> None:
    client, settings = app_client
    app = client._transport.app
    async with app.state.sessions() as db:
        repository = Repository(
            name="fair-repository",
            root="/fair-repository",
            format_version=1,
            compression="none",
            encryption="none",
        )
        source = Source(
            name="fair-source",
            kind="local",
            public_config={"root": "/fair-source"},
            secret_refs=[],
        )
        db.add_all([repository, source])
        await db.flush()
        job = Job(
            name="fair-job",
            source_id=source.id,
            repository_id=repository.id,
            exclusions=[],
            retention={},
            requested_mode="full",
        )
        db.add(job)
        await db.flush()
        execution = Execution(job_id=job.id, trigger="manual", progress={})
        db.add(execution)
        await db.commit()
        execution_id = execution.id

    called = False

    async def dispatched(*_args, **_kwargs) -> bool:
        nonlocal called
        called = True
        return True

    monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched)
    worker = Worker(settings, owner="fair-worker")
    worker._execution_turns = 1
    try:
        assert await worker.run_once()
    finally:
        await worker.engine.dispose()
    async with app.state.sessions() as db:
        queued = await db.get(Execution, execution_id)
    assert called
    assert queued is not None and queued.state == "queued"