from __future__ import annotations from datetime import UTC, datetime, timedelta import pytest from backup_tool.db.models import ( Execution, Job, NotificationDelivery, NotificationDeliveryAttempt, NotificationEmailSettings, NotificationSubscription, Repository, Secret, Source, ) from backup_tool.ids import new_uuid7 from backup_tool.notifications.dispatcher import ( dispatch_one, recover_notification_leases, ) from backup_tool.notifications.email import EmailResult, EmailTransportError from backup_tool.notifications.events import emit_event from backup_tool.worker import Worker from sqlalchemy import select @pytest.mark.asyncio async def test_transient_smtp_failure_retries_and_lease_recovers(app_client, monkeypatch) -> None: client, settings = app_client app = client._transport.app async with app.state.sessions() as db: ciphertext, key_id = app.state.cipher.encrypt( "smtp-password", purpose="notification_smtp", version=1 ) secret = Secret(ciphertext=ciphertext, key_id=key_id, purpose="notification_smtp") db.add(secret) await db.flush() db.add( NotificationEmailSettings( id=1, host="smtp.example.test", port=587, username="operator", password_secret_id=secret.id, sender="sender@example.test", max_attempts=2, rate_limit_per_minute=60, ) ) subscription = NotificationSubscription( channel="email", event_filters=["execution.queued"], destination_config={"recipients": ["operator@example.test"]}, rate_limit_per_minute=60, rate_tokens=60.0, ) db.add(subscription) await db.flush() event = await emit_event( db, "execution.queued", correlation_id=str(new_uuid7()), resource={}, deduplication_key="retry-test", ) await db.commit() async def transient(*_args, **_kwargs): raise EmailTransportError("smtp_421", transient=True) monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient) assert await dispatch_one(db, settings, app.state.cipher, "worker-test") delivery = await db.scalar( select(NotificationDelivery).where(NotificationDelivery.event_id == event.id) ) assert delivery is not None assert delivery.state == "retry" assert delivery.attempt_count == 1 delivery.state = "leased" delivery.attempt_count = 2 delivery.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1) db.add( NotificationDeliveryAttempt( delivery_id=delivery.id, number=2, started_at=datetime.now(UTC), outcome="started", ) ) await db.commit() assert await recover_notification_leases(db) == 1 await db.refresh(delivery) abandoned = await db.scalar( select(NotificationDeliveryAttempt).where( NotificationDeliveryAttempt.delivery_id == delivery.id, NotificationDeliveryAttempt.number == 2, ) ) assert delivery.state == "retry" assert abandoned is not None assert abandoned.outcome == "retry" assert abandoned.diagnostic == "abandoned_lease" async def succeeded(*_args, **_kwargs): return EmailResult(response_class="smtp_2xx") monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", succeeded) delivery.due_at = datetime.now(UTC) - timedelta(seconds=1) await db.commit() assert await dispatch_one(db, settings, app.state.cipher, "worker-test") await db.refresh(delivery) assert delivery.state == "delivered" assert delivery.attempt_count == 3 max_event = await emit_event( db, "execution.queued", correlation_id=str(new_uuid7()), resource={}, deduplication_key="smtp-max-attempts", ) await db.commit() monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", transient) assert await dispatch_one(db, settings, app.state.cipher, "worker-test") max_delivery = await db.scalar( select(NotificationDelivery).where(NotificationDelivery.event_id == max_event.id) ) assert max_delivery is not None and max_delivery.state == "retry" max_delivery.due_at = datetime.now(UTC) - timedelta(seconds=1) await db.commit() assert await dispatch_one(db, settings, app.state.cipher, "worker-test") await db.refresh(max_delivery) assert max_delivery.state == "failed" assert max_delivery.attempt_count == 2 permanent_event = await emit_event( db, "execution.queued", correlation_id=str(new_uuid7()), resource={}, deduplication_key="smtp-permanent", ) await db.commit() async def permanent(*_args, **_kwargs): raise EmailTransportError("smtp_550", transient=False) monkeypatch.setattr("backup_tool.notifications.dispatcher.deliver_email", permanent) assert await dispatch_one(db, settings, app.state.cipher, "worker-test") permanent_delivery = await db.scalar( select(NotificationDelivery).where(NotificationDelivery.event_id == permanent_event.id) ) assert permanent_delivery is not None assert permanent_delivery.state == "failed" assert permanent_delivery.attempt_count == 1 @pytest.mark.asyncio async def test_notification_dispatch_gets_a_turn_during_execution_backlog( app_client, monkeypatch ) -> None: client, settings = app_client app = client._transport.app async with app.state.sessions() as db: repository = Repository( name="fair-repository", root="/fair-repository", format_version=1, compression="none", encryption="none", ) source = Source( name="fair-source", kind="local", public_config={"root": "/fair-source"}, secret_refs=[], ) db.add_all([repository, source]) await db.flush() job = Job( name="fair-job", source_id=source.id, repository_id=repository.id, exclusions=[], retention={}, requested_mode="full", ) db.add(job) await db.flush() execution = Execution(job_id=job.id, trigger="manual", progress={}) db.add(execution) await db.commit() execution_id = execution.id called = False async def dispatched(*_args, **_kwargs) -> bool: nonlocal called called = True return True monkeypatch.setattr("backup_tool.worker.dispatch_one", dispatched) worker = Worker(settings, owner="fair-worker") worker._execution_turns = 1 try: assert await worker.run_once() finally: await worker.engine.dispose() async with app.state.sessions() as db: queued = await db.get(Execution, execution_id) assert called assert queued is not None and queued.state == "queued"