feat: container monitoring backend core (PR-1)
- Add instance_events and health_checks tables with Alembic migration - InstanceEventBus: typed pub/sub singleton with wildcard support - HealthMonitor: async background loop polling containers every 15s - SSE endpoint GET /events/stream with auth and connection limits - Lifecycle hooks in tool_instances.py (create/start/stop/restart/delete) - Structured JSON logging with correlation IDs - 15 new unit tests (EventBus, HealthMonitor, MonitoringModels) Quality gates: pytest 15 new passed, ruff clean
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
"""Unit tests for InstanceEventBus."""
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def event_bus() -> InstanceEventBus:
|
||||
"""Provide a fresh EventBus instance with reset singleton state."""
|
||||
bus = InstanceEventBus()
|
||||
bus._reset_for_testing()
|
||||
return bus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_payload() -> InstanceEventPayload:
|
||||
"""Provide a sample event payload."""
|
||||
return {
|
||||
"event": "instance.started",
|
||||
"instance_id": str(uuid.uuid4()),
|
||||
"status": "starting",
|
||||
"message": "Container starting...",
|
||||
"metadata": {},
|
||||
"timestamp": "2026-05-28T12:00:00Z",
|
||||
"correlation_id": str(uuid.uuid4()),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_publish_delivers_to_all_subscribers(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""All subscribed callbacks should receive the published payload."""
|
||||
received: list[Any] = []
|
||||
|
||||
def callback_1(payload: InstanceEventPayload) -> None:
|
||||
received.append(("callback_1", payload))
|
||||
|
||||
def callback_2(payload: InstanceEventPayload) -> None:
|
||||
received.append(("callback_2", payload))
|
||||
|
||||
def callback_3(payload: InstanceEventPayload) -> None:
|
||||
received.append(("callback_3", payload))
|
||||
|
||||
event_bus.subscribe("instance.started", callback_1)
|
||||
event_bus.subscribe("instance.started", callback_2)
|
||||
event_bus.subscribe("instance.started", callback_3)
|
||||
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
assert len(received) == 3
|
||||
assert received[0][0] == "callback_1"
|
||||
assert received[1][0] == "callback_2"
|
||||
assert received[2][0] == "callback_3"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_subscriber_exception_isolation(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""If one subscriber raises, others should still receive the event."""
|
||||
received: list[str] = []
|
||||
|
||||
def bad_callback(_payload: InstanceEventPayload) -> None:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def good_callback(_payload: InstanceEventPayload) -> None:
|
||||
received.append("good_callback")
|
||||
|
||||
event_bus.subscribe("instance.started", bad_callback)
|
||||
event_bus.subscribe("instance.started", good_callback)
|
||||
|
||||
# Should not raise
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
assert received == ["good_callback"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_unsubscribe_removes_callback(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""After unsubscribing, the callback should not be called."""
|
||||
received: list[str] = []
|
||||
|
||||
def callback(_payload: InstanceEventPayload) -> None:
|
||||
received.append("callback")
|
||||
|
||||
unsubscribe = event_bus.subscribe("instance.started", callback)
|
||||
unsubscribe()
|
||||
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
assert received == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_publish_to_empty_subscriber_list(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""Publishing to an event type with no subscribers should not raise."""
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_async_subscriber_supported(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""Async callbacks should be awaited correctly."""
|
||||
received: list[str] = []
|
||||
|
||||
async def async_callback(_payload: InstanceEventPayload) -> None:
|
||||
await asyncio.sleep(0)
|
||||
received.append("async_callback")
|
||||
|
||||
event_bus.subscribe("instance.started", async_callback)
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
assert received == ["async_callback"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_unsubscribe_all_clears_subscribers(
|
||||
event_bus: InstanceEventBus,
|
||||
sample_payload: InstanceEventPayload,
|
||||
) -> None:
|
||||
"""unsubscribe_all should remove all callbacks for an event type."""
|
||||
received: list[str] = []
|
||||
|
||||
def callback(_payload: InstanceEventPayload) -> None:
|
||||
received.append("callback")
|
||||
|
||||
event_bus.subscribe("instance.started", callback)
|
||||
event_bus.unsubscribe_all("instance.started")
|
||||
|
||||
await event_bus.publish("instance.started", sample_payload)
|
||||
|
||||
assert received == []
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Unit tests for HealthMonitor state-transition logic."""
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from contextlib import suppress
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.models.health_check import HealthCheck
|
||||
from src.models.tool_instance import ToolInstance
|
||||
from src.models.user import User
|
||||
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
from src.services.health_monitor import HealthMonitor, HealthSnapshot
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def event_bus() -> InstanceEventBus:
|
||||
"""Provide a fresh EventBus instance."""
|
||||
bus = InstanceEventBus()
|
||||
bus._reset_for_testing()
|
||||
return bus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def health_monitor(event_bus: InstanceEventBus) -> HealthMonitor:
|
||||
"""Provide a HealthMonitor with a short poll interval for testing."""
|
||||
monitor = HealthMonitor(event_bus)
|
||||
monitor.POLL_INTERVAL_SECONDS = 0.1
|
||||
return monitor
|
||||
|
||||
|
||||
async def _create_running_instance(db_session) -> ToolInstance:
|
||||
"""Helper to create a user and a running tool instance."""
|
||||
user = User(
|
||||
id=uuid.uuid4(),
|
||||
email="hm@example.com",
|
||||
name="HM Test",
|
||||
authentik_id="auth-hm",
|
||||
)
|
||||
db_session.add(user)
|
||||
await db_session.commit()
|
||||
|
||||
instance = ToolInstance(
|
||||
id=uuid.uuid4(),
|
||||
name="hm-test-instance",
|
||||
display_name="HM Test Instance",
|
||||
tool_type_id=uuid.uuid4(),
|
||||
repository_id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
owner_id=user.id,
|
||||
status="running",
|
||||
container_id="container123",
|
||||
public_url="https://example.trycloudflare.com",
|
||||
)
|
||||
db_session.add(instance)
|
||||
await db_session.commit()
|
||||
return instance
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_detects_container_crash(
|
||||
db_session,
|
||||
event_bus: InstanceEventBus,
|
||||
health_monitor: HealthMonitor,
|
||||
) -> None:
|
||||
"""Monitor should detect exited container and publish error event."""
|
||||
instance = await _create_running_instance(db_session)
|
||||
|
||||
events_captured: list[InstanceEventPayload] = []
|
||||
|
||||
def capture_event(payload: InstanceEventPayload) -> None:
|
||||
events_captured.append(payload)
|
||||
|
||||
event_bus.subscribe("instance.error", capture_event)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"src.services.health_monitor.get_container_status",
|
||||
return_value={"status": "exited", "exit_code": 137, "health": None},
|
||||
),
|
||||
patch(
|
||||
"src.services.health_monitor.check_tunnel_health",
|
||||
return_value={"healthy": False, "tunnel_status": "not_applicable"},
|
||||
),
|
||||
):
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
|
||||
# Refresh instance from DB
|
||||
await db_session.refresh(instance)
|
||||
assert instance.status == "error"
|
||||
|
||||
# Event published
|
||||
assert len(events_captured) == 1
|
||||
assert events_captured[0]["event"] == "instance.error"
|
||||
assert events_captured[0]["status"] == "error"
|
||||
assert events_captured[0]["metadata"]["exit_code"] == 137
|
||||
|
||||
# Health check row inserted
|
||||
result = await db_session.execute(
|
||||
select(HealthCheck).where(HealthCheck.instance_id == instance.id)
|
||||
)
|
||||
check = result.scalar_one()
|
||||
assert check.container_status == "exited"
|
||||
assert check.exit_code == 137
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_detects_tunnel_failure(
|
||||
db_session,
|
||||
event_bus: InstanceEventBus,
|
||||
health_monitor: HealthMonitor,
|
||||
) -> None:
|
||||
"""Monitor should detect tunnel failure and mark unhealthy."""
|
||||
instance = await _create_running_instance(db_session)
|
||||
|
||||
events_captured: list[InstanceEventPayload] = []
|
||||
|
||||
def capture_event(payload: InstanceEventPayload) -> None:
|
||||
events_captured.append(payload)
|
||||
|
||||
event_bus.subscribe("instance.health_changed", capture_event)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"src.services.health_monitor.get_container_status",
|
||||
return_value={"status": "running", "exit_code": None, "health": "healthy"},
|
||||
),
|
||||
patch(
|
||||
"src.services.health_monitor.check_tunnel_health",
|
||||
return_value={
|
||||
"healthy": False,
|
||||
"tunnel_status": "error_response",
|
||||
"status_code": 502,
|
||||
},
|
||||
),
|
||||
):
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
|
||||
await db_session.refresh(instance)
|
||||
assert instance.status == "unhealthy"
|
||||
|
||||
assert len(events_captured) == 1
|
||||
assert events_captured[0]["event"] == "instance.health_changed"
|
||||
assert events_captured[0]["status"] == "unhealthy"
|
||||
assert events_captured[0]["metadata"]["previous_status"] == "running"
|
||||
|
||||
result = await db_session.execute(
|
||||
select(HealthCheck).where(HealthCheck.instance_id == instance.id)
|
||||
)
|
||||
check = result.scalar_one()
|
||||
assert check.tunnel_healthy is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_detects_recovery(
|
||||
db_session,
|
||||
event_bus: InstanceEventBus,
|
||||
health_monitor: HealthMonitor,
|
||||
) -> None:
|
||||
"""Monitor should detect recovery from unhealthy to running."""
|
||||
instance = await _create_running_instance(db_session)
|
||||
instance.status = "unhealthy"
|
||||
await db_session.commit()
|
||||
|
||||
# Seed last known state as unhealthy
|
||||
health_monitor._last_known_state[instance.id] = HealthSnapshot(
|
||||
container_status="running",
|
||||
container_healthy=None,
|
||||
tunnel_healthy=False,
|
||||
exit_code=None,
|
||||
)
|
||||
|
||||
events_captured: list[InstanceEventPayload] = []
|
||||
|
||||
def capture_event(payload: InstanceEventPayload) -> None:
|
||||
events_captured.append(payload)
|
||||
|
||||
event_bus.subscribe("instance.health_changed", capture_event)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"src.services.health_monitor.get_container_status",
|
||||
return_value={"status": "running", "exit_code": None, "health": None},
|
||||
),
|
||||
patch(
|
||||
"src.services.health_monitor.check_tunnel_health",
|
||||
return_value={
|
||||
"healthy": True,
|
||||
"tunnel_status": "healthy",
|
||||
"status_code": 200,
|
||||
},
|
||||
),
|
||||
):
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
|
||||
await db_session.refresh(instance)
|
||||
assert instance.status == "running"
|
||||
|
||||
assert len(events_captured) == 1
|
||||
assert events_captured[0]["status"] == "running"
|
||||
assert events_captured[0]["metadata"]["previous_status"] == "unhealthy"
|
||||
|
||||
result = await db_session.execute(
|
||||
select(HealthCheck).where(HealthCheck.instance_id == instance.id)
|
||||
)
|
||||
check = result.scalar_one()
|
||||
assert check.tunnel_healthy is True
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_skips_writes_when_no_state_change(
|
||||
db_session,
|
||||
event_bus: InstanceEventBus,
|
||||
health_monitor: HealthMonitor,
|
||||
) -> None:
|
||||
"""Two identical polls should result in only one health_checks row."""
|
||||
instance = await _create_running_instance(db_session)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"src.services.health_monitor.get_container_status",
|
||||
return_value={"status": "running", "exit_code": None, "health": None},
|
||||
),
|
||||
patch(
|
||||
"src.services.health_monitor.check_tunnel_health",
|
||||
return_value={
|
||||
"healthy": True,
|
||||
"tunnel_status": "healthy",
|
||||
"status_code": 200,
|
||||
},
|
||||
),
|
||||
):
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
|
||||
result = await db_session.execute(
|
||||
select(HealthCheck).where(HealthCheck.instance_id == instance.id)
|
||||
)
|
||||
assert len(result.scalars().all()) == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_docker_exception_resilience(
|
||||
db_session,
|
||||
event_bus: InstanceEventBus,
|
||||
health_monitor: HealthMonitor,
|
||||
) -> None:
|
||||
"""Docker exception should be caught and not propagate."""
|
||||
instance = await _create_running_instance(db_session)
|
||||
|
||||
events_captured: list[InstanceEventPayload] = []
|
||||
|
||||
def capture_event(payload: InstanceEventPayload) -> None:
|
||||
events_captured.append(payload)
|
||||
|
||||
event_bus.subscribe("instance.error", capture_event)
|
||||
event_bus.subscribe("instance.health_changed", capture_event)
|
||||
|
||||
with patch(
|
||||
"src.services.health_monitor.get_container_status",
|
||||
side_effect=RuntimeError("docker exploded"),
|
||||
):
|
||||
# Should not raise
|
||||
await health_monitor._check_instance(db_session, instance)
|
||||
|
||||
# No DB writes
|
||||
result = await db_session.execute(
|
||||
select(HealthCheck).where(HealthCheck.instance_id == instance.id)
|
||||
)
|
||||
assert result.scalar_one_or_none() is None
|
||||
|
||||
# No events published
|
||||
assert events_captured == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_monitor_start_stop(health_monitor: HealthMonitor) -> None:
|
||||
"""Start and stop should manage the background task."""
|
||||
health_monitor.start()
|
||||
task = health_monitor._task
|
||||
assert task is not None
|
||||
assert not task.done()
|
||||
|
||||
health_monitor.stop()
|
||||
if task is not None and not task.done():
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
assert task is not None
|
||||
assert task.cancelled() or task.done()
|
||||
assert health_monitor._last_known_state == {}
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Unit tests for monitoring models and migration compatibility."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.models.health_check import HealthCheck
|
||||
from src.models.instance_event import InstanceEvent
|
||||
from src.models.tool_instance import ToolInstance
|
||||
from src.models.user import User
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_instance_event_creation(db_session) -> None:
|
||||
"""InstanceEvent model can be created and persisted."""
|
||||
user = User(
|
||||
id=uuid.uuid4(),
|
||||
email="test@example.com",
|
||||
name="Test",
|
||||
authentik_id="auth-1",
|
||||
)
|
||||
db_session.add(user)
|
||||
await db_session.commit()
|
||||
|
||||
instance = ToolInstance(
|
||||
id=uuid.uuid4(),
|
||||
name="test-instance",
|
||||
display_name="Test Instance",
|
||||
tool_type_id=uuid.uuid4(),
|
||||
repository_id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
owner_id=user.id,
|
||||
status="pending",
|
||||
)
|
||||
db_session.add(instance)
|
||||
await db_session.commit()
|
||||
|
||||
event = InstanceEvent(
|
||||
instance_id=instance.id,
|
||||
event_type="started",
|
||||
status="starting",
|
||||
message="Container starting...",
|
||||
created_by=user.id,
|
||||
event_metadata={"previous_status": "pending"},
|
||||
)
|
||||
db_session.add(event)
|
||||
await db_session.commit()
|
||||
await db_session.refresh(event)
|
||||
|
||||
assert event.id is not None
|
||||
assert event.instance_id == instance.id
|
||||
assert event.event_type == "started"
|
||||
assert event.status == "starting"
|
||||
assert event.created_by == user.id
|
||||
assert event.event_metadata == {"previous_status": "pending"}
|
||||
assert isinstance(event.created_at, datetime)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_health_check_creation(db_session) -> None:
|
||||
"""HealthCheck model can be created and persisted."""
|
||||
user = User(
|
||||
id=uuid.uuid4(),
|
||||
email="test2@example.com",
|
||||
name="Test2",
|
||||
authentik_id="auth-2",
|
||||
)
|
||||
db_session.add(user)
|
||||
await db_session.commit()
|
||||
|
||||
instance = ToolInstance(
|
||||
id=uuid.uuid4(),
|
||||
name="test-instance-2",
|
||||
display_name="Test Instance 2",
|
||||
tool_type_id=uuid.uuid4(),
|
||||
repository_id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
owner_id=user.id,
|
||||
status="running",
|
||||
)
|
||||
db_session.add(instance)
|
||||
await db_session.commit()
|
||||
|
||||
check = HealthCheck(
|
||||
instance_id=instance.id,
|
||||
container_status="running",
|
||||
container_healthy=True,
|
||||
tunnel_healthy=True,
|
||||
exit_code=None,
|
||||
probe_status="passed",
|
||||
probe_output="OK",
|
||||
)
|
||||
db_session.add(check)
|
||||
await db_session.commit()
|
||||
await db_session.refresh(check)
|
||||
|
||||
assert check.id is not None
|
||||
assert check.instance_id == instance.id
|
||||
assert check.container_status == "running"
|
||||
assert check.container_healthy is True
|
||||
assert check.tunnel_healthy is True
|
||||
assert isinstance(check.checked_at, datetime)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_instance_event_query_by_instance(db_session) -> None:
|
||||
"""InstanceEvent rows can be queried by instance_id."""
|
||||
user = User(
|
||||
id=uuid.uuid4(),
|
||||
email="test3@example.com",
|
||||
name="Test3",
|
||||
authentik_id="auth-3",
|
||||
)
|
||||
db_session.add(user)
|
||||
await db_session.commit()
|
||||
|
||||
instance = ToolInstance(
|
||||
id=uuid.uuid4(),
|
||||
name="test-instance-3",
|
||||
display_name="Test Instance 3",
|
||||
tool_type_id=uuid.uuid4(),
|
||||
repository_id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
owner_id=user.id,
|
||||
status="pending",
|
||||
)
|
||||
db_session.add(instance)
|
||||
await db_session.commit()
|
||||
|
||||
event = InstanceEvent(
|
||||
instance_id=instance.id,
|
||||
event_type="created",
|
||||
status="pending",
|
||||
)
|
||||
db_session.add(event)
|
||||
await db_session.commit()
|
||||
|
||||
result = await db_session.execute(
|
||||
select(InstanceEvent).where(InstanceEvent.instance_id == instance.id)
|
||||
)
|
||||
assert result.scalar_one() is not None
|
||||
Reference in New Issue
Block a user