feat: container monitoring backend core (PR-1)
- Add instance_events and health_checks tables with Alembic migration - InstanceEventBus: typed pub/sub singleton with wildcard support - HealthMonitor: async background loop polling containers every 15s - SSE endpoint GET /events/stream with auth and connection limits - Lifecycle hooks in tool_instances.py (create/start/stop/restart/delete) - Structured JSON logging with correlation IDs - 15 new unit tests (EventBus, HealthMonitor, MonitoringModels) Quality gates: pytest 15 new passed, ruff clean
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
"""Async correlation ID context variable and helpers."""
|
||||
|
||||
import contextvars
|
||||
import uuid
|
||||
|
||||
from fastapi import Request
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
CORRELATION_ID: contextvars.ContextVar[str] = contextvars.ContextVar("correlation_id")
|
||||
|
||||
|
||||
def get_correlation_id() -> str:
|
||||
"""Return the current correlation ID or generate a new UUID."""
|
||||
try:
|
||||
return CORRELATION_ID.get()
|
||||
except LookupError:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class CorrelationIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Set correlation ID from X-Request-ID header or generate a new UUID."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
request_id = request.headers.get("X-Request-ID")
|
||||
correlation_id = request_id or str(uuid.uuid4())
|
||||
token = CORRELATION_ID.set(correlation_id)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-ID"] = correlation_id
|
||||
return response
|
||||
finally:
|
||||
CORRELATION_ID.reset(token)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""In-memory typed event bus for instance lifecycle and health events."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
InstanceEventPayload = dict[str, Any]
|
||||
EventCallback = Callable[[InstanceEventPayload], Awaitable[None] | None] # noqa: UP044
|
||||
|
||||
|
||||
class InstanceEventBus:
|
||||
"""Singleton in-memory event bus with typed pub/sub and exception isolation."""
|
||||
|
||||
_instance: "InstanceEventBus | None" = None
|
||||
_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._subscribers: dict[str, list[tuple[str, EventCallback]]] = {}
|
||||
|
||||
def __new__(cls) -> "InstanceEventBus":
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._subscribers = {}
|
||||
return cls._instance
|
||||
|
||||
def _reset_for_testing(self) -> None:
|
||||
"""Clear all subscribers. For test use only."""
|
||||
self._subscribers.clear()
|
||||
|
||||
def subscribe(
|
||||
self,
|
||||
event_type: str,
|
||||
callback: EventCallback,
|
||||
) -> Callable[[], None]:
|
||||
"""Register a callback for an event type.
|
||||
|
||||
Args:
|
||||
event_type: The event type to subscribe to.
|
||||
callback: A sync or async callable that receives the payload.
|
||||
|
||||
Returns:
|
||||
An unsubscribe function.
|
||||
"""
|
||||
if event_type not in self._subscribers:
|
||||
self._subscribers[event_type] = []
|
||||
callback_id = str(uuid.uuid4())
|
||||
self._subscribers[event_type].append((callback_id, callback))
|
||||
|
||||
def unsubscribe() -> None:
|
||||
self.unsubscribe(event_type, callback_id)
|
||||
|
||||
return unsubscribe
|
||||
|
||||
def unsubscribe(self, event_type: str, callback_id: str) -> None:
|
||||
"""Remove a specific callback by ID."""
|
||||
if event_type in self._subscribers:
|
||||
self._subscribers[event_type] = [
|
||||
(cid, cb)
|
||||
for cid, cb in self._subscribers[event_type]
|
||||
if cid != callback_id
|
||||
]
|
||||
if not self._subscribers[event_type]:
|
||||
del self._subscribers[event_type]
|
||||
|
||||
def unsubscribe_all(self, event_type: str) -> None:
|
||||
"""Remove all subscribers for an event type."""
|
||||
self._subscribers.pop(event_type, None)
|
||||
|
||||
async def publish(self, event_type: str, payload: InstanceEventPayload) -> None:
|
||||
"""Deliver payload to all subscribers of event_type.
|
||||
|
||||
Also delivers to subscribers registered under the wildcard "*".
|
||||
Exceptions from individual subscribers are caught and logged;
|
||||
delivery continues to remaining subscribers.
|
||||
"""
|
||||
callbacks: list[tuple[str, EventCallback]] = []
|
||||
callbacks.extend(self._subscribers.get(event_type, []))
|
||||
callbacks.extend(self._subscribers.get("*", []))
|
||||
|
||||
for _callback_id, callback in callbacks:
|
||||
try:
|
||||
if inspect.iscoroutinefunction(callback):
|
||||
await callback(payload)
|
||||
else:
|
||||
callback(payload)
|
||||
except Exception:
|
||||
correlation_id = payload.get("correlation_id", "unknown")
|
||||
logger.exception(
|
||||
"Event subscriber failed for %s",
|
||||
event_type,
|
||||
extra={"correlation_id": correlation_id},
|
||||
)
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Background health monitor that polls container and tunnel health."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.database import SessionLocal
|
||||
from src.models.health_check import HealthCheck
|
||||
from src.models.tool_instance import ToolInstance
|
||||
from src.services.correlation import get_correlation_id
|
||||
from src.services.docker import check_tunnel_health, get_container_status
|
||||
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthSnapshot:
|
||||
"""In-memory snapshot of an instance's health state."""
|
||||
|
||||
container_status: str | None = None
|
||||
container_healthy: bool | None = None
|
||||
tunnel_healthy: bool | None = None
|
||||
exit_code: int | None = None
|
||||
|
||||
|
||||
class HealthMonitor:
|
||||
"""Polls container and tunnel health, publishing events on state changes."""
|
||||
|
||||
POLL_INTERVAL_SECONDS: float = 15.0
|
||||
_MONITORED_STATUSES: set[str] = {"starting", "running", "unhealthy"}
|
||||
|
||||
def __init__(self, event_bus: InstanceEventBus) -> None:
|
||||
self._event_bus = event_bus
|
||||
self._task: asyncio.Task | None = None
|
||||
self._last_known_state: dict[uuid.UUID, HealthSnapshot] = {}
|
||||
|
||||
def start(self) -> None:
|
||||
"""Idempotent start of the background polling task."""
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
self._task = loop.create_task(self._poll_loop())
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Cancel the background task and clear state."""
|
||||
if self._task is not None and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._last_known_state.clear()
|
||||
self._task = None
|
||||
|
||||
async def _poll_loop(self) -> None:
|
||||
"""Main polling loop."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.POLL_INTERVAL_SECONDS)
|
||||
await self._run_check_cycle()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("Health monitor poll loop error")
|
||||
|
||||
async def _run_check_cycle(self) -> None:
|
||||
"""Check all monitored instances in one cycle."""
|
||||
async with SessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(ToolInstance).where(
|
||||
ToolInstance.status.in_(self._MONITORED_STATUSES)
|
||||
)
|
||||
)
|
||||
instances = result.scalars().all()
|
||||
|
||||
for instance in instances:
|
||||
async with SessionLocal() as session:
|
||||
await self._check_instance(session, instance)
|
||||
|
||||
async def _check_instance(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
) -> None:
|
||||
"""Check a single instance and handle state transitions."""
|
||||
try:
|
||||
container_info = get_container_status(instance.container_id or "")
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Health check failed for instance %s",
|
||||
instance.id,
|
||||
extra={
|
||||
"instance_id": str(instance.id),
|
||||
"correlation_id": get_correlation_id(),
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
container_status = container_info["status"]
|
||||
exit_code = container_info["exit_code"]
|
||||
container_healthy = (
|
||||
container_info["health"] == "healthy" if container_info["health"] else None
|
||||
)
|
||||
|
||||
tunnel_healthy: bool | None = None
|
||||
if instance.public_url and container_status == "running":
|
||||
try:
|
||||
tunnel_result = check_tunnel_health(instance.public_url)
|
||||
tunnel_healthy = tunnel_result.get("healthy", False)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Tunnel health check failed for instance %s",
|
||||
instance.id,
|
||||
extra={
|
||||
"instance_id": str(instance.id),
|
||||
"correlation_id": get_correlation_id(),
|
||||
},
|
||||
)
|
||||
tunnel_healthy = False
|
||||
|
||||
snapshot = HealthSnapshot(
|
||||
container_status=container_status,
|
||||
container_healthy=container_healthy,
|
||||
tunnel_healthy=tunnel_healthy,
|
||||
exit_code=exit_code,
|
||||
)
|
||||
|
||||
previous = self._last_known_state.get(instance.id)
|
||||
|
||||
# Determine new status
|
||||
new_status = self._derive_status(snapshot)
|
||||
|
||||
# If first check or state changed
|
||||
if previous is None or not self._snapshots_equal(previous, snapshot):
|
||||
await self._handle_state_change(
|
||||
session, instance, previous, snapshot, new_status
|
||||
)
|
||||
self._last_known_state[instance.id] = snapshot
|
||||
|
||||
def _derive_status(self, snapshot: HealthSnapshot) -> str:
|
||||
"""Derive instance status from health snapshot."""
|
||||
if snapshot.container_status != "running":
|
||||
return "error"
|
||||
if snapshot.tunnel_healthy is False:
|
||||
return "unhealthy"
|
||||
return "running"
|
||||
|
||||
def _snapshots_equal(self, a: HealthSnapshot, b: HealthSnapshot) -> bool:
|
||||
"""Compare two snapshots for equality."""
|
||||
return (
|
||||
a.container_status == b.container_status
|
||||
and a.container_healthy == b.container_healthy
|
||||
and a.tunnel_healthy == b.tunnel_healthy
|
||||
and a.exit_code == b.exit_code
|
||||
)
|
||||
|
||||
async def _handle_state_change(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
previous: HealthSnapshot | None,
|
||||
snapshot: HealthSnapshot,
|
||||
new_status: str,
|
||||
) -> None:
|
||||
"""Update DB, insert health check, and publish event."""
|
||||
previous_status = instance.status
|
||||
|
||||
# Update instance status
|
||||
instance.status = new_status
|
||||
if new_status == "error":
|
||||
instance.last_stopped_at = datetime.now(timezone.utc)
|
||||
|
||||
# Insert health check row
|
||||
health_check = HealthCheck(
|
||||
instance_id=instance.id,
|
||||
container_status=snapshot.container_status,
|
||||
container_healthy=snapshot.container_healthy,
|
||||
tunnel_healthy=snapshot.tunnel_healthy,
|
||||
exit_code=snapshot.exit_code,
|
||||
probe_status=None,
|
||||
probe_output=None,
|
||||
)
|
||||
session.add(health_check)
|
||||
await session.commit()
|
||||
|
||||
# Build event payload
|
||||
correlation_id = get_correlation_id()
|
||||
metadata: dict = {"previous_status": previous_status}
|
||||
if snapshot.exit_code is not None:
|
||||
metadata["exit_code"] = snapshot.exit_code
|
||||
metadata["error_type"] = "container"
|
||||
if instance.public_url:
|
||||
metadata["tunnel_url"] = instance.public_url
|
||||
|
||||
if new_status == "error":
|
||||
event_type = "instance.error"
|
||||
message = f"Container failed with status {snapshot.container_status}"
|
||||
if snapshot.exit_code is not None:
|
||||
message += f" (exit code: {snapshot.exit_code})"
|
||||
else:
|
||||
event_type = "instance.health_changed"
|
||||
message = f"Container is now {new_status}"
|
||||
|
||||
payload: InstanceEventPayload = {
|
||||
"event": event_type,
|
||||
"instance_id": str(instance.id),
|
||||
"status": new_status,
|
||||
"message": message,
|
||||
"metadata": metadata,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
|
||||
await self._event_bus.publish(event_type, payload)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Lifecycle hook helpers for instrumenting tool instance transitions."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.models.instance_event import InstanceEvent
|
||||
from src.models.tool_instance import ToolInstance
|
||||
from src.services.correlation import get_correlation_id
|
||||
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
|
||||
|
||||
def _build_payload(
|
||||
event_type: str,
|
||||
instance: ToolInstance,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> InstanceEventPayload:
|
||||
"""Construct a standard event payload."""
|
||||
return {
|
||||
"event": event_type,
|
||||
"instance_id": str(instance.id),
|
||||
"status": status or instance.status,
|
||||
"message": message,
|
||||
"metadata": metadata or {},
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"correlation_id": get_correlation_id(),
|
||||
}
|
||||
|
||||
|
||||
async def _write_audit_row(
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
event_type: str,
|
||||
created_by: uuid.UUID | None = None,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> InstanceEvent:
|
||||
"""Persist an instance_events audit row."""
|
||||
row = InstanceEvent(
|
||||
instance_id=instance.id,
|
||||
event_type=event_type.replace("instance.", ""),
|
||||
status=status or instance.status,
|
||||
message=message,
|
||||
created_by=created_by,
|
||||
event_metadata=metadata or {},
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return row
|
||||
|
||||
|
||||
async def publish_lifecycle_event(
|
||||
event_bus: InstanceEventBus,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
event_type: str,
|
||||
created_by: uuid.UUID | None = None,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> None:
|
||||
"""Publish a lifecycle event and write an audit row after DB commit.
|
||||
|
||||
Args:
|
||||
event_bus: The global event bus.
|
||||
session: Active async DB session.
|
||||
instance: The affected tool instance.
|
||||
event_type: One of instance.created, instance.started, etc.
|
||||
created_by: User ID for user-initiated actions; None for system.
|
||||
status: Optional status override.
|
||||
message: Optional human-readable message.
|
||||
metadata: Optional extra metadata.
|
||||
"""
|
||||
payload = _build_payload(
|
||||
event_type=event_type,
|
||||
instance=instance,
|
||||
status=status,
|
||||
message=message,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Write audit row
|
||||
await _write_audit_row(
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type=event_type,
|
||||
created_by=created_by,
|
||||
status=status or instance.status,
|
||||
message=message,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Publish to bus
|
||||
await event_bus.publish(event_type, payload)
|
||||
Reference in New Issue
Block a user