feat: container monitoring backend core (PR-1)

- Add instance_events and health_checks tables with Alembic migration
- InstanceEventBus: typed pub/sub singleton with wildcard support
- HealthMonitor: async background loop polling containers every 15s
- SSE endpoint GET /events/stream with auth and connection limits
- Lifecycle hooks in tool_instances.py (create/start/stop/restart/delete)
- Structured JSON logging with correlation IDs
- 15 new unit tests (EventBus, HealthMonitor, MonitoringModels)

Quality gates: pytest 15 new passed, ruff clean

apps/api/alembic/versions/2026_05_28_add_monitoring_tables.py

@@ -0,0 +1,122 @@
+"""add monitoring tables
+
+Revision ID: 2026_05_28_add_monitoring_tables
+Revises: 2026_05_28_drop_tool_configs_and_config_folders
+Create Date: 2026-05-28
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "2026_05_28_add_monitoring_tables"
+down_revision: str | None = "2026_05_28_drop_tool_configs_and_config_folders"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "instance_events",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "instance_id",
+            sa.Uuid(),
+            nullable=False,
+        ),
+        sa.Column("event_type", sa.String(length=50), nullable=False),
+        sa.Column("status", sa.String(length=50), nullable=True),
+        sa.Column("message", sa.Text(), nullable=True),
+        sa.Column("created_by", sa.Uuid(), nullable=True),
+        sa.Column(
+            "metadata",
+            sa.JSON(),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["instance_id"],
+            ["tool_instances.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["created_by"],
+            ["users.id"],
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "idx_instance_events_instance_id",
+        "instance_events",
+        ["instance_id"],
+    )
+    op.create_index(
+        "idx_instance_events_created_at",
+        "instance_events",
+        ["created_at"],
+        postgresql_using="btree",
+    )
+    op.create_index(
+        "idx_instance_events_event_type",
+        "instance_events",
+        ["event_type"],
+    )
+
+    op.create_table(
+        "health_checks",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "instance_id",
+            sa.Uuid(),
+            nullable=False,
+        ),
+        sa.Column("container_status", sa.String(length=50), nullable=True),
+        sa.Column("container_healthy", sa.Boolean(), nullable=True),
+        sa.Column("tunnel_healthy", sa.Boolean(), nullable=True),
+        sa.Column("exit_code", sa.Integer(), nullable=True),
+        sa.Column("probe_status", sa.String(length=50), nullable=True),
+        sa.Column("probe_output", sa.Text(), nullable=True),
+        sa.Column(
+            "checked_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["instance_id"],
+            ["tool_instances.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "idx_health_checks_instance_id",
+        "health_checks",
+        ["instance_id"],
+    )
+    op.create_index(
+        "idx_health_checks_checked_at",
+        "health_checks",
+        ["checked_at"],
+        postgresql_using="btree",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("idx_health_checks_checked_at", table_name="health_checks")
+    op.drop_index("idx_health_checks_instance_id", table_name="health_checks")
+    op.drop_table("health_checks")
+    op.drop_index("idx_instance_events_event_type", table_name="instance_events")
+    op.drop_index("idx_instance_events_created_at", table_name="instance_events")
+    op.drop_index("idx_instance_events_instance_id", table_name="instance_events")
+    op.drop_table("instance_events")

apps/api/src/api/__init__.py

@@ -1,4 +1,5 @@
 from src.api.auth import router as auth_router
+from src.api.events import router as events_router
 from src.api.users import router as users_router
 
-__all__ = ["auth_router", "users_router"]
+__all__ = ["auth_router", "events_router", "users_router"]

apps/api/src/api/events.py

@@ -0,0 +1,80 @@
+"""SSE streaming endpoint for instance events."""
+
+import asyncio
+import contextlib
+import json
+import uuid
+from collections.abc import AsyncGenerator
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+from fastapi.responses import StreamingResponse
+
+from src.auth.dependencies import get_current_user_id
+from src.services.event_bus import InstanceEventBus, InstanceEventPayload
+
+router = APIRouter(prefix="/events", tags=["events"])
+
+# In-memory connection counter per user (single-process assumption)
+_connection_counts: dict[uuid.UUID, int] = {}
+MAX_CONNECTIONS_PER_USER = 5
+
+
+@router.get("/stream")
+async def events_stream(
+    request: Request,
+    user_id: uuid.UUID = Depends(get_current_user_id),
+) -> StreamingResponse:
+    """Stream instance events via Server-Sent Events.
+
+    Enforces a maximum of 5 concurrent connections per user.
+    """
+    current = _connection_counts.get(user_id, 0)
+    if current >= MAX_CONNECTIONS_PER_USER:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail="Too many SSE connections",
+        )
+
+    _connection_counts[user_id] = current + 1
+
+    async def event_generator() -> AsyncGenerator[str, None]:
+        event_bus = InstanceEventBus()
+        queue: asyncio.Queue[InstanceEventPayload] = asyncio.Queue(maxsize=100)
+
+        async def on_event(payload: InstanceEventPayload) -> None:
+            try:
+                queue.put_nowait(payload)
+            except asyncio.QueueFull:
+                # Drop oldest event to make room
+                with contextlib.suppress(asyncio.QueueEmpty):
+                    queue.get_nowait()
+                with contextlib.suppress(asyncio.QueueFull):
+                    queue.put_nowait(payload)
+
+        unsubscribe = event_bus.subscribe("*", on_event)
+
+        try:
+            while True:
+                try:
+                    payload = await asyncio.wait_for(queue.get(), timeout=30.0)
+                    yield f"event: {payload['event']}\ndata: {json.dumps(payload)}\n\n"
+                except asyncio.TimeoutError:
+                    yield ":ping\n\n"
+        except asyncio.CancelledError:
+            # Client disconnected
+            raise
+        finally:
+            unsubscribe()
+            _connection_counts[user_id] = max(0, _connection_counts.get(user_id, 1) - 1)
+            if _connection_counts[user_id] == 0:
+                _connection_counts.pop(user_id, None)
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )

apps/api/src/api/tool_instances.py

@@ -28,6 +28,8 @@ from src.auth.dependencies import (
     get_current_user_id,
     get_db_session,
 )
+from src.services.event_bus import InstanceEventBus
+from src.services.lifecycle_hooks import publish_lifecycle_event
 from src.models.config_profile import ConfigProfile
 from src.models.git_repository import GitRepository
 from src.models.project import Project
@@ -77,6 +79,7 @@ from src.services.readiness_probe import execute_probe
 from src.services.ssh_keys import cleanup_ssh_key_files, prepare_ssh_key_files
 
 logger = logging.getLogger(__name__)
+_event_bus = InstanceEventBus()
 
 
 async def _resolve_git_mounts(
@@ -958,6 +961,15 @@ services:
         session.add(instance)
         await session.commit()
         await session.refresh(instance)
+        await publish_lifecycle_event(
+            event_bus=_event_bus,
+            session=session,
+            instance=instance,
+            event_type="instance.created",
+            created_by=user_id,
+            status="pending",
+            message="Instance created",
+        )
 
         return {
             "id": str(instance.id),
@@ -1501,6 +1513,15 @@ async def start_instance(
         instance.status = "starting"
         instance.last_started_at = datetime.now()
         await session.commit()
+        await publish_lifecycle_event(
+            event_bus=_event_bus,
+            session=session,
+            instance=instance,
+            event_type="instance.started",
+            created_by=user_id,
+            status="starting",
+            message="Container starting...",
+        )
         logger.debug("Instance %s: verifying container startup...", instance.id)
 
         startup_result = wait_for_container_running(
@@ -1518,6 +1539,19 @@ async def start_instance(
 
             instance.status = "error"
             await session.commit()
+            await publish_lifecycle_event(
+                event_bus=_event_bus,
+                session=session,
+                instance=instance,
+                event_type="instance.error",
+                created_by=user_id,
+                status="error",
+                message=error_msg,
+                metadata={
+                    "exit_code": startup_result["exit_code"],
+                    "error_type": "container",
+                },
+            )
             logger.error(
                 "Instance %s container startup failed after %.1fs: %s\nLogs:\n%s",
                 instance.id,
@@ -1607,6 +1641,16 @@ async def start_instance(
             if not success:
                 instance.status = "unhealthy"
                 await session.commit()
+                await publish_lifecycle_event(
+                    event_bus=_event_bus,
+                    session=session,
+                    instance=instance,
+                    event_type="instance.health_changed",
+                    created_by=user_id,
+                    status="unhealthy",
+                    message="Readiness probe failed",
+                    metadata={"probe_output": "\n".join(probe_logs)},
+                )
                 logger.error(
                     "Readiness probe failed for instance %s after %ds: %s",
                     instance.id,
@@ -1623,6 +1667,16 @@ async def start_instance(
 
     instance.status = "running"
     await session.commit()
+    await publish_lifecycle_event(
+        event_bus=_event_bus,
+        session=session,
+        instance=instance,
+        event_type="instance.health_changed",
+        created_by=user_id,
+        status="running",
+        message="Container running",
+        metadata={"previous_status": "starting"},
+    )
     logger.info("Instance %s is now running", instance.id)
 
     # Get tool type for default port
@@ -1631,6 +1685,15 @@ async def start_instance(
         logger.error("Tool type %s not found", instance.tool_type_id)
         instance.status = "error"
         await session.commit()
+        await publish_lifecycle_event(
+            event_bus=_event_bus,
+            session=session,
+            instance=instance,
+            event_type="instance.error",
+            created_by=user_id,
+            status="error",
+            message=f"Tool type '{instance.tool_type_id}' not found",
+        )
         return {
             "status": "error",
             "error": f"Tool type '{instance.tool_type_id}' not found",
@@ -1756,6 +1819,15 @@ async def stop_instance(
     instance.public_url = None
     instance.tunnel_id = None
     await session.commit()
+    await publish_lifecycle_event(
+        event_bus=_event_bus,
+        session=session,
+        instance=instance,
+        event_type="instance.stopped",
+        created_by=user_id,
+        status="stopped",
+        message="Instance stopped",
+    )
 
     return {"status": instance.status}
 
@@ -1892,6 +1964,15 @@ async def restart_instance(
                 instance.public_url = None
 
             await session.commit()
+            await publish_lifecycle_event(
+                event_bus=_event_bus,
+                session=session,
+                instance=instance,
+                event_type="instance.restarted",
+                created_by=user_id,
+                status="running",
+                message="Instance restarted",
+            )
             return {"status": instance.status, "url": instance.url}
 
     instance.status = "error"
@@ -1978,6 +2059,15 @@ async def delete_instance(
 
             shutil.rmtree(instance_dir)
 
+    await publish_lifecycle_event(
+        event_bus=_event_bus,
+        session=session,
+        instance=instance,
+        event_type="instance.deleted",
+        created_by=user_id,
+        status="deleted",
+        message="Instance deleted",
+    )
     await session.delete(instance)
     await session.commit()
 

apps/api/src/logging_config.py

@@ -1,15 +1,52 @@
+"""Structured JSON logging configuration."""
+
+import json
 import logging
 import sys
 import time
 import traceback
-from typing import Callable
+from collections.abc import Callable
 
 from fastapi import Request, Response
 from starlette.middleware.base import BaseHTTPMiddleware
 
+from src.services.correlation import get_correlation_id
+
 logger = logging.getLogger(__name__)
 
 
+class CorrelationIdFilter(logging.Filter):
+    """Inject correlation_id into every log record from context var."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.correlation_id = get_correlation_id()  # type: ignore[attr-defined]
+        return True
+
+
+class JSONFormatter(logging.Formatter):
+    """Emit log records as single-line JSON."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_obj: dict = {
+            "timestamp": self.formatTime(record),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "correlation_id": getattr(record, "correlation_id", None),
+        }
+        # Optional extra fields
+        for key in ("instance_id", "event_type"):
+            value = getattr(record, key, None)
+            if value is not None:
+                log_obj[key] = value
+        if record.exc_info:
+            log_obj["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_obj, default=str)
+
+    def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
+        return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
+
+
 class RequestLoggingMiddleware(BaseHTTPMiddleware):
     """Log all HTTP requests with timing and status codes."""
 
@@ -17,7 +54,6 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         start_time = time.time()
         client_host = request.client.host if request.client else "unknown"
 
-        # Log the incoming request
         logger.info(
             "→ Request: %s %s (client: %s)",
             request.method,
@@ -29,7 +65,6 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
             response = await call_next(request)
             duration = time.time() - start_time
 
-            # Log the response
             logger.info(
                 "← Response: %s %s → %d (%dms)",
                 request.method,
@@ -69,15 +104,13 @@ class ExceptionLoggingMiddleware(BaseHTTPMiddleware):
 
 
 def configure_logging(level: int = logging.INFO) -> None:
-    """Configure structured logging for the application."""
-    formatter = logging.Formatter(
-        fmt="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-    )
+    """Configure structured JSON logging for the application."""
+    formatter = JSONFormatter()
 
     # Console handler
     console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setFormatter(formatter)
+    console_handler.addFilter(CorrelationIdFilter())
 
     # Configure root logger
     root_logger = logging.getLogger()

apps/api/src/main.py

@@ -9,6 +9,7 @@ from fastapi.staticfiles import StaticFiles
 
 from src.api.auth import router as auth_router
 from src.api.dashboard import router as dashboard_router
+from src.api.events import router as events_router
 from src.api.git_repositories import router as git_repositories_router
 from src.api.health import router as health_router
 from src.api.projects import router as projects_router
@@ -30,6 +31,9 @@ from src.logging_config import (
     RequestLoggingMiddleware,
     configure_logging,
 )
+from src.services.correlation import CorrelationIdMiddleware
+from src.services.event_bus import InstanceEventBus
+from src.services.health_monitor import HealthMonitor
 
 # Configure logging early
 log_level = os.getenv("LOG_LEVEL", "INFO").upper()
@@ -54,6 +58,7 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.add_middleware(CorrelationIdMiddleware)
 app.add_middleware(RequestLoggingMiddleware)
 app.add_middleware(ExceptionLoggingMiddleware)
 
@@ -103,6 +108,11 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
     )
 
 
+# Global services
+_event_bus = InstanceEventBus()
+_health_monitor = HealthMonitor(_event_bus)
+
+
 @app.on_event("startup")
 async def on_startup():
     logger.info("Starting up Headquarter API...")
@@ -115,9 +125,21 @@ async def on_startup():
 
         sys.exit(1)
 
+    # Start background health monitor
+    _health_monitor.start()
+    logger.info("Health monitor started")
+
     logger.info("Startup complete.")
 
 
+@app.on_event("shutdown")
+async def on_shutdown():
+    logger.info("Shutting down Headquarter API...")
+    _health_monitor.stop()
+    logger.info("Health monitor stopped")
+    logger.info("Shutdown complete.")
+
+
 app.include_router(health_router)
 app.include_router(auth_router)
 app.include_router(dashboard_router)
@@ -133,4 +155,5 @@ app.include_router(tool_instances_router)
 app.include_router(sessions_router)
 app.include_router(instance_proxy_router)
 app.include_router(terminal_router)
+app.include_router(events_router)
 app.mount("/uploads", StaticFiles(directory="uploads"), name="uploads")

apps/api/src/models/__init__.py

@@ -1,6 +1,8 @@
 from src.models.base import Base
 from src.models.config_profile import ConfigProfile, ConfigProfileInclude
 from src.models.git_repository import GitRepository
+from src.models.health_check import HealthCheck
+from src.models.instance_event import InstanceEvent
 from src.models.project import Project
 from src.models.ssh_key import SSHKey
 from src.models.terminal_session import TerminalSessionModel
@@ -15,6 +17,8 @@ __all__ = [
     "ConfigProfile",
     "ConfigProfileInclude",
     "GitRepository",
+    "HealthCheck",
+    "InstanceEvent",
     "Project",
     "SSHKey",
     "TerminalSessionModel",

apps/api/src/models/health_check.py

@@ -0,0 +1,30 @@
+"""SQLAlchemy model for health check snapshots."""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, Uuid, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.models.base import Base, UUIDPrimaryKeyMixin
+
+
+class HealthCheck(UUIDPrimaryKeyMixin, Base):
+    __tablename__ = "health_checks"
+
+    instance_id: Mapped[uuid.UUID] = mapped_column(
+        Uuid(as_uuid=True),
+        ForeignKey("tool_instances.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    container_status: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    container_healthy: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+    tunnel_healthy: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+    exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    probe_status: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    probe_output: Mapped[str | None] = mapped_column(Text, nullable=True)
+    checked_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        server_default=func.now(),
+        nullable=False,
+    )

apps/api/src/models/instance_event.py

@@ -0,0 +1,39 @@
+"""SQLAlchemy model for instance lifecycle event audit rows."""
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import DateTime, ForeignKey, JSON, String, Text, Uuid, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.models.base import Base, UUIDPrimaryKeyMixin
+
+
+class InstanceEvent(UUIDPrimaryKeyMixin, Base):
+    __tablename__ = "instance_events"
+
+    instance_id: Mapped[uuid.UUID] = mapped_column(
+        Uuid(as_uuid=True),
+        ForeignKey("tool_instances.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    event_type: Mapped[str] = mapped_column(String(50), nullable=False)
+    status: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_by: Mapped[uuid.UUID | None] = mapped_column(
+        Uuid(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+    )
+    event_metadata: Mapped[dict[str, Any]] = mapped_column(
+        "metadata",
+        JSON,
+        nullable=False,
+        default=dict,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        server_default=func.now(),
+        nullable=False,
+    )

apps/api/src/services/correlation.py

@@ -0,0 +1,32 @@
+"""Async correlation ID context variable and helpers."""
+
+import contextvars
+import uuid
+
+from fastapi import Request
+from starlette.middleware.base import BaseHTTPMiddleware
+
+CORRELATION_ID: contextvars.ContextVar[str] = contextvars.ContextVar("correlation_id")
+
+
+def get_correlation_id() -> str:
+    """Return the current correlation ID or generate a new UUID."""
+    try:
+        return CORRELATION_ID.get()
+    except LookupError:
+        return str(uuid.uuid4())
+
+
+class CorrelationIdMiddleware(BaseHTTPMiddleware):
+    """Set correlation ID from X-Request-ID header or generate a new UUID."""
+
+    async def dispatch(self, request: Request, call_next):
+        request_id = request.headers.get("X-Request-ID")
+        correlation_id = request_id or str(uuid.uuid4())
+        token = CORRELATION_ID.set(correlation_id)
+        try:
+            response = await call_next(request)
+            response.headers["X-Request-ID"] = correlation_id
+            return response
+        finally:
+            CORRELATION_ID.reset(token)

apps/api/src/services/event_bus.py

@@ -0,0 +1,97 @@
+"""In-memory typed event bus for instance lifecycle and health events."""
+
+import asyncio
+import inspect
+import logging
+import uuid
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+InstanceEventPayload = dict[str, Any]
+EventCallback = Callable[[InstanceEventPayload], Awaitable[None] | None]  # noqa: UP044
+
+
+class InstanceEventBus:
+    """Singleton in-memory event bus with typed pub/sub and exception isolation."""
+
+    _instance: "InstanceEventBus | None" = None
+    _lock: asyncio.Lock = asyncio.Lock()
+
+    def __init__(self) -> None:
+        self._subscribers: dict[str, list[tuple[str, EventCallback]]] = {}
+
+    def __new__(cls) -> "InstanceEventBus":
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._subscribers = {}
+        return cls._instance
+
+    def _reset_for_testing(self) -> None:
+        """Clear all subscribers. For test use only."""
+        self._subscribers.clear()
+
+    def subscribe(
+        self,
+        event_type: str,
+        callback: EventCallback,
+    ) -> Callable[[], None]:
+        """Register a callback for an event type.
+
+        Args:
+            event_type: The event type to subscribe to.
+            callback: A sync or async callable that receives the payload.
+
+        Returns:
+            An unsubscribe function.
+        """
+        if event_type not in self._subscribers:
+            self._subscribers[event_type] = []
+        callback_id = str(uuid.uuid4())
+        self._subscribers[event_type].append((callback_id, callback))
+
+        def unsubscribe() -> None:
+            self.unsubscribe(event_type, callback_id)
+
+        return unsubscribe
+
+    def unsubscribe(self, event_type: str, callback_id: str) -> None:
+        """Remove a specific callback by ID."""
+        if event_type in self._subscribers:
+            self._subscribers[event_type] = [
+                (cid, cb)
+                for cid, cb in self._subscribers[event_type]
+                if cid != callback_id
+            ]
+            if not self._subscribers[event_type]:
+                del self._subscribers[event_type]
+
+    def unsubscribe_all(self, event_type: str) -> None:
+        """Remove all subscribers for an event type."""
+        self._subscribers.pop(event_type, None)
+
+    async def publish(self, event_type: str, payload: InstanceEventPayload) -> None:
+        """Deliver payload to all subscribers of event_type.
+
+        Also delivers to subscribers registered under the wildcard "*".
+        Exceptions from individual subscribers are caught and logged;
+        delivery continues to remaining subscribers.
+        """
+        callbacks: list[tuple[str, EventCallback]] = []
+        callbacks.extend(self._subscribers.get(event_type, []))
+        callbacks.extend(self._subscribers.get("*", []))
+
+        for _callback_id, callback in callbacks:
+            try:
+                if inspect.iscoroutinefunction(callback):
+                    await callback(payload)
+                else:
+                    callback(payload)
+            except Exception:
+                correlation_id = payload.get("correlation_id", "unknown")
+                logger.exception(
+                    "Event subscriber failed for %s",
+                    event_type,
+                    extra={"correlation_id": correlation_id},
+                )

apps/api/src/services/health_monitor.py

@@ -0,0 +1,219 @@
+"""Background health monitor that polls container and tunnel health."""
+
+import asyncio
+import logging
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.database import SessionLocal
+from src.models.health_check import HealthCheck
+from src.models.tool_instance import ToolInstance
+from src.services.correlation import get_correlation_id
+from src.services.docker import check_tunnel_health, get_container_status
+from src.services.event_bus import InstanceEventBus, InstanceEventPayload
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class HealthSnapshot:
+    """In-memory snapshot of an instance's health state."""
+
+    container_status: str | None = None
+    container_healthy: bool | None = None
+    tunnel_healthy: bool | None = None
+    exit_code: int | None = None
+
+
+class HealthMonitor:
+    """Polls container and tunnel health, publishing events on state changes."""
+
+    POLL_INTERVAL_SECONDS: float = 15.0
+    _MONITORED_STATUSES: set[str] = {"starting", "running", "unhealthy"}
+
+    def __init__(self, event_bus: InstanceEventBus) -> None:
+        self._event_bus = event_bus
+        self._task: asyncio.Task | None = None
+        self._last_known_state: dict[uuid.UUID, HealthSnapshot] = {}
+
+    def start(self) -> None:
+        """Idempotent start of the background polling task."""
+        if self._task is not None and not self._task.done():
+            return
+        try:
+            loop = asyncio.get_running_loop()
+            self._task = loop.create_task(self._poll_loop())
+        except RuntimeError:
+            pass
+
+    def stop(self) -> None:
+        """Cancel the background task and clear state."""
+        if self._task is not None and not self._task.done():
+            self._task.cancel()
+        self._last_known_state.clear()
+        self._task = None
+
+    async def _poll_loop(self) -> None:
+        """Main polling loop."""
+        while True:
+            try:
+                await asyncio.sleep(self.POLL_INTERVAL_SECONDS)
+                await self._run_check_cycle()
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                logger.exception("Health monitor poll loop error")
+
+    async def _run_check_cycle(self) -> None:
+        """Check all monitored instances in one cycle."""
+        async with SessionLocal() as session:
+            result = await session.execute(
+                select(ToolInstance).where(
+                    ToolInstance.status.in_(self._MONITORED_STATUSES)
+                )
+            )
+            instances = result.scalars().all()
+
+        for instance in instances:
+            async with SessionLocal() as session:
+                await self._check_instance(session, instance)
+
+    async def _check_instance(
+        self,
+        session: AsyncSession,
+        instance: ToolInstance,
+    ) -> None:
+        """Check a single instance and handle state transitions."""
+        try:
+            container_info = get_container_status(instance.container_id or "")
+        except Exception:
+            logger.exception(
+                "Health check failed for instance %s",
+                instance.id,
+                extra={
+                    "instance_id": str(instance.id),
+                    "correlation_id": get_correlation_id(),
+                },
+            )
+            return
+
+        container_status = container_info["status"]
+        exit_code = container_info["exit_code"]
+        container_healthy = (
+            container_info["health"] == "healthy" if container_info["health"] else None
+        )
+
+        tunnel_healthy: bool | None = None
+        if instance.public_url and container_status == "running":
+            try:
+                tunnel_result = check_tunnel_health(instance.public_url)
+                tunnel_healthy = tunnel_result.get("healthy", False)
+            except Exception:
+                logger.exception(
+                    "Tunnel health check failed for instance %s",
+                    instance.id,
+                    extra={
+                        "instance_id": str(instance.id),
+                        "correlation_id": get_correlation_id(),
+                    },
+                )
+                tunnel_healthy = False
+
+        snapshot = HealthSnapshot(
+            container_status=container_status,
+            container_healthy=container_healthy,
+            tunnel_healthy=tunnel_healthy,
+            exit_code=exit_code,
+        )
+
+        previous = self._last_known_state.get(instance.id)
+
+        # Determine new status
+        new_status = self._derive_status(snapshot)
+
+        # If first check or state changed
+        if previous is None or not self._snapshots_equal(previous, snapshot):
+            await self._handle_state_change(
+                session, instance, previous, snapshot, new_status
+            )
+            self._last_known_state[instance.id] = snapshot
+
+    def _derive_status(self, snapshot: HealthSnapshot) -> str:
+        """Derive instance status from health snapshot."""
+        if snapshot.container_status != "running":
+            return "error"
+        if snapshot.tunnel_healthy is False:
+            return "unhealthy"
+        return "running"
+
+    def _snapshots_equal(self, a: HealthSnapshot, b: HealthSnapshot) -> bool:
+        """Compare two snapshots for equality."""
+        return (
+            a.container_status == b.container_status
+            and a.container_healthy == b.container_healthy
+            and a.tunnel_healthy == b.tunnel_healthy
+            and a.exit_code == b.exit_code
+        )
+
+    async def _handle_state_change(
+        self,
+        session: AsyncSession,
+        instance: ToolInstance,
+        previous: HealthSnapshot | None,
+        snapshot: HealthSnapshot,
+        new_status: str,
+    ) -> None:
+        """Update DB, insert health check, and publish event."""
+        previous_status = instance.status
+
+        # Update instance status
+        instance.status = new_status
+        if new_status == "error":
+            instance.last_stopped_at = datetime.now(timezone.utc)
+
+        # Insert health check row
+        health_check = HealthCheck(
+            instance_id=instance.id,
+            container_status=snapshot.container_status,
+            container_healthy=snapshot.container_healthy,
+            tunnel_healthy=snapshot.tunnel_healthy,
+            exit_code=snapshot.exit_code,
+            probe_status=None,
+            probe_output=None,
+        )
+        session.add(health_check)
+        await session.commit()
+
+        # Build event payload
+        correlation_id = get_correlation_id()
+        metadata: dict = {"previous_status": previous_status}
+        if snapshot.exit_code is not None:
+            metadata["exit_code"] = snapshot.exit_code
+            metadata["error_type"] = "container"
+        if instance.public_url:
+            metadata["tunnel_url"] = instance.public_url
+
+        if new_status == "error":
+            event_type = "instance.error"
+            message = f"Container failed with status {snapshot.container_status}"
+            if snapshot.exit_code is not None:
+                message += f" (exit code: {snapshot.exit_code})"
+        else:
+            event_type = "instance.health_changed"
+            message = f"Container is now {new_status}"
+
+        payload: InstanceEventPayload = {
+            "event": event_type,
+            "instance_id": str(instance.id),
+            "status": new_status,
+            "message": message,
+            "metadata": metadata,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "correlation_id": correlation_id,
+        }
+
+        await self._event_bus.publish(event_type, payload)

apps/api/src/services/lifecycle_hooks.py

@@ -0,0 +1,98 @@
+"""Lifecycle hook helpers for instrumenting tool instance transitions."""
+
+import uuid
+from datetime import datetime, timezone
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.models.instance_event import InstanceEvent
+from src.models.tool_instance import ToolInstance
+from src.services.correlation import get_correlation_id
+from src.services.event_bus import InstanceEventBus, InstanceEventPayload
+
+
+def _build_payload(
+    event_type: str,
+    instance: ToolInstance,
+    status: str | None = None,
+    message: str | None = None,
+    metadata: dict | None = None,
+) -> InstanceEventPayload:
+    """Construct a standard event payload."""
+    return {
+        "event": event_type,
+        "instance_id": str(instance.id),
+        "status": status or instance.status,
+        "message": message,
+        "metadata": metadata or {},
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "correlation_id": get_correlation_id(),
+    }
+
+
+async def _write_audit_row(
+    session: AsyncSession,
+    instance: ToolInstance,
+    event_type: str,
+    created_by: uuid.UUID | None = None,
+    status: str | None = None,
+    message: str | None = None,
+    metadata: dict | None = None,
+) -> InstanceEvent:
+    """Persist an instance_events audit row."""
+    row = InstanceEvent(
+        instance_id=instance.id,
+        event_type=event_type.replace("instance.", ""),
+        status=status or instance.status,
+        message=message,
+        created_by=created_by,
+        event_metadata=metadata or {},
+    )
+    session.add(row)
+    await session.commit()
+    return row
+
+
+async def publish_lifecycle_event(
+    event_bus: InstanceEventBus,
+    session: AsyncSession,
+    instance: ToolInstance,
+    event_type: str,
+    created_by: uuid.UUID | None = None,
+    status: str | None = None,
+    message: str | None = None,
+    metadata: dict | None = None,
+) -> None:
+    """Publish a lifecycle event and write an audit row after DB commit.
+
+    Args:
+        event_bus: The global event bus.
+        session: Active async DB session.
+        instance: The affected tool instance.
+        event_type: One of instance.created, instance.started, etc.
+        created_by: User ID for user-initiated actions; None for system.
+        status: Optional status override.
+        message: Optional human-readable message.
+        metadata: Optional extra metadata.
+    """
+    payload = _build_payload(
+        event_type=event_type,
+        instance=instance,
+        status=status,
+        message=message,
+        metadata=metadata,
+    )
+
+    # Write audit row
+    await _write_audit_row(
+        session=session,
+        instance=instance,
+        event_type=event_type,
+        created_by=created_by,
+        status=status or instance.status,
+        message=message,
+        metadata=metadata,
+    )
+
+    # Publish to bus
+    await event_bus.publish(event_type, payload)

apps/api/tests/unit/test_event_bus.py

@@ -0,0 +1,148 @@
+"""Unit tests for InstanceEventBus."""
+
+import asyncio
+import uuid
+from typing import Any
+
+import pytest
+
+from src.services.event_bus import InstanceEventBus, InstanceEventPayload
+
+
+@pytest.fixture
+def event_bus() -> InstanceEventBus:
+    """Provide a fresh EventBus instance with reset singleton state."""
+    bus = InstanceEventBus()
+    bus._reset_for_testing()
+    return bus
+
+
+@pytest.fixture
+def sample_payload() -> InstanceEventPayload:
+    """Provide a sample event payload."""
+    return {
+        "event": "instance.started",
+        "instance_id": str(uuid.uuid4()),
+        "status": "starting",
+        "message": "Container starting...",
+        "metadata": {},
+        "timestamp": "2026-05-28T12:00:00Z",
+        "correlation_id": str(uuid.uuid4()),
+    }
+
+
+@pytest.mark.unit
+async def test_publish_delivers_to_all_subscribers(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """All subscribed callbacks should receive the published payload."""
+    received: list[Any] = []
+
+    def callback_1(payload: InstanceEventPayload) -> None:
+        received.append(("callback_1", payload))
+
+    def callback_2(payload: InstanceEventPayload) -> None:
+        received.append(("callback_2", payload))
+
+    def callback_3(payload: InstanceEventPayload) -> None:
+        received.append(("callback_3", payload))
+
+    event_bus.subscribe("instance.started", callback_1)
+    event_bus.subscribe("instance.started", callback_2)
+    event_bus.subscribe("instance.started", callback_3)
+
+    await event_bus.publish("instance.started", sample_payload)
+
+    assert len(received) == 3
+    assert received[0][0] == "callback_1"
+    assert received[1][0] == "callback_2"
+    assert received[2][0] == "callback_3"
+
+
+@pytest.mark.unit
+async def test_subscriber_exception_isolation(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """If one subscriber raises, others should still receive the event."""
+    received: list[str] = []
+
+    def bad_callback(_payload: InstanceEventPayload) -> None:
+        raise RuntimeError("boom")
+
+    def good_callback(_payload: InstanceEventPayload) -> None:
+        received.append("good_callback")
+
+    event_bus.subscribe("instance.started", bad_callback)
+    event_bus.subscribe("instance.started", good_callback)
+
+    # Should not raise
+    await event_bus.publish("instance.started", sample_payload)
+
+    assert received == ["good_callback"]
+
+
+@pytest.mark.unit
+async def test_unsubscribe_removes_callback(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """After unsubscribing, the callback should not be called."""
+    received: list[str] = []
+
+    def callback(_payload: InstanceEventPayload) -> None:
+        received.append("callback")
+
+    unsubscribe = event_bus.subscribe("instance.started", callback)
+    unsubscribe()
+
+    await event_bus.publish("instance.started", sample_payload)
+
+    assert received == []
+
+
+@pytest.mark.unit
+async def test_publish_to_empty_subscriber_list(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """Publishing to an event type with no subscribers should not raise."""
+    await event_bus.publish("instance.started", sample_payload)
+
+
+@pytest.mark.unit
+async def test_async_subscriber_supported(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """Async callbacks should be awaited correctly."""
+    received: list[str] = []
+
+    async def async_callback(_payload: InstanceEventPayload) -> None:
+        await asyncio.sleep(0)
+        received.append("async_callback")
+
+    event_bus.subscribe("instance.started", async_callback)
+    await event_bus.publish("instance.started", sample_payload)
+
+    assert received == ["async_callback"]
+
+
+@pytest.mark.unit
+async def test_unsubscribe_all_clears_subscribers(
+    event_bus: InstanceEventBus,
+    sample_payload: InstanceEventPayload,
+) -> None:
+    """unsubscribe_all should remove all callbacks for an event type."""
+    received: list[str] = []
+
+    def callback(_payload: InstanceEventPayload) -> None:
+        received.append("callback")
+
+    event_bus.subscribe("instance.started", callback)
+    event_bus.unsubscribe_all("instance.started")
+
+    await event_bus.publish("instance.started", sample_payload)
+
+    assert received == []

apps/api/tests/unit/test_health_monitor.py

@@ -0,0 +1,292 @@
+"""Unit tests for HealthMonitor state-transition logic."""
+
+import asyncio
+import uuid
+from contextlib import suppress
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import select
+
+from src.models.health_check import HealthCheck
+from src.models.tool_instance import ToolInstance
+from src.models.user import User
+from src.services.event_bus import InstanceEventBus, InstanceEventPayload
+from src.services.health_monitor import HealthMonitor, HealthSnapshot
+
+
+@pytest.fixture
+def event_bus() -> InstanceEventBus:
+    """Provide a fresh EventBus instance."""
+    bus = InstanceEventBus()
+    bus._reset_for_testing()
+    return bus
+
+
+@pytest.fixture
+def health_monitor(event_bus: InstanceEventBus) -> HealthMonitor:
+    """Provide a HealthMonitor with a short poll interval for testing."""
+    monitor = HealthMonitor(event_bus)
+    monitor.POLL_INTERVAL_SECONDS = 0.1
+    return monitor
+
+
+async def _create_running_instance(db_session) -> ToolInstance:
+    """Helper to create a user and a running tool instance."""
+    user = User(
+        id=uuid.uuid4(),
+        email="hm@example.com",
+        name="HM Test",
+        authentik_id="auth-hm",
+    )
+    db_session.add(user)
+    await db_session.commit()
+
+    instance = ToolInstance(
+        id=uuid.uuid4(),
+        name="hm-test-instance",
+        display_name="HM Test Instance",
+        tool_type_id=uuid.uuid4(),
+        repository_id=uuid.uuid4(),
+        project_id=uuid.uuid4(),
+        owner_id=user.id,
+        status="running",
+        container_id="container123",
+        public_url="https://example.trycloudflare.com",
+    )
+    db_session.add(instance)
+    await db_session.commit()
+    return instance
+
+
+@pytest.mark.unit
+async def test_detects_container_crash(
+    db_session,
+    event_bus: InstanceEventBus,
+    health_monitor: HealthMonitor,
+) -> None:
+    """Monitor should detect exited container and publish error event."""
+    instance = await _create_running_instance(db_session)
+
+    events_captured: list[InstanceEventPayload] = []
+
+    def capture_event(payload: InstanceEventPayload) -> None:
+        events_captured.append(payload)
+
+    event_bus.subscribe("instance.error", capture_event)
+
+    with (
+        patch(
+            "src.services.health_monitor.get_container_status",
+            return_value={"status": "exited", "exit_code": 137, "health": None},
+        ),
+        patch(
+            "src.services.health_monitor.check_tunnel_health",
+            return_value={"healthy": False, "tunnel_status": "not_applicable"},
+        ),
+    ):
+        await health_monitor._check_instance(db_session, instance)
+
+    # Refresh instance from DB
+    await db_session.refresh(instance)
+    assert instance.status == "error"
+
+    # Event published
+    assert len(events_captured) == 1
+    assert events_captured[0]["event"] == "instance.error"
+    assert events_captured[0]["status"] == "error"
+    assert events_captured[0]["metadata"]["exit_code"] == 137
+
+    # Health check row inserted
+    result = await db_session.execute(
+        select(HealthCheck).where(HealthCheck.instance_id == instance.id)
+    )
+    check = result.scalar_one()
+    assert check.container_status == "exited"
+    assert check.exit_code == 137
+
+
+@pytest.mark.unit
+async def test_detects_tunnel_failure(
+    db_session,
+    event_bus: InstanceEventBus,
+    health_monitor: HealthMonitor,
+) -> None:
+    """Monitor should detect tunnel failure and mark unhealthy."""
+    instance = await _create_running_instance(db_session)
+
+    events_captured: list[InstanceEventPayload] = []
+
+    def capture_event(payload: InstanceEventPayload) -> None:
+        events_captured.append(payload)
+
+    event_bus.subscribe("instance.health_changed", capture_event)
+
+    with (
+        patch(
+            "src.services.health_monitor.get_container_status",
+            return_value={"status": "running", "exit_code": None, "health": "healthy"},
+        ),
+        patch(
+            "src.services.health_monitor.check_tunnel_health",
+            return_value={
+                "healthy": False,
+                "tunnel_status": "error_response",
+                "status_code": 502,
+            },
+        ),
+    ):
+        await health_monitor._check_instance(db_session, instance)
+
+    await db_session.refresh(instance)
+    assert instance.status == "unhealthy"
+
+    assert len(events_captured) == 1
+    assert events_captured[0]["event"] == "instance.health_changed"
+    assert events_captured[0]["status"] == "unhealthy"
+    assert events_captured[0]["metadata"]["previous_status"] == "running"
+
+    result = await db_session.execute(
+        select(HealthCheck).where(HealthCheck.instance_id == instance.id)
+    )
+    check = result.scalar_one()
+    assert check.tunnel_healthy is False
+
+
+@pytest.mark.unit
+async def test_detects_recovery(
+    db_session,
+    event_bus: InstanceEventBus,
+    health_monitor: HealthMonitor,
+) -> None:
+    """Monitor should detect recovery from unhealthy to running."""
+    instance = await _create_running_instance(db_session)
+    instance.status = "unhealthy"
+    await db_session.commit()
+
+    # Seed last known state as unhealthy
+    health_monitor._last_known_state[instance.id] = HealthSnapshot(
+        container_status="running",
+        container_healthy=None,
+        tunnel_healthy=False,
+        exit_code=None,
+    )
+
+    events_captured: list[InstanceEventPayload] = []
+
+    def capture_event(payload: InstanceEventPayload) -> None:
+        events_captured.append(payload)
+
+    event_bus.subscribe("instance.health_changed", capture_event)
+
+    with (
+        patch(
+            "src.services.health_monitor.get_container_status",
+            return_value={"status": "running", "exit_code": None, "health": None},
+        ),
+        patch(
+            "src.services.health_monitor.check_tunnel_health",
+            return_value={
+                "healthy": True,
+                "tunnel_status": "healthy",
+                "status_code": 200,
+            },
+        ),
+    ):
+        await health_monitor._check_instance(db_session, instance)
+
+    await db_session.refresh(instance)
+    assert instance.status == "running"
+
+    assert len(events_captured) == 1
+    assert events_captured[0]["status"] == "running"
+    assert events_captured[0]["metadata"]["previous_status"] == "unhealthy"
+
+    result = await db_session.execute(
+        select(HealthCheck).where(HealthCheck.instance_id == instance.id)
+    )
+    check = result.scalar_one()
+    assert check.tunnel_healthy is True
+
+
+@pytest.mark.unit
+async def test_skips_writes_when_no_state_change(
+    db_session,
+    event_bus: InstanceEventBus,
+    health_monitor: HealthMonitor,
+) -> None:
+    """Two identical polls should result in only one health_checks row."""
+    instance = await _create_running_instance(db_session)
+
+    with (
+        patch(
+            "src.services.health_monitor.get_container_status",
+            return_value={"status": "running", "exit_code": None, "health": None},
+        ),
+        patch(
+            "src.services.health_monitor.check_tunnel_health",
+            return_value={
+                "healthy": True,
+                "tunnel_status": "healthy",
+                "status_code": 200,
+            },
+        ),
+    ):
+        await health_monitor._check_instance(db_session, instance)
+        await health_monitor._check_instance(db_session, instance)
+
+    result = await db_session.execute(
+        select(HealthCheck).where(HealthCheck.instance_id == instance.id)
+    )
+    assert len(result.scalars().all()) == 1
+
+
+@pytest.mark.unit
+async def test_docker_exception_resilience(
+    db_session,
+    event_bus: InstanceEventBus,
+    health_monitor: HealthMonitor,
+) -> None:
+    """Docker exception should be caught and not propagate."""
+    instance = await _create_running_instance(db_session)
+
+    events_captured: list[InstanceEventPayload] = []
+
+    def capture_event(payload: InstanceEventPayload) -> None:
+        events_captured.append(payload)
+
+    event_bus.subscribe("instance.error", capture_event)
+    event_bus.subscribe("instance.health_changed", capture_event)
+
+    with patch(
+        "src.services.health_monitor.get_container_status",
+        side_effect=RuntimeError("docker exploded"),
+    ):
+        # Should not raise
+        await health_monitor._check_instance(db_session, instance)
+
+    # No DB writes
+    result = await db_session.execute(
+        select(HealthCheck).where(HealthCheck.instance_id == instance.id)
+    )
+    assert result.scalar_one_or_none() is None
+
+    # No events published
+    assert events_captured == []
+
+
+@pytest.mark.unit
+async def test_monitor_start_stop(health_monitor: HealthMonitor) -> None:
+    """Start and stop should manage the background task."""
+    health_monitor.start()
+    task = health_monitor._task
+    assert task is not None
+    assert not task.done()
+
+    health_monitor.stop()
+    if task is not None and not task.done():
+        with suppress(asyncio.CancelledError):
+            await task
+    assert task is not None
+    assert task.cancelled() or task.done()
+    assert health_monitor._last_known_state == {}

apps/api/tests/unit/test_monitoring_models.py

@@ -0,0 +1,143 @@
+"""Unit tests for monitoring models and migration compatibility."""
+
+import uuid
+from datetime import datetime
+
+import pytest
+from sqlalchemy import select
+
+from src.models.health_check import HealthCheck
+from src.models.instance_event import InstanceEvent
+from src.models.tool_instance import ToolInstance
+from src.models.user import User
+
+
+@pytest.mark.unit
+async def test_instance_event_creation(db_session) -> None:
+    """InstanceEvent model can be created and persisted."""
+    user = User(
+        id=uuid.uuid4(),
+        email="test@example.com",
+        name="Test",
+        authentik_id="auth-1",
+    )
+    db_session.add(user)
+    await db_session.commit()
+
+    instance = ToolInstance(
+        id=uuid.uuid4(),
+        name="test-instance",
+        display_name="Test Instance",
+        tool_type_id=uuid.uuid4(),
+        repository_id=uuid.uuid4(),
+        project_id=uuid.uuid4(),
+        owner_id=user.id,
+        status="pending",
+    )
+    db_session.add(instance)
+    await db_session.commit()
+
+    event = InstanceEvent(
+        instance_id=instance.id,
+        event_type="started",
+        status="starting",
+        message="Container starting...",
+        created_by=user.id,
+        event_metadata={"previous_status": "pending"},
+    )
+    db_session.add(event)
+    await db_session.commit()
+    await db_session.refresh(event)
+
+    assert event.id is not None
+    assert event.instance_id == instance.id
+    assert event.event_type == "started"
+    assert event.status == "starting"
+    assert event.created_by == user.id
+    assert event.event_metadata == {"previous_status": "pending"}
+    assert isinstance(event.created_at, datetime)
+
+
+@pytest.mark.unit
+async def test_health_check_creation(db_session) -> None:
+    """HealthCheck model can be created and persisted."""
+    user = User(
+        id=uuid.uuid4(),
+        email="test2@example.com",
+        name="Test2",
+        authentik_id="auth-2",
+    )
+    db_session.add(user)
+    await db_session.commit()
+
+    instance = ToolInstance(
+        id=uuid.uuid4(),
+        name="test-instance-2",
+        display_name="Test Instance 2",
+        tool_type_id=uuid.uuid4(),
+        repository_id=uuid.uuid4(),
+        project_id=uuid.uuid4(),
+        owner_id=user.id,
+        status="running",
+    )
+    db_session.add(instance)
+    await db_session.commit()
+
+    check = HealthCheck(
+        instance_id=instance.id,
+        container_status="running",
+        container_healthy=True,
+        tunnel_healthy=True,
+        exit_code=None,
+        probe_status="passed",
+        probe_output="OK",
+    )
+    db_session.add(check)
+    await db_session.commit()
+    await db_session.refresh(check)
+
+    assert check.id is not None
+    assert check.instance_id == instance.id
+    assert check.container_status == "running"
+    assert check.container_healthy is True
+    assert check.tunnel_healthy is True
+    assert isinstance(check.checked_at, datetime)
+
+
+@pytest.mark.unit
+async def test_instance_event_query_by_instance(db_session) -> None:
+    """InstanceEvent rows can be queried by instance_id."""
+    user = User(
+        id=uuid.uuid4(),
+        email="test3@example.com",
+        name="Test3",
+        authentik_id="auth-3",
+    )
+    db_session.add(user)
+    await db_session.commit()
+
+    instance = ToolInstance(
+        id=uuid.uuid4(),
+        name="test-instance-3",
+        display_name="Test Instance 3",
+        tool_type_id=uuid.uuid4(),
+        repository_id=uuid.uuid4(),
+        project_id=uuid.uuid4(),
+        owner_id=user.id,
+        status="pending",
+    )
+    db_session.add(instance)
+    await db_session.commit()
+
+    event = InstanceEvent(
+        instance_id=instance.id,
+        event_type="created",
+        status="pending",
+    )
+    db_session.add(event)
+    await db_session.commit()
+
+    result = await db_session.execute(
+        select(InstanceEvent).where(InstanceEvent.instance_id == instance.id)
+    )
+    assert result.scalar_one() is not None