From 2682e0268c36b19c5a105cb8f4391c665f693c60 Mon Sep 17 00:00:00 2001 From: Alex Blank Date: Fri, 29 May 2026 10:23:28 +0200 Subject: [PATCH] feat: container monitoring integration + polish (PR-3) - Integration tests: SSE auth, connection limits, lifecycle hooks, event persistence (6 tests) - Instance events history API: GET /instances/{id}/events - Documentation updates: terminal.md, backend.md, frontend.md - Performance: SSE max 5 connections, health monitor write-on-change Quality gates: pytest 21 monitoring passed, 172 unit passed (4 pre-existing), vitest 14 passed, tsc clean, eslint clean, ruff clean --- apps/api/src/api/tool_instances.py | 59 ++++ apps/api/tests/integration/test_events.py | 268 ++++++++++++++++++ docs/architecture/backend.md | 41 +++ docs/architecture/frontend.md | 35 ++- docs/features/terminal.md | 21 ++ .../apply-pr3.md | 61 ++++ 6 files changed, 481 insertions(+), 4 deletions(-) create mode 100644 apps/api/tests/integration/test_events.py create mode 100644 openspec/changes/container-monitoring-notifications/apply-pr3.md diff --git a/apps/api/src/api/tool_instances.py b/apps/api/src/api/tool_instances.py index c5b12cf..8267ba8 100644 --- a/apps/api/src/api/tool_instances.py +++ b/apps/api/src/api/tool_instances.py @@ -2284,6 +2284,65 @@ async def check_instance_tunnel_health( return response +@router.get( + "/{project_id}/repositories/{repo_id}/instances/{instance_id}/events", + summary="Get instance events history", + description="Get lifecycle event history for a tool instance.", +) +async def get_instance_events( + project_id: uuid.UUID, + repo_id: uuid.UUID, + instance_id: uuid.UUID, + limit: int = 50, + user_id: uuid.UUID = Depends(get_current_user_id), + session: AsyncSession = Depends(get_db_session), +) -> list[dict]: + """Get lifecycle event history for an instance. + + Args: + project_id: UUID of the project. + repo_id: UUID of the repository. + instance_id: UUID of the instance. + limit: Maximum number of events to return (default: 50). + user_id: ID of the authenticated user. + session: Database session. + + Returns: + List of event dictionaries. + """ + from sqlalchemy import select + from src.models.instance_event import InstanceEvent + + _user = await _get_user(session, user_id) + _project = await _get_owned_project(project_id, user_id, session) + + instance = await session.get(ToolInstance, instance_id) + if instance is None or instance.repository_id != repo_id: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="instance not found" + ) + + result = await session.execute( + select(InstanceEvent) + .where(InstanceEvent.instance_id == instance_id) + .order_by(InstanceEvent.created_at.desc()) + .limit(limit) + ) + rows = result.scalars().all() + + return [ + { + "id": str(row.id), + "event_type": row.event_type, + "status": row.status, + "message": row.message, + "metadata": row.event_metadata, + "created_at": row.created_at.isoformat() if row.created_at else None, + } + for row in rows + ] + + @router.get( "/{project_id}/repositories/{repo_id}/instances/{instance_id}/proxy/{path:path}", summary="Proxy to instance", diff --git a/apps/api/tests/integration/test_events.py b/apps/api/tests/integration/test_events.py new file mode 100644 index 0000000..a490b76 --- /dev/null +++ b/apps/api/tests/integration/test_events.py @@ -0,0 +1,268 @@ +"""Integration tests for SSE endpoint and lifecycle event flow.""" + +import asyncio +import uuid +from collections.abc import Generator +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api import events as events_module +from src.auth.session import decode_session_cookie +from src.config import Settings +from src.models.git_repository import GitRepository +from src.models.instance_event import InstanceEvent +from src.models.project import Project +from src.models.tool_instance import ToolInstance +from src.models.tool_type import ToolType +from src.services.event_bus import InstanceEventBus, InstanceEventPayload + + +@pytest.fixture +def event_bus() -> Generator[InstanceEventBus, None, None]: + bus = InstanceEventBus() + bus._reset_for_testing() + yield bus + bus._reset_for_testing() + + +@pytest.fixture +def sample_payload() -> InstanceEventPayload: + return { + "event": "instance.started", + "instance_id": str(uuid.uuid4()), + "status": "starting", + "message": "Container starting...", + "metadata": {}, + "timestamp": "2026-05-28T12:00:00Z", + "correlation_id": str(uuid.uuid4()), + } + + +def _get_user_id_from_client(client: TestClient) -> uuid.UUID | None: + settings = Settings() + cookie = client.cookies.get("session") + if not cookie: + return None + session = decode_session_cookie(settings=settings, cookie_value=cookie) + if session and "user_id" in session: + return uuid.UUID(session["user_id"]) + return None + + +@pytest.mark.integration +def test_sse_requires_auth(test_client: TestClient) -> None: + response = test_client.get("/events/stream") + assert response.status_code == 401 + + +@pytest.mark.integration +def test_sse_enforces_connection_limit(authenticated_client: TestClient) -> None: + user_id = _get_user_id_from_client(authenticated_client) + assert user_id is not None + + events_module._connection_counts[user_id] = events_module.MAX_CONNECTIONS_PER_USER + try: + response = authenticated_client.get("/events/stream") + assert response.status_code == 429 + finally: + events_module._connection_counts.pop(user_id, None) + + +@pytest.mark.integration +def test_sse_event_generator_format() -> None: + """Test the SSE endpoint is registered.""" + from src.api.events import router + + route_paths = [getattr(r, "path", "") for r in router.routes] + assert any("/stream" in str(p) for p in route_paths) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_lifecycle_hook_publishes_event_and_persists( + authenticated_client: TestClient, + db_session: AsyncSession, + event_bus: InstanceEventBus, +) -> None: + """Test that the lifecycle hook publishes an event and persists an audit row.""" + user_id = _get_user_id_from_client(authenticated_client) + assert user_id is not None + + project = Project( + id=uuid.uuid4(), + name="test-project", + description="Test", + owner_id=user_id, + ) + repo = GitRepository( + id=uuid.uuid4(), + name="test-repo", + path="/tmp/test-repo", + project_id=project.id, + owner_id=user_id, + remote_url="https://github.com/test/repo.git", + ) + tool_type = ToolType( + id=uuid.uuid4(), + name="test-tool", + display_name="Test Tool", + category="other", + interface_type="web", + requires_port=True, + default_port=8080, + definition_type="legacy", + compose_template="version: '3.8'\nservices:\n app:\n image: alpine\n command: sleep 3600\n", + ) + db_session.add_all([project, repo, tool_type]) + await db_session.commit() + + instance = ToolInstance( + id=uuid.uuid4(), + name="test-instance", + display_name="Test Instance", + tool_type_id=tool_type.id, + repository_id=repo.id, + project_id=project.id, + owner_id=user_id, + status="pending", + compose_path="/tmp/test-compose.yml", + port=8080, + ) + db_session.add(instance) + await db_session.commit() + + received: list[Any] = [] + + def subscriber(payload: InstanceEventPayload) -> None: + received.append(payload) + + event_bus.subscribe("instance.created", subscriber) + + from src.services.lifecycle_hooks import publish_lifecycle_event + + await publish_lifecycle_event( + event_bus=event_bus, + session=db_session, + instance=instance, + event_type="instance.created", + created_by=user_id, + status="pending", + message="Instance created", + ) + + assert len(received) == 1 + assert received[0]["event"] == "instance.created" + + result = await db_session.execute( + select(InstanceEvent).where(InstanceEvent.instance_id == instance.id) + ) + rows = result.scalars().all() + assert len(rows) == 1 + assert rows[0].event_type == "created" + assert rows[0].created_by == user_id + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_lifecycle_event_persists_audit_row( + authenticated_client: TestClient, + db_session: AsyncSession, + event_bus: InstanceEventBus, +) -> None: + """Test that publishing a lifecycle event persists an audit row.""" + user_id = _get_user_id_from_client(authenticated_client) + assert user_id is not None + + project = Project( + id=uuid.uuid4(), + name="test-project", + description="Test", + owner_id=user_id, + ) + repo = GitRepository( + id=uuid.uuid4(), + name="test-repo", + path="/tmp/test-repo", + project_id=project.id, + owner_id=user_id, + remote_url="https://github.com/test/repo.git", + ) + tool_type = ToolType( + id=uuid.uuid4(), + name="test-tool-2", + display_name="Test Tool 2", + category="other", + interface_type="web", + requires_port=True, + default_port=8080, + definition_type="legacy", + compose_template="version: '3.8'\nservices:\n app:\n image: alpine\n command: sleep 3600\n", + ) + db_session.add_all([project, repo, tool_type]) + await db_session.commit() + + instance = ToolInstance( + id=uuid.uuid4(), + name="test-instance", + display_name="Test Instance", + tool_type_id=tool_type.id, + repository_id=repo.id, + project_id=project.id, + owner_id=user_id, + status="running", + compose_path="/tmp/test-compose.yml", + port=8080, + ) + db_session.add(instance) + await db_session.commit() + + from src.services.lifecycle_hooks import publish_lifecycle_event + + await publish_lifecycle_event( + event_bus=event_bus, + session=db_session, + instance=instance, + event_type="instance.stopped", + created_by=user_id, + status="stopped", + message="Instance stopped", + ) + + result = await db_session.execute( + select(InstanceEvent).where(InstanceEvent.instance_id == instance.id) + ) + rows = result.scalars().all() + assert len(rows) == 1 + assert rows[0].event_type == "stopped" + assert rows[0].status == "stopped" + assert rows[0].created_by == user_id + + +@pytest.mark.integration +def test_event_bus_pubsub(event_bus: InstanceEventBus) -> None: + """Test that the event bus delivers events to subscribers.""" + received: list[InstanceEventPayload] = [] + + def handler(payload: InstanceEventPayload) -> None: + received.append(payload) + + event_bus.subscribe("test.event", handler) + + payload: InstanceEventPayload = { + "event": "test.event", + "instance_id": str(uuid.uuid4()), + "status": "running", + "message": "Test", + "metadata": {}, + "timestamp": "2026-05-28T12:00:00Z", + "correlation_id": str(uuid.uuid4()), + } + + asyncio.run(event_bus.publish("test.event", payload)) + + assert len(received) == 1 + assert received[0]["event"] == "test.event" diff --git a/docs/architecture/backend.md b/docs/architecture/backend.md index db82501..3adc532 100644 --- a/docs/architecture/backend.md +++ b/docs/architecture/backend.md @@ -17,6 +17,10 @@ The Headquarter backend is built with **FastAPI** and follows a layered architec │ │ Auth │ │ Projects │ │ Users │ │ Git │ │ │ │ Routes │ │ Routes │ │ Routes │ │ Repos │ │ │ └────┬────┘ └────┬─────┘ └───┬────┘ └────┬─────┘ │ +│ ┌──────────┐ ┌──────────┐ │ +│ │ ToolInst │ │ Events │ │ +│ │ Routes │ │ Routes │ │ +│ └────┬─────┘ └────┬─────┘ │ ├───────┼───────────┼───────────┼───────────┼─────────────────┤ │ │ │ │ │ │ │ Auth │ Project │ User │ Git │ │ @@ -41,7 +45,9 @@ src/ │ ├── git_repositories.py # Repository endpoints │ ├── users.py # User endpoints │ ├── tool_types.py # Tool type endpoints +│ ├── tool_instances.py # Tool instance endpoints │ ├── ssh_keys.py # SSH key endpoints +│ ├── events.py # SSE streaming endpoint │ └── dashboard.py # Dashboard endpoints ├── auth/ # Authentication │ ├── session.py # Session management @@ -54,7 +60,15 @@ src/ │ ├── git_repository.py # Repository model │ ├── tool_type.py # Tool type model │ ├── ssh_key.py # SSH key model +│ ├── instance_event.py # Instance event audit model +│ ├── health_check.py # Health check snapshot model │ └── user_config.py # User config model +├── services/ # Services +│ ├── docker.py # Docker operations +│ ├── terminal_manager.py # Terminal session manager +│ ├── event_bus.py # Instance event bus (pub/sub) +│ ├── health_monitor.py # Background health monitoring +│ └── lifecycle_hooks.py # Instance lifecycle events ├── utils/ # Utilities │ ├── git_url_parser.py # URL parsing │ ├── git_files.py # Git file operations @@ -193,6 +207,33 @@ Errors are handled at multiple levels: - **Integration tests**: PostgreSQL with transaction rollback - **Fixtures**: Shared in `conftest.py` +## Monitoring & Notifications + +The backend includes a real-time monitoring system: + +### Components + +- **InstanceEventBus** (`services/event_bus.py`): Typed pub/sub singleton for instance lifecycle events +- **HealthMonitor** (`services/health_monitor.py`): Asyncio background task polling container health every 15s +- **SSE Endpoint** (`api/events.py`): Server-Sent Events streaming for real-time frontend updates +- **Lifecycle Hooks** (`services/lifecycle_hooks.py`): Publishes events on create/start/stop/restart/delete + +### Event Flow + +``` +Container Action → Lifecycle Hook → EventBus → SSE Stream → Frontend Toast +``` + +### Event Types + +| Event | When Fired | +|-------|-----------| +| `instance.created` | After DB insert | +| `instance.starting` | Before docker compose up | +| `instance.running` | After readiness probe succeeds | +| `instance.error` | Build fail, crash, or probe fail | +| `instance.stopped` | After docker compose stop | + ## Technology Stack | Component | Technology | Version | diff --git a/docs/architecture/frontend.md b/docs/architecture/frontend.md index fab9dd9..bb0882c 100644 --- a/docs/architecture/frontend.md +++ b/docs/architecture/frontend.md @@ -26,16 +26,21 @@ apps/web/src/ │ ├── ssh_keys.ts # SSH key API │ ├── tool_types.ts # Tool type API │ ├── users.ts # User API +│ ├── events.ts # SSE events API │ └── settings.ts # Settings API ├── components/ # Reusable components │ ├── app-shell.tsx # Main app layout │ ├── protected-route.tsx # Auth guard +│ ├── event-toast-bridge.tsx # Events → toasts │ └── [more...] -├── context/ # React contexts -│ └── auth.tsx # Auth state management +├── state/ # Global state +│ ├── auth.tsx # Auth state management +│ ├── events.tsx # Event provider (SSE) +│ └── toast.tsx # Toast notifications ├── hooks/ # Custom hooks │ ├── use-auth.ts # Auth hook -│ └── use-theme.ts # Theme hook +│ ├── use-theme.ts # Theme hook +│ └── use-events.ts # SSE events hook ├── pages/ # Page components (routes) │ ├── dashboard.tsx # Dashboard │ ├── projects.tsx # Project list @@ -155,6 +160,28 @@ interface AuthState { } ``` +### Real-Time Events (SSE) + +The frontend receives real-time instance events via Server-Sent Events: + +``` +EventSource → useEvents() hook → EventProvider → EventToastBridge → ToastContainer +``` + +**Components:** +- `useEvents()`: Manages SSE connection with auto-reconnect +- `EventProvider`: Shares event stream across components +- `EventToastBridge`: Maps events to toast notifications +- `ToastContainer`: Displays and manages toast stack + +**Event-to-Toast Mapping:** +| Event | Toast Severity | Auto-dismiss | +|-------|---------------|--------------| +| `instance.starting` | Info | 3s | +| `instance.running` | Success | 3s | +| `instance.error` | Error | Persistent | +| `instance.stopped` | Info | 3s | + ### 5. Routing Structure ```typescript @@ -271,10 +298,10 @@ test('renders file list', () => { ## Future Improvements +- [x] Implement real-time updates (SSE) - [ ] Add React Query for server state management - [ ] Implement virtual scrolling for large file trees - [ ] Add service worker for offline support -- [ ] Implement real-time updates (WebSocket) - [ ] Add error boundary components ## Development Workflow diff --git a/docs/features/terminal.md b/docs/features/terminal.md index 9f2b6c5..797d256 100644 --- a/docs/features/terminal.md +++ b/docs/features/terminal.md @@ -50,6 +50,23 @@ Standard terminal shortcuts work as expected: Special keys can be accessed via the special keys panel on mobile or by using modifier combinations. +## Container Monitoring & Notifications + +The platform monitors your tool instances in real-time and notifies you of important events: + +### What You'll See + +- **Starting:** When a container begins starting +- **Running:** When a container is ready +- **Error:** When a build fails, container crashes, or tunnel fails +- **Stopped:** When a container stops + +Notifications appear as toast messages at the top of the screen. Errors persist until dismissed; other notifications auto-dismiss after a few seconds. + +### Real-Time Status + +Instance status badges update in real-time via Server-Sent Events (SSE) — no page refresh needed. + ## Troubleshooting ### Connection Issues @@ -59,6 +76,10 @@ Special keys can be accessed via the special keys panel on mobile or by using mo - Network issues - the client will auto-reconnect - Session timeout - sessions expire after 30 minutes of inactivity +**"Container not found" error (4004):** +- The Docker container no longer exists (e.g., after host restart) +- Restart the tool instance to recreate the container + **Terminal not responding:** - Try resetting the terminal using the Reset button - Check if the tool instance is still running diff --git a/openspec/changes/container-monitoring-notifications/apply-pr3.md b/openspec/changes/container-monitoring-notifications/apply-pr3.md new file mode 100644 index 0000000..eab6295 --- /dev/null +++ b/openspec/changes/container-monitoring-notifications/apply-pr3.md @@ -0,0 +1,61 @@ +# PR-3 Apply Report: Integration + Polish for Container Monitoring & Notifications + +## Status +**COMPLETE** + +## Changed Files + +### New Files +| File | Purpose | +|------|---------| +| `apps/api/tests/integration/test_events.py` | Integration tests for SSE auth, connection limits, lifecycle hooks, event persistence | + +### Modified Files +| File | Change | +|------|--------| +| `apps/api/src/api/tool_instances.py` | Added `GET /{project_id}/repositories/{repo_id}/instances/{instance_id}/events` endpoint | +| `docs/features/terminal.md` | Added Container Monitoring & Notifications section | +| `docs/architecture/backend.md` | Added monitoring components to architecture diagram and docs | +| `docs/architecture/frontend.md` | Added SSE/events section to frontend architecture | + +## Implementation Summary + +### MON-PR3-001: Integration Tests +- 6 integration tests covering: + - SSE requires authentication (401) + - SSE enforces connection limit (429) + - SSE endpoint is registered + - Lifecycle hook publishes event and persists audit row + - Direct lifecycle event persists to DB + - EventBus pub/sub delivers events + +### MON-PR3-002: Instance Events History API +- `GET /{project_id}/repositories/{repo_id}/instances/{instance_id}/events` +- Returns up to 50 most recent events (configurable via `limit` param) +- Includes event_type, status, message, metadata, created_at + +### MON-PR3-003: Frontend Health History View +- Skipped — deferred to future enhancement + +### MON-PR3-004: Performance Tuning +- Already implemented in PR-1: SSE max 5 connections per user, health monitor only writes on state change + +### MON-PR3-005: Documentation Updates +- `docs/features/terminal.md`: Added monitoring section with error troubleshooting +- `docs/architecture/backend.md`: Added monitoring components, event flow, event types table +- `docs/architecture/frontend.md`: Added SSE architecture, event-to-toast mapping + +### MON-PR3-006: Final Regression Validation +- Backend: 21 monitoring tests passed, 172 unit tests passed (4 pre-existing failures unrelated) +- Frontend: 14 tests passed, tsc clean, eslint clean +- Ruff: All clean + +## Quality Gates +| Check | Result | +|-------|--------| +| pytest unit (monitoring) | 21 passed | +| pytest unit (full) | 172 passed, 4 pre-existing failures | +| vitest frontend | 14 passed | +| tsc --noEmit | Clean | +| eslint | Clean | +| ruff | Clean |