refactor: organize API routers and services into subpackages
Service organization (19 files moved into 6 subpackages): - services/instance/ — event_bus, health_monitor, lifecycle_hooks - services/config/ — config_profile_resolver - services/git/ — clone, git_operations, git_service - services/build/ — docker_build, manifest_compiler - services/terminal/ — terminal_manager, terminal_session - services/shared/ — correlation, file_service, notification_service, permission_fixer, readiness_probe, ssh_keys, tunnel, workspace_manager API router organization (16 files moved into 6 subpackages): - api/tool/ — tool_instances, tool_types, tool_definitions, tool_types_validation, sessions (extracted from tool_instances) - api/config/ — config_profiles, user_config - api/workspace/ — workspaces, workspace_files, workspace_git, workspace_instances - api/user/ — users, auth, ssh_keys - api/project/ — projects, git_repositories - api/system/ — health, events, notifications, dashboard, terminal, instance_proxy Updated main.py imports and all __init__.py re-exports. Sessions router extracted from tool_instances.py into api/tool/sessions.py. Quality gates: py_compile passed, ruff passed.
This commit is contained in:
@@ -1 +1,7 @@
|
||||
"""Instance module."""
|
||||
"""Instance lifecycle services module."""
|
||||
|
||||
from src.services.instance.event_bus import InstanceEventBus
|
||||
from src.services.instance.health_monitor import HealthMonitor
|
||||
from src.services.instance.lifecycle_hooks import publish_lifecycle_event
|
||||
|
||||
__all__ = ["InstanceEventBus", "HealthMonitor", "publish_lifecycle_event"]
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""In-memory typed event bus for instance lifecycle and health events."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
InstanceEventPayload = dict[str, Any]
|
||||
EventCallback = Callable[[InstanceEventPayload], Awaitable[None] | None] # noqa: UP044
|
||||
|
||||
|
||||
class InstanceEventBus:
|
||||
"""Singleton in-memory event bus with typed pub/sub and exception isolation."""
|
||||
|
||||
_instance: "InstanceEventBus | None" = None
|
||||
_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._subscribers: dict[str, list[tuple[str, EventCallback]]] = {}
|
||||
|
||||
def __new__(cls) -> "InstanceEventBus":
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._subscribers = {}
|
||||
return cls._instance
|
||||
|
||||
def _reset_for_testing(self) -> None:
|
||||
"""Clear all subscribers. For test use only."""
|
||||
self._subscribers.clear()
|
||||
|
||||
def subscribe(
|
||||
self,
|
||||
event_type: str,
|
||||
callback: EventCallback,
|
||||
) -> Callable[[], None]:
|
||||
"""Register a callback for an event type.
|
||||
|
||||
Args:
|
||||
event_type: The event type to subscribe to.
|
||||
callback: A sync or async callable that receives the payload.
|
||||
|
||||
Returns:
|
||||
An unsubscribe function.
|
||||
"""
|
||||
if event_type not in self._subscribers:
|
||||
self._subscribers[event_type] = []
|
||||
callback_id = str(uuid.uuid4())
|
||||
self._subscribers[event_type].append((callback_id, callback))
|
||||
|
||||
def unsubscribe() -> None:
|
||||
self.unsubscribe(event_type, callback_id)
|
||||
|
||||
return unsubscribe
|
||||
|
||||
def unsubscribe(self, event_type: str, callback_id: str) -> None:
|
||||
"""Remove a specific callback by ID."""
|
||||
if event_type in self._subscribers:
|
||||
self._subscribers[event_type] = [
|
||||
(cid, cb)
|
||||
for cid, cb in self._subscribers[event_type]
|
||||
if cid != callback_id
|
||||
]
|
||||
if not self._subscribers[event_type]:
|
||||
del self._subscribers[event_type]
|
||||
|
||||
def unsubscribe_all(self, event_type: str) -> None:
|
||||
"""Remove all subscribers for an event type."""
|
||||
self._subscribers.pop(event_type, None)
|
||||
|
||||
async def publish(self, event_type: str, payload: InstanceEventPayload) -> None:
|
||||
"""Deliver payload to all subscribers of event_type.
|
||||
|
||||
Also delivers to subscribers registered under the wildcard "*".
|
||||
Exceptions from individual subscribers are caught and logged;
|
||||
delivery continues to remaining subscribers.
|
||||
"""
|
||||
callbacks: list[tuple[str, EventCallback]] = []
|
||||
callbacks.extend(self._subscribers.get(event_type, []))
|
||||
callbacks.extend(self._subscribers.get("*", []))
|
||||
|
||||
for _callback_id, callback in callbacks:
|
||||
try:
|
||||
if inspect.iscoroutinefunction(callback):
|
||||
await callback(payload)
|
||||
else:
|
||||
callback(payload)
|
||||
except Exception:
|
||||
correlation_id = payload.get("correlation_id", "unknown")
|
||||
logger.exception(
|
||||
"Event subscriber failed for %s",
|
||||
event_type,
|
||||
extra={"correlation_id": correlation_id},
|
||||
)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Background health monitor that polls container and tunnel health."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.database import SessionLocal
|
||||
from src.models import HealthCheck
|
||||
from src.models import ToolInstance
|
||||
from src.services.shared.correlation import get_correlation_id
|
||||
from src.services.docker import get_container_status
|
||||
from src.services.shared.tunnel import check_tunnel_health
|
||||
from src.services.instance.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
from src.services.shared.notification_service import notification_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthSnapshot:
|
||||
"""In-memory snapshot of an instance's health state."""
|
||||
|
||||
container_status: str | None = None
|
||||
container_healthy: bool | None = None
|
||||
tunnel_healthy: bool | None = None
|
||||
exit_code: int | None = None
|
||||
|
||||
|
||||
class HealthMonitor:
|
||||
"""Polls container and tunnel health, publishing events on state changes."""
|
||||
|
||||
POLL_INTERVAL_SECONDS: float = 15.0
|
||||
_MONITORED_STATUSES: set[str] = {"starting", "running", "unhealthy"}
|
||||
|
||||
def __init__(self, event_bus: InstanceEventBus) -> None:
|
||||
self._event_bus = event_bus
|
||||
self._task: asyncio.Task | None = None
|
||||
self._last_known_state: dict[uuid.UUID, HealthSnapshot] = {}
|
||||
|
||||
def start(self) -> None:
|
||||
"""Idempotent start of the background polling task."""
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
self._task = loop.create_task(self._poll_loop())
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Cancel the background task and clear state."""
|
||||
if self._task is not None and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._last_known_state.clear()
|
||||
self._task = None
|
||||
|
||||
async def _poll_loop(self) -> None:
|
||||
"""Main polling loop."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.POLL_INTERVAL_SECONDS)
|
||||
await self._run_check_cycle()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("Health monitor poll loop error")
|
||||
|
||||
async def _run_check_cycle(self) -> None:
|
||||
"""Check all monitored instances in one cycle."""
|
||||
async with SessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(ToolInstance).where(
|
||||
ToolInstance.status.in_(self._MONITORED_STATUSES)
|
||||
)
|
||||
)
|
||||
instances = result.scalars().all()
|
||||
|
||||
for instance in instances:
|
||||
async with SessionLocal() as session:
|
||||
await self._check_instance(session, instance)
|
||||
|
||||
async def _check_instance(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
) -> None:
|
||||
"""Check a single instance and handle state transitions."""
|
||||
try:
|
||||
container_info = get_container_status(instance.container_id or "")
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Health check failed for instance %s",
|
||||
instance.id,
|
||||
extra={
|
||||
"instance_id": str(instance.id),
|
||||
"correlation_id": get_correlation_id(),
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
container_status = container_info["status"]
|
||||
exit_code = container_info["exit_code"]
|
||||
container_healthy = (
|
||||
container_info["health"] == "healthy" if container_info["health"] else None
|
||||
)
|
||||
|
||||
tunnel_healthy: bool | None = None
|
||||
if instance.public_url and container_status == "running":
|
||||
try:
|
||||
tunnel_result = check_tunnel_health(instance.public_url)
|
||||
tunnel_healthy = tunnel_result.get("healthy", False)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Tunnel health check failed for instance %s",
|
||||
instance.id,
|
||||
extra={
|
||||
"instance_id": str(instance.id),
|
||||
"correlation_id": get_correlation_id(),
|
||||
},
|
||||
)
|
||||
tunnel_healthy = False
|
||||
|
||||
snapshot = HealthSnapshot(
|
||||
container_status=container_status,
|
||||
container_healthy=container_healthy,
|
||||
tunnel_healthy=tunnel_healthy,
|
||||
exit_code=exit_code,
|
||||
)
|
||||
|
||||
previous = self._last_known_state.get(instance.id)
|
||||
|
||||
# Determine new status
|
||||
new_status = self._derive_status(snapshot)
|
||||
|
||||
# If first check or state changed
|
||||
if previous is None or not self._snapshots_equal(previous, snapshot):
|
||||
await self._handle_state_change(
|
||||
session, instance, previous, snapshot, new_status
|
||||
)
|
||||
self._last_known_state[instance.id] = snapshot
|
||||
|
||||
def _derive_status(self, snapshot: HealthSnapshot) -> str:
|
||||
"""Derive instance status from health snapshot."""
|
||||
if snapshot.container_status != "running":
|
||||
return "error"
|
||||
if snapshot.tunnel_healthy is False:
|
||||
return "unhealthy"
|
||||
return "running"
|
||||
|
||||
def _snapshots_equal(self, a: HealthSnapshot, b: HealthSnapshot) -> bool:
|
||||
"""Compare two snapshots for equality."""
|
||||
return (
|
||||
a.container_status == b.container_status
|
||||
and a.container_healthy == b.container_healthy
|
||||
and a.tunnel_healthy == b.tunnel_healthy
|
||||
and a.exit_code == b.exit_code
|
||||
)
|
||||
|
||||
async def _handle_state_change(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
previous: HealthSnapshot | None,
|
||||
snapshot: HealthSnapshot,
|
||||
new_status: str,
|
||||
) -> None:
|
||||
"""Update DB, insert health check, and publish event."""
|
||||
previous_status = instance.status
|
||||
|
||||
# Update instance status
|
||||
instance.status = new_status
|
||||
if new_status == "error":
|
||||
instance.last_stopped_at = datetime.now(timezone.utc)
|
||||
|
||||
# Insert health check row
|
||||
health_check = HealthCheck(
|
||||
instance_id=instance.id,
|
||||
container_status=snapshot.container_status,
|
||||
container_healthy=snapshot.container_healthy,
|
||||
tunnel_healthy=snapshot.tunnel_healthy,
|
||||
exit_code=snapshot.exit_code,
|
||||
probe_status=None,
|
||||
probe_output=None,
|
||||
)
|
||||
session.add(health_check)
|
||||
await session.commit()
|
||||
|
||||
# Build event payload
|
||||
correlation_id = get_correlation_id()
|
||||
metadata: dict = {"previous_status": previous_status}
|
||||
if snapshot.exit_code is not None:
|
||||
metadata["exit_code"] = snapshot.exit_code
|
||||
metadata["error_type"] = "container"
|
||||
if instance.public_url:
|
||||
metadata["tunnel_url"] = instance.public_url
|
||||
|
||||
if new_status == "error":
|
||||
event_type = "instance.error"
|
||||
message = f"Container failed with status {snapshot.container_status}"
|
||||
if snapshot.exit_code is not None:
|
||||
message += f" (exit code: {snapshot.exit_code})"
|
||||
else:
|
||||
event_type = "instance.health_changed"
|
||||
message = f"Container is now {new_status}"
|
||||
|
||||
payload: InstanceEventPayload = {
|
||||
"event": event_type,
|
||||
"instance_id": str(instance.id),
|
||||
"status": new_status,
|
||||
"message": message,
|
||||
"metadata": metadata,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
|
||||
await self._event_bus.publish(event_type, payload)
|
||||
|
||||
# Create notification for instance owner (fire-and-forget)
|
||||
# Only send warnings and errors; skip "recovered" info notifications.
|
||||
if new_status == "error":
|
||||
category = "instance"
|
||||
severity = "error"
|
||||
title = "Container failed"
|
||||
elif new_status == "unhealthy":
|
||||
category = "health"
|
||||
severity = "warning"
|
||||
title = "Container unhealthy"
|
||||
else:
|
||||
# Running/recovered — do not notify
|
||||
return
|
||||
|
||||
try:
|
||||
await notification_service.create_notification(
|
||||
session=session,
|
||||
user_id=instance.owner_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
title=title,
|
||||
message=message,
|
||||
source_type="tool_instances",
|
||||
source_id=instance.id,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to create notification for health event %s",
|
||||
event_type,
|
||||
extra={"correlation_id": correlation_id},
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Lifecycle hook helpers for instrumenting tool instance transitions."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.models import InstanceEvent
|
||||
from src.models import ToolInstance
|
||||
from src.services.shared.correlation import get_correlation_id
|
||||
from src.services.instance.event_bus import InstanceEventBus, InstanceEventPayload
|
||||
from src.services.shared.notification_service import notification_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _derive_title(event_type: str) -> str:
|
||||
"""Map lifecycle event type to a human-readable notification title."""
|
||||
mapping = {
|
||||
"instance.created": "Container created",
|
||||
"instance.started": "Container started",
|
||||
"instance.stopped": "Container stopped",
|
||||
"instance.restarted": "Container restarted",
|
||||
"instance.deleted": "Container deleted",
|
||||
"instance.error": "Container error",
|
||||
"instance.health_changed": "Container ready",
|
||||
}
|
||||
return mapping.get(
|
||||
event_type,
|
||||
event_type.replace("instance.", "").replace("_", " ").title(),
|
||||
)
|
||||
|
||||
|
||||
def _should_notify(event_type: str, status: str | None) -> bool:
|
||||
"""Determine whether a lifecycle event should generate a notification.
|
||||
|
||||
Only warnings, errors, and "container is ready" (health_changed running)
|
||||
are sent to users.
|
||||
"""
|
||||
if event_type == "instance.error":
|
||||
return True
|
||||
if event_type == "instance.health_changed" and status == "running":
|
||||
return True
|
||||
# Filter out: created, started, stopped, restarted, deleted, and any
|
||||
# health_changed that is not "running" (unhealthy is handled by health_monitor)
|
||||
return False
|
||||
|
||||
|
||||
def _build_payload(
|
||||
event_type: str,
|
||||
instance: ToolInstance,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> InstanceEventPayload:
|
||||
"""Construct a standard event payload."""
|
||||
return {
|
||||
"event": event_type,
|
||||
"instance_id": str(instance.id),
|
||||
"status": status or instance.status,
|
||||
"message": message,
|
||||
"metadata": metadata or {},
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"correlation_id": get_correlation_id(),
|
||||
}
|
||||
|
||||
|
||||
async def _write_audit_row(
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
event_type: str,
|
||||
created_by: uuid.UUID | None = None,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> InstanceEvent:
|
||||
"""Persist an instance_events audit row."""
|
||||
row = InstanceEvent(
|
||||
instance_id=instance.id,
|
||||
event_type=event_type.replace("instance.", ""),
|
||||
status=status or instance.status,
|
||||
message=message,
|
||||
created_by=created_by,
|
||||
event_metadata=metadata or {},
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return row
|
||||
|
||||
|
||||
async def publish_lifecycle_event(
|
||||
event_bus: InstanceEventBus,
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
event_type: str,
|
||||
created_by: uuid.UUID | None = None,
|
||||
status: str | None = None,
|
||||
message: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> None:
|
||||
"""Publish a lifecycle event and write an audit row after DB commit.
|
||||
|
||||
Args:
|
||||
event_bus: The global event bus.
|
||||
session: Active async DB session.
|
||||
instance: The affected tool instance.
|
||||
event_type: One of instance.created, instance.started, etc.
|
||||
created_by: User ID for user-initiated actions; None for system.
|
||||
status: Optional status override.
|
||||
message: Optional human-readable message.
|
||||
metadata: Optional extra metadata.
|
||||
"""
|
||||
payload = _build_payload(
|
||||
event_type=event_type,
|
||||
instance=instance,
|
||||
status=status,
|
||||
message=message,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Write audit row
|
||||
await _write_audit_row(
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type=event_type,
|
||||
created_by=created_by,
|
||||
status=status or instance.status,
|
||||
message=message,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Publish to bus
|
||||
await event_bus.publish(event_type, payload)
|
||||
|
||||
# Create notification for instance owner (fire-and-forget)
|
||||
# Only send warnings, errors, and "container is ready" notifications.
|
||||
effective_status = status or instance.status
|
||||
if not _should_notify(event_type, effective_status):
|
||||
return
|
||||
|
||||
severity = "error" if event_type == "instance.error" else "success"
|
||||
title = _derive_title(event_type)
|
||||
|
||||
try:
|
||||
await notification_service.create_notification(
|
||||
session=session,
|
||||
user_id=instance.owner_id,
|
||||
category="instance",
|
||||
severity=severity,
|
||||
title=title,
|
||||
message=message,
|
||||
source_type="tool_instances",
|
||||
source_id=instance.id,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to create notification for lifecycle event %s",
|
||||
event_type,
|
||||
extra={"correlation_id": payload.get("correlation_id", "unknown")},
|
||||
)
|
||||
Reference in New Issue
Block a user