fix: reduce false container-failed notifications, add error details to UI

Backend (health_monitor.py):
- Skip health checks for instances with no container_id
- Treat 'not_found' as error only when container was previously running
- Skip duplicate error notifications when already in error state
- Skip 'not_found' notifications for containers that never ran

Frontend (notification-item.tsx):
- Display notification.message (detailed error text)
- Add expandable Details section showing metadata (exit_code, previous_status, etc.)
- New CSS styles for message and metadata display

Quality gates: py_compile, tsc --noEmit, 80/80 tests pass
This commit is contained in:
Developer
2026-06-06 09:01:01 +00:00
parent 2169b24875
commit 1ef9d66eed
6 changed files with 697 additions and 475 deletions
@@ -90,8 +90,16 @@ class HealthMonitor:
instance: ToolInstance,
) -> None:
"""Check a single instance and handle state transitions."""
# Skip instances that have never been assigned a container.
if not instance.container_id:
logger.debug(
"Skipping health check for instance %s: no container_id",
instance.id,
)
return
try:
container_info = get_container_status(instance.container_id or "")
container_info = get_container_status(instance.container_id)
except Exception:
logger.exception(
"Health check failed for instance %s",
@@ -135,7 +143,11 @@ class HealthMonitor:
previous = self._last_known_state.get(instance.id)
# Determine new status
new_status = self._derive_status(snapshot)
new_status = self._derive_status(
snapshot,
previous,
instance.status,
)
# If first check or state changed
if previous is None or not self._snapshots_equal(previous, snapshot):
@@ -144,10 +156,34 @@ class HealthMonitor:
)
self._last_known_state[instance.id] = snapshot
def _derive_status(self, snapshot: HealthSnapshot) -> str:
"""Derive instance status from health snapshot."""
def _derive_status(
self,
snapshot: HealthSnapshot,
previous: HealthSnapshot | None,
current_status: str | None,
) -> str:
"""Derive instance status from health snapshot.
Treats missing containers as an error only when the container was
previously known to be running. This avoids false "container failed"
alerts for instances that are still starting or have no container yet.
"""
if snapshot.container_status == "not_found":
# If the container was never seen running, assume it's still
# starting or was deleted intentionally; don't flag as error.
if previous is None and current_status == "starting":
return "starting"
if previous is not None and previous.container_status == "running":
return "error"
# Fall back to current status to avoid spurious errors.
return current_status or "error"
if snapshot.container_status == "exited":
return "error"
if snapshot.container_status != "running":
return "error"
if snapshot.tunnel_healthy is False:
return "unhealthy"
return "running"
@@ -223,6 +259,16 @@ class HealthMonitor:
# Create notification for instance owner (fire-and-forget)
# Only send warnings and errors; skip "recovered" info notifications.
if new_status == "error":
# Skip duplicate error notifications if already in error state.
if previous_status == "error":
return
# Skip "not_found" errors for containers that were never running
# (e.g. still starting, or intentionally stopped/deleted).
if (
snapshot.container_status == "not_found"
and (previous is None or previous.container_status != "running")
):
return
category = "instance"
severity = "error"
title = "Container failed"