fix: reduce false container-failed notifications, add error details to UI
Backend (health_monitor.py): - Skip health checks for instances with no container_id - Treat 'not_found' as error only when container was previously running - Skip duplicate error notifications when already in error state - Skip 'not_found' notifications for containers that never ran Frontend (notification-item.tsx): - Display notification.message (detailed error text) - Add expandable Details section showing metadata (exit_code, previous_status, etc.) - New CSS styles for message and metadata display Quality gates: py_compile, tsc --noEmit, 80/80 tests pass
This commit is contained in:
@@ -90,8 +90,16 @@ class HealthMonitor:
|
||||
instance: ToolInstance,
|
||||
) -> None:
|
||||
"""Check a single instance and handle state transitions."""
|
||||
# Skip instances that have never been assigned a container.
|
||||
if not instance.container_id:
|
||||
logger.debug(
|
||||
"Skipping health check for instance %s: no container_id",
|
||||
instance.id,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
container_info = get_container_status(instance.container_id or "")
|
||||
container_info = get_container_status(instance.container_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Health check failed for instance %s",
|
||||
@@ -135,7 +143,11 @@ class HealthMonitor:
|
||||
previous = self._last_known_state.get(instance.id)
|
||||
|
||||
# Determine new status
|
||||
new_status = self._derive_status(snapshot)
|
||||
new_status = self._derive_status(
|
||||
snapshot,
|
||||
previous,
|
||||
instance.status,
|
||||
)
|
||||
|
||||
# If first check or state changed
|
||||
if previous is None or not self._snapshots_equal(previous, snapshot):
|
||||
@@ -144,10 +156,34 @@ class HealthMonitor:
|
||||
)
|
||||
self._last_known_state[instance.id] = snapshot
|
||||
|
||||
def _derive_status(self, snapshot: HealthSnapshot) -> str:
|
||||
"""Derive instance status from health snapshot."""
|
||||
def _derive_status(
|
||||
self,
|
||||
snapshot: HealthSnapshot,
|
||||
previous: HealthSnapshot | None,
|
||||
current_status: str | None,
|
||||
) -> str:
|
||||
"""Derive instance status from health snapshot.
|
||||
|
||||
Treats missing containers as an error only when the container was
|
||||
previously known to be running. This avoids false "container failed"
|
||||
alerts for instances that are still starting or have no container yet.
|
||||
"""
|
||||
if snapshot.container_status == "not_found":
|
||||
# If the container was never seen running, assume it's still
|
||||
# starting or was deleted intentionally; don't flag as error.
|
||||
if previous is None and current_status == "starting":
|
||||
return "starting"
|
||||
if previous is not None and previous.container_status == "running":
|
||||
return "error"
|
||||
# Fall back to current status to avoid spurious errors.
|
||||
return current_status or "error"
|
||||
|
||||
if snapshot.container_status == "exited":
|
||||
return "error"
|
||||
|
||||
if snapshot.container_status != "running":
|
||||
return "error"
|
||||
|
||||
if snapshot.tunnel_healthy is False:
|
||||
return "unhealthy"
|
||||
return "running"
|
||||
@@ -223,6 +259,16 @@ class HealthMonitor:
|
||||
# Create notification for instance owner (fire-and-forget)
|
||||
# Only send warnings and errors; skip "recovered" info notifications.
|
||||
if new_status == "error":
|
||||
# Skip duplicate error notifications if already in error state.
|
||||
if previous_status == "error":
|
||||
return
|
||||
# Skip "not_found" errors for containers that were never running
|
||||
# (e.g. still starting, or intentionally stopped/deleted).
|
||||
if (
|
||||
snapshot.container_status == "not_found"
|
||||
and (previous is None or previous.container_status != "running")
|
||||
):
|
||||
return
|
||||
category = "instance"
|
||||
severity = "error"
|
||||
title = "Container failed"
|
||||
|
||||
Reference in New Issue
Block a user