feat(frontend): update sessions page for enhanced health monitoring

- Add new status badges: starting, probing, unhealthy
- Show tunnel error only when tunnel_status is unreachable
- Show app error badge with status code for error_response
- Add collapsible probe output section for diagnostics
- Update health polling to check all active instances
- Only show Recreate Tunnel button for unreachable tunnels

apps/api/src/api/tool_instances.py

@@ -30,11 +30,14 @@ from src.services.docker import (
     execute_compose_command,
     find_free_port,
     get_container_id,
+    get_container_logs,
     get_container_name,
+    get_container_status,
     recreate_tunnel,
     render_compose_template,
     start_cloudflared_tunnel,
     stop_cloudflared_tunnel,
+    wait_for_container_running,
     write_compose_file,
     write_config_files,
     write_env_file,
@@ -552,20 +555,67 @@ async def start_instance(
         else:
             logger.warning("Failed to connect %s to backend network", container_name)
 
-    instance.status = "starting"
-    instance.last_started_at = datetime.now()
-    await session.commit()
-    logger.info("Instance %s container is running, checking readiness", instance.id)
-
+    # Verify container reached running state
+    if instance.container_id:
+        instance.status = "starting"
+        instance.last_started_at = datetime.now()
+        await session.commit()
+        logger.info("Instance %s: verifying container startup...", instance.id)
+        
+        startup_result = wait_for_container_running(instance.container_id, timeout=30, interval=2.0)
+        
+        if not startup_result["success"]:
+            # Container failed to start
+            error_msg = f"Container failed to start: status={startup_result['status']}"
+            if startup_result["exit_code"] is not None:
+                error_msg += f", exit_code={startup_result['exit_code']}"
+            
+            # Get logs for debugging
+            logs = get_container_logs(instance.container_id, tail=50)
+            
+            instance.status = "error"
+            await session.commit()
+            logger.error(
+                "Instance %s container startup failed after %.1fs: %s\nLogs:\n%s",
+                instance.id,
+                startup_result["waited_seconds"],
+                error_msg,
+                logs,
+            )
+            return {
+                "status": "error",
+                "error": error_msg,
+                "logs": logs,
+            }
+        
+        logger.info(
+            "Instance %s container started successfully after %.1fs",
+            instance.id,
+            startup_result["waited_seconds"],
+        )
+    
     # Execute readiness probe if configured
     tool_type = await session.get(ToolType, instance.tool_type_id)
-    if tool_type and tool_type.readiness_probe:
-        probe_config = tool_type.readiness_probe
-        probe_command = probe_config.get("command", "")
-        probe_timeout = probe_config.get("timeout", 30)
-        probe_interval = probe_config.get("interval", 2)
+    if tool_type and instance.container_id:
+        # Determine probe command
+        probe_command = None
+        probe_timeout = 30
+        probe_interval = 2
         
-        if probe_command and instance.container_id:
+        if tool_type.readiness_probe:
+            probe_config = tool_type.readiness_probe
+            probe_command = probe_config.get("command", "")
+            probe_timeout = probe_config.get("timeout", 30)
+            probe_interval = probe_config.get("interval", 2)
+        elif "web" in (tool_type.interfaces or []):
+            # Default probe for web tools
+            probe_command = f"curl -f http://localhost:{tool_type.default_port or 8080}"
+            probe_timeout = 30
+            probe_interval = 2
+        
+        if probe_command:
+            instance.status = "probing"
+            await session.commit()
             logger.info(
                 "Executing readiness probe for instance %s: command='%s', timeout=%d, interval=%d",
                 instance.id, probe_command, probe_timeout, probe_interval
@@ -578,14 +628,25 @@ async def start_instance(
                 interval=probe_interval,
             )
             
+            # Store probe result
+            instance.probe_result = {
+                "success": success,
+                "command": probe_command,
+                "logs": probe_logs,
+                "timestamp": datetime.now().isoformat(),
+            }
+            
             if not success:
-                instance.status = "failed"
-                instance.url = None
-                instance.public_url = None
+                instance.status = "unhealthy"
                 await session.commit()
-                logger.error("Readiness probe failed for instance %s: %s", instance.id, "\n".join(probe_logs))
+                logger.error(
+                    "Readiness probe failed for instance %s after %ds: %s",
+                    instance.id,
+                    probe_timeout,
+                    "\n".join(probe_logs),
+                )
                 return {
-                    "status": "failed",
+                    "status": "unhealthy",
                     "error": f"Readiness probe failed after {probe_timeout}s",
                     "probe_logs": probe_logs,
                 }
@@ -956,6 +1017,17 @@ async def recreate_tunnel_endpoint(
             detail="instance must be running to recreate tunnel",
         )
 
+    # Validate tunnel is actually broken before recreating
+    if instance.url:
+        tunnel_health = check_tunnel_health(instance.url)
+        if tunnel_health["tunnel_status"] == "error_response":
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Tunnel is working but application returned HTTP {tunnel_health.get('status_code')}. Recreating the tunnel will not fix this issue.",
+            )
+        elif tunnel_health["tunnel_status"] == "healthy":
+            return {"status": "healthy", "url": instance.url, "message": "Tunnel is already healthy"}
+
     # Get tool type for default port
     tool_type = await session.get(ToolType, instance.tool_type_id)
     instance_port = tool_type.default_port if tool_type and tool_type.default_port else 8080
@@ -987,8 +1059,8 @@ async def recreate_tunnel_endpoint(
 
 @router.get(
     "/{project_id}/repositories/{repo_id}/instances/{instance_id}/health",
-    summary="Check tunnel health",
-    description="Check if the temporary Cloudflare tunnel for an instance is healthy.",
+    summary="Check instance health",
+    description="Check container and tunnel health for an instance.",
 )
 async def check_instance_tunnel_health(
     project_id: uuid.UUID,
@@ -997,7 +1069,7 @@ async def check_instance_tunnel_health(
     user_id: uuid.UUID = Depends(get_current_user_id),
     session: AsyncSession = Depends(get_db_session),
 ) -> dict:
-    """Check tunnel health for an instance.
+    """Check health for an instance (container + tunnel).
 
     Args:
         project_id: UUID of the project.
@@ -1007,7 +1079,7 @@ async def check_instance_tunnel_health(
         session: Database session.
 
     Returns:
-        Dictionary with health status.
+        Dictionary with container_status, tunnel_status, probe_status, and overall healthy flag.
     """
     _user = await _get_user(session, user_id)
     _project = await _get_owned_project(project_id, user_id, session)
@@ -1018,11 +1090,50 @@ async def check_instance_tunnel_health(
             status_code=status.HTTP_404_NOT_FOUND, detail="instance not found"
         )
 
-    if not instance.url or instance.status != "running":
-        return {"healthy": False, "status_code": None, "error": "instance not running"}
+    # Check container status
+    container_info = {"status": "not_found", "exit_code": None, "health": None}
+    if instance.container_id:
+        container_info = get_container_status(instance.container_id)
 
-    health = check_tunnel_health(instance.url)
-    return health
+    # Build response
+    response = {
+        "healthy": False,
+        "container_status": container_info["status"],
+        "container_health": container_info["health"],
+        "tunnel_status": "not_applicable",
+        "tunnel_status_code": None,
+        "probe_status": "not_applicable",
+        "last_probe_output": None,
+        "error": None,
+    }
+
+    # Determine probe status
+    if instance.status == "probing":
+        response["probe_status"] = "pending"
+    elif instance.probe_result:
+        response["probe_status"] = "success" if instance.probe_result.get("success") else "failed"
+        response["last_probe_output"] = "\n".join(instance.probe_result.get("logs", []))[:500]
+
+    # Check tunnel health if instance has a URL and is web-enabled
+    if instance.url and instance.status in ("running", "unhealthy"):
+        tunnel_health = check_tunnel_health(instance.url)
+        response["tunnel_status"] = tunnel_health["tunnel_status"]
+        response["tunnel_status_code"] = tunnel_health.get("status_code")
+        if tunnel_health.get("error"):
+            response["error"] = tunnel_health["error"]
+
+    # Overall healthy only if container is running AND tunnel is healthy
+    container_healthy = container_info["status"] == "running"
+    tunnel_healthy = response["tunnel_status"] == "healthy"
+    response["healthy"] = container_healthy and tunnel_healthy
+
+    # If container is not running, override error message
+    if not container_healthy:
+        response["error"] = f"Container is {container_info['status']}"
+        if container_info["exit_code"] is not None:
+            response["error"] += f" (exit code: {container_info['exit_code']})"
+
+    return response
 
 
 @router.get(

apps/api/src/models/tool_instance.py

@@ -2,7 +2,7 @@ import uuid
 from datetime import datetime
 from typing import TYPE_CHECKING
 
-from sqlalchemy import DateTime, ForeignKey, Integer, String
+from sqlalchemy import DateTime, ForeignKey, Integer, JSON, String
 from sqlalchemy import Uuid as UUID
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
@@ -62,6 +62,9 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base):
     last_stopped_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )
+    probe_result: Mapped[dict | None] = mapped_column(
+        JSON, nullable=True
+    )
 
     tool_type: Mapped["ToolType"] = relationship()
     repository: Mapped["GitRepository"] = relationship()

apps/api/src/services/docker.py

@@ -244,24 +244,94 @@ def connect_container_to_network(container_name: str, network_name: str = "backe
     return result.returncode == 0
 
 
-def get_container_status(container_id: str) -> str:
+def get_container_status(container_id: str) -> dict[str, Any]:
     """Get the status of a Docker container.
 
     Args:
         container_id: Docker container ID
 
     Returns:
-        Container status string (running, exited, etc.)
+        Dict with 'status' (running, exited, restarting, not_found), 
+        'exit_code' (int or None), and 'health' (health status or None)
     """
     result = subprocess.run(
-        ["docker", "inspect", "-f", "{{.State.Status}}", container_id],
+        [
+            "docker", "inspect", "-f",
+            "{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}",
+            container_id,
+        ],
         capture_output=True,
         text=True,
     )
 
-    if result.returncode == 0:
-        return result.stdout.strip()
-    return "unknown"
+    if result.returncode != 0:
+        return {"status": "not_found", "exit_code": None, "health": None}
+    
+    parts = result.stdout.strip().split("|")
+    status = parts[0] if parts else "unknown"
+    exit_code = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
+    health = parts[2] if len(parts) > 2 and parts[2] != "none" else None
+    
+    return {"status": status, "exit_code": exit_code, "health": health}
+
+
+def wait_for_container_running(
+    container_id: str, timeout: int = 30, interval: float = 2.0
+) -> dict[str, Any]:
+    """Wait for a container to reach the running state.
+
+    Polls docker inspect until the container status is "running" or timeout.
+
+    Args:
+        container_id: Docker container ID
+        timeout: Maximum seconds to wait
+        interval: Seconds between polls
+
+    Returns:
+        Dict with 'success' (bool), 'status' (str), 'exit_code' (int or None),
+        and 'waited_seconds' (float)
+    """
+    import time
+
+    start_time = time.time()
+    
+    while time.time() - start_time < timeout:
+        info = get_container_status(container_id)
+        
+        if info["status"] == "running":
+            return {
+                "success": True,
+                "status": "running",
+                "exit_code": None,
+                "waited_seconds": time.time() - start_time,
+            }
+        
+        if info["status"] == "exited":
+            return {
+                "success": False,
+                "status": "exited",
+                "exit_code": info["exit_code"],
+                "waited_seconds": time.time() - start_time,
+            }
+        
+        if info["status"] == "not_found":
+            return {
+                "success": False,
+                "status": "not_found",
+                "exit_code": None,
+                "waited_seconds": time.time() - start_time,
+            }
+        
+        time.sleep(interval)
+    
+    # Timeout reached
+    info = get_container_status(container_id)
+    return {
+        "success": False,
+        "status": info["status"],
+        "exit_code": info["exit_code"],
+        "waited_seconds": time.time() - start_time,
+    }
 
 
 def get_container_logs(container_id: str, tail: int = 100) -> str:
@@ -424,14 +494,15 @@ def recreate_tunnel(
 
 
 def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
-    """Check if a tunnel URL is healthy.
+    """Check if a tunnel URL is healthy with smart error classification.
 
     Args:
         url: The tunnel URL to check
         timeout: Request timeout in seconds
 
     Returns:
-        Dict with 'healthy' (bool) and 'status_code' (int or None)
+        Dict with 'tunnel_status' (healthy, unreachable, error_response, not_applicable),
+        'status_code' (int or None), 'healthy' (bool), and 'error' (str or None)
     """
     import subprocess
 
@@ -444,13 +515,49 @@ def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
             timeout=timeout + 5,
         )
         status_code = int(result.stdout.strip())
+        
+        if 200 <= status_code < 400:
+            return {
+                "tunnel_status": "healthy",
+                "status_code": status_code,
+                "healthy": True,
+                "error": None,
+            }
+        elif status_code in (502, 503, 504):
+            # Application error, not tunnel error
+            return {
+                "tunnel_status": "error_response",
+                "status_code": status_code,
+                "healthy": False,
+                "error": f"Application returned HTTP {status_code}",
+            }
+        else:
+            return {
+                "tunnel_status": "error_response",
+                "status_code": status_code,
+                "healthy": False,
+                "error": f"HTTP {status_code}",
+            }
+    except subprocess.TimeoutExpired:
         return {
-            "healthy": 200 <= status_code < 400,
-            "status_code": status_code,
-        }
-    except (ValueError, subprocess.TimeoutExpired, Exception) as e:
-        return {
-            "healthy": False,
+            "tunnel_status": "unreachable",
             "status_code": None,
+            "healthy": False,
+            "error": "Tunnel request timed out",
+        }
+    except (ValueError, Exception) as e:
+        error_str = str(e).lower()
+        # Classify connection errors
+        if any(err in error_str for err in ["connection refused", "econnrefused", "could not resolve", "nodename"]):
+            return {
+                "tunnel_status": "unreachable",
+                "status_code": None,
+                "healthy": False,
+                "error": f"Tunnel unreachable: {e}",
+            }
+        return {
+            "tunnel_status": "unreachable",
+            "status_code": None,
+            "healthy": False,
             "error": str(e),
         }

apps/web/src/api/sessions.ts

@@ -25,6 +25,8 @@ export interface Session {
   project_id: string;
   status: string;
   url: string | null;
+  container_status?: string;
+  probe_status?: string;
 }
 
 export async function listInstances(
@@ -101,11 +103,23 @@ export async function getUserSessions(): Promise<Session[]> {
   return response.data.sessions;
 }
 
+export interface InstanceHealth {
+  healthy: boolean;
+  container_status: string;
+  container_health?: string;
+  container_exit_code?: number | null;
+  tunnel_status: string;
+  tunnel_status_code: number | null;
+  probe_status: string;
+  last_probe_output?: string;
+  error?: string;
+}
+
 export async function checkInstanceHealth(
   projectId: string,
   repoId: string,
   instanceId: string
-): Promise<{ healthy: boolean; status_code: number | null; error?: string }> {
+): Promise<InstanceHealth> {
   const response = await apiClient.get(
     `/projects/${projectId}/repositories/${repoId}/instances/${instanceId}/health`
   );

apps/web/src/pages/sessions.tsx

@@ -40,8 +40,18 @@ export const SessionsPage = () => {
 
   const [deleteConfirmId, setDeleteConfirmId] = useState<string | null>(null);
   const [stopConfirmId, setStopConfirmId] = useState<string | null>(null);
-  const [tunnelHealth, setTunnelHealth] = useState<Record<string, { healthy: boolean; status_code: number | null; error?: string }>>({});
+  const [tunnelHealth, setTunnelHealth] = useState<Record<string, {
+    healthy: boolean;
+    container_status: string;
+    container_health: string | null;
+    tunnel_status: string;
+    tunnel_status_code: number | null;
+    probe_status: string;
+    last_probe_output: string | null;
+    error: string | null;
+  }>>({});
   const [recreatingId, setRecreatingId] = useState<string | null>(null);
+  const [expandedProbeId, setExpandedProbeId] = useState<string | null>(null);
 
   const loadSessions = useCallback(async () => {
     setStatus("loading");
@@ -86,13 +96,13 @@ export const SessionsPage = () => {
     void loadToolTypes();
   }, []);
 
-  // Poll tunnel health every 30 seconds for running instances
+  // Poll health every 30 seconds for active instances
   useEffect(() => {
     const checkHealth = async () => {
-      const runningSessions = sessions.filter(
-        (s) => s.status === "running" && s.url
+      const activeSessions = sessions.filter(
+        (s) => ["running", "starting", "unhealthy", "probing"].includes(s.status)
       );
-      for (const session of runningSessions) {
+      for (const session of activeSessions) {
         try {
           const health = await checkInstanceHealth(
             session.project_id,
@@ -106,7 +116,14 @@ export const SessionsPage = () => {
         } catch {
           setTunnelHealth((prev) => ({
             ...prev,
-            [session.id]: { healthy: false, status_code: null, error: "check failed" },
+            [session.id]: {
+              healthy: false,
+              container_status: "unknown",
+              tunnel_status: "unreachable",
+              tunnel_status_code: null,
+              probe_status: "unknown",
+              error: "check failed",
+            },
           }));
         }
       }
@@ -135,7 +152,7 @@ export const SessionsPage = () => {
   }, [selectedProject]);
 
   const activeSessions = useMemo(
-    () => sessions.filter((s) => ["running", "building", "pending"].includes(s.status)),
+    () => sessions.filter((s) => ["running", "building", "pending", "starting", "probing", "unhealthy"].includes(s.status)),
     [sessions]
   );
 
@@ -328,9 +345,35 @@ export const SessionsPage = () => {
                         </p>
                       )}
                       <span className={`status-badge ${session.status}`}>{session.status}</span>
-                      {tunnelHealth[session.id] && !tunnelHealth[session.id].healthy && (
+                      {session.status === "starting" && (
+                        <span className="status-badge starting">starting...</span>
+                      )}
+                      {session.status === "probing" && (
+                        <span className="status-badge probing">checking...</span>
+                      )}
+                      {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "unreachable" && (
                         <span className="status-badge error">tunnel error</span>
                       )}
+                      {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "error_response" && (
+                        <span className="status-badge warning">app error ({tunnelHealth[session.id].tunnel_status_code})</span>
+                      )}
+                      {tunnelHealth[session.id]?.last_probe_output && (
+                        <div className="probe-output-section">
+                          <button
+                            className="probe-toggle"
+                            onClick={() => setExpandedProbeId(expandedProbeId === session.id ? null : session.id)}
+                            type="button"
+                          >
+                            <Icon name="info" size="sm" />
+                            {expandedProbeId === session.id ? "Hide probe output" : "Show probe output"}
+                          </button>
+                          {expandedProbeId === session.id && (
+                            <pre className="probe-output">
+                              {tunnelHealth[session.id].last_probe_output}
+                            </pre>
+                          )}
+                        </div>
+                      )}
                     </div>
                     <div className="session-actions">
                       {session.url ? (
@@ -353,7 +396,7 @@ export const SessionsPage = () => {
                           Open
                         </button>
                       )}
-                      {tunnelHealth[session.id] && !tunnelHealth[session.id].healthy && (
+                      {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "unreachable" && (
                         <button
                           className="secondary-button small"
                           onClick={() => void handleRecreateTunnel(session)}

openspec/changes/instance-health-monitoring/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-05-22

openspec/changes/instance-health-monitoring/design.md

@@ -0,0 +1,79 @@
+## Context
+
+The current instance management has critical gaps in health monitoring that lead to poor user experience:
+
+1. **Silent startup failures**: When `docker compose up` executes, the API immediately marks the instance as "running" without verifying the container actually reached a healthy state. Containers that crash on startup or fail to bind to their port appear "running" in the UI but serve 502 errors.
+
+2. **Tunnel-only health checks**: The existing health check at `GET /instances/{id}/health` only performs an HTTP HEAD request to the tunnel URL. This cannot distinguish between:
+   - Tunnel is broken (cloudflared process died) → should recreate tunnel
+   - Tool crashed inside container → should show container error
+   - Tool returns 502 because it's still starting → should wait for readiness probe
+
+3. **Unused readiness probes**: The `readiness_probe.py` service was built during the tool-workshop change but is never called during instance startup. Tool types can configure readiness probes (e.g., `curl -f http://localhost:8080/health`) but these are ignored.
+
+4. **Blind auto-recovery**: The frontend shows a "Recreate Tunnel" button when the health check fails, but this recreates the tunnel even when the application itself is returning 502 errors, wasting time and confusing users.
+
+## Goals / Non-Goals
+
+**Goals:**
+- Verify containers actually start successfully before marking instances as "running"
+- Distinguish container health from tunnel health in monitoring
+- Integrate readiness probes into the instance startup flow
+- Only recreate tunnels when the tunnel itself is broken, not when the tool returns errors
+- Provide clear error messages when instances fail to start
+
+**Non-Goals:**
+- Persistent tunnels (keeping temporary cloudflared tunnels)
+- Automatic restart of crashed containers (Docker already does this with restart policies)
+- Health check WebSocket push (polling is sufficient)
+- Changing the Docker compose architecture
+
+## Decisions
+
+**1. Startup verification via Docker API**
+- After `docker compose up`, poll `docker ps` for 30 seconds to verify container state transitions to "running"
+- If container exits or stays in "restarting" loop, mark instance as "error" with exit code
+- Rationale: Direct Docker API check is more reliable than HTTP checks during startup when ports may not be bound yet
+
+**2. Readiness probe as gate to "running" status**
+- Instance status flow: `pending` → `starting` (container up) → `running` (probe passed)
+- If probe fails after timeout, status becomes `unhealthy` (not `error` - container is still up)
+- Rationale: Distinguishes "container won't start" from "container started but app isn't ready yet"
+
+**3. Container + Tunnel dual health checks**
+- Health endpoint returns both `container_status` (from Docker API) and `tunnel_status` (HTTP check)
+- Frontend shows different badges: "container unhealthy" vs "tunnel error"
+- Rationale: Users need to know if they should wait (app starting) or recreate tunnel
+
+**4. Smart tunnel failure detection**
+- Connection errors (ECONNREFUSED, ETIMEDOUT, DNS failure) → tunnel is broken → allow recreate
+- HTTP 502/503/504 → application error → show "app error" badge, don't recreate
+- HTTP 200-399 → healthy
+- Rationale: 502 from the tool means the tunnel is working fine, the tool just isn't responding
+
+**5. Readiness probe configuration from ToolType**
+- Use existing `readiness_probe` JSON field on ToolType model
+- Default probe for web tools: `curl -f http://localhost:{port}`
+- Default probe for terminal tools: none (skip probe, mark running immediately)
+- Rationale: Leverages existing infrastructure, provides sensible defaults
+
+## Risks / Trade-offs
+
+**[Risk] Startup polling adds latency** → Mitigation: Poll every 2 seconds with 30 second max timeout. Most containers start in <5 seconds.
+
+**[Risk] Docker API calls from API container** → Mitigation: API container already has Docker CLI access for managing instances. Using `docker ps` is consistent with existing patterns.
+
+**[Risk] False "unhealthy" from slow-starting tools** → Mitigation: 30 second default timeout with configurable override per tool type. Frontend shows "starting..." status during probe.
+
+**[Risk] Probe commands may not exist in container** → Mitigation: Probe failures log stderr. If probe command missing, container still starts but marked as running without probe validation.
+
+## Migration Plan
+
+No database migration needed. This change:
+1. Adds new status values ("starting", "unhealthy") to existing `status` enum
+2. Uses existing `readiness_probe` column on `tool_types` table
+3. Changes health check API response format (adds fields, doesn't remove)
+
+## Open Questions
+
+None.

openspec/changes/instance-health-monitoring/proposal.md

@@ -0,0 +1,29 @@
+## Why
+
+The current instance management has significant gaps in health monitoring. When starting instances, there's no verification that containers actually boot successfully - failures only surface when users try to access broken tunnels. The existing health check only validates tunnel URLs, not container health, leading to false positives where a "healthy" tunnel serves 502 errors from a crashed tool. Additionally, readiness probes exist as unused infrastructure, and auto-recovery blindly recreates tunnels on any HTTP error including legitimate 502s from the application itself.
+
+## What Changes
+
+- **Startup health checks**: Verify containers reach a running state after `docker compose up`, with clear failure messages when containers crash or fail to start
+- **Container health checks**: Check container status via Docker API (`docker ps`, `docker inspect`) in addition to tunnel URL checks
+- **Readiness probe integration**: Wire the existing `execute_probe()` service into the instance startup flow, using tool type configured probes
+- **Smart auto-recovery**: Only recreate tunnels when the tunnel endpoint itself is unreachable (connection refused, timeout, DNS failure), NOT when the tool returns 502/503/504 errors
+- **Instance status granularity**: Distinguish between "starting" (container booting), "running" (healthy), "unhealthy" (container up but probe failing), and "error" (failed to start)
+
+## Capabilities
+
+### New Capabilities
+- `instance-startup-health`: Container startup verification and failure detection
+- `instance-runtime-health`: Continuous health monitoring combining container and tunnel checks
+- `readiness-probe-integration`: Tool-type configured readiness probes during instance startup
+- `smart-tunnel-recovery`: Context-aware tunnel recreation that distinguishes tunnel failures from application errors
+
+### Modified Capabilities
+- `session-management-fixes`: Update health check endpoint to include container status, modify tunnel health logic to be smarter about error codes
+
+## Impact
+
+- **Backend**: `api/tool_instances.py` (start_instance, health check, recreate tunnel), `services/docker.py` (container status checks), `services/readiness_probe.py` (integration into startup flow)
+- **Frontend**: `pages/sessions.tsx` (display new status states, show startup errors, smarter health badges)
+- **Database**: No schema changes - uses existing `status` field with new state values
+- **API**: New response fields in health check endpoint (container_status, probe_result, last_probe_at)

openspec/changes/instance-health-monitoring/specs/instance-runtime-health/spec.md

@@ -0,0 +1,57 @@
+## ADDED Requirements
+
+### Requirement: Runtime health endpoint
+The system SHALL provide a health endpoint that checks both container and tunnel health.
+
+#### Scenario: Full health check
+- **GIVEN** a running web-enabled instance
+- **WHEN** `GET /instances/{id}/health` is called
+- **THEN** the response includes:
+  - `container_status`: "running", "exited", "restarting", or "not_found"
+  - `container_health`: "healthy", "unhealthy", or null (if no Docker healthcheck)
+  - `tunnel_status`: "healthy", "unreachable", or "error_response"
+  - `tunnel_status_code`: the HTTP status code from the tunnel URL, or null
+  - `probe_status`: "passed", "failed", "pending", or "not_configured"
+  - `healthy`: true only if container is running AND tunnel is healthy
+
+#### Scenario: Health check for terminal-only instance
+- **GIVEN** a running terminal-only instance
+- **WHEN** `GET /instances/{id}/health` is called
+- **THEN** the response includes `container_status: "running"`
+- **AND** `tunnel_status: "not_applicable"`
+- **AND** `healthy: true` if container is running
+
+### Requirement: Continuous health polling
+The system SHALL support periodic health checks from the frontend.
+
+#### Scenario: Frontend health polling
+- **GIVEN** active instances in the UI
+- **WHEN** the frontend polls health every 30 seconds
+- **THEN** the health status is displayed as a badge
+- **AND** the badge shows "tunnel error" only when tunnel is unreachable
+- **AND** the badge shows "app error" when tunnel returns 502/503/504
+- **AND** the badge shows "starting" when container is up but probe is pending
+
+### Requirement: Container state synchronization
+The system SHALL update instance status when container state changes unexpectedly.
+
+#### Scenario: Container crashes
+- **GIVEN** an instance with status "running"
+- **WHEN** the container exits (crash or OOM)
+- **AND** a health check is performed
+- **THEN** the instance status is updated to "error"
+- **AND** the container exit code and logs are captured
+
+#### Scenario: Container stopped externally
+- **GIVEN** an instance with status "running"
+- **WHEN** the container is stopped via docker command outside the system
+- **AND** a health check is performed
+- **THEN** the instance status is updated to "stopped"
+
+## MODIFIED Requirements
+
+None.
+
+## REMOVED Requirements
+
+None.

openspec/changes/instance-health-monitoring/specs/instance-startup-health/spec.md

@@ -0,0 +1,83 @@
+## ADDED Requirements
+
+### Requirement: Container startup verification
+The system SHALL verify that containers reach a running state before marking instances as "running".
+
+#### Scenario: Container starts successfully
+- **WHEN** `docker compose up` completes
+- **THEN** the system polls `docker ps` every 2 seconds for up to 30 seconds
+- **AND** when the container state is "running", the instance status becomes "starting"
+- **AND** the readiness probe begins execution
+
+#### Scenario: Container fails to start
+- **WHEN** `docker compose up` completes
+- **AND** the container exits within 30 seconds
+- **THEN** the instance status becomes "error"
+- **AND** the container exit code is stored in the error message
+
+#### Scenario: Container stays in restarting loop
+- **WHEN** `docker compose up` completes
+- **AND** the container remains in "restarting" state after 30 seconds
+- **THEN** the instance status becomes "error"
+- **AND** the error message indicates the container is stuck restarting
+
+### Requirement: Readiness probe execution
+The system SHALL execute readiness probes for web-enabled tool instances before marking them as "running".
+
+#### Scenario: Probe succeeds
+- **GIVEN** a tool instance with status "starting"
+- **AND** the tool type has a readiness probe configured
+- **WHEN** the probe command returns exit code 0 within the timeout
+- **THEN** the instance status becomes "running"
+- **AND** the tunnel is created (for web tools)
+
+#### Scenario: Probe times out
+- **GIVEN** a tool instance with status "starting"
+- **AND** the tool type has a readiness probe configured
+- **WHEN** the probe does not succeed within the configured timeout (default 30s)
+- **THEN** the instance status becomes "unhealthy"
+- **AND** the tunnel is still created (the container is running)
+- **AND** the last probe output is stored for diagnostics
+
+#### Scenario: Terminal tool skips probe
+- **GIVEN** a tool instance for a terminal-only tool type
+- **WHEN** the container reaches "running" state
+- **THEN** the instance status immediately becomes "running"
+- **AND** no readiness probe is executed
+
+### Requirement: Container health monitoring
+The system SHALL check container health in addition to tunnel health.
+
+#### Scenario: Container is healthy
+- **GIVEN** a running instance
+- **WHEN** the health endpoint is queried
+- **THEN** the response includes `container_status: "running"`
+- **AND** the response includes `container_health: "healthy"` if Docker healthcheck exists
+
+#### Scenario: Container has crashed
+- **GIVEN** a running instance
+- **WHEN** the container exits or is stopped externally
+- **AND** the health endpoint is queried
+- **THEN** the response includes `container_status: "exited"`
+- **AND** the response includes `healthy: false`
+- **AND** the instance status in the database is updated to "error"
+
+## MODIFIED Requirements
+
+### Requirement: Status Monitoring
+The system SHALL track tool status with startup and health states.
+
+#### Scenario: Status check with health details
+- **GIVEN** a tool instance
+- **WHEN** status is queried
+- **THEN** the real-time container status is returned:
+  - `pending`: Instance created, container not yet started
+  - `starting`: Container is running, readiness probe in progress
+  - `running`: Container is running and probe passed (or terminal tool)
+  - `unhealthy`: Container is running but probe failed/timed out
+  - `stopped`: Container was stopped by user
+  - `error`: Container failed to start or crashed
+
+## REMOVED Requirements
+
+None.

openspec/changes/instance-health-monitoring/specs/readiness-probe-integration/spec.md

@@ -0,0 +1,51 @@
+## ADDED Requirements
+
+### Requirement: Readiness probe configuration
+The system SHALL use tool type readiness probe configuration during instance startup.
+
+#### Scenario: Web tool with custom probe
+- **GIVEN** a tool type with `readiness_probe` configured as:
+  - `command: "curl -f http://localhost:8080/api/health"`
+  - `timeout: 60`
+  - `interval: 5`
+- **WHEN** an instance of this type starts
+- **THEN** the system executes the probe command inside the container
+- **AND** retries every 5 seconds for up to 60 seconds
+- **AND** the instance remains in "starting" status until probe succeeds
+
+#### Scenario: Web tool with default probe
+- **GIVEN** a web-enabled tool type with no `readiness_probe` configured
+- **WHEN** an instance of this type starts
+- **THEN** the system uses the default probe: `curl -f http://localhost:{port}`
+- **AND** retries every 2 seconds for up to 30 seconds
+
+#### Scenario: Probe command execution
+- **GIVEN** a readiness probe command
+- **WHEN** the system executes it inside the container
+- **THEN** it runs via `docker exec {container_id} sh -c "{command}"`
+- **AND** stdout/stderr are captured for diagnostics
+- **AND** exit code 0 indicates success
+
+### Requirement: Probe result storage
+The system SHALL store readiness probe results for diagnostics.
+
+#### Scenario: Successful probe logged
+- **GIVEN** a readiness probe that succeeds
+- **WHEN** the probe returns exit code 0
+- **THEN** the success is logged with timestamp
+- **AND** the instance status changes to "running"
+
+#### Scenario: Failed probe logged
+- **GIVEN** a readiness probe that fails or times out
+- **WHEN** the probe reaches timeout
+- **THEN** the failure is logged with last stdout/stderr output
+- **AND** the instance status changes to "unhealthy"
+- **AND** the probe output is available via the health endpoint
+
+## MODIFIED Requirements
+
+None.
+
+## REMOVED Requirements
+
+None.

openspec/changes/instance-health-monitoring/specs/smart-tunnel-recovery/spec.md

@@ -0,0 +1,45 @@
+## ADDED Requirements
+
+### Requirement: Tunnel failure classification
+The system SHALL distinguish tunnel failures from application errors when determining whether to recreate a tunnel.
+
+#### Scenario: Tunnel is broken
+- **GIVEN** a running instance with a tunnel URL
+- **WHEN** the health check receives one of:
+  - Connection refused (ECONNREFUSED)
+  - Connection timeout (ETIMEDOUT)
+  - DNS resolution failure (ENOTFOUND)
+  - Empty response
+- **THEN** the tunnel status is "unreachable"
+- **AND** the frontend shows a "tunnel error" badge
+- **AND** the "Recreate Tunnel" button is enabled
+
+#### Scenario: Application returns error
+- **GIVEN** a running instance with a tunnel URL
+- **WHEN** the health check receives HTTP 502, 503, or 504
+- **THEN** the tunnel status is "error_response"
+- **AND** the frontend shows an "app error" badge
+- **AND** the "Recreate Tunnel" button is NOT shown
+- **AND** the status code is displayed for diagnostics
+
+#### Scenario: Application is healthy
+- **GIVEN** a running instance with a tunnel URL
+- **WHEN** the health check receives HTTP 200-399
+- **THEN** the tunnel status is "healthy"
+- **AND** no error badge is shown
+
+#### Scenario: Tunnel recreates successfully
+- **GIVEN** an instance with a broken tunnel (status "unreachable")
+- **WHEN** the user clicks "Recreate Tunnel"
+- **THEN** the old cloudflared process is stopped
+- **AND** a new cloudflared process is started
+- **AND** the instance URL is updated
+- **AND** the tunnel status becomes "healthy" (after verification)
+
+## MODIFIED Requirements
+
+None.
+
+## REMOVED Requirements
+
+None.

openspec/changes/instance-health-monitoring/specs/tool-instances/spec.md

@@ -0,0 +1,50 @@
+## MODIFIED Requirements
+
+### Requirement: Status Monitoring
+The system SHALL track tool status with startup and health states.
+
+#### Scenario: Status check with health details
+- **GIVEN** a tool instance
+- **WHEN** status is queried
+- **THEN** the real-time container status is returned:
+  - `pending`: Instance created, container not yet started
+  - `starting`: Container is running, readiness probe in progress
+  - `running`: Container is running and probe passed (or terminal tool)
+  - `unhealthy`: Container is running but probe failed/timed out
+  - `stopped`: Container was stopped by user
+  - `error`: Container failed to start or crashed
+
+## ADDED Requirements
+
+### Requirement: Health check endpoint enhancement
+The system SHALL provide detailed health information through the health check endpoint.
+
+#### Scenario: Health check with container and tunnel status
+- **GIVEN** a running instance
+- **WHEN** `GET /instances/{id}/health` is called
+- **THEN** the response includes:
+  - `healthy`: boolean - overall health
+  - `container_status`: "running", "exited", "restarting", or "not_found"
+  - `tunnel_status`: "healthy", "unreachable", "error_response", or "not_applicable"
+  - `tunnel_status_code`: HTTP status code or null
+  - `probe_status`: "passed", "failed", "pending", or "not_configured"
+  - `last_probe_output`: string or null
+
+### Requirement: Smart tunnel recreation
+The system SHALL only allow tunnel recreation when the tunnel itself is broken.
+
+#### Scenario: Recreate tunnel for unreachable tunnel
+- **GIVEN** an instance with `tunnel_status: "unreachable"`
+- **WHEN** the recreate tunnel endpoint is called
+- **THEN** the tunnel is recreated
+- **AND** the new URL is returned
+
+#### Scenario: Block recreation for application errors
+- **GIVEN** an instance with `tunnel_status: "error_response"` (e.g., HTTP 502)
+- **WHEN** the recreate tunnel endpoint is called
+- **THEN** the request is rejected with 400 Bad Request
+- **AND** the error message explains the tunnel is working but the application is returning errors
+
+## REMOVED Requirements
+
+None.

openspec/changes/instance-health-monitoring/tasks.md

@@ -0,0 +1,56 @@
+## 1. Backend - Container Startup Verification
+
+- [x] 1.1 Implement `wait_for_container_running()` in `services/docker.py` - polls `docker ps` until container reaches "running" state or timeout
+- [x] 1.2 Implement `get_container_status()` in `services/docker.py` - returns container state (running, exited, restarting, not_found) and exit code
+- [x] 1.3 Update `start_instance()` in `api/tool_instances.py` to call startup verification after `docker compose up`
+- [x] 1.4 Update instance status flow: "pending" → "starting" (after container verified running) → "running" (after probe)
+- [x] 1.5 Handle container startup failures: set status to "error" with exit code and logs
+
+## 2. Backend - Readiness Probe Integration
+
+- [x] 2.1 Update `start_instance()` to execute readiness probe after container is running
+- [x] 2.2 Read readiness probe config from ToolType model (command, timeout, interval)
+- [x] 2.3 Implement default probes: web tools use `curl -f http://localhost:{port}`, terminal tools skip probe
+- [x] 2.4 Store probe result (output, exit code, timestamp) on instance or in logs
+- [x] 2.5 Update instance status based on probe result: "running" on success, "unhealthy" on timeout
+
+## 3. Backend - Health Check Enhancement
+
+- [x] 3.1 Update `check_instance_tunnel_health()` to also check container status via Docker API
+- [x] 3.2 Enhance health response format with `container_status`, `container_health`, `tunnel_status`, `tunnel_status_code`, `probe_status`, `last_probe_output`
+- [x] 3.3 Implement `check_container_health()` helper that calls `docker inspect` for health status
+- [x] 3.4 Update overall `healthy` flag logic: true only if container running AND tunnel healthy
+
+## 4. Backend - Smart Tunnel Recovery
+
+- [x] 4.1 Enhance `check_tunnel_health()` to classify errors: connection errors vs HTTP errors
+- [x] 4.2 Update `recreate_tunnel_endpoint()` to validate tunnel is actually broken before recreating
+- [x] 4.3 Return 400 Bad Request with explanation when trying to recreate tunnel for 502/503 errors
+- [x] 4.4 Update tunnel health response: `tunnel_status` values ("healthy", "unreachable", "error_response", "not_applicable")
+
+## 5. Frontend - Status Display
+
+- [x] 5.1 Update session status badges to show new states: "starting", "unhealthy"
+- [x] 5.2 Show container error messages when instance fails to start
+- [x] 5.3 Display "tunnel error" badge only when `tunnel_status === "unreachable"`
+- [x] 5.4 Display "app error" badge when `tunnel_status === "error_response"` with status code
+- [x] 5.5 Show "starting..." badge when `container_status === "running"` but `probe_status === "pending"`
+
+## 6. Frontend - Health Polling
+
+- [x] 6.1 Update health polling to use enhanced health endpoint response
+- [x] 6.2 Store full health state (container + tunnel) in component state
+- [x] 6.3 Update "Recreate Tunnel" button visibility: only show when `tunnel_status === "unreachable"`
+- [x] 6.4 Show probe output in a collapsible section for diagnostics
+
+## 7. Testing and Quality Gates
+
+- [ ] 7.1 Test container startup verification with fast-starting container
+- [ ] 7.2 Test container startup failure (container exits immediately)
+- [ ] 7.3 Test readiness probe success and timeout scenarios
+- [ ] 7.4 Test health endpoint with various container states
+- [ ] 7.5 Test smart tunnel recovery (connection error vs 502)
+- [ ] 7.6 Run backend linting (ruff)
+- [ ] 7.7 Run backend type checking (mypy)
+- [ ] 7.8 Run frontend type checking (tsc)
+- [ ] 7.9 Build frontend and verify no errors