diff --git a/apps/api/src/api/tool_instances.py b/apps/api/src/api/tool_instances.py
index 65150f3..e8805b4 100644
--- a/apps/api/src/api/tool_instances.py
+++ b/apps/api/src/api/tool_instances.py
@@ -30,11 +30,14 @@ from src.services.docker import (
execute_compose_command,
find_free_port,
get_container_id,
+ get_container_logs,
get_container_name,
+ get_container_status,
recreate_tunnel,
render_compose_template,
start_cloudflared_tunnel,
stop_cloudflared_tunnel,
+ wait_for_container_running,
write_compose_file,
write_config_files,
write_env_file,
@@ -552,20 +555,67 @@ async def start_instance(
else:
logger.warning("Failed to connect %s to backend network", container_name)
- instance.status = "starting"
- instance.last_started_at = datetime.now()
- await session.commit()
- logger.info("Instance %s container is running, checking readiness", instance.id)
-
+ # Verify container reached running state
+ if instance.container_id:
+ instance.status = "starting"
+ instance.last_started_at = datetime.now()
+ await session.commit()
+ logger.info("Instance %s: verifying container startup...", instance.id)
+
+ startup_result = wait_for_container_running(instance.container_id, timeout=30, interval=2.0)
+
+ if not startup_result["success"]:
+ # Container failed to start
+ error_msg = f"Container failed to start: status={startup_result['status']}"
+ if startup_result["exit_code"] is not None:
+ error_msg += f", exit_code={startup_result['exit_code']}"
+
+ # Get logs for debugging
+ logs = get_container_logs(instance.container_id, tail=50)
+
+ instance.status = "error"
+ await session.commit()
+ logger.error(
+ "Instance %s container startup failed after %.1fs: %s\nLogs:\n%s",
+ instance.id,
+ startup_result["waited_seconds"],
+ error_msg,
+ logs,
+ )
+ return {
+ "status": "error",
+ "error": error_msg,
+ "logs": logs,
+ }
+
+ logger.info(
+ "Instance %s container started successfully after %.1fs",
+ instance.id,
+ startup_result["waited_seconds"],
+ )
+
# Execute readiness probe if configured
tool_type = await session.get(ToolType, instance.tool_type_id)
- if tool_type and tool_type.readiness_probe:
- probe_config = tool_type.readiness_probe
- probe_command = probe_config.get("command", "")
- probe_timeout = probe_config.get("timeout", 30)
- probe_interval = probe_config.get("interval", 2)
+ if tool_type and instance.container_id:
+ # Determine probe command
+ probe_command = None
+ probe_timeout = 30
+ probe_interval = 2
- if probe_command and instance.container_id:
+ if tool_type.readiness_probe:
+ probe_config = tool_type.readiness_probe
+ probe_command = probe_config.get("command", "")
+ probe_timeout = probe_config.get("timeout", 30)
+ probe_interval = probe_config.get("interval", 2)
+ elif "web" in (tool_type.interfaces or []):
+ # Default probe for web tools
+ probe_command = f"curl -f http://localhost:{tool_type.default_port or 8080}"
+ probe_timeout = 30
+ probe_interval = 2
+
+ if probe_command:
+ instance.status = "probing"
+ await session.commit()
logger.info(
"Executing readiness probe for instance %s: command='%s', timeout=%d, interval=%d",
instance.id, probe_command, probe_timeout, probe_interval
@@ -578,14 +628,25 @@ async def start_instance(
interval=probe_interval,
)
+ # Store probe result
+ instance.probe_result = {
+ "success": success,
+ "command": probe_command,
+ "logs": probe_logs,
+ "timestamp": datetime.now().isoformat(),
+ }
+
if not success:
- instance.status = "failed"
- instance.url = None
- instance.public_url = None
+ instance.status = "unhealthy"
await session.commit()
- logger.error("Readiness probe failed for instance %s: %s", instance.id, "\n".join(probe_logs))
+ logger.error(
+ "Readiness probe failed for instance %s after %ds: %s",
+ instance.id,
+ probe_timeout,
+ "\n".join(probe_logs),
+ )
return {
- "status": "failed",
+ "status": "unhealthy",
"error": f"Readiness probe failed after {probe_timeout}s",
"probe_logs": probe_logs,
}
@@ -956,6 +1017,17 @@ async def recreate_tunnel_endpoint(
detail="instance must be running to recreate tunnel",
)
+ # Validate tunnel is actually broken before recreating
+ if instance.url:
+ tunnel_health = check_tunnel_health(instance.url)
+ if tunnel_health["tunnel_status"] == "error_response":
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail=f"Tunnel is working but application returned HTTP {tunnel_health.get('status_code')}. Recreating the tunnel will not fix this issue.",
+ )
+ elif tunnel_health["tunnel_status"] == "healthy":
+ return {"status": "healthy", "url": instance.url, "message": "Tunnel is already healthy"}
+
# Get tool type for default port
tool_type = await session.get(ToolType, instance.tool_type_id)
instance_port = tool_type.default_port if tool_type and tool_type.default_port else 8080
@@ -987,8 +1059,8 @@ async def recreate_tunnel_endpoint(
@router.get(
"/{project_id}/repositories/{repo_id}/instances/{instance_id}/health",
- summary="Check tunnel health",
- description="Check if the temporary Cloudflare tunnel for an instance is healthy.",
+ summary="Check instance health",
+ description="Check container and tunnel health for an instance.",
)
async def check_instance_tunnel_health(
project_id: uuid.UUID,
@@ -997,7 +1069,7 @@ async def check_instance_tunnel_health(
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
- """Check tunnel health for an instance.
+ """Check health for an instance (container + tunnel).
Args:
project_id: UUID of the project.
@@ -1007,7 +1079,7 @@ async def check_instance_tunnel_health(
session: Database session.
Returns:
- Dictionary with health status.
+ Dictionary with container_status, tunnel_status, probe_status, and overall healthy flag.
"""
_user = await _get_user(session, user_id)
_project = await _get_owned_project(project_id, user_id, session)
@@ -1018,11 +1090,50 @@ async def check_instance_tunnel_health(
status_code=status.HTTP_404_NOT_FOUND, detail="instance not found"
)
- if not instance.url or instance.status != "running":
- return {"healthy": False, "status_code": None, "error": "instance not running"}
+ # Check container status
+ container_info = {"status": "not_found", "exit_code": None, "health": None}
+ if instance.container_id:
+ container_info = get_container_status(instance.container_id)
- health = check_tunnel_health(instance.url)
- return health
+ # Build response
+ response = {
+ "healthy": False,
+ "container_status": container_info["status"],
+ "container_health": container_info["health"],
+ "tunnel_status": "not_applicable",
+ "tunnel_status_code": None,
+ "probe_status": "not_applicable",
+ "last_probe_output": None,
+ "error": None,
+ }
+
+ # Determine probe status
+ if instance.status == "probing":
+ response["probe_status"] = "pending"
+ elif instance.probe_result:
+ response["probe_status"] = "success" if instance.probe_result.get("success") else "failed"
+ response["last_probe_output"] = "\n".join(instance.probe_result.get("logs", []))[:500]
+
+ # Check tunnel health if instance has a URL and is web-enabled
+ if instance.url and instance.status in ("running", "unhealthy"):
+ tunnel_health = check_tunnel_health(instance.url)
+ response["tunnel_status"] = tunnel_health["tunnel_status"]
+ response["tunnel_status_code"] = tunnel_health.get("status_code")
+ if tunnel_health.get("error"):
+ response["error"] = tunnel_health["error"]
+
+ # Overall healthy only if container is running AND tunnel is healthy
+ container_healthy = container_info["status"] == "running"
+ tunnel_healthy = response["tunnel_status"] == "healthy"
+ response["healthy"] = container_healthy and tunnel_healthy
+
+ # If container is not running, override error message
+ if not container_healthy:
+ response["error"] = f"Container is {container_info['status']}"
+ if container_info["exit_code"] is not None:
+ response["error"] += f" (exit code: {container_info['exit_code']})"
+
+ return response
@router.get(
diff --git a/apps/api/src/models/tool_instance.py b/apps/api/src/models/tool_instance.py
index b556bab..162144e 100644
--- a/apps/api/src/models/tool_instance.py
+++ b/apps/api/src/models/tool_instance.py
@@ -2,7 +2,7 @@ import uuid
from datetime import datetime
from typing import TYPE_CHECKING
-from sqlalchemy import DateTime, ForeignKey, Integer, String
+from sqlalchemy import DateTime, ForeignKey, Integer, JSON, String
from sqlalchemy import Uuid as UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -62,6 +62,9 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base):
last_stopped_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
+ probe_result: Mapped[dict | None] = mapped_column(
+ JSON, nullable=True
+ )
tool_type: Mapped["ToolType"] = relationship()
repository: Mapped["GitRepository"] = relationship()
diff --git a/apps/api/src/services/docker.py b/apps/api/src/services/docker.py
index bb01fee..083ed57 100644
--- a/apps/api/src/services/docker.py
+++ b/apps/api/src/services/docker.py
@@ -244,24 +244,94 @@ def connect_container_to_network(container_name: str, network_name: str = "backe
return result.returncode == 0
-def get_container_status(container_id: str) -> str:
+def get_container_status(container_id: str) -> dict[str, Any]:
"""Get the status of a Docker container.
Args:
container_id: Docker container ID
Returns:
- Container status string (running, exited, etc.)
+ Dict with 'status' (running, exited, restarting, not_found),
+ 'exit_code' (int or None), and 'health' (health status or None)
"""
result = subprocess.run(
- ["docker", "inspect", "-f", "{{.State.Status}}", container_id],
+ [
+ "docker", "inspect", "-f",
+ "{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}",
+ container_id,
+ ],
capture_output=True,
text=True,
)
- if result.returncode == 0:
- return result.stdout.strip()
- return "unknown"
+ if result.returncode != 0:
+ return {"status": "not_found", "exit_code": None, "health": None}
+
+ parts = result.stdout.strip().split("|")
+ status = parts[0] if parts else "unknown"
+ exit_code = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
+ health = parts[2] if len(parts) > 2 and parts[2] != "none" else None
+
+ return {"status": status, "exit_code": exit_code, "health": health}
+
+
+def wait_for_container_running(
+ container_id: str, timeout: int = 30, interval: float = 2.0
+) -> dict[str, Any]:
+ """Wait for a container to reach the running state.
+
+ Polls docker inspect until the container status is "running" or timeout.
+
+ Args:
+ container_id: Docker container ID
+ timeout: Maximum seconds to wait
+ interval: Seconds between polls
+
+ Returns:
+ Dict with 'success' (bool), 'status' (str), 'exit_code' (int or None),
+ and 'waited_seconds' (float)
+ """
+ import time
+
+ start_time = time.time()
+
+ while time.time() - start_time < timeout:
+ info = get_container_status(container_id)
+
+ if info["status"] == "running":
+ return {
+ "success": True,
+ "status": "running",
+ "exit_code": None,
+ "waited_seconds": time.time() - start_time,
+ }
+
+ if info["status"] == "exited":
+ return {
+ "success": False,
+ "status": "exited",
+ "exit_code": info["exit_code"],
+ "waited_seconds": time.time() - start_time,
+ }
+
+ if info["status"] == "not_found":
+ return {
+ "success": False,
+ "status": "not_found",
+ "exit_code": None,
+ "waited_seconds": time.time() - start_time,
+ }
+
+ time.sleep(interval)
+
+ # Timeout reached
+ info = get_container_status(container_id)
+ return {
+ "success": False,
+ "status": info["status"],
+ "exit_code": info["exit_code"],
+ "waited_seconds": time.time() - start_time,
+ }
def get_container_logs(container_id: str, tail: int = 100) -> str:
@@ -424,14 +494,15 @@ def recreate_tunnel(
def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
- """Check if a tunnel URL is healthy.
+ """Check if a tunnel URL is healthy with smart error classification.
Args:
url: The tunnel URL to check
timeout: Request timeout in seconds
Returns:
- Dict with 'healthy' (bool) and 'status_code' (int or None)
+ Dict with 'tunnel_status' (healthy, unreachable, error_response, not_applicable),
+ 'status_code' (int or None), 'healthy' (bool), and 'error' (str or None)
"""
import subprocess
@@ -444,13 +515,49 @@ def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
timeout=timeout + 5,
)
status_code = int(result.stdout.strip())
+
+ if 200 <= status_code < 400:
+ return {
+ "tunnel_status": "healthy",
+ "status_code": status_code,
+ "healthy": True,
+ "error": None,
+ }
+ elif status_code in (502, 503, 504):
+ # Application error, not tunnel error
+ return {
+ "tunnel_status": "error_response",
+ "status_code": status_code,
+ "healthy": False,
+ "error": f"Application returned HTTP {status_code}",
+ }
+ else:
+ return {
+ "tunnel_status": "error_response",
+ "status_code": status_code,
+ "healthy": False,
+ "error": f"HTTP {status_code}",
+ }
+ except subprocess.TimeoutExpired:
return {
- "healthy": 200 <= status_code < 400,
- "status_code": status_code,
- }
- except (ValueError, subprocess.TimeoutExpired, Exception) as e:
- return {
- "healthy": False,
+ "tunnel_status": "unreachable",
"status_code": None,
+ "healthy": False,
+ "error": "Tunnel request timed out",
+ }
+ except (ValueError, Exception) as e:
+ error_str = str(e).lower()
+ # Classify connection errors
+ if any(err in error_str for err in ["connection refused", "econnrefused", "could not resolve", "nodename"]):
+ return {
+ "tunnel_status": "unreachable",
+ "status_code": None,
+ "healthy": False,
+ "error": f"Tunnel unreachable: {e}",
+ }
+ return {
+ "tunnel_status": "unreachable",
+ "status_code": None,
+ "healthy": False,
"error": str(e),
}
diff --git a/apps/web/src/api/sessions.ts b/apps/web/src/api/sessions.ts
index a404760..bedb600 100644
--- a/apps/web/src/api/sessions.ts
+++ b/apps/web/src/api/sessions.ts
@@ -25,6 +25,8 @@ export interface Session {
project_id: string;
status: string;
url: string | null;
+ container_status?: string;
+ probe_status?: string;
}
export async function listInstances(
@@ -101,11 +103,23 @@ export async function getUserSessions(): Promise {
return response.data.sessions;
}
+export interface InstanceHealth {
+ healthy: boolean;
+ container_status: string;
+ container_health?: string;
+ container_exit_code?: number | null;
+ tunnel_status: string;
+ tunnel_status_code: number | null;
+ probe_status: string;
+ last_probe_output?: string;
+ error?: string;
+}
+
export async function checkInstanceHealth(
projectId: string,
repoId: string,
instanceId: string
-): Promise<{ healthy: boolean; status_code: number | null; error?: string }> {
+): Promise {
const response = await apiClient.get(
`/projects/${projectId}/repositories/${repoId}/instances/${instanceId}/health`
);
diff --git a/apps/web/src/pages/sessions.tsx b/apps/web/src/pages/sessions.tsx
index a548d19..fe44e91 100644
--- a/apps/web/src/pages/sessions.tsx
+++ b/apps/web/src/pages/sessions.tsx
@@ -40,8 +40,18 @@ export const SessionsPage = () => {
const [deleteConfirmId, setDeleteConfirmId] = useState(null);
const [stopConfirmId, setStopConfirmId] = useState(null);
- const [tunnelHealth, setTunnelHealth] = useState>({});
+ const [tunnelHealth, setTunnelHealth] = useState>({});
const [recreatingId, setRecreatingId] = useState(null);
+ const [expandedProbeId, setExpandedProbeId] = useState(null);
const loadSessions = useCallback(async () => {
setStatus("loading");
@@ -86,13 +96,13 @@ export const SessionsPage = () => {
void loadToolTypes();
}, []);
- // Poll tunnel health every 30 seconds for running instances
+ // Poll health every 30 seconds for active instances
useEffect(() => {
const checkHealth = async () => {
- const runningSessions = sessions.filter(
- (s) => s.status === "running" && s.url
+ const activeSessions = sessions.filter(
+ (s) => ["running", "starting", "unhealthy", "probing"].includes(s.status)
);
- for (const session of runningSessions) {
+ for (const session of activeSessions) {
try {
const health = await checkInstanceHealth(
session.project_id,
@@ -106,7 +116,14 @@ export const SessionsPage = () => {
} catch {
setTunnelHealth((prev) => ({
...prev,
- [session.id]: { healthy: false, status_code: null, error: "check failed" },
+ [session.id]: {
+ healthy: false,
+ container_status: "unknown",
+ tunnel_status: "unreachable",
+ tunnel_status_code: null,
+ probe_status: "unknown",
+ error: "check failed",
+ },
}));
}
}
@@ -135,7 +152,7 @@ export const SessionsPage = () => {
}, [selectedProject]);
const activeSessions = useMemo(
- () => sessions.filter((s) => ["running", "building", "pending"].includes(s.status)),
+ () => sessions.filter((s) => ["running", "building", "pending", "starting", "probing", "unhealthy"].includes(s.status)),
[sessions]
);
@@ -328,9 +345,35 @@ export const SessionsPage = () => {
)}
{session.status}
- {tunnelHealth[session.id] && !tunnelHealth[session.id].healthy && (
+ {session.status === "starting" && (
+ starting...
+ )}
+ {session.status === "probing" && (
+ checking...
+ )}
+ {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "unreachable" && (
tunnel error
)}
+ {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "error_response" && (
+ app error ({tunnelHealth[session.id].tunnel_status_code})
+ )}
+ {tunnelHealth[session.id]?.last_probe_output && (
+
+
+ {expandedProbeId === session.id && (
+
+ {tunnelHealth[session.id].last_probe_output}
+
+ )}
+
+ )}
{session.url ? (
@@ -353,7 +396,7 @@ export const SessionsPage = () => {
Open
)}
- {tunnelHealth[session.id] && !tunnelHealth[session.id].healthy && (
+ {tunnelHealth[session.id] && tunnelHealth[session.id].tunnel_status === "unreachable" && (