From fdf78353ad338a73d61cb3509a3172a1e6f82f6b Mon Sep 17 00:00:00 2001 From: Alex Blank Date: Sat, 30 May 2026 14:42:25 +0200 Subject: [PATCH] debug: add extensive logging to recreate_tunnel_endpoint Adds INFO-level logging to trace exactly what happens during tunnel recreation: container lookup, network membership, target IP/URL, tunnel creation result, health check, and direct curl probe from API. This will help diagnose why recreated tunnels return 502 while original tunnels work. Quality gates: ruff clean --- apps/api/src/api/tool_instances.py | 105 ++++++++++++++++++++--------- apps/api/src/services/docker.py | 4 +- 2 files changed, 74 insertions(+), 35 deletions(-) diff --git a/apps/api/src/api/tool_instances.py b/apps/api/src/api/tool_instances.py index 3a86a2a..59f702c 100644 --- a/apps/api/src/api/tool_instances.py +++ b/apps/api/src/api/tool_instances.py @@ -2462,49 +2462,73 @@ async def recreate_tunnel_endpoint( detail="Tool type not found for this instance", ) - # Find the tool container — try stored ID first, then fall back to name lookup - tool_container_id = instance.container_id - if not tool_container_id: - tool_container_id = get_container_id(instance.name.lower()) + expected_name = instance.name.lower() + logger.info( + "Recreate tunnel for instance %s (expected container name: %s, default_port: %s)", + instance.id, + expected_name, + tool_type.default_port, + ) - if not tool_container_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Could not find running container for this instance", + # Find the tool container — try stored ID first, then fall back to exact name + tool_container_id = instance.container_id + if tool_container_id: + logger.info("Using stored container_id: %s", tool_container_id) + else: + # Use exact name match via inspect to avoid substring collisions with tunnel containers + inspect_result = subprocess.run( + ["docker", "inspect", "-f", "{{.Id}}", expected_name], + capture_output=True, + text=True, ) + if inspect_result.returncode == 0 and inspect_result.stdout.strip(): + tool_container_id = inspect_result.stdout.strip() + logger.info("Found container by exact name: %s", tool_container_id) + else: + logger.error( + "Container %s not found via docker inspect: %s", + expected_name, + inspect_result.stderr, + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Could not find running container for this instance", + ) # Ensure the tool container is on the backend network so the tunnel can reach it network_name = get_backend_network_name() - if not is_container_on_network(tool_container_id, network_name): - logger.debug( - "Connecting container %s to network %s for tunnel access", + on_network = is_container_on_network(tool_container_id, network_name) + logger.info( + "Container %s on network %s: %s", + tool_container_id, + network_name, + on_network, + ) + if not on_network: + logger.info( + "Connecting container %s to network %s", tool_container_id, network_name, ) connected = connect_container_to_network(tool_container_id, network_name) - if not connected: - logger.warning( - "Failed to connect container %s to network %s", - tool_container_id, - network_name, - ) + logger.info("Network connect result: %s", connected) - # Use the container's IP on the backend network as the tunnel target. - # This is more reliable than name-based DNS, especially for old instances - # whose container name may differ from instance.name.lower(). + # Get the container's IP on the backend network target_ip = get_container_ip_on_network(tool_container_id, network_name) if target_ip: target_url = f"http://{target_ip}:{tool_type.default_port or 0}" - logger.debug( - "Using container IP %s as tunnel target for instance %s", - target_ip, + logger.info( + "Tunnel target for instance %s: %s (IP %s on %s)", instance.id, + target_url, + target_ip, + network_name, ) else: - target_url = None + target_url = f"http://{expected_name}:{tool_type.default_port or 0}" logger.warning( - "Could not get container IP for %s, falling back to name resolution", - tool_container_id, + "Could not get container IP, falling back to name-based target: %s", + target_url, ) try: @@ -2513,16 +2537,33 @@ async def recreate_tunnel_endpoint( container_port=tool_type.default_port or 0, target_url=target_url, ) + logger.info( + "Tunnel recreated: container=%s, url=%s", + tunnel_info["container_name"], + tunnel_info["url"], + ) + + # Verify the tunnel can actually reach the origin + health = check_tunnel_health(tunnel_info["url"], timeout=10) + logger.info( + "Tunnel health check: status=%s, code=%s, error=%s", + health.get("tunnel_status"), + health.get("status_code"), + health.get("error"), + ) + + # Also probe from inside the API container directly to the target + probe = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", target_url], + capture_output=True, + text=True, + ) + logger.info("Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip()) + instance.tunnel_id = tunnel_info["container_name"] instance.public_url = tunnel_info["url"] instance.url = tunnel_info["url"] await session.commit() - logger.debug( - "Recreated tunnel for instance %s: container=%s, url=%s", - instance.id, - tunnel_info["container_name"], - tunnel_info["url"], - ) return {"status": "healthy", "url": instance.url} except Exception as exc: logger.exception("Failed to recreate tunnel for instance %s", instance.id) diff --git a/apps/api/src/services/docker.py b/apps/api/src/services/docker.py index fa482ea..8025ed1 100644 --- a/apps/api/src/services/docker.py +++ b/apps/api/src/services/docker.py @@ -318,9 +318,7 @@ def get_container_ip_on_network( return None -def is_container_on_network( - container_id: str, network_name: str | None = None -) -> bool: +def is_container_on_network(container_id: str, network_name: str | None = None) -> bool: """Check whether a container is already attached to a Docker network. Args: