debug: add extensive logging to recreate_tunnel_endpoint

Adds INFO-level logging to trace exactly what happens during tunnel
recreation: container lookup, network membership, target IP/URL,
tunnel creation result, health check, and direct curl probe from API.

This will help diagnose why recreated tunnels return 502 while
original tunnels work.

Quality gates: ruff clean
This commit is contained in:
2026-05-30 14:42:25 +02:00
parent 4cc433a1b8
commit fdf78353ad
2 changed files with 74 additions and 35 deletions
+73 -32
View File
@@ -2462,49 +2462,73 @@ async def recreate_tunnel_endpoint(
detail="Tool type not found for this instance",
)
# Find the tool container — try stored ID first, then fall back to name lookup
tool_container_id = instance.container_id
if not tool_container_id:
tool_container_id = get_container_id(instance.name.lower())
expected_name = instance.name.lower()
logger.info(
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
instance.id,
expected_name,
tool_type.default_port,
)
if not tool_container_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Could not find running container for this instance",
# Find the tool container — try stored ID first, then fall back to exact name
tool_container_id = instance.container_id
if tool_container_id:
logger.info("Using stored container_id: %s", tool_container_id)
else:
# Use exact name match via inspect to avoid substring collisions with tunnel containers
inspect_result = subprocess.run(
["docker", "inspect", "-f", "{{.Id}}", expected_name],
capture_output=True,
text=True,
)
if inspect_result.returncode == 0 and inspect_result.stdout.strip():
tool_container_id = inspect_result.stdout.strip()
logger.info("Found container by exact name: %s", tool_container_id)
else:
logger.error(
"Container %s not found via docker inspect: %s",
expected_name,
inspect_result.stderr,
)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Could not find running container for this instance",
)
# Ensure the tool container is on the backend network so the tunnel can reach it
network_name = get_backend_network_name()
if not is_container_on_network(tool_container_id, network_name):
logger.debug(
"Connecting container %s to network %s for tunnel access",
on_network = is_container_on_network(tool_container_id, network_name)
logger.info(
"Container %s on network %s: %s",
tool_container_id,
network_name,
on_network,
)
if not on_network:
logger.info(
"Connecting container %s to network %s",
tool_container_id,
network_name,
)
connected = connect_container_to_network(tool_container_id, network_name)
if not connected:
logger.warning(
"Failed to connect container %s to network %s",
tool_container_id,
network_name,
)
logger.info("Network connect result: %s", connected)
# Use the container's IP on the backend network as the tunnel target.
# This is more reliable than name-based DNS, especially for old instances
# whose container name may differ from instance.name.lower().
# Get the container's IP on the backend network
target_ip = get_container_ip_on_network(tool_container_id, network_name)
if target_ip:
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
logger.debug(
"Using container IP %s as tunnel target for instance %s",
target_ip,
logger.info(
"Tunnel target for instance %s: %s (IP %s on %s)",
instance.id,
target_url,
target_ip,
network_name,
)
else:
target_url = None
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
logger.warning(
"Could not get container IP for %s, falling back to name resolution",
tool_container_id,
"Could not get container IP, falling back to name-based target: %s",
target_url,
)
try:
@@ -2513,16 +2537,33 @@ async def recreate_tunnel_endpoint(
container_port=tool_type.default_port or 0,
target_url=target_url,
)
logger.info(
"Tunnel recreated: container=%s, url=%s",
tunnel_info["container_name"],
tunnel_info["url"],
)
# Verify the tunnel can actually reach the origin
health = check_tunnel_health(tunnel_info["url"], timeout=10)
logger.info(
"Tunnel health check: status=%s, code=%s, error=%s",
health.get("tunnel_status"),
health.get("status_code"),
health.get("error"),
)
# Also probe from inside the API container directly to the target
probe = subprocess.run(
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", target_url],
capture_output=True,
text=True,
)
logger.info("Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip())
instance.tunnel_id = tunnel_info["container_name"]
instance.public_url = tunnel_info["url"]
instance.url = tunnel_info["url"]
await session.commit()
logger.debug(
"Recreated tunnel for instance %s: container=%s, url=%s",
instance.id,
tunnel_info["container_name"],
tunnel_info["url"],
)
return {"status": "healthy", "url": instance.url}
except Exception as exc:
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
+1 -3
View File
@@ -318,9 +318,7 @@ def get_container_ip_on_network(
return None
def is_container_on_network(
container_id: str, network_name: str | None = None
) -> bool:
def is_container_on_network(container_id: str, network_name: str | None = None) -> bool:
"""Check whether a container is already attached to a Docker network.
Args: