debug: add extensive logging to recreate_tunnel_endpoint
Adds INFO-level logging to trace exactly what happens during tunnel recreation: container lookup, network membership, target IP/URL, tunnel creation result, health check, and direct curl probe from API. This will help diagnose why recreated tunnels return 502 while original tunnels work. Quality gates: ruff clean
This commit is contained in:
@@ -2462,49 +2462,73 @@ async def recreate_tunnel_endpoint(
|
||||
detail="Tool type not found for this instance",
|
||||
)
|
||||
|
||||
# Find the tool container — try stored ID first, then fall back to name lookup
|
||||
tool_container_id = instance.container_id
|
||||
if not tool_container_id:
|
||||
tool_container_id = get_container_id(instance.name.lower())
|
||||
expected_name = instance.name.lower()
|
||||
logger.info(
|
||||
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
|
||||
instance.id,
|
||||
expected_name,
|
||||
tool_type.default_port,
|
||||
)
|
||||
|
||||
if not tool_container_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Could not find running container for this instance",
|
||||
# Find the tool container — try stored ID first, then fall back to exact name
|
||||
tool_container_id = instance.container_id
|
||||
if tool_container_id:
|
||||
logger.info("Using stored container_id: %s", tool_container_id)
|
||||
else:
|
||||
# Use exact name match via inspect to avoid substring collisions with tunnel containers
|
||||
inspect_result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.Id}}", expected_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if inspect_result.returncode == 0 and inspect_result.stdout.strip():
|
||||
tool_container_id = inspect_result.stdout.strip()
|
||||
logger.info("Found container by exact name: %s", tool_container_id)
|
||||
else:
|
||||
logger.error(
|
||||
"Container %s not found via docker inspect: %s",
|
||||
expected_name,
|
||||
inspect_result.stderr,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Could not find running container for this instance",
|
||||
)
|
||||
|
||||
# Ensure the tool container is on the backend network so the tunnel can reach it
|
||||
network_name = get_backend_network_name()
|
||||
if not is_container_on_network(tool_container_id, network_name):
|
||||
logger.debug(
|
||||
"Connecting container %s to network %s for tunnel access",
|
||||
on_network = is_container_on_network(tool_container_id, network_name)
|
||||
logger.info(
|
||||
"Container %s on network %s: %s",
|
||||
tool_container_id,
|
||||
network_name,
|
||||
on_network,
|
||||
)
|
||||
if not on_network:
|
||||
logger.info(
|
||||
"Connecting container %s to network %s",
|
||||
tool_container_id,
|
||||
network_name,
|
||||
)
|
||||
connected = connect_container_to_network(tool_container_id, network_name)
|
||||
if not connected:
|
||||
logger.warning(
|
||||
"Failed to connect container %s to network %s",
|
||||
tool_container_id,
|
||||
network_name,
|
||||
)
|
||||
logger.info("Network connect result: %s", connected)
|
||||
|
||||
# Use the container's IP on the backend network as the tunnel target.
|
||||
# This is more reliable than name-based DNS, especially for old instances
|
||||
# whose container name may differ from instance.name.lower().
|
||||
# Get the container's IP on the backend network
|
||||
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
||||
if target_ip:
|
||||
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
||||
logger.debug(
|
||||
"Using container IP %s as tunnel target for instance %s",
|
||||
target_ip,
|
||||
logger.info(
|
||||
"Tunnel target for instance %s: %s (IP %s on %s)",
|
||||
instance.id,
|
||||
target_url,
|
||||
target_ip,
|
||||
network_name,
|
||||
)
|
||||
else:
|
||||
target_url = None
|
||||
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
|
||||
logger.warning(
|
||||
"Could not get container IP for %s, falling back to name resolution",
|
||||
tool_container_id,
|
||||
"Could not get container IP, falling back to name-based target: %s",
|
||||
target_url,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -2513,16 +2537,33 @@ async def recreate_tunnel_endpoint(
|
||||
container_port=tool_type.default_port or 0,
|
||||
target_url=target_url,
|
||||
)
|
||||
logger.info(
|
||||
"Tunnel recreated: container=%s, url=%s",
|
||||
tunnel_info["container_name"],
|
||||
tunnel_info["url"],
|
||||
)
|
||||
|
||||
# Verify the tunnel can actually reach the origin
|
||||
health = check_tunnel_health(tunnel_info["url"], timeout=10)
|
||||
logger.info(
|
||||
"Tunnel health check: status=%s, code=%s, error=%s",
|
||||
health.get("tunnel_status"),
|
||||
health.get("status_code"),
|
||||
health.get("error"),
|
||||
)
|
||||
|
||||
# Also probe from inside the API container directly to the target
|
||||
probe = subprocess.run(
|
||||
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", target_url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
logger.info("Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip())
|
||||
|
||||
instance.tunnel_id = tunnel_info["container_name"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
instance.url = tunnel_info["url"]
|
||||
await session.commit()
|
||||
logger.debug(
|
||||
"Recreated tunnel for instance %s: container=%s, url=%s",
|
||||
instance.id,
|
||||
tunnel_info["container_name"],
|
||||
tunnel_info["url"],
|
||||
)
|
||||
return {"status": "healthy", "url": instance.url}
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
||||
|
||||
@@ -318,9 +318,7 @@ def get_container_ip_on_network(
|
||||
return None
|
||||
|
||||
|
||||
def is_container_on_network(
|
||||
container_id: str, network_name: str | None = None
|
||||
) -> bool:
|
||||
def is_container_on_network(container_id: str, network_name: str | None = None) -> bool:
|
||||
"""Check whether a container is already attached to a Docker network.
|
||||
|
||||
Args:
|
||||
|
||||
Reference in New Issue
Block a user