debug: add extensive logging to recreate_tunnel_endpoint
Adds INFO-level logging to trace exactly what happens during tunnel recreation: container lookup, network membership, target IP/URL, tunnel creation result, health check, and direct curl probe from API. This will help diagnose why recreated tunnels return 502 while original tunnels work. Quality gates: ruff clean
This commit is contained in:
@@ -2462,49 +2462,73 @@ async def recreate_tunnel_endpoint(
|
|||||||
detail="Tool type not found for this instance",
|
detail="Tool type not found for this instance",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Find the tool container — try stored ID first, then fall back to name lookup
|
expected_name = instance.name.lower()
|
||||||
tool_container_id = instance.container_id
|
logger.info(
|
||||||
if not tool_container_id:
|
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
|
||||||
tool_container_id = get_container_id(instance.name.lower())
|
instance.id,
|
||||||
|
expected_name,
|
||||||
|
tool_type.default_port,
|
||||||
|
)
|
||||||
|
|
||||||
if not tool_container_id:
|
# Find the tool container — try stored ID first, then fall back to exact name
|
||||||
raise HTTPException(
|
tool_container_id = instance.container_id
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
if tool_container_id:
|
||||||
detail="Could not find running container for this instance",
|
logger.info("Using stored container_id: %s", tool_container_id)
|
||||||
|
else:
|
||||||
|
# Use exact name match via inspect to avoid substring collisions with tunnel containers
|
||||||
|
inspect_result = subprocess.run(
|
||||||
|
["docker", "inspect", "-f", "{{.Id}}", expected_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
)
|
)
|
||||||
|
if inspect_result.returncode == 0 and inspect_result.stdout.strip():
|
||||||
|
tool_container_id = inspect_result.stdout.strip()
|
||||||
|
logger.info("Found container by exact name: %s", tool_container_id)
|
||||||
|
else:
|
||||||
|
logger.error(
|
||||||
|
"Container %s not found via docker inspect: %s",
|
||||||
|
expected_name,
|
||||||
|
inspect_result.stderr,
|
||||||
|
)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="Could not find running container for this instance",
|
||||||
|
)
|
||||||
|
|
||||||
# Ensure the tool container is on the backend network so the tunnel can reach it
|
# Ensure the tool container is on the backend network so the tunnel can reach it
|
||||||
network_name = get_backend_network_name()
|
network_name = get_backend_network_name()
|
||||||
if not is_container_on_network(tool_container_id, network_name):
|
on_network = is_container_on_network(tool_container_id, network_name)
|
||||||
logger.debug(
|
logger.info(
|
||||||
"Connecting container %s to network %s for tunnel access",
|
"Container %s on network %s: %s",
|
||||||
|
tool_container_id,
|
||||||
|
network_name,
|
||||||
|
on_network,
|
||||||
|
)
|
||||||
|
if not on_network:
|
||||||
|
logger.info(
|
||||||
|
"Connecting container %s to network %s",
|
||||||
tool_container_id,
|
tool_container_id,
|
||||||
network_name,
|
network_name,
|
||||||
)
|
)
|
||||||
connected = connect_container_to_network(tool_container_id, network_name)
|
connected = connect_container_to_network(tool_container_id, network_name)
|
||||||
if not connected:
|
logger.info("Network connect result: %s", connected)
|
||||||
logger.warning(
|
|
||||||
"Failed to connect container %s to network %s",
|
|
||||||
tool_container_id,
|
|
||||||
network_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Use the container's IP on the backend network as the tunnel target.
|
# Get the container's IP on the backend network
|
||||||
# This is more reliable than name-based DNS, especially for old instances
|
|
||||||
# whose container name may differ from instance.name.lower().
|
|
||||||
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
||||||
if target_ip:
|
if target_ip:
|
||||||
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
||||||
logger.debug(
|
logger.info(
|
||||||
"Using container IP %s as tunnel target for instance %s",
|
"Tunnel target for instance %s: %s (IP %s on %s)",
|
||||||
target_ip,
|
|
||||||
instance.id,
|
instance.id,
|
||||||
|
target_url,
|
||||||
|
target_ip,
|
||||||
|
network_name,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
target_url = None
|
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Could not get container IP for %s, falling back to name resolution",
|
"Could not get container IP, falling back to name-based target: %s",
|
||||||
tool_container_id,
|
target_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -2513,16 +2537,33 @@ async def recreate_tunnel_endpoint(
|
|||||||
container_port=tool_type.default_port or 0,
|
container_port=tool_type.default_port or 0,
|
||||||
target_url=target_url,
|
target_url=target_url,
|
||||||
)
|
)
|
||||||
|
logger.info(
|
||||||
|
"Tunnel recreated: container=%s, url=%s",
|
||||||
|
tunnel_info["container_name"],
|
||||||
|
tunnel_info["url"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Verify the tunnel can actually reach the origin
|
||||||
|
health = check_tunnel_health(tunnel_info["url"], timeout=10)
|
||||||
|
logger.info(
|
||||||
|
"Tunnel health check: status=%s, code=%s, error=%s",
|
||||||
|
health.get("tunnel_status"),
|
||||||
|
health.get("status_code"),
|
||||||
|
health.get("error"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Also probe from inside the API container directly to the target
|
||||||
|
probe = subprocess.run(
|
||||||
|
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", target_url],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
logger.info("Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip())
|
||||||
|
|
||||||
instance.tunnel_id = tunnel_info["container_name"]
|
instance.tunnel_id = tunnel_info["container_name"]
|
||||||
instance.public_url = tunnel_info["url"]
|
instance.public_url = tunnel_info["url"]
|
||||||
instance.url = tunnel_info["url"]
|
instance.url = tunnel_info["url"]
|
||||||
await session.commit()
|
await session.commit()
|
||||||
logger.debug(
|
|
||||||
"Recreated tunnel for instance %s: container=%s, url=%s",
|
|
||||||
instance.id,
|
|
||||||
tunnel_info["container_name"],
|
|
||||||
tunnel_info["url"],
|
|
||||||
)
|
|
||||||
return {"status": "healthy", "url": instance.url}
|
return {"status": "healthy", "url": instance.url}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
||||||
|
|||||||
@@ -318,9 +318,7 @@ def get_container_ip_on_network(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def is_container_on_network(
|
def is_container_on_network(container_id: str, network_name: str | None = None) -> bool:
|
||||||
container_id: str, network_name: str | None = None
|
|
||||||
) -> bool:
|
|
||||||
"""Check whether a container is already attached to a Docker network.
|
"""Check whether a container is already attached to a Docker network.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
Reference in New Issue
Block a user