Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d2b6bba15c | |||
| b7396d58d2 | |||
| 351e76c00d | |||
| 2c2c4f3683 | |||
| fdf78353ad | |||
| 4cc433a1b8 | |||
| 321b4e3d0e | |||
| 2e156fc534 | |||
| cc52811522 | |||
| 9cab8c7bc7 | |||
| eeb7d9a1b2 | |||
| 6cf06d2380 | |||
| 401ad2e65d |
@@ -45,24 +45,29 @@ from src.services.config_profile_resolver import (
|
|||||||
resolve_profile,
|
resolve_profile,
|
||||||
)
|
)
|
||||||
from src.services.docker import (
|
from src.services.docker import (
|
||||||
check_tunnel_health,
|
|
||||||
connect_container_to_network,
|
connect_container_to_network,
|
||||||
ensure_instance_directory,
|
ensure_instance_directory,
|
||||||
execute_compose_command,
|
execute_compose_command,
|
||||||
find_free_port,
|
find_free_port,
|
||||||
|
get_backend_network_name,
|
||||||
get_container_id,
|
get_container_id,
|
||||||
|
get_container_ip_on_network,
|
||||||
get_container_logs,
|
get_container_logs,
|
||||||
get_container_status,
|
get_container_status,
|
||||||
recreate_tunnel,
|
is_container_on_network,
|
||||||
render_compose_template,
|
render_compose_template,
|
||||||
sort_volumes_by_specificity,
|
sort_volumes_by_specificity,
|
||||||
start_cloudflared_tunnel,
|
|
||||||
stop_cloudflared_tunnel,
|
|
||||||
wait_for_container_running,
|
wait_for_container_running,
|
||||||
write_compose_file,
|
write_compose_file,
|
||||||
write_config_files,
|
write_config_files,
|
||||||
write_env_file,
|
write_env_file,
|
||||||
)
|
)
|
||||||
|
from src.services.tunnel import (
|
||||||
|
check_tunnel_health,
|
||||||
|
recreate_tunnel,
|
||||||
|
start_tunnel,
|
||||||
|
stop_tunnel,
|
||||||
|
)
|
||||||
from src.services.docker_build import build_image
|
from src.services.docker_build import build_image
|
||||||
from src.services.manifest_compiler import (
|
from src.services.manifest_compiler import (
|
||||||
compile_compose,
|
compile_compose,
|
||||||
@@ -748,6 +753,49 @@ def _ensure_web_bind_address(
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_backend_network_in_compose(compose_path: str) -> None:
|
||||||
|
"""Inject the backend network into the compose file so compose up attaches it.
|
||||||
|
|
||||||
|
Instead of running 'docker network connect' after container creation (which
|
||||||
|
is prone to race conditions and silent failures), we declare the network in
|
||||||
|
the compose file itself. Docker Compose then connects the container to the
|
||||||
|
network atomically during 'docker compose up'.
|
||||||
|
"""
|
||||||
|
import yaml
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
compose_file = Path(compose_path)
|
||||||
|
if not compose_file.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
content = compose_file.read_text()
|
||||||
|
compose_data = yaml.safe_load(content)
|
||||||
|
|
||||||
|
if not compose_data or "services" not in compose_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
network_name = get_backend_network_name()
|
||||||
|
modified = False
|
||||||
|
|
||||||
|
for svc_config in compose_data["services"].values():
|
||||||
|
existing = svc_config.get("networks", [])
|
||||||
|
if network_name not in existing:
|
||||||
|
svc_config["networks"] = existing + [network_name]
|
||||||
|
modified = True
|
||||||
|
break # Only modify first service
|
||||||
|
|
||||||
|
# Declare the network as external at the top level
|
||||||
|
if "networks" not in compose_data:
|
||||||
|
compose_data["networks"] = {}
|
||||||
|
if network_name not in compose_data["networks"]:
|
||||||
|
compose_data["networks"][network_name] = {"external": True}
|
||||||
|
modified = True
|
||||||
|
|
||||||
|
if modified:
|
||||||
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
||||||
|
logger.info("Injected backend network '%s' into compose file", network_name)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/{project_id}/repositories/{repo_id}/instances",
|
"/{project_id}/repositories/{repo_id}/instances",
|
||||||
summary="Create tool instance",
|
summary="Create tool instance",
|
||||||
@@ -1688,6 +1736,7 @@ async def start_instance(
|
|||||||
|
|
||||||
# Ensure predictable container name for tunnel connectivity
|
# Ensure predictable container name for tunnel connectivity
|
||||||
_ensure_container_name_in_compose(instance.compose_path, instance.name)
|
_ensure_container_name_in_compose(instance.compose_path, instance.name)
|
||||||
|
_ensure_backend_network_in_compose(instance.compose_path)
|
||||||
|
|
||||||
# Execute docker compose up with env file
|
# Execute docker compose up with env file
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -1723,15 +1772,9 @@ async def start_instance(
|
|||||||
logger.debug("Container ID for instance %s: %s", instance.id, container_id)
|
logger.debug("Container ID for instance %s: %s", instance.id, container_id)
|
||||||
|
|
||||||
instance.container_name = expected_container_name
|
instance.container_name = expected_container_name
|
||||||
logger.debug("Container name for instance %s: %s", instance.id, expected_container_name)
|
logger.debug(
|
||||||
|
"Container name for instance %s: %s", instance.id, expected_container_name
|
||||||
# Connect container to backend network so API can reach it
|
)
|
||||||
logger.debug("Connecting container %s to backend network...", expected_container_name)
|
|
||||||
connected = connect_container_to_network(expected_container_name, "backend")
|
|
||||||
if connected:
|
|
||||||
logger.debug("Successfully connected %s to backend network", expected_container_name)
|
|
||||||
else:
|
|
||||||
logger.warning("Failed to connect %s to backend network", expected_container_name)
|
|
||||||
|
|
||||||
# Verify container reached running state
|
# Verify container reached running state
|
||||||
if instance.container_id:
|
if instance.container_id:
|
||||||
@@ -1952,12 +1995,11 @@ async def start_instance(
|
|||||||
"error": f"Tool type '{instance.tool_type_id}' not found",
|
"error": f"Tool type '{instance.tool_type_id}' not found",
|
||||||
}
|
}
|
||||||
|
|
||||||
instance_port = tool_type.default_port or 0
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Tool type for instance %s: name=%s, default_port=%s, interface_type=%s",
|
"Tool type for instance %s: name=%s, container_port=%s, interface_type=%s",
|
||||||
instance.id,
|
instance.id,
|
||||||
tool_type.name,
|
tool_type.name,
|
||||||
instance_port,
|
tool_type.default_port or 0,
|
||||||
tool_type.interface_type,
|
tool_type.interface_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1966,23 +2008,22 @@ async def start_instance(
|
|||||||
# Create temporary Cloudflare tunnel for public access
|
# Create temporary Cloudflare tunnel for public access
|
||||||
try:
|
try:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Creating temporary tunnel for instance %s (container=%s, port=%d)",
|
"Creating tunnel for instance %s (container_port=%d)",
|
||||||
instance.id,
|
instance.id,
|
||||||
instance.container_name,
|
tool_type.default_port or 0,
|
||||||
instance_port,
|
|
||||||
)
|
)
|
||||||
tunnel_info = start_cloudflared_tunnel(
|
tunnel_info = start_tunnel(
|
||||||
container_name=instance.container_name or instance.name,
|
instance_name=instance.name,
|
||||||
port=instance_port,
|
container_port=tool_type.default_port or 0,
|
||||||
)
|
)
|
||||||
instance.tunnel_id = tunnel_info["pid"]
|
instance.tunnel_id = tunnel_info["container_name"]
|
||||||
instance.public_url = tunnel_info["url"]
|
instance.public_url = tunnel_info["url"]
|
||||||
instance.url = tunnel_info["url"]
|
instance.url = tunnel_info["url"]
|
||||||
await session.commit()
|
await session.commit()
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Created temporary tunnel for instance %s: pid=%s, url=%s",
|
"Created tunnel for instance %s: container=%s, url=%s",
|
||||||
instance.id,
|
instance.id,
|
||||||
tunnel_info["pid"],
|
tunnel_info["container_name"],
|
||||||
tunnel_info["url"],
|
tunnel_info["url"],
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2052,9 +2093,9 @@ async def stop_instance(
|
|||||||
# Stop Cloudflare tunnel if exists
|
# Stop Cloudflare tunnel if exists
|
||||||
if instance.tunnel_id:
|
if instance.tunnel_id:
|
||||||
try:
|
try:
|
||||||
stop_cloudflared_tunnel(instance.tunnel_id)
|
stop_tunnel(instance.name)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Stopped tunnel for instance %s (pid=%s)",
|
"Stopped tunnel for instance %s (container=%s)",
|
||||||
instance.id,
|
instance.id,
|
||||||
instance.tunnel_id,
|
instance.tunnel_id,
|
||||||
)
|
)
|
||||||
@@ -2121,9 +2162,9 @@ async def restart_instance(
|
|||||||
# Stop old tunnel if exists
|
# Stop old tunnel if exists
|
||||||
if instance.tunnel_id:
|
if instance.tunnel_id:
|
||||||
try:
|
try:
|
||||||
stop_cloudflared_tunnel(instance.tunnel_id)
|
stop_tunnel(instance.name)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Stopped old tunnel for instance %s (pid=%s)",
|
"Stopped old tunnel for instance %s (container=%s)",
|
||||||
instance.id,
|
instance.id,
|
||||||
instance.tunnel_id,
|
instance.tunnel_id,
|
||||||
)
|
)
|
||||||
@@ -2166,6 +2207,7 @@ async def restart_instance(
|
|||||||
instance.compose_path, tool_type.name, tool_type.default_port
|
instance.compose_path, tool_type.name, tool_type.default_port
|
||||||
)
|
)
|
||||||
_ensure_container_name_in_compose(instance.compose_path, instance.name)
|
_ensure_container_name_in_compose(instance.compose_path, instance.name)
|
||||||
|
_ensure_backend_network_in_compose(instance.compose_path)
|
||||||
|
|
||||||
returncode, stdout, stderr = execute_compose_command(
|
returncode, stdout, stderr = execute_compose_command(
|
||||||
instance.compose_path, "restart"
|
instance.compose_path, "restart"
|
||||||
@@ -2189,17 +2231,15 @@ async def restart_instance(
|
|||||||
"error": f"Tool type '{tool_type.name if tool_type else 'unknown'}' has no port configured",
|
"error": f"Tool type '{tool_type.name if tool_type else 'unknown'}' has no port configured",
|
||||||
}
|
}
|
||||||
|
|
||||||
instance_port = tool_type.default_port
|
|
||||||
|
|
||||||
# Only create tunnel for web-enabled tools
|
# Only create tunnel for web-enabled tools
|
||||||
if tool_type.interface_type == "web":
|
if tool_type.interface_type == "web":
|
||||||
# Create new temporary tunnel
|
# Create new tunnel
|
||||||
try:
|
try:
|
||||||
tunnel_info = start_cloudflared_tunnel(
|
tunnel_info = start_tunnel(
|
||||||
container_name=instance.name.lower(),
|
instance_name=instance.name,
|
||||||
port=instance_port,
|
container_port=tool_type.default_port or 0,
|
||||||
)
|
)
|
||||||
instance.tunnel_id = tunnel_info["pid"]
|
instance.tunnel_id = tunnel_info["container_name"]
|
||||||
instance.public_url = tunnel_info["url"]
|
instance.public_url = tunnel_info["url"]
|
||||||
instance.url = tunnel_info["url"]
|
instance.url = tunnel_info["url"]
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -2298,9 +2338,9 @@ async def delete_instance(
|
|||||||
# Stop Cloudflare tunnel if exists
|
# Stop Cloudflare tunnel if exists
|
||||||
if instance.tunnel_id:
|
if instance.tunnel_id:
|
||||||
try:
|
try:
|
||||||
stop_cloudflared_tunnel(instance.tunnel_id)
|
stop_tunnel(instance.name)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Stopped tunnel for instance %s (pid=%s)",
|
"Stopped tunnel for instance %s (container=%s)",
|
||||||
instance.id,
|
instance.id,
|
||||||
instance.tunnel_id,
|
instance.tunnel_id,
|
||||||
)
|
)
|
||||||
@@ -2415,43 +2455,117 @@ async def recreate_tunnel_endpoint(
|
|||||||
detail="instance must be running to recreate tunnel",
|
detail="instance must be running to recreate tunnel",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Validate tunnel is actually broken before recreating
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||||
if instance.url:
|
if not tool_type:
|
||||||
tunnel_health = check_tunnel_health(instance.url)
|
raise HTTPException(
|
||||||
if tunnel_health["tunnel_status"] == "error_response":
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="Tool type not found for this instance",
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_name = instance.name.lower()
|
||||||
|
logger.info(
|
||||||
|
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
|
||||||
|
instance.id,
|
||||||
|
expected_name,
|
||||||
|
tool_type.default_port,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Find the tool container — try stored ID first, then fall back to name lookup
|
||||||
|
tool_container_id = instance.container_id
|
||||||
|
if tool_container_id:
|
||||||
|
logger.info("Using stored container_id: %s", tool_container_id)
|
||||||
|
else:
|
||||||
|
tool_container_id = get_container_id(expected_name)
|
||||||
|
if tool_container_id:
|
||||||
|
logger.info("Found container by name: %s", tool_container_id)
|
||||||
|
else:
|
||||||
|
logger.error("Container %s not found", expected_name)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=f"Tunnel is working but application returned HTTP {tunnel_health.get('status_code')}. Recreating the tunnel will not fix this issue.",
|
detail="Could not find running container for this instance",
|
||||||
)
|
)
|
||||||
elif tunnel_health["tunnel_status"] == "healthy":
|
|
||||||
return {
|
|
||||||
"status": "healthy",
|
|
||||||
"url": instance.url,
|
|
||||||
"message": "Tunnel is already healthy",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Get tool type for default port
|
# Ensure the tool container is on the backend network so the tunnel can reach it
|
||||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
network_name = get_backend_network_name()
|
||||||
instance_port = (
|
on_network = is_container_on_network(tool_container_id, network_name)
|
||||||
tool_type.default_port if tool_type and tool_type.default_port else 8080
|
logger.info(
|
||||||
|
"Container %s on network %s: %s",
|
||||||
|
tool_container_id,
|
||||||
|
network_name,
|
||||||
|
on_network,
|
||||||
)
|
)
|
||||||
|
if not on_network:
|
||||||
|
logger.info(
|
||||||
|
"Connecting container %s to network %s",
|
||||||
|
tool_container_id,
|
||||||
|
network_name,
|
||||||
|
)
|
||||||
|
connected = connect_container_to_network(tool_container_id, network_name)
|
||||||
|
logger.info("Network connect result: %s", connected)
|
||||||
|
|
||||||
|
# Get the container's IP on the backend network
|
||||||
|
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
||||||
|
if target_ip:
|
||||||
|
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
||||||
|
logger.info(
|
||||||
|
"Tunnel target for instance %s: %s (IP %s on %s)",
|
||||||
|
instance.id,
|
||||||
|
target_url,
|
||||||
|
target_ip,
|
||||||
|
network_name,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
|
||||||
|
logger.warning(
|
||||||
|
"Could not get container IP, falling back to name-based target: %s",
|
||||||
|
target_url,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
tunnel_info = recreate_tunnel(
|
tunnel_info = recreate_tunnel(
|
||||||
container_name=instance.container_name or instance.name,
|
instance_name=instance.name,
|
||||||
port=instance_port,
|
container_port=tool_type.default_port or 0,
|
||||||
old_pid=instance.tunnel_id,
|
target_url=target_url,
|
||||||
)
|
)
|
||||||
instance.tunnel_id = tunnel_info["pid"]
|
logger.info(
|
||||||
|
"Tunnel recreated: container=%s, url=%s",
|
||||||
|
tunnel_info["container_name"],
|
||||||
|
tunnel_info["url"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Verify the tunnel can actually reach the origin
|
||||||
|
health = check_tunnel_health(tunnel_info["url"], timeout=10)
|
||||||
|
logger.info(
|
||||||
|
"Tunnel health check: status=%s, code=%s, error=%s",
|
||||||
|
health.get("tunnel_status"),
|
||||||
|
health.get("status_code"),
|
||||||
|
health.get("error"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Also probe from inside the API container directly to the target
|
||||||
|
probe = subprocess.run(
|
||||||
|
[
|
||||||
|
"curl",
|
||||||
|
"-s",
|
||||||
|
"-o",
|
||||||
|
"/dev/null",
|
||||||
|
"-w",
|
||||||
|
"%{http_code}",
|
||||||
|
"--max-time",
|
||||||
|
"5",
|
||||||
|
target_url,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
instance.tunnel_id = tunnel_info["container_name"]
|
||||||
instance.public_url = tunnel_info["url"]
|
instance.public_url = tunnel_info["url"]
|
||||||
instance.url = tunnel_info["url"]
|
instance.url = tunnel_info["url"]
|
||||||
await session.commit()
|
await session.commit()
|
||||||
logger.debug(
|
|
||||||
"Recreated tunnel for instance %s: pid=%s, url=%s",
|
|
||||||
instance.id,
|
|
||||||
tunnel_info["pid"],
|
|
||||||
tunnel_info["url"],
|
|
||||||
)
|
|
||||||
return {"status": "healthy", "url": instance.url}
|
return {"status": "healthy", "url": instance.url}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
||||||
|
|||||||
+124
-357
@@ -1,8 +1,6 @@
|
|||||||
"""Docker service for managing tool instances."""
|
"""Docker service for managing tool instances."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
@@ -181,69 +179,113 @@ def execute_compose_command(
|
|||||||
def get_container_id(instance_name: str) -> str | None:
|
def get_container_id(instance_name: str) -> str | None:
|
||||||
"""Get the container ID for a compose service.
|
"""Get the container ID for a compose service.
|
||||||
|
|
||||||
Searches all containers including stopped/exited ones.
|
Uses exact name matching to avoid substring collisions with tunnel
|
||||||
|
containers (e.g. tunnel-code-server-... matching code-server-...).
|
||||||
|
Falls back to case-insensitive matching since Docker DNS is case-
|
||||||
|
insensitive but docker inspect is case-sensitive.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
instance_name: The service name in compose
|
instance_name: The expected container name.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Container ID or None if not found
|
Container ID or None if not found.
|
||||||
"""
|
"""
|
||||||
# Docker container names are lowercase internally; normalize to ensure match
|
expected = instance_name.lower()
|
||||||
|
|
||||||
|
# Fast path: exact match via docker inspect
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "ps", "-a", "-q", "--filter", f"name={instance_name.lower()}"],
|
["docker", "inspect", "-f", "{{.Id}}", expected],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
return result.stdout.strip().split("\n")[0]
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
# Fallback: list all containers and do case-insensitive exact match
|
||||||
|
ps_result = subprocess.run(
|
||||||
|
["docker", "ps", "-a", "--format", "{{.Names}}\t{{.ID}}"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if ps_result.returncode == 0:
|
||||||
|
for line in ps_result.stdout.strip().splitlines():
|
||||||
|
parts = line.split("\t")
|
||||||
|
if len(parts) == 2:
|
||||||
|
name, cid = parts
|
||||||
|
if name.lower() == expected:
|
||||||
|
return cid
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_container_name(instance_name: str) -> str | None:
|
def get_container_name(instance_name: str) -> str | None:
|
||||||
"""Get the full container name for a compose service.
|
"""Get the full container name for a compose service.
|
||||||
|
|
||||||
Searches all containers including stopped/exited ones.
|
Uses exact name matching via docker inspect to avoid substring collisions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
instance_name: The service name in compose
|
instance_name: The exact container name (case-insensitive for Docker).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Container name or None if not found
|
Container name or None if not found.
|
||||||
"""
|
"""
|
||||||
# Docker container names are lowercase internally; normalize to ensure match
|
result = subprocess.run(
|
||||||
|
["docker", "inspect", "-f", "{{.Name}}", instance_name.lower()],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
|
return result.stdout.strip().lstrip("/")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_backend_network_name() -> str:
|
||||||
|
"""Auto-detect the actual Docker network name for the backend network.
|
||||||
|
|
||||||
|
Docker Compose prefixes network names with the project directory name
|
||||||
|
(e.g. 'headquarter_backend' instead of 'backend'). We inspect the API
|
||||||
|
container itself to find the real network name it's connected to.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The actual Docker network name, or 'backend' as fallback.
|
||||||
|
"""
|
||||||
|
# Try to find the API container by its known name
|
||||||
|
api_container = "hq-api"
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
"ps",
|
"inspect",
|
||||||
"-a",
|
"-f",
|
||||||
"--format",
|
"{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}",
|
||||||
"{{.Names}}",
|
api_container,
|
||||||
"--filter",
|
|
||||||
f"name={instance_name.lower()}",
|
|
||||||
],
|
],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
return result.stdout.strip().split("\n")[0]
|
networks = result.stdout.strip().split()
|
||||||
return None
|
for net in networks:
|
||||||
|
if "backend" in net.lower():
|
||||||
|
return net
|
||||||
|
# API container is on some network — return the first one
|
||||||
|
return networks[0]
|
||||||
|
return "backend"
|
||||||
|
|
||||||
|
|
||||||
def connect_container_to_network(
|
def connect_container_to_network(
|
||||||
container_name: str, network_name: str = "backend"
|
container_name: str, network_name: str | None = None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Connect a Docker container to an existing network.
|
"""Connect a Docker container to an existing network.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
container_name: Name or ID of the container
|
container_name: Name or ID of the container
|
||||||
network_name: Name of the Docker network (default: backend)
|
network_name: Name of the Docker network. If None, auto-detects
|
||||||
|
from the API container's own network membership.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if successful, False otherwise
|
True if successful, False otherwise
|
||||||
"""
|
"""
|
||||||
|
if network_name is None:
|
||||||
|
network_name = get_backend_network_name()
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "network", "connect", network_name, container_name],
|
["docker", "network", "connect", network_name, container_name],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
@@ -252,6 +294,64 @@ def connect_container_to_network(
|
|||||||
return result.returncode == 0
|
return result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
def get_container_ip_on_network(
|
||||||
|
container_id: str, network_name: str | None = None
|
||||||
|
) -> str | None:
|
||||||
|
"""Get a container's IP address on a specific Docker network.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
container_id: Docker container ID or name.
|
||||||
|
network_name: Network name. If None, auto-detects from the API container.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
IP address string, or None if the container is not on that network.
|
||||||
|
"""
|
||||||
|
if network_name is None:
|
||||||
|
network_name = get_backend_network_name()
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"inspect",
|
||||||
|
"-f",
|
||||||
|
f"{{{{.NetworkSettings.Networks.{network_name}.IPAddress}}}}",
|
||||||
|
container_id,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
ip = result.stdout.strip()
|
||||||
|
if ip and ip != "<no value>":
|
||||||
|
return ip
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_container_on_network(container_id: str, network_name: str | None = None) -> bool:
|
||||||
|
"""Check whether a container is already attached to a Docker network.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
container_id: Docker container ID or name.
|
||||||
|
network_name: Network name. If None, auto-detects from the API container.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the container is on the network.
|
||||||
|
"""
|
||||||
|
if network_name is None:
|
||||||
|
network_name = get_backend_network_name()
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"inspect",
|
||||||
|
"-f",
|
||||||
|
f"{{{{.NetworkSettings.Networks.{network_name}}}}}",
|
||||||
|
container_id,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
return result.returncode == 0 and "<no value>" not in result.stdout
|
||||||
|
|
||||||
|
|
||||||
def get_container_status(container_id: str) -> dict[str, Any]:
|
def get_container_status(container_id: str) -> dict[str, Any]:
|
||||||
"""Get the status of a Docker container.
|
"""Get the status of a Docker container.
|
||||||
|
|
||||||
@@ -382,336 +482,3 @@ def find_free_port(start: int = 10000, end: int = 20000) -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
raise RuntimeError(f"No free port found in range {start}-{end}")
|
raise RuntimeError(f"No free port found in range {start}-{end}")
|
||||||
|
|
||||||
|
|
||||||
def _check_app_binding(container_name: str, port: int) -> dict[str, str | bool]:
|
|
||||||
"""Diagnose whether the app is bound to 127.0.0.1 or 0.0.0.0.
|
|
||||||
|
|
||||||
Checks from both inside the container (localhost) and outside
|
|
||||||
(via Docker network) to detect binding issues.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with 'internal_ok', 'external_ok', 'internal_status',
|
|
||||||
'external_status', and 'diagnosis'.
|
|
||||||
"""
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
result: dict[str, Any] = {
|
|
||||||
"internal_ok": False,
|
|
||||||
"external_ok": False,
|
|
||||||
"internal_status": None,
|
|
||||||
"external_status": None,
|
|
||||||
"diagnosis": "unknown",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check from inside the container (loopback)
|
|
||||||
internal = subprocess.run(
|
|
||||||
[
|
|
||||||
"docker",
|
|
||||||
"exec",
|
|
||||||
container_name,
|
|
||||||
"sh",
|
|
||||||
"-c",
|
|
||||||
f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{port}",
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=5,
|
|
||||||
)
|
|
||||||
if internal.returncode == 0:
|
|
||||||
try:
|
|
||||||
result["internal_status"] = int(internal.stdout.strip())
|
|
||||||
result["internal_ok"] = result["internal_status"] > 0
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Check from outside the container (Docker network)
|
|
||||||
external = subprocess.run(
|
|
||||||
[
|
|
||||||
"curl",
|
|
||||||
"-s",
|
|
||||||
"-o",
|
|
||||||
"/dev/null",
|
|
||||||
"-w",
|
|
||||||
"%{http_code}",
|
|
||||||
f"http://{container_name}:{port}",
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=5,
|
|
||||||
)
|
|
||||||
if external.returncode == 0:
|
|
||||||
try:
|
|
||||||
result["external_status"] = int(external.stdout.strip())
|
|
||||||
result["external_ok"] = result["external_status"] > 0
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Diagnose binding issue
|
|
||||||
if result["internal_ok"] and not result["external_ok"]:
|
|
||||||
result["diagnosis"] = (
|
|
||||||
f"App appears to be bound to 127.0.0.1:{port} inside the container. "
|
|
||||||
f"It must bind to 0.0.0.0:{port} to be accessible from the tunnel."
|
|
||||||
)
|
|
||||||
elif result["internal_ok"] and result["external_ok"]:
|
|
||||||
result["diagnosis"] = "App is accessible on both interfaces."
|
|
||||||
elif not result["internal_ok"] and not result["external_ok"]:
|
|
||||||
result["diagnosis"] = f"App is not responding on port {port} at all."
|
|
||||||
else:
|
|
||||||
result["diagnosis"] = "Unexpected binding state."
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def start_cloudflared_tunnel(
|
|
||||||
container_name: str, port: int, timeout: int = 30
|
|
||||||
) -> dict[str, str]:
|
|
||||||
"""Start a temporary Cloudflare tunnel for a container.
|
|
||||||
|
|
||||||
Uses 'cloudflared tunnel --url' to create a temporary tunnel
|
|
||||||
with a random trycloudflare.com URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
container_name: Name of the Docker container to tunnel to
|
|
||||||
port: Port number the container listens on
|
|
||||||
timeout: Maximum seconds to wait for tunnel URL
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with 'url' (the public tunnel URL) and 'pid' (process ID)
|
|
||||||
"""
|
|
||||||
import subprocess
|
|
||||||
import logging
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# First verify the container is accessible from the Docker network
|
|
||||||
logger.info("Checking connectivity to %s:%d...", container_name, port)
|
|
||||||
accessible = False
|
|
||||||
last_status = None
|
|
||||||
for attempt in range(30): # 30 attempts × 1s = 30s max wait for app startup
|
|
||||||
check = subprocess.run(
|
|
||||||
[
|
|
||||||
"curl",
|
|
||||||
"-s",
|
|
||||||
"-o",
|
|
||||||
"/dev/null",
|
|
||||||
"-w",
|
|
||||||
"%{http_code}",
|
|
||||||
"--max-time",
|
|
||||||
"3",
|
|
||||||
f"http://{container_name}:{port}",
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=5,
|
|
||||||
)
|
|
||||||
status_str = check.stdout.strip()
|
|
||||||
logger.info(
|
|
||||||
"Connectivity check %d/%d: http_code=%s (rc=%d)",
|
|
||||||
attempt + 1,
|
|
||||||
30,
|
|
||||||
status_str,
|
|
||||||
check.returncode,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
last_status = int(status_str)
|
|
||||||
# Accept 2xx, 3xx, 401, 403 as "app is listening"
|
|
||||||
if last_status in (401, 403) or 200 <= last_status < 400:
|
|
||||||
accessible = True
|
|
||||||
logger.info(
|
|
||||||
"App on %s:%d is ready (HTTP %d)",
|
|
||||||
container_name,
|
|
||||||
port,
|
|
||||||
last_status,
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if check.returncode != 0:
|
|
||||||
logger.debug(
|
|
||||||
"curl failed: stderr=%s", check.stderr.strip() if check.stderr else ""
|
|
||||||
)
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
if not accessible:
|
|
||||||
logger.warning(
|
|
||||||
"Container %s:%d not responding after 30s (last status: %s). "
|
|
||||||
"Running binding diagnostics...",
|
|
||||||
container_name,
|
|
||||||
port,
|
|
||||||
last_status,
|
|
||||||
)
|
|
||||||
diagnosis = _check_app_binding(container_name, port)
|
|
||||||
logger.warning(
|
|
||||||
"Binding diagnosis: internal=%s (HTTP %s), external=%s (HTTP %s). %s",
|
|
||||||
diagnosis["internal_ok"],
|
|
||||||
diagnosis["internal_status"],
|
|
||||||
diagnosis["external_ok"],
|
|
||||||
diagnosis["external_status"],
|
|
||||||
diagnosis["diagnosis"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run cloudflared in background, capture output
|
|
||||||
logger.info("Starting cloudflared tunnel to http://%s:%d", container_name, port)
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
["cloudflared", "tunnel", "--url", f"http://{container_name}:{port}"],
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Wait for the URL to appear in output
|
|
||||||
url_pattern = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com")
|
|
||||||
start_time = time.time()
|
|
||||||
url = None
|
|
||||||
|
|
||||||
if proc.stdout is None:
|
|
||||||
proc.terminate()
|
|
||||||
proc.wait(timeout=5)
|
|
||||||
raise RuntimeError("Failed to capture cloudflared output")
|
|
||||||
|
|
||||||
while time.time() - start_time < timeout:
|
|
||||||
# Read available output
|
|
||||||
import select
|
|
||||||
|
|
||||||
readable, _, _ = select.select([proc.stdout], [], [], 1.0)
|
|
||||||
if readable:
|
|
||||||
line = proc.stdout.readline()
|
|
||||||
if line:
|
|
||||||
match = url_pattern.search(line)
|
|
||||||
if match:
|
|
||||||
url = match.group(0)
|
|
||||||
break
|
|
||||||
|
|
||||||
if not url:
|
|
||||||
proc.terminate()
|
|
||||||
proc.wait(timeout=5)
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Failed to get tunnel URL within {timeout}s. "
|
|
||||||
f"cloudflared output may contain errors."
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"url": url, "pid": str(proc.pid)}
|
|
||||||
|
|
||||||
|
|
||||||
def stop_cloudflared_tunnel(pid: str) -> None:
|
|
||||||
"""Stop a cloudflared tunnel process.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
pid: Process ID of the cloudflared tunnel
|
|
||||||
"""
|
|
||||||
import signal
|
|
||||||
|
|
||||||
try:
|
|
||||||
os.kill(int(pid), signal.SIGTERM)
|
|
||||||
except ProcessLookupError:
|
|
||||||
pass # Already stopped
|
|
||||||
|
|
||||||
|
|
||||||
def recreate_tunnel(
|
|
||||||
container_name: str, port: int, old_pid: str | None = None
|
|
||||||
) -> dict[str, str]:
|
|
||||||
"""Recreate a temporary Cloudflare tunnel.
|
|
||||||
|
|
||||||
Stops the old tunnel (if pid provided) and starts a new one.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
container_name: Name of the Docker container to tunnel to
|
|
||||||
port: Port number the container listens on
|
|
||||||
old_pid: Optional PID of the old tunnel process to stop
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with 'url' and 'pid' for the new tunnel
|
|
||||||
"""
|
|
||||||
if old_pid:
|
|
||||||
stop_cloudflared_tunnel(old_pid)
|
|
||||||
|
|
||||||
return start_cloudflared_tunnel(container_name, port)
|
|
||||||
|
|
||||||
|
|
||||||
def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
|
|
||||||
"""Check if a tunnel URL is healthy with smart error classification.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: The tunnel URL to check
|
|
||||||
timeout: Request timeout in seconds
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with 'tunnel_status' (healthy, unreachable, error_response, not_applicable),
|
|
||||||
'status_code' (int or None), 'healthy' (bool), and 'error' (str or None)
|
|
||||||
"""
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
[
|
|
||||||
"curl",
|
|
||||||
"-s",
|
|
||||||
"-o",
|
|
||||||
"/dev/null",
|
|
||||||
"-w",
|
|
||||||
"%{http_code}",
|
|
||||||
"--max-time",
|
|
||||||
str(timeout),
|
|
||||||
url,
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=timeout + 5,
|
|
||||||
)
|
|
||||||
status_code = int(result.stdout.strip())
|
|
||||||
|
|
||||||
if 200 <= status_code < 400:
|
|
||||||
return {
|
|
||||||
"tunnel_status": "healthy",
|
|
||||||
"status_code": status_code,
|
|
||||||
"healthy": True,
|
|
||||||
"error": None,
|
|
||||||
}
|
|
||||||
elif status_code in (502, 503, 504):
|
|
||||||
# Application error, not tunnel error
|
|
||||||
return {
|
|
||||||
"tunnel_status": "error_response",
|
|
||||||
"status_code": status_code,
|
|
||||||
"healthy": False,
|
|
||||||
"error": f"Application returned HTTP {status_code}",
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
return {
|
|
||||||
"tunnel_status": "error_response",
|
|
||||||
"status_code": status_code,
|
|
||||||
"healthy": False,
|
|
||||||
"error": f"HTTP {status_code}",
|
|
||||||
}
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
return {
|
|
||||||
"tunnel_status": "unreachable",
|
|
||||||
"status_code": None,
|
|
||||||
"healthy": False,
|
|
||||||
"error": "Tunnel request timed out",
|
|
||||||
}
|
|
||||||
except (ValueError, Exception) as e:
|
|
||||||
error_str = str(e).lower()
|
|
||||||
# Classify connection errors
|
|
||||||
if any(
|
|
||||||
err in error_str
|
|
||||||
for err in [
|
|
||||||
"connection refused",
|
|
||||||
"econnrefused",
|
|
||||||
"could not resolve",
|
|
||||||
"nodename",
|
|
||||||
]
|
|
||||||
):
|
|
||||||
return {
|
|
||||||
"tunnel_status": "unreachable",
|
|
||||||
"status_code": None,
|
|
||||||
"healthy": False,
|
|
||||||
"error": f"Tunnel unreachable: {e}",
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
"tunnel_status": "unreachable",
|
|
||||||
"status_code": None,
|
|
||||||
"healthy": False,
|
|
||||||
"error": str(e),
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ from src.database import SessionLocal
|
|||||||
from src.models.health_check import HealthCheck
|
from src.models.health_check import HealthCheck
|
||||||
from src.models.tool_instance import ToolInstance
|
from src.models.tool_instance import ToolInstance
|
||||||
from src.services.correlation import get_correlation_id
|
from src.services.correlation import get_correlation_id
|
||||||
from src.services.docker import check_tunnel_health, get_container_status
|
from src.services.docker import get_container_status
|
||||||
|
from src.services.tunnel import check_tunnel_health
|
||||||
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
from src.services.event_bus import InstanceEventBus, InstanceEventPayload
|
||||||
from src.services.notification_service import notification_service
|
from src.services.notification_service import notification_service
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,281 @@
|
|||||||
|
"""Clean tunnel service using cloudflared containers on the backend network.
|
||||||
|
|
||||||
|
Design:
|
||||||
|
- Each tunnel runs as a Docker container on the same 'backend' network as the API.
|
||||||
|
- cloudflared connects to the tool container by its Docker Compose service name
|
||||||
|
(e.g. http://code-server-headquarter-34837cd3:8443).
|
||||||
|
- This avoids host port conflicts and DNS resolution issues.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from src.services.docker import get_backend_network_name
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
TUNNEL_IMAGE = "cloudflare/cloudflared:latest"
|
||||||
|
|
||||||
|
|
||||||
|
def _tunnel_container_name(instance_name: str) -> str:
|
||||||
|
return f"tunnel-{instance_name.lower()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_image() -> None:
|
||||||
|
"""Pull cloudflared image if not already present."""
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "images", "-q", TUNNEL_IMAGE],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if not result.stdout.strip():
|
||||||
|
logger.info("Pulling %s ...", TUNNEL_IMAGE)
|
||||||
|
pull = subprocess.run(
|
||||||
|
["docker", "pull", TUNNEL_IMAGE],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if pull.returncode != 0:
|
||||||
|
logger.warning("Failed to pull %s: %s", TUNNEL_IMAGE, pull.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_stale_tunnel(tunnel_name: str) -> None:
|
||||||
|
"""Remove any existing tunnel container with this name."""
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "stop", "-t", "3", tunnel_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "-f", tunnel_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_container_logs(tunnel_name: str) -> tuple[str, str]:
|
||||||
|
"""Get stdout and stderr logs from a container."""
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "logs", tunnel_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
return result.stdout, result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def _get_container_exit_code(tunnel_name: str) -> int | None:
|
||||||
|
"""Get exit code of a container if it has exited."""
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "inspect", "-f", "{{.State.ExitCode}}", tunnel_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
try:
|
||||||
|
return int(result.stdout.strip())
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def start_tunnel(
|
||||||
|
instance_name: str,
|
||||||
|
container_port: int,
|
||||||
|
timeout: int = 30,
|
||||||
|
target_url: str | None = None,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
"""Start a temporary Cloudflare tunnel for an instance.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
instance_name: The tool instance name (used for tunnel naming).
|
||||||
|
container_port: The port the tool container listens on internally.
|
||||||
|
timeout: Seconds to wait for the tunnel URL.
|
||||||
|
target_url: Optional explicit URL to proxy to. If omitted, derives
|
||||||
|
http://{instance_name.lower()}:{container_port}.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'url' and 'container_name'.
|
||||||
|
"""
|
||||||
|
_ensure_image()
|
||||||
|
|
||||||
|
tunnel_name = _tunnel_container_name(instance_name)
|
||||||
|
_cleanup_stale_tunnel(tunnel_name)
|
||||||
|
|
||||||
|
# Target the tool container by name on the backend network
|
||||||
|
if target_url is None:
|
||||||
|
target_url = f"http://{instance_name.lower()}:{container_port}"
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--network",
|
||||||
|
get_backend_network_name(),
|
||||||
|
"--name",
|
||||||
|
tunnel_name,
|
||||||
|
TUNNEL_IMAGE,
|
||||||
|
"tunnel",
|
||||||
|
"--no-autoupdate",
|
||||||
|
"--url",
|
||||||
|
target_url,
|
||||||
|
]
|
||||||
|
|
||||||
|
logger.debug("Running: %s", " ".join(cmd))
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Failed to start tunnel container {tunnel_name}: {proc.stderr}"
|
||||||
|
)
|
||||||
|
|
||||||
|
container_id = proc.stdout.strip()
|
||||||
|
logger.debug("Tunnel container started: %s", container_id)
|
||||||
|
|
||||||
|
# Wait for URL to appear in logs
|
||||||
|
url_pattern = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com")
|
||||||
|
start_time = __import__("time").time()
|
||||||
|
url: str | None = None
|
||||||
|
combined_logs = ""
|
||||||
|
|
||||||
|
while __import__("time").time() - start_time < timeout:
|
||||||
|
stdout, stderr = _get_container_logs(tunnel_name)
|
||||||
|
combined_logs = stdout + "\n" + stderr
|
||||||
|
|
||||||
|
match = url_pattern.search(combined_logs)
|
||||||
|
if match:
|
||||||
|
url = match.group(0)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Check if container exited early
|
||||||
|
exit_code = _get_container_exit_code(tunnel_name)
|
||||||
|
if exit_code is not None and exit_code != 0:
|
||||||
|
_cleanup_stale_tunnel(tunnel_name)
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Tunnel container {tunnel_name} exited with code {exit_code}. "
|
||||||
|
f"Logs:\n{combined_logs[-3000:]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
__import__("time").sleep(0.5)
|
||||||
|
|
||||||
|
if not url:
|
||||||
|
stdout, stderr = _get_container_logs(tunnel_name)
|
||||||
|
combined_logs = stdout + "\n" + stderr
|
||||||
|
exit_code = _get_container_exit_code(tunnel_name)
|
||||||
|
|
||||||
|
_cleanup_stale_tunnel(tunnel_name)
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Tunnel {tunnel_name} did not produce a URL within {timeout}s. "
|
||||||
|
f"Exit code: {exit_code}. Logs:\n{combined_logs[-3000:]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Wait a moment for Cloudflare DNS edge to propagate the new tunnel subdomain
|
||||||
|
__import__("time").sleep(2)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Tunnel %s started for %s → %s (%s)",
|
||||||
|
tunnel_name,
|
||||||
|
instance_name,
|
||||||
|
target_url,
|
||||||
|
url,
|
||||||
|
)
|
||||||
|
return {"url": url, "container_name": tunnel_name}
|
||||||
|
|
||||||
|
|
||||||
|
def stop_tunnel(instance_name: str) -> None:
|
||||||
|
"""Stop and remove the tunnel container for an instance."""
|
||||||
|
tunnel_name = _tunnel_container_name(instance_name)
|
||||||
|
_cleanup_stale_tunnel(tunnel_name)
|
||||||
|
logger.debug("Stopped and removed tunnel container %s", tunnel_name)
|
||||||
|
|
||||||
|
|
||||||
|
def recreate_tunnel(
|
||||||
|
instance_name: str, container_port: int, target_url: str | None = None
|
||||||
|
) -> dict[str, str]:
|
||||||
|
"""Recreate a tunnel for an instance.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
instance_name: The tool instance name.
|
||||||
|
container_port: The port the tool container listens on internally.
|
||||||
|
target_url: Optional explicit origin URL. If omitted, derives
|
||||||
|
http://{instance_name.lower()}:{container_port}.
|
||||||
|
"""
|
||||||
|
stop_tunnel(instance_name)
|
||||||
|
return start_tunnel(instance_name, container_port, target_url=target_url)
|
||||||
|
|
||||||
|
|
||||||
|
def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
|
||||||
|
"""Check if a tunnel URL is healthy.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'tunnel_status', 'status_code', 'healthy', 'error'.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"curl",
|
||||||
|
"-s",
|
||||||
|
"-o",
|
||||||
|
"/dev/null",
|
||||||
|
"-w",
|
||||||
|
"%{http_code}",
|
||||||
|
"--max-time",
|
||||||
|
str(timeout),
|
||||||
|
url,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=timeout + 5,
|
||||||
|
)
|
||||||
|
status_code = int(result.stdout.strip())
|
||||||
|
|
||||||
|
if 200 <= status_code < 400:
|
||||||
|
return {
|
||||||
|
"tunnel_status": "healthy",
|
||||||
|
"status_code": status_code,
|
||||||
|
"healthy": True,
|
||||||
|
"error": None,
|
||||||
|
}
|
||||||
|
if status_code in (502, 503, 504):
|
||||||
|
return {
|
||||||
|
"tunnel_status": "error_response",
|
||||||
|
"status_code": status_code,
|
||||||
|
"healthy": False,
|
||||||
|
"error": f"Application returned HTTP {status_code}",
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"tunnel_status": "error_response",
|
||||||
|
"status_code": status_code,
|
||||||
|
"healthy": False,
|
||||||
|
"error": f"HTTP {status_code}",
|
||||||
|
}
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return {
|
||||||
|
"tunnel_status": "unreachable",
|
||||||
|
"status_code": None,
|
||||||
|
"healthy": False,
|
||||||
|
"error": "Tunnel request timed out",
|
||||||
|
}
|
||||||
|
except (ValueError, Exception) as exc:
|
||||||
|
error_str = str(exc).lower()
|
||||||
|
if any(
|
||||||
|
err in error_str
|
||||||
|
for err in [
|
||||||
|
"connection refused",
|
||||||
|
"econnrefused",
|
||||||
|
"could not resolve",
|
||||||
|
"nodename",
|
||||||
|
]
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"tunnel_status": "unreachable",
|
||||||
|
"status_code": None,
|
||||||
|
"healthy": False,
|
||||||
|
"error": f"Tunnel unreachable: {exc}",
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"tunnel_status": "unreachable",
|
||||||
|
"status_code": None,
|
||||||
|
"healthy": False,
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
@@ -229,12 +229,13 @@ export function SessionCard({
|
|||||||
</button>
|
</button>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{hasTunnelError && onRecreateTunnel && (
|
{!isTerminalOnly && onRecreateTunnel && (
|
||||||
<button
|
<button
|
||||||
className="secondary-button small"
|
className="ghost-button small"
|
||||||
onClick={() => onRecreateTunnel(session)}
|
onClick={() => onRecreateTunnel(session)}
|
||||||
type="button"
|
type="button"
|
||||||
disabled={isBusy}
|
disabled={isBusy}
|
||||||
|
title="Recreate Cloudflare tunnel"
|
||||||
>
|
>
|
||||||
<Icon name="refresh" size="sm" />
|
<Icon name="refresh" size="sm" />
|
||||||
<span className="action-label">Tunnel</span>
|
<span className="action-label">Tunnel</span>
|
||||||
@@ -323,7 +324,7 @@ export function SessionCard({
|
|||||||
onClose={() => setShowActionSheet(false)}
|
onClose={() => setShowActionSheet(false)}
|
||||||
title={session.display_name}
|
title={session.display_name}
|
||||||
actions={[
|
actions={[
|
||||||
...(isActive && hasTunnelError && onRecreateTunnel
|
...(isActive && !isTerminalOnly && onRecreateTunnel
|
||||||
? [
|
? [
|
||||||
{
|
{
|
||||||
id: "tunnel",
|
id: "tunnel",
|
||||||
|
|||||||
@@ -1,158 +1,187 @@
|
|||||||
import { useState, useCallback } from "react";
|
import { useState, useCallback } from "react";
|
||||||
import {
|
import {
|
||||||
stopInstance,
|
stopInstance,
|
||||||
deleteInstance,
|
deleteInstance,
|
||||||
startInstance,
|
startInstance,
|
||||||
recreateInstanceTunnel,
|
recreateInstanceTunnel,
|
||||||
} from "../api/sessions";
|
} from "../api/sessions";
|
||||||
import type { Session } from "../api/sessions";
|
import type { Session } from "../api/sessions";
|
||||||
|
|
||||||
interface UseInstanceActionsOptions {
|
interface UseInstanceActionsOptions {
|
||||||
onRefresh: () => Promise<void>;
|
onRefresh: () => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface UseInstanceActionsReturn {
|
interface UseInstanceActionsReturn {
|
||||||
loadingSessionId: string | null;
|
loadingSessionId: string | null;
|
||||||
dirtyDeleteSession: Session | null;
|
dirtyDeleteSession: Session | null;
|
||||||
dirtyDeleteFiles: string[];
|
dirtyDeleteFiles: string[];
|
||||||
handleOpen: (session: Session) => void;
|
handleOpen: (session: Session) => void;
|
||||||
handleStart: (session: Session) => Promise<void>;
|
handleStart: (session: Session) => Promise<void>;
|
||||||
handleStop: (session: Session) => Promise<void>;
|
handleStop: (session: Session) => Promise<void>;
|
||||||
handleDelete: (session: Session) => Promise<void>;
|
handleDelete: (session: Session) => Promise<void>;
|
||||||
handleForceDelete: (session: Session) => Promise<void>;
|
handleForceDelete: (session: Session) => Promise<void>;
|
||||||
handleRecreateTunnel: (session: Session) => Promise<void>;
|
handleRecreateTunnel: (session: Session) => Promise<void>;
|
||||||
clearDirtyDelete: () => void;
|
clearDirtyDelete: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useInstanceActions(
|
export function useInstanceActions(
|
||||||
options: UseInstanceActionsOptions
|
options: UseInstanceActionsOptions,
|
||||||
): UseInstanceActionsReturn {
|
): UseInstanceActionsReturn {
|
||||||
const { onRefresh } = options;
|
const { onRefresh } = options;
|
||||||
const [loadingSessionId, setLoadingSessionId] = useState<string | null>(null);
|
const [loadingSessionId, setLoadingSessionId] = useState<string | null>(null);
|
||||||
const [dirtyDeleteSession, setDirtyDeleteSession] = useState<Session | null>(null);
|
const [dirtyDeleteSession, setDirtyDeleteSession] = useState<Session | null>(
|
||||||
const [dirtyDeleteFiles, setDirtyDeleteFiles] = useState<string[]>([]);
|
null,
|
||||||
|
);
|
||||||
|
const [dirtyDeleteFiles, setDirtyDeleteFiles] = useState<string[]>([]);
|
||||||
|
|
||||||
const handleOpen = useCallback((session: Session) => {
|
const handleOpen = useCallback((session: Session) => {
|
||||||
if (session.url) {
|
if (session.url) {
|
||||||
window.open(session.url, "_blank", "noopener,noreferrer");
|
window.open(session.url, "_blank", "noopener,noreferrer");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (session.tool_type_interfaces?.includes("terminal")) {
|
if (session.tool_type_interfaces?.includes("terminal")) {
|
||||||
window.location.href = `/instances/${session.id}/terminal`;
|
window.location.href = `/instances/${session.id}/terminal`;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
window.location.href = `/projects/${session.project_id}`;
|
window.location.href = `/projects/${session.project_id}`;
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleStart = useCallback(
|
const handleStart = useCallback(
|
||||||
async (session: Session) => {
|
async (session: Session) => {
|
||||||
if (loadingSessionId === session.id) return;
|
if (loadingSessionId === session.id) return;
|
||||||
setLoadingSessionId(session.id);
|
setLoadingSessionId(session.id);
|
||||||
try {
|
try {
|
||||||
await startInstance(session.project_id, session.repository_id, session.id);
|
await startInstance(
|
||||||
await onRefresh();
|
session.project_id,
|
||||||
} catch {
|
session.repository_id,
|
||||||
// ignore
|
session.id,
|
||||||
} finally {
|
);
|
||||||
setLoadingSessionId(null);
|
await onRefresh();
|
||||||
}
|
} catch {
|
||||||
},
|
// ignore
|
||||||
[loadingSessionId, onRefresh]
|
} finally {
|
||||||
);
|
setLoadingSessionId(null);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[loadingSessionId, onRefresh],
|
||||||
|
);
|
||||||
|
|
||||||
const handleStop = useCallback(
|
const handleStop = useCallback(
|
||||||
async (session: Session) => {
|
async (session: Session) => {
|
||||||
if (loadingSessionId === session.id) return;
|
if (loadingSessionId === session.id) return;
|
||||||
setLoadingSessionId(session.id);
|
setLoadingSessionId(session.id);
|
||||||
try {
|
try {
|
||||||
await stopInstance(session.project_id, session.repository_id, session.id);
|
await stopInstance(
|
||||||
await onRefresh();
|
session.project_id,
|
||||||
} catch {
|
session.repository_id,
|
||||||
// ignore
|
session.id,
|
||||||
} finally {
|
);
|
||||||
setLoadingSessionId(null);
|
await onRefresh();
|
||||||
}
|
} catch {
|
||||||
},
|
// ignore
|
||||||
[loadingSessionId, onRefresh]
|
} finally {
|
||||||
);
|
setLoadingSessionId(null);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[loadingSessionId, onRefresh],
|
||||||
|
);
|
||||||
|
|
||||||
const handleDelete = useCallback(
|
const handleDelete = useCallback(
|
||||||
async (session: Session) => {
|
async (session: Session) => {
|
||||||
if (loadingSessionId === session.id) return;
|
if (loadingSessionId === session.id) return;
|
||||||
setLoadingSessionId(session.id);
|
setLoadingSessionId(session.id);
|
||||||
try {
|
try {
|
||||||
await deleteInstance(session.project_id, session.repository_id, session.id);
|
await deleteInstance(
|
||||||
setDirtyDeleteSession(null);
|
session.project_id,
|
||||||
setDirtyDeleteFiles([]);
|
session.repository_id,
|
||||||
await onRefresh();
|
session.id,
|
||||||
} catch (error) {
|
);
|
||||||
const axiosError = error as {
|
setDirtyDeleteSession(null);
|
||||||
response?: { status?: number; data?: { detail?: { changed_files?: string[] } } };
|
setDirtyDeleteFiles([]);
|
||||||
};
|
await onRefresh();
|
||||||
if (axiosError.response?.status === 409) {
|
} catch (error) {
|
||||||
const detail = axiosError.response.data?.detail;
|
const axiosError = error as {
|
||||||
if (detail?.changed_files) {
|
response?: {
|
||||||
setDirtyDeleteSession(session);
|
status?: number;
|
||||||
setDirtyDeleteFiles(detail.changed_files);
|
data?: { detail?: { changed_files?: string[] } };
|
||||||
return;
|
};
|
||||||
}
|
};
|
||||||
}
|
if (axiosError.response?.status === 409) {
|
||||||
} finally {
|
const detail = axiosError.response.data?.detail;
|
||||||
setLoadingSessionId(null);
|
if (detail?.changed_files) {
|
||||||
}
|
setDirtyDeleteSession(session);
|
||||||
},
|
setDirtyDeleteFiles(detail.changed_files);
|
||||||
[loadingSessionId, onRefresh]
|
return;
|
||||||
);
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
setLoadingSessionId(null);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[loadingSessionId, onRefresh],
|
||||||
|
);
|
||||||
|
|
||||||
const handleForceDelete = useCallback(
|
const handleForceDelete = useCallback(
|
||||||
async (session: Session) => {
|
async (session: Session) => {
|
||||||
if (loadingSessionId === session.id) return;
|
if (loadingSessionId === session.id) return;
|
||||||
setLoadingSessionId(session.id);
|
setLoadingSessionId(session.id);
|
||||||
try {
|
try {
|
||||||
await deleteInstance(session.project_id, session.repository_id, session.id, true);
|
await deleteInstance(
|
||||||
setDirtyDeleteSession(null);
|
session.project_id,
|
||||||
setDirtyDeleteFiles([]);
|
session.repository_id,
|
||||||
await onRefresh();
|
session.id,
|
||||||
} catch {
|
true,
|
||||||
// ignore
|
);
|
||||||
} finally {
|
setDirtyDeleteSession(null);
|
||||||
setLoadingSessionId(null);
|
setDirtyDeleteFiles([]);
|
||||||
}
|
await onRefresh();
|
||||||
},
|
} catch {
|
||||||
[loadingSessionId, onRefresh]
|
// ignore
|
||||||
);
|
} finally {
|
||||||
|
setLoadingSessionId(null);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[loadingSessionId, onRefresh],
|
||||||
|
);
|
||||||
|
|
||||||
const handleRecreateTunnel = useCallback(
|
const handleRecreateTunnel = useCallback(
|
||||||
async (session: Session) => {
|
async (session: Session) => {
|
||||||
if (loadingSessionId === session.id) return;
|
if (loadingSessionId === session.id) return;
|
||||||
setLoadingSessionId(session.id);
|
setLoadingSessionId(session.id);
|
||||||
try {
|
try {
|
||||||
await recreateInstanceTunnel(session.project_id, session.repository_id, session.id);
|
await recreateInstanceTunnel(
|
||||||
await onRefresh();
|
session.project_id,
|
||||||
} catch {
|
session.repository_id,
|
||||||
// ignore
|
session.id,
|
||||||
} finally {
|
);
|
||||||
setLoadingSessionId(null);
|
await onRefresh();
|
||||||
}
|
} catch (err) {
|
||||||
},
|
const message =
|
||||||
[loadingSessionId, onRefresh]
|
(err as { response?: { data?: { detail?: string } } })?.response?.data
|
||||||
);
|
?.detail || "Failed to recreate tunnel";
|
||||||
|
alert(message);
|
||||||
|
} finally {
|
||||||
|
setLoadingSessionId(null);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[loadingSessionId, onRefresh],
|
||||||
|
);
|
||||||
|
|
||||||
const clearDirtyDelete = useCallback(() => {
|
const clearDirtyDelete = useCallback(() => {
|
||||||
setDirtyDeleteSession(null);
|
setDirtyDeleteSession(null);
|
||||||
setDirtyDeleteFiles([]);
|
setDirtyDeleteFiles([]);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
loadingSessionId,
|
loadingSessionId,
|
||||||
dirtyDeleteSession,
|
dirtyDeleteSession,
|
||||||
dirtyDeleteFiles,
|
dirtyDeleteFiles,
|
||||||
handleOpen,
|
handleOpen,
|
||||||
handleStart,
|
handleStart,
|
||||||
handleStop,
|
handleStop,
|
||||||
handleDelete,
|
handleDelete,
|
||||||
handleForceDelete,
|
handleForceDelete,
|
||||||
handleRecreateTunnel,
|
handleRecreateTunnel,
|
||||||
clearDirtyDelete,
|
clearDirtyDelete,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ services:
|
|||||||
AUTHENTIK_AUTHORIZE_URL: ${AUTHENTIK_AUTHORIZE_URL:-}
|
AUTHENTIK_AUTHORIZE_URL: ${AUTHENTIK_AUTHORIZE_URL:-}
|
||||||
AUTHENTIK_TOKEN_URL: ${AUTHENTIK_TOKEN_URL:-}
|
AUTHENTIK_TOKEN_URL: ${AUTHENTIK_TOKEN_URL:-}
|
||||||
volumes:
|
volumes:
|
||||||
- repo_data:/data/repos
|
- /data/repos:/data/repos
|
||||||
- /data/instances:/data/instances
|
- /data/instances:/data/instances
|
||||||
- avatar_uploads:/app/uploads
|
- avatar_uploads:/app/uploads
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
@@ -116,7 +116,6 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
redis_data:
|
redis_data:
|
||||||
repo_data:
|
|
||||||
avatar_uploads:
|
avatar_uploads:
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
|
|||||||
+1
-2
@@ -57,7 +57,7 @@ services:
|
|||||||
REPO_BASE_PATH: /data/repos
|
REPO_BASE_PATH: /data/repos
|
||||||
INSTANCE_BASE_PATH: /data/instances
|
INSTANCE_BASE_PATH: /data/instances
|
||||||
volumes:
|
volumes:
|
||||||
- repo_data:/data/repos
|
- /data/repos:/data/repos
|
||||||
- /data/instances:/data/instances
|
- /data/instances:/data/instances
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "8000:8000"
|
||||||
@@ -91,7 +91,6 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
redis_data:
|
redis_data:
|
||||||
repo_data:
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
backend:
|
backend:
|
||||||
|
|||||||
Reference in New Issue
Block a user