Merge branch 'dev' of ssh://git.commumedia.org:2222/alex/headquarter into dev
This commit is contained in:
@@ -1834,7 +1834,8 @@ async def start_instance(
|
||||
instance_port,
|
||||
)
|
||||
tunnel_info = start_cloudflared_tunnel(
|
||||
host_port=instance.port,
|
||||
container_name=instance.container_name or instance.name,
|
||||
port=instance_port,
|
||||
)
|
||||
instance.tunnel_id = tunnel_info["pid"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
@@ -2041,12 +2042,15 @@ async def restart_instance(
|
||||
"error": f"Tool type '{tool_type.name if tool_type else 'unknown'}' has no port configured",
|
||||
}
|
||||
|
||||
instance_port = tool_type.default_port
|
||||
|
||||
# Only create tunnel for web-enabled tools
|
||||
if tool_type.interface_type == "web":
|
||||
# Create new temporary tunnel
|
||||
try:
|
||||
tunnel_info = start_cloudflared_tunnel(
|
||||
host_port=instance.port or 0,
|
||||
container_name=instance.container_name or instance.name,
|
||||
port=instance_port,
|
||||
)
|
||||
instance.tunnel_id = tunnel_info["pid"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
@@ -2279,9 +2283,16 @@ async def recreate_tunnel_endpoint(
|
||||
"message": "Tunnel is already healthy",
|
||||
}
|
||||
|
||||
# Get tool type for default port
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
instance_port = (
|
||||
tool_type.default_port if tool_type and tool_type.default_port else 8080
|
||||
)
|
||||
|
||||
try:
|
||||
tunnel_info = recreate_tunnel(
|
||||
host_port=instance.port or 0,
|
||||
container_name=instance.container_name or instance.name,
|
||||
port=instance_port,
|
||||
old_pid=instance.tunnel_id,
|
||||
)
|
||||
instance.tunnel_id = tunnel_info["pid"]
|
||||
|
||||
+140
-17
@@ -384,16 +384,98 @@ def find_free_port(start: int = 10000, end: int = 20000) -> int:
|
||||
raise RuntimeError(f"No free port found in range {start}-{end}")
|
||||
|
||||
|
||||
def _check_app_binding(
|
||||
container_name: str, port: int
|
||||
) -> dict[str, str | bool]:
|
||||
"""Diagnose whether the app is bound to 127.0.0.1 or 0.0.0.0.
|
||||
|
||||
Checks from both inside the container (localhost) and outside
|
||||
(via Docker network) to detect binding issues.
|
||||
|
||||
Returns:
|
||||
Dict with 'internal_ok', 'external_ok', 'internal_status',
|
||||
'external_status', and 'diagnosis'.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"internal_ok": False,
|
||||
"external_ok": False,
|
||||
"internal_status": None,
|
||||
"external_status": None,
|
||||
"diagnosis": "unknown",
|
||||
}
|
||||
|
||||
# Check from inside the container (loopback)
|
||||
internal = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
container_name,
|
||||
"sh",
|
||||
"-c",
|
||||
f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{port}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if internal.returncode == 0:
|
||||
try:
|
||||
result["internal_status"] = int(internal.stdout.strip())
|
||||
result["internal_ok"] = result["internal_status"] > 0
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Check from outside the container (Docker network)
|
||||
external = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-s",
|
||||
"-o",
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
f"http://{container_name}:{port}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if external.returncode == 0:
|
||||
try:
|
||||
result["external_status"] = int(external.stdout.strip())
|
||||
result["external_ok"] = result["external_status"] > 0
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Diagnose binding issue
|
||||
if result["internal_ok"] and not result["external_ok"]:
|
||||
result["diagnosis"] = (
|
||||
f"App appears to be bound to 127.0.0.1:{port} inside the container. "
|
||||
f"It must bind to 0.0.0.0:{port} to be accessible from the tunnel."
|
||||
)
|
||||
elif result["internal_ok"] and result["external_ok"]:
|
||||
result["diagnosis"] = "App is accessible on both interfaces."
|
||||
elif not result["internal_ok"] and not result["external_ok"]:
|
||||
result["diagnosis"] = f"App is not responding on port {port} at all."
|
||||
else:
|
||||
result["diagnosis"] = "Unexpected binding state."
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def start_cloudflared_tunnel(
|
||||
host_port: int, timeout: int = 30
|
||||
container_name: str, port: int, timeout: int = 30
|
||||
) -> dict[str, str]:
|
||||
"""Start a temporary Cloudflare tunnel to localhost.
|
||||
"""Start a temporary Cloudflare tunnel for a container.
|
||||
|
||||
Uses 'cloudflared tunnel --url' to create a temporary tunnel
|
||||
with a random trycloudflare.com URL.
|
||||
|
||||
Args:
|
||||
host_port: Host-mapped port number (e.g. from find_free_port)
|
||||
container_name: Name of the Docker container to tunnel to
|
||||
port: Port number the container listens on
|
||||
timeout: Maximum seconds to wait for tunnel URL
|
||||
|
||||
Returns:
|
||||
@@ -404,9 +486,11 @@ def start_cloudflared_tunnel(
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# First verify the container is accessible via the host-mapped port
|
||||
logger.info("Checking connectivity to localhost:%d...", host_port)
|
||||
for attempt in range(10):
|
||||
# First verify the container is accessible from the Docker network
|
||||
logger.info("Checking connectivity to %s:%d...", container_name, port)
|
||||
accessible = False
|
||||
last_status = None
|
||||
for attempt in range(30): # 30 attempts × 1s = 30s max wait for app startup
|
||||
check = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
@@ -415,27 +499,65 @@ def start_cloudflared_tunnel(
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
f"http://localhost:{host_port}",
|
||||
"--max-time",
|
||||
"3",
|
||||
f"http://{container_name}:{port}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
status_str = check.stdout.strip()
|
||||
logger.info(
|
||||
"Connectivity check %d: http_code=%s", attempt + 1, check.stdout.strip()
|
||||
"Connectivity check %d/%d: http_code=%s (rc=%d)",
|
||||
attempt + 1,
|
||||
30,
|
||||
status_str,
|
||||
check.returncode,
|
||||
)
|
||||
if check.returncode == 0:
|
||||
break
|
||||
try:
|
||||
last_status = int(status_str)
|
||||
# Accept 2xx, 3xx, 401, 403 as "app is listening"
|
||||
if last_status in (401, 403) or 200 <= last_status < 400:
|
||||
accessible = True
|
||||
logger.info(
|
||||
"App on %s:%d is ready (HTTP %d)",
|
||||
container_name,
|
||||
port,
|
||||
last_status,
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if check.returncode != 0:
|
||||
logger.debug(
|
||||
"curl failed: stderr=%s", check.stderr.strip() if check.stderr else ""
|
||||
)
|
||||
time.sleep(1)
|
||||
else:
|
||||
|
||||
if not accessible:
|
||||
logger.warning(
|
||||
"localhost:%d not responding to curl checks", host_port
|
||||
"Container %s:%d not responding after 30s (last status: %s). "
|
||||
"Running binding diagnostics...",
|
||||
container_name,
|
||||
port,
|
||||
last_status,
|
||||
)
|
||||
diagnosis = _check_app_binding(container_name, port)
|
||||
logger.warning(
|
||||
"Binding diagnosis: internal=%s (HTTP %s), external=%s (HTTP %s). %s",
|
||||
diagnosis["internal_ok"],
|
||||
diagnosis["internal_status"],
|
||||
diagnosis["external_ok"],
|
||||
diagnosis["external_status"],
|
||||
diagnosis["diagnosis"],
|
||||
)
|
||||
|
||||
# Run cloudflared in background, capture output
|
||||
logger.info("Starting cloudflared tunnel to http://localhost:%d", host_port)
|
||||
logger.info("Starting cloudflared tunnel to http://%s:%d", container_name, port)
|
||||
proc = subprocess.Popen(
|
||||
["cloudflared", "tunnel", "--url", f"http://localhost:{host_port}"],
|
||||
["cloudflared", "tunnel", "--url", f"http://{container_name}:{port}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
@@ -490,14 +612,15 @@ def stop_cloudflared_tunnel(pid: str) -> None:
|
||||
|
||||
|
||||
def recreate_tunnel(
|
||||
host_port: int, old_pid: str | None = None
|
||||
container_name: str, port: int, old_pid: str | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Recreate a temporary Cloudflare tunnel.
|
||||
|
||||
Stops the old tunnel (if pid provided) and starts a new one.
|
||||
|
||||
Args:
|
||||
host_port: Host-mapped port number (e.g. from find_free_port)
|
||||
container_name: Name of the Docker container to tunnel to
|
||||
port: Port number the container listens on
|
||||
old_pid: Optional PID of the old tunnel process to stop
|
||||
|
||||
Returns:
|
||||
@@ -506,7 +629,7 @@ def recreate_tunnel(
|
||||
if old_pid:
|
||||
stop_cloudflared_tunnel(old_pid)
|
||||
|
||||
return start_cloudflared_tunnel(host_port)
|
||||
return start_cloudflared_tunnel(container_name, port)
|
||||
|
||||
|
||||
def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
|
||||
|
||||
@@ -120,9 +120,9 @@ async def publish_lifecycle_event(
|
||||
# Create notification for instance owner (fire-and-forget)
|
||||
# Skip intermediate "starting" notifications — only notify on terminal states
|
||||
# (failed or successful attempts)
|
||||
_is_starting_intermediate = (
|
||||
event_type == "instance.started" and (status or instance.status) == "starting"
|
||||
)
|
||||
_is_starting_intermediate = event_type == "instance.started" and (
|
||||
status or instance.status
|
||||
) == "starting"
|
||||
if _is_starting_intermediate:
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user