351e76c00d
Docker container names are case-sensitive for 'docker inspect' but case- insensitive for Docker DNS. Compose templates may render container names with mixed case (e.g. code-server-Headquarter-abc123), causing exact-name docker inspect to fail while DNS resolution in tunnels works fine. - apps/api/src/services/docker.py: get_container_id now tries exact match first, then falls back to case-insensitive exact match via 'docker ps' - apps/api/src/api/tool_instances.py: recreate_tunnel_endpoint uses get_container_id instead of its own docker inspect call Quality gates: ruff clean
485 lines
14 KiB
Python
485 lines
14 KiB
Python
"""Docker service for managing tool instances."""
|
|
|
|
import logging
|
|
import subprocess
|
|
import time
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def sort_volumes_by_specificity(volumes: list[str]) -> list[str]:
|
|
"""Sort volume strings so parent paths come before child paths.
|
|
|
|
Docker Compose mounts volumes in array order. A later mount at a parent
|
|
path hides earlier mounts at child paths. By sorting shallow paths first
|
|
and deep paths last, deeper (more specific) mounts overlay correctly.
|
|
|
|
Volume format: source:target or source:target:type
|
|
|
|
Args:
|
|
volumes: List of Docker volume mount strings.
|
|
|
|
Returns:
|
|
Sorted list with parent paths before child paths.
|
|
"""
|
|
|
|
def _target_depth(vol: str) -> int:
|
|
parts = vol.split(":")
|
|
if len(parts) < 2:
|
|
return 0
|
|
target = parts[1].rstrip("/")
|
|
if not target or target == "/":
|
|
return 0
|
|
return target.count("/")
|
|
|
|
# Detect duplicate targets and warn
|
|
targets = []
|
|
for vol in volumes:
|
|
parts = vol.split(":")
|
|
targets.append(parts[1] if len(parts) > 1 else "")
|
|
dupes = [t for t, c in Counter(targets).items() if c > 1]
|
|
if dupes:
|
|
logger.warning("Duplicate mount targets detected: %s", dupes)
|
|
|
|
# Stable sort: parent paths first, child paths last
|
|
return sorted(volumes, key=_target_depth)
|
|
|
|
|
|
def render_compose_template(template: str, variables: dict[str, Any]) -> str:
|
|
"""Render a Docker Compose template with variable substitution.
|
|
|
|
Args:
|
|
template: The compose template string
|
|
variables: Dictionary of variable names to values
|
|
|
|
Returns:
|
|
Rendered compose file content
|
|
"""
|
|
result = template
|
|
for key, value in variables.items():
|
|
placeholder = f"{{{{{key}}}}}"
|
|
result = result.replace(placeholder, str(value))
|
|
return result
|
|
|
|
|
|
def ensure_instance_directory(instance_id: str, base_path: str | None = None) -> str:
|
|
"""Create and return the instance directory path.
|
|
|
|
Args:
|
|
instance_id: Unique instance identifier
|
|
base_path: Base directory for all instances (defaults to Settings.instance_base_path)
|
|
|
|
Returns:
|
|
Absolute path to instance directory
|
|
"""
|
|
if base_path is None:
|
|
from src.config import Settings
|
|
|
|
base_path = Settings().instance_base_path
|
|
instance_dir = Path(base_path) / instance_id
|
|
instance_dir.mkdir(parents=True, exist_ok=True)
|
|
return str(instance_dir.absolute())
|
|
|
|
|
|
def write_compose_file(instance_dir: str, content: str) -> str:
|
|
"""Write the rendered compose file to the instance directory.
|
|
|
|
Args:
|
|
instance_dir: Path to instance directory
|
|
content: Rendered compose content
|
|
|
|
Returns:
|
|
Path to the compose file
|
|
"""
|
|
compose_path = Path(instance_dir) / "docker-compose.yml"
|
|
compose_path.write_text(content)
|
|
return str(compose_path)
|
|
|
|
|
|
def write_env_file(instance_dir: str, env_vars: dict[str, str]) -> str:
|
|
"""Write environment variables to a .env file.
|
|
|
|
Args:
|
|
instance_dir: Path to instance directory
|
|
env_vars: Dictionary of env var names to values
|
|
|
|
Returns:
|
|
Path to the env file
|
|
"""
|
|
env_path = Path(instance_dir) / ".env"
|
|
lines = [f'{key}="{value}"' for key, value in env_vars.items()]
|
|
env_path.write_text("\n".join(lines) + "\n")
|
|
return str(env_path)
|
|
|
|
|
|
def write_config_files(instance_dir: str, files: dict[str, str]) -> None:
|
|
"""Write config files to the instance directory.
|
|
|
|
Args:
|
|
instance_dir: Path to instance directory
|
|
files: Dictionary of file paths (relative to instance dir) to content
|
|
"""
|
|
instance_path = Path(instance_dir)
|
|
for file_path, content in files.items():
|
|
# Ensure the path is within the instance directory (security)
|
|
full_path = instance_path / file_path
|
|
try:
|
|
full_path.resolve().relative_to(instance_path.resolve())
|
|
except ValueError:
|
|
raise ValueError(f"File path '{file_path}' escapes instance directory")
|
|
|
|
full_path.parent.mkdir(parents=True, exist_ok=True)
|
|
full_path.write_text(content)
|
|
|
|
|
|
def execute_compose_command(
|
|
compose_path: str, action: str, timeout: int = 60, env_file: str | None = None
|
|
) -> tuple[int, str, str]:
|
|
"""Execute a docker compose command.
|
|
|
|
Args:
|
|
compose_path: Path to docker-compose.yml
|
|
action: The compose action (up, down, start, stop, restart)
|
|
timeout: Command timeout in seconds
|
|
env_file: Optional path to .env file for environment variables
|
|
|
|
Returns:
|
|
Tuple of (returncode, stdout, stderr)
|
|
"""
|
|
instance_dir = Path(compose_path).parent
|
|
|
|
cmd = ["docker", "compose", "-f", compose_path]
|
|
|
|
if env_file:
|
|
cmd.extend(["--env-file", env_file])
|
|
|
|
if action == "up":
|
|
cmd.extend(["up", "-d", "--force-recreate"])
|
|
elif action == "down":
|
|
cmd.extend(["down", "-v"])
|
|
elif action in ("start", "stop", "restart"):
|
|
cmd.append(action)
|
|
else:
|
|
raise ValueError(f"Unknown compose action: {action}")
|
|
|
|
result = subprocess.run(
|
|
cmd,
|
|
cwd=str(instance_dir),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
return result.returncode, result.stdout, result.stderr
|
|
|
|
|
|
def get_container_id(instance_name: str) -> str | None:
|
|
"""Get the container ID for a compose service.
|
|
|
|
Uses exact name matching to avoid substring collisions with tunnel
|
|
containers (e.g. tunnel-code-server-... matching code-server-...).
|
|
Falls back to case-insensitive matching since Docker DNS is case-
|
|
insensitive but docker inspect is case-sensitive.
|
|
|
|
Args:
|
|
instance_name: The expected container name.
|
|
|
|
Returns:
|
|
Container ID or None if not found.
|
|
"""
|
|
expected = instance_name.lower()
|
|
|
|
# Fast path: exact match via docker inspect
|
|
result = subprocess.run(
|
|
["docker", "inspect", "-f", "{{.Id}}", expected],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip():
|
|
return result.stdout.strip()
|
|
|
|
# Fallback: list all containers and do case-insensitive exact match
|
|
ps_result = subprocess.run(
|
|
["docker", "ps", "-a", "--format", "{{.Names}}\t{{.ID}}"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if ps_result.returncode == 0:
|
|
for line in ps_result.stdout.strip().splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) == 2:
|
|
name, cid = parts
|
|
if name.lower() == expected:
|
|
return cid
|
|
return None
|
|
|
|
|
|
def get_container_name(instance_name: str) -> str | None:
|
|
"""Get the full container name for a compose service.
|
|
|
|
Uses exact name matching via docker inspect to avoid substring collisions.
|
|
|
|
Args:
|
|
instance_name: The exact container name (case-insensitive for Docker).
|
|
|
|
Returns:
|
|
Container name or None if not found.
|
|
"""
|
|
result = subprocess.run(
|
|
["docker", "inspect", "-f", "{{.Name}}", instance_name.lower()],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip():
|
|
return result.stdout.strip().lstrip("/")
|
|
return None
|
|
|
|
|
|
def get_backend_network_name() -> str:
|
|
"""Auto-detect the actual Docker network name for the backend network.
|
|
|
|
Docker Compose prefixes network names with the project directory name
|
|
(e.g. 'headquarter_backend' instead of 'backend'). We inspect the API
|
|
container itself to find the real network name it's connected to.
|
|
|
|
Returns:
|
|
The actual Docker network name, or 'backend' as fallback.
|
|
"""
|
|
# Try to find the API container by its known name
|
|
api_container = "hq-api"
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"-f",
|
|
"{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}",
|
|
api_container,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip():
|
|
networks = result.stdout.strip().split()
|
|
for net in networks:
|
|
if "backend" in net.lower():
|
|
return net
|
|
# API container is on some network — return the first one
|
|
return networks[0]
|
|
return "backend"
|
|
|
|
|
|
def connect_container_to_network(
|
|
container_name: str, network_name: str | None = None
|
|
) -> bool:
|
|
"""Connect a Docker container to an existing network.
|
|
|
|
Args:
|
|
container_name: Name or ID of the container
|
|
network_name: Name of the Docker network. If None, auto-detects
|
|
from the API container's own network membership.
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
if network_name is None:
|
|
network_name = get_backend_network_name()
|
|
result = subprocess.run(
|
|
["docker", "network", "connect", network_name, container_name],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return result.returncode == 0
|
|
|
|
|
|
def get_container_ip_on_network(
|
|
container_id: str, network_name: str | None = None
|
|
) -> str | None:
|
|
"""Get a container's IP address on a specific Docker network.
|
|
|
|
Args:
|
|
container_id: Docker container ID or name.
|
|
network_name: Network name. If None, auto-detects from the API container.
|
|
|
|
Returns:
|
|
IP address string, or None if the container is not on that network.
|
|
"""
|
|
if network_name is None:
|
|
network_name = get_backend_network_name()
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"-f",
|
|
f"{{{{.NetworkSettings.Networks.{network_name}.IPAddress}}}}",
|
|
container_id,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 0:
|
|
ip = result.stdout.strip()
|
|
if ip and ip != "<no value>":
|
|
return ip
|
|
return None
|
|
|
|
|
|
def is_container_on_network(container_id: str, network_name: str | None = None) -> bool:
|
|
"""Check whether a container is already attached to a Docker network.
|
|
|
|
Args:
|
|
container_id: Docker container ID or name.
|
|
network_name: Network name. If None, auto-detects from the API container.
|
|
|
|
Returns:
|
|
True if the container is on the network.
|
|
"""
|
|
if network_name is None:
|
|
network_name = get_backend_network_name()
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"-f",
|
|
f"{{{{.NetworkSettings.Networks.{network_name}}}}}",
|
|
container_id,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return result.returncode == 0 and "<no value>" not in result.stdout
|
|
|
|
|
|
def get_container_status(container_id: str) -> dict[str, Any]:
|
|
"""Get the status of a Docker container.
|
|
|
|
Args:
|
|
container_id: Docker container ID
|
|
|
|
Returns:
|
|
Dict with 'status' (running, exited, restarting, not_found),
|
|
'exit_code' (int or None), and 'health' (health status or None)
|
|
"""
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"-f",
|
|
"{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}",
|
|
container_id,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
return {"status": "not_found", "exit_code": None, "health": None}
|
|
|
|
parts = result.stdout.strip().split("|")
|
|
status = parts[0] if parts else "unknown"
|
|
exit_code = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
|
health = parts[2] if len(parts) > 2 and parts[2] != "none" else None
|
|
|
|
return {"status": status, "exit_code": exit_code, "health": health}
|
|
|
|
|
|
def wait_for_container_running(
|
|
container_id: str, timeout: int = 30, interval: float = 2.0
|
|
) -> dict[str, Any]:
|
|
"""Wait for a container to reach the running state.
|
|
|
|
Polls docker inspect until the container status is "running" or timeout.
|
|
|
|
Args:
|
|
container_id: Docker container ID
|
|
timeout: Maximum seconds to wait
|
|
interval: Seconds between polls
|
|
|
|
Returns:
|
|
Dict with 'success' (bool), 'status' (str), 'exit_code' (int or None),
|
|
and 'waited_seconds' (float)
|
|
"""
|
|
|
|
start_time = time.time()
|
|
|
|
while time.time() - start_time < timeout:
|
|
info = get_container_status(container_id)
|
|
|
|
if info["status"] == "running":
|
|
return {
|
|
"success": True,
|
|
"status": "running",
|
|
"exit_code": None,
|
|
"waited_seconds": time.time() - start_time,
|
|
}
|
|
|
|
if info["status"] == "exited":
|
|
return {
|
|
"success": False,
|
|
"status": "exited",
|
|
"exit_code": info["exit_code"],
|
|
"waited_seconds": time.time() - start_time,
|
|
}
|
|
|
|
if info["status"] == "not_found":
|
|
return {
|
|
"success": False,
|
|
"status": "not_found",
|
|
"exit_code": None,
|
|
"waited_seconds": time.time() - start_time,
|
|
}
|
|
|
|
time.sleep(interval)
|
|
|
|
# Timeout reached
|
|
info = get_container_status(container_id)
|
|
return {
|
|
"success": False,
|
|
"status": info["status"],
|
|
"exit_code": info["exit_code"],
|
|
"waited_seconds": time.time() - start_time,
|
|
}
|
|
|
|
|
|
def get_container_logs(container_id: str, tail: int = 100) -> str:
|
|
"""Get the logs of a Docker container.
|
|
|
|
Args:
|
|
container_id: Docker container ID
|
|
tail: Number of lines to return
|
|
|
|
Returns:
|
|
Container logs
|
|
"""
|
|
result = subprocess.run(
|
|
["docker", "logs", "--tail", str(tail), container_id],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
return result.stdout
|
|
return f"Failed to get logs: {result.stderr}"
|
|
|
|
|
|
def find_free_port(start: int = 10000, end: int = 20000) -> int:
|
|
"""Find a free TCP port in the given range.
|
|
|
|
Args:
|
|
start: Start of port range
|
|
end: End of port range
|
|
|
|
Returns:
|
|
Free port number
|
|
"""
|
|
import socket
|
|
|
|
for port in range(start, end):
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
if s.connect_ex(("localhost", port)) != 0:
|
|
return port
|
|
|
|
raise RuntimeError(f"No free port found in range {start}-{end}")
|