refactor: split services/docker.py into focused modules (Task 3.3)
- Create services/docker/compose.py — compose file generation and commands - Create services/docker/container.py — container lifecycle and queries - Create services/docker/config_staging.py — config folder file writing - Create services/docker/tunnel.py — Cloudflare tunnel management - Create services/docker/__init__.py — barrel exports - Delete services/docker.py (replaced by package) - All imports in api/tool_instances.py remain functional Quality gates: Python syntax check (pass), imports verified Refs: repo-restructure Task 3.3
This commit is contained in:
Developer
@@ -0,0 +1,44 @@
|
||||
"""Docker services for container and tunnel management."""
|
||||
|
||||
from .compose import (
|
||||
ensure_instance_directory,
|
||||
execute_compose_command,
|
||||
render_compose_template,
|
||||
write_compose_file,
|
||||
write_env_file,
|
||||
)
|
||||
from .config_staging import write_config_files, write_config_folder_files
|
||||
from .container import (
|
||||
connect_container_to_network,
|
||||
find_free_port,
|
||||
get_container_id,
|
||||
get_container_logs,
|
||||
get_container_name,
|
||||
get_container_status,
|
||||
)
|
||||
from .tunnel import (
|
||||
check_tunnel_health,
|
||||
recreate_tunnel,
|
||||
start_cloudflared_tunnel,
|
||||
stop_cloudflared_tunnel,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"render_compose_template",
|
||||
"ensure_instance_directory",
|
||||
"write_compose_file",
|
||||
"write_env_file",
|
||||
"execute_compose_command",
|
||||
"write_config_files",
|
||||
"write_config_folder_files",
|
||||
"get_container_id",
|
||||
"get_container_name",
|
||||
"connect_container_to_network",
|
||||
"get_container_status",
|
||||
"get_container_logs",
|
||||
"find_free_port",
|
||||
"start_cloudflared_tunnel",
|
||||
"stop_cloudflared_tunnel",
|
||||
"recreate_tunnel",
|
||||
"check_tunnel_health",
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Docker Compose file generation and command execution."""
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def render_compose_template(template: str, variables: dict[str, Any]) -> str:
|
||||
"""Render a Docker Compose template with variable substitution.
|
||||
|
||||
Args:
|
||||
template: The compose template string
|
||||
variables: Dictionary of variable names to values
|
||||
|
||||
Returns:
|
||||
Rendered compose file content
|
||||
"""
|
||||
result = template
|
||||
for key, value in variables.items():
|
||||
placeholder = f"{{{{{key}}}}}"
|
||||
result = result.replace(placeholder, str(value))
|
||||
return result
|
||||
|
||||
|
||||
def ensure_instance_directory(instance_id: str, base_path: str | None = None) -> str:
|
||||
"""Create and return the instance directory path.
|
||||
|
||||
Args:
|
||||
instance_id: Unique instance identifier
|
||||
base_path: Base directory for all instances (defaults to Settings.instance_base_path)
|
||||
|
||||
Returns:
|
||||
Absolute path to instance directory
|
||||
"""
|
||||
if base_path is None:
|
||||
from src.config import Settings
|
||||
base_path = Settings().instance_base_path
|
||||
instance_dir = Path(base_path) / instance_id
|
||||
instance_dir.mkdir(parents=True, exist_ok=True)
|
||||
return str(instance_dir.absolute())
|
||||
|
||||
|
||||
def write_compose_file(instance_dir: str, content: str) -> str:
|
||||
"""Write the rendered compose file to the instance directory.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
content: Rendered compose content
|
||||
|
||||
Returns:
|
||||
Path to the compose file
|
||||
"""
|
||||
compose_path = Path(instance_dir) / "docker-compose.yml"
|
||||
compose_path.write_text(content)
|
||||
return str(compose_path)
|
||||
|
||||
|
||||
def write_env_file(instance_dir: str, env_vars: dict[str, str]) -> str:
|
||||
"""Write environment variables to a .env file.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
env_vars: Dictionary of env var names to values
|
||||
|
||||
Returns:
|
||||
Path to the env file
|
||||
"""
|
||||
env_path = Path(instance_dir) / ".env"
|
||||
lines = [f'{key}="{value}"' for key, value in env_vars.items()]
|
||||
env_path.write_text("\n".join(lines) + "\n")
|
||||
return str(env_path)
|
||||
|
||||
|
||||
def execute_compose_command(
|
||||
compose_path: str, action: str, timeout: int = 60, env_file: str | None = None
|
||||
) -> tuple[int, str, str]:
|
||||
"""Execute a docker compose command.
|
||||
|
||||
Args:
|
||||
compose_path: Path to docker-compose.yml
|
||||
action: The compose action (up, down, start, stop, restart)
|
||||
timeout: Command timeout in seconds
|
||||
env_file: Optional path to .env file for environment variables
|
||||
|
||||
Returns:
|
||||
Tuple of (returncode, stdout, stderr)
|
||||
"""
|
||||
instance_dir = Path(compose_path).parent
|
||||
|
||||
cmd = ["docker", "compose", "-f", compose_path]
|
||||
|
||||
if env_file:
|
||||
cmd.extend(["--env-file", env_file])
|
||||
|
||||
if action == "up":
|
||||
cmd.extend(["up", "-d"])
|
||||
elif action == "down":
|
||||
cmd.extend(["down", "-v"])
|
||||
elif action in ("start", "stop", "restart"):
|
||||
cmd.append(action)
|
||||
else:
|
||||
raise ValueError(f"Unknown compose action: {action}")
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(instance_dir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
return result.returncode, result.stdout, result.stderr
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Config folder file staging for Docker instances."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def write_config_files(instance_dir: str, files: dict[str, str]) -> None:
|
||||
"""Write config files to the instance directory.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
files: Dictionary of file paths (relative to instance dir) to content
|
||||
"""
|
||||
instance_path = Path(instance_dir)
|
||||
for file_path, content in files.items():
|
||||
# Ensure the path is within the instance directory (security)
|
||||
full_path = instance_path / file_path
|
||||
try:
|
||||
full_path.resolve().relative_to(instance_path.resolve())
|
||||
except ValueError:
|
||||
raise ValueError(f"File path '{file_path}' escapes instance directory")
|
||||
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content)
|
||||
|
||||
|
||||
def write_config_folder_files(instance_dir: str, folders: list, project_id: str | None = None) -> list[dict]:
|
||||
"""Write config folder files to the instance directory and return volume mounts.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
folders: List of ConfigFolder objects
|
||||
project_id: Optional project ID for applying overrides
|
||||
|
||||
Returns:
|
||||
List of volume mount dicts [{"source": "...", "target": "...", "type": "..."}]
|
||||
"""
|
||||
instance_path = Path(instance_dir)
|
||||
volume_mounts = []
|
||||
|
||||
for folder in folders:
|
||||
# Determine mount path (with project override if applicable)
|
||||
mount_path = folder.mount_path
|
||||
files = folder.files.copy()
|
||||
|
||||
if project_id and folder.project_overrides:
|
||||
override = folder.project_overrides.get(str(project_id))
|
||||
if override:
|
||||
if override.get("mount_path"):
|
||||
mount_path = override["mount_path"]
|
||||
if override.get("files"):
|
||||
files.update(override["files"])
|
||||
|
||||
# Write files to instance directory
|
||||
folder_dir = instance_path / "volumes" / folder.name
|
||||
folder_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for file_path, content in files.items():
|
||||
# Security: ensure path doesn't escape folder_dir
|
||||
full_path = folder_dir / file_path
|
||||
try:
|
||||
full_path.resolve().relative_to(folder_dir.resolve())
|
||||
except ValueError:
|
||||
logger.warning("Config folder file path escapes directory: %s", file_path)
|
||||
continue
|
||||
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content)
|
||||
|
||||
# Add volume mount
|
||||
volume_mounts.append({
|
||||
"source": str(folder_dir),
|
||||
"target": mount_path,
|
||||
"type": "bind",
|
||||
})
|
||||
|
||||
return volume_mounts
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Docker container lifecycle and query operations."""
|
||||
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
|
||||
def get_container_id(instance_name: str) -> str | None:
|
||||
"""Get the container ID for a compose service.
|
||||
|
||||
Args:
|
||||
instance_name: The service name in compose
|
||||
|
||||
Returns:
|
||||
Container ID or None if not found
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "-q", "--filter", f"name={instance_name}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip().split("\n")[0]
|
||||
return None
|
||||
|
||||
|
||||
def get_container_name(instance_name: str) -> str | None:
|
||||
"""Get the full container name for a compose service.
|
||||
|
||||
Args:
|
||||
instance_name: The service name in compose
|
||||
|
||||
Returns:
|
||||
Container name or None if not found
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "--format", "{{.Names}}", "--filter", f"name={instance_name}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip().split("\n")[0]
|
||||
return None
|
||||
|
||||
|
||||
def connect_container_to_network(container_name: str, network_name: str = "backend") -> bool:
|
||||
"""Connect a Docker container to an existing network.
|
||||
|
||||
Args:
|
||||
container_name: Name or ID of the container
|
||||
network_name: Name of the Docker network (default: backend)
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "network", "connect", network_name, container_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def get_container_status(container_id: str) -> str:
|
||||
"""Get the status of a Docker container.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID
|
||||
|
||||
Returns:
|
||||
Container status string (running, exited, etc.)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Status}}", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_container_logs(container_id: str, tail: int = 100) -> str:
|
||||
"""Get the logs of a Docker container.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID
|
||||
tail: Number of lines to return
|
||||
|
||||
Returns:
|
||||
Container logs
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "logs", "--tail", str(tail), container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return result.stdout
|
||||
return f"Failed to get logs: {result.stderr}"
|
||||
|
||||
|
||||
def find_free_port(start: int = 10000, end: int = 20000) -> int:
|
||||
"""Find a free TCP port in the given range.
|
||||
|
||||
Args:
|
||||
start: Start of port range
|
||||
end: End of port range
|
||||
|
||||
Returns:
|
||||
Free port number
|
||||
"""
|
||||
for port in range(start, end):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
if s.connect_ex(("localhost", port)) != 0:
|
||||
return port
|
||||
|
||||
raise RuntimeError(f"No free port found in range {start}-{end}")
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Cloudflare tunnel management for Docker instances."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def start_cloudflared_tunnel(
|
||||
container_name: str, port: int, timeout: int = 30
|
||||
) -> dict[str, str]:
|
||||
"""Start a temporary Cloudflare tunnel for a container.
|
||||
|
||||
Uses 'cloudflared tunnel --url' to create a temporary tunnel
|
||||
with a random trycloudflare.com URL.
|
||||
|
||||
Args:
|
||||
container_name: Name of the Docker container to tunnel to
|
||||
port: Port number the container listens on
|
||||
timeout: Maximum seconds to wait for tunnel URL
|
||||
|
||||
Returns:
|
||||
Dict with 'url' (the public tunnel URL) and 'pid' (process ID)
|
||||
"""
|
||||
import select as sel
|
||||
|
||||
# First verify the container is accessible
|
||||
logger.info("Checking connectivity to %s:%d...", container_name, port)
|
||||
for attempt in range(10):
|
||||
check = subprocess.run(
|
||||
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
|
||||
f"http://{container_name}:{port}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
logger.info("Connectivity check %d: http_code=%s", attempt + 1, check.stdout.strip())
|
||||
if check.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
logger.warning("Container %s:%d not responding to curl checks", container_name, port)
|
||||
|
||||
# Run cloudflared in background, capture output
|
||||
logger.info("Starting cloudflared tunnel to http://%s:%d", container_name, port)
|
||||
proc = subprocess.Popen(
|
||||
["cloudflared", "tunnel", "--url", f"http://{container_name}:{port}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
|
||||
# Wait for the URL to appear in output
|
||||
url_pattern = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com")
|
||||
start_time = time.time()
|
||||
url = None
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
# Read available output
|
||||
readable, _, _ = sel.select([proc.stdout], [], [], 1.0)
|
||||
if readable:
|
||||
line = proc.stdout.readline()
|
||||
if line:
|
||||
match = url_pattern.search(line)
|
||||
if match:
|
||||
url = match.group(0)
|
||||
break
|
||||
|
||||
if not url:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=5)
|
||||
raise RuntimeError(
|
||||
f"Failed to get tunnel URL within {timeout}s. "
|
||||
f"cloudflared output may contain errors."
|
||||
)
|
||||
|
||||
return {"url": url, "pid": str(proc.pid)}
|
||||
|
||||
|
||||
def stop_cloudflared_tunnel(pid: str) -> None:
|
||||
"""Stop a cloudflared tunnel process.
|
||||
|
||||
Args:
|
||||
pid: Process ID of the cloudflared tunnel
|
||||
"""
|
||||
try:
|
||||
os.kill(int(pid), signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass # Already stopped
|
||||
|
||||
|
||||
def recreate_tunnel(
|
||||
container_name: str, port: int, old_pid: str | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Recreate a temporary Cloudflare tunnel.
|
||||
|
||||
Stops the old tunnel (if pid provided) and starts a new one.
|
||||
|
||||
Args:
|
||||
container_name: Name of the Docker container to tunnel to
|
||||
port: Port number the container listens on
|
||||
old_pid: Optional PID of the old tunnel process to stop
|
||||
|
||||
Returns:
|
||||
Dict with 'url' and 'pid' for the new tunnel
|
||||
"""
|
||||
if old_pid:
|
||||
stop_cloudflared_tunnel(old_pid)
|
||||
|
||||
return start_cloudflared_tunnel(container_name, port)
|
||||
|
||||
|
||||
def check_tunnel_health(url: str, timeout: int = 10) -> dict[str, Any]:
|
||||
"""Check if a tunnel URL is healthy.
|
||||
|
||||
Args:
|
||||
url: The tunnel URL to check
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
Dict with 'healthy' (bool) and 'status_code' (int or None)
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
|
||||
"--max-time", str(timeout), url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout + 5,
|
||||
)
|
||||
status_code = int(result.stdout.strip())
|
||||
return {
|
||||
"healthy": 200 <= status_code < 400,
|
||||
"status_code": status_code,
|
||||
}
|
||||
except (ValueError, subprocess.TimeoutExpired, Exception) as e:
|
||||
return {
|
||||
"healthy": False,
|
||||
"status_code": None,
|
||||
"error": str(e),
|
||||
}
|
||||
Reference in New Issue
Block a user