"""High-level tool instance lifecycle orchestration. Coordinates Docker compose, container, tunnel, and config staging services to create, start, stop, restart, and delete tool instances. """ import logging import os import shutil from datetime import datetime from typing import Any from fastapi import HTTPException, status from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from src.models.config_folder import ConfigFolder from src.models.config_profile import ConfigProfile from src.models.git_repository import GitRepository from src.models.project import Project from src.models.tool_config import ToolConfig from src.models.tool_instance import ToolInstance from src.models.tool_type import ToolType from src.models.user import User from src.services.docker import compose as compose_svc from src.services.docker import config_staging from src.services.docker import container as container_svc from src.services.docker import tunnel as tunnel_svc from src.services.docker_build import build_image from src.services.readiness_probe import execute_probe logger = logging.getLogger(__name__) async def create_new_instance( session: AsyncSession, project: Project, repo: GitRepository, tool_type: ToolType, user: User, display_name: str | None, selected_profile: ConfigProfile | None, ) -> ToolInstance: """Create a new tool instance record and its compose file.""" instance_name = await compose_svc._generate_instance_name( session, project.name, tool_type.name ) instance_dir = compose_svc.ensure_instance_directory(instance_name) tool_port = container_svc.find_free_port() compose_path = await _build_or_render_compose( tool_type, instance_name, instance_dir, repo, user, project.id, tool_port ) instance = ToolInstance( name=instance_name, display_name=display_name or f"{project.name} / {repo.name} / {tool_type.display_name}", tool_type_id=tool_type.id, repository_id=repo.id, project_id=project.id, owner_id=user.id, status="pending", compose_path=compose_path, port=tool_port, selected_profile_id=selected_profile.id if selected_profile else None, ) session.add(instance) await session.commit() await session.refresh(instance) return instance async def start_existing_instance( session: AsyncSession, instance: ToolInstance, user: User, project_id: Any, ) -> dict: """Start an existing instance: stage configs, compose up, probe, tunnel.""" if not instance.compose_path or not os.path.exists(instance.compose_path): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="compose file not found" ) instance.status = "building" await session.commit() env_vars, config_files, port_override, start_command, working_directory, _extra_env, extra_volumes = await _fetch_tool_configs( session, user.id, instance.tool_type_id, project_id ) selected_profile = None if instance.selected_profile_id: selected_profile = await session.get(ConfigProfile, instance.selected_profile_id) if selected_profile and selected_profile.user_id == user.id: instance_dir = os.path.dirname(instance.compose_path) env_vars, port_override, start_command, working_directory, extra_volumes = await compose_svc._apply_resolved_profile( selected_profile, instance_dir, env_vars, port_override, start_command, working_directory, extra_volumes, ) env_file_path, extra_volumes = await _stage_configs_and_folders( session, user.id, project_id, os.path.dirname(instance.compose_path), env_vars, config_files, extra_volumes ) if port_override or start_command or working_directory or extra_volumes: compose_svc._modify_compose_file( instance.compose_path, port_override, start_command, working_directory, extra_volumes ) returncode, _stdout, stderr = compose_svc.execute_compose_command( instance.compose_path, "up", env_file=env_file_path ) if returncode != 0: instance.status = "error" await session.commit() raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"failed to start instance: {stderr}", ) container_id = container_svc.get_container_id(instance.name) if container_id: instance.container_id = container_id container_name = container_svc.get_container_name(instance.name) if container_name: instance.container_name = container_name container_svc.connect_container_to_network(container_name, "backend") instance.status = "starting" instance.last_started_at = datetime.now() await session.commit() tool_type = await session.get(ToolType, instance.tool_type_id) success, probe_logs = await _run_readiness_probe(instance, tool_type) if not success: instance.status = "failed" instance.url = None instance.public_url = None await session.commit() return { "status": "failed", "error": f"Readiness probe failed: {' '.join(probe_logs)}", } instance.status = "running" await session.commit() await _start_tunnel_if_web(instance, tool_type) await session.commit() return {"status": instance.status, "url": instance.url} async def restart_existing_instance( session: AsyncSession, instance: ToolInstance, user: User, project_id: Any, ) -> dict: """Restart an instance: re-stage configs, compose restart, tunnel.""" if instance.tunnel_id: try: tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id) except Exception as exc: logger.warning("Failed to stop old tunnel: %s", exc) if not instance.compose_path or not os.path.exists(instance.compose_path): instance.status = "error" await session.commit() return {"status": instance.status} env_vars, config_files, port_override, start_command, working_directory, _extra_env, extra_volumes = await _fetch_tool_configs( session, user.id, instance.tool_type_id, project_id ) stored_profile = None if instance.selected_profile_id: stored_profile = await session.get(ConfigProfile, instance.selected_profile_id) if stored_profile and stored_profile.user_id == user.id: instance_dir = os.path.dirname(instance.compose_path) env_vars, port_override, start_command, working_directory, extra_volumes = await compose_svc._apply_resolved_profile( stored_profile, instance_dir, env_vars, port_override, start_command, working_directory, extra_volumes ) env_file_path, extra_volumes = await _stage_configs_and_folders( session, user.id, project_id, os.path.dirname(instance.compose_path), env_vars, config_files, extra_volumes ) if port_override or start_command or working_directory or extra_volumes: compose_svc._modify_compose_file( instance.compose_path, port_override, start_command, working_directory, extra_volumes ) returncode, _stdout, _stderr = compose_svc.execute_compose_command( instance.compose_path, "restart", env_file=env_file_path ) if returncode != 0: instance.status = "error" await session.commit() return {"status": instance.status} instance.status = "running" instance.last_started_at = datetime.now() tool_type = await session.get(ToolType, instance.tool_type_id) await _start_tunnel_if_web(instance, tool_type) await session.commit() return {"status": instance.status, "url": instance.url} async def stop_existing_instance(session: AsyncSession, instance: ToolInstance) -> None: """Stop an instance and its tunnel.""" if instance.tunnel_id: try: tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id) except Exception as exc: logger.warning("Failed to stop tunnel: %s", exc) if instance.compose_path and os.path.exists(instance.compose_path): compose_svc.execute_compose_command(instance.compose_path, "stop") instance.status = "stopped" instance.last_stopped_at = datetime.now() instance.url = None instance.public_url = None instance.tunnel_id = None await session.commit() async def delete_existing_instance(session: AsyncSession, instance: ToolInstance) -> None: """Delete an instance, its containers, and its directory.""" if instance.tunnel_id: try: tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id) except Exception as exc: logger.warning("Failed to stop tunnel: %s", exc) if instance.compose_path and os.path.exists(instance.compose_path): compose_svc.execute_compose_command(instance.compose_path, "down") instance_dir = os.path.dirname(instance.compose_path) if os.path.exists(instance_dir): shutil.rmtree(instance_dir) await session.delete(instance) await session.commit() # ── Internal helpers ─────────────────────────────────────────────────────── async def _build_or_render_compose( tool_type: ToolType, instance_name: str, instance_dir: str, repo: GitRepository, user: User, project_id: Any, tool_port: int, ) -> str: """Build Dockerfile or render compose template.""" if tool_type.definition_type == "dockerfile": image_tag = f"headquarter/{instance_name}:latest" if tool_type.dockerfile_template: returncode, _stdout, stderr = build_image( instance_dir=instance_dir, dockerfile=tool_type.dockerfile_template, tag=image_tag, build_context=tool_type.build_context, ) if returncode != 0: logger.error("Build failed for %s: %s", instance_name, stderr) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to build Docker image: {stderr[:500]}", ) compose_content = ( f'version: "3.8"\nservices:\n app:\n' f' image: {image_tag}\n' f' container_name: {instance_name}\n' f' ports:\n - "{tool_port}:{tool_type.default_port}"\n' f' volumes:\n - {repo.path}:/workspace\n' f' restart: unless-stopped\n' ) else: variables = { "REPO_PATH": repo.path, "INSTANCE_NAME": instance_name, "INSTANCE_ID": instance_name, "TOOL_NAME": instance_name, "TOOL_PORT": tool_port, "USER_ID": str(user.id), "PROJECT_ID": str(project_id), } compose_content = compose_svc.render_compose_template( tool_type.compose_template, variables ) compose_svc.write_compose_file(instance_dir, compose_content) return os.path.join(instance_dir, "docker-compose.yml") async def _fetch_tool_configs( session: AsyncSession, user_id: Any, tool_type_id: Any, project_id: Any, ) -> tuple[dict, dict, Any, Any, Any, dict, list]: """Fetch tool configs and return parsed values.""" env_vars: dict[str, str] = {} config_files: dict[str, str] = {} port_override = None start_command = None working_directory = None extra_env_vars: dict[str, str] = {} extra_volumes: list[dict] = [] query = ( select(ToolConfig) .where(ToolConfig.user_id == user_id, ToolConfig.tool_type_id == tool_type_id) .where((ToolConfig.project_id == project_id) | (ToolConfig.project_id.is_(None))) ) configs = (await session.execute(query)).scalars().all() for cfg in configs: if cfg.config_type == "env": env_vars[cfg.key] = cfg.value elif cfg.config_type == "file" and cfg.file_path: config_files[cfg.file_path] = cfg.value if cfg.port_override: port_override = cfg.port_override if cfg.start_command: start_command = cfg.start_command if cfg.working_directory: working_directory = cfg.working_directory if cfg.environment_variables: extra_env_vars.update(cfg.environment_variables) if cfg.volumes: extra_volumes.extend(cfg.volumes) env_vars.update(extra_env_vars) return env_vars, config_files, port_override, start_command, working_directory, extra_env_vars, extra_volumes async def _stage_configs_and_folders( session: AsyncSession, user_id: Any, project_id: Any, instance_dir: str, env_vars: dict[str, str], config_files: dict[str, str], extra_volumes: list[dict], ) -> tuple[str | None, list[dict]]: """Write env/config files and config folders.""" env_file_path: str | None = None if env_vars: env_file_path = compose_svc.write_env_file(instance_dir, env_vars) if config_files: config_staging.write_config_files(instance_dir, config_files) folder_query = select(ConfigFolder).where( ConfigFolder.user_id == user_id, ConfigFolder.is_active.is_(True) ) folders = (await session.execute(folder_query)).scalars().all() if folders: folder_volumes = config_staging.write_config_folder_files( instance_dir, folders, str(project_id) ) extra_volumes.extend(folder_volumes) return env_file_path, extra_volumes async def _start_tunnel_if_web(instance: ToolInstance, tool_type: ToolType) -> None: """Create Cloudflare tunnel for web-enabled tools.""" if "web" not in tool_type.interfaces or not tool_type.default_port: instance.url = None instance.public_url = None return try: tunnel_info = tunnel_svc.start_cloudflared_tunnel( container_name=instance.container_name or instance.name, port=tool_type.default_port, ) instance.tunnel_id = tunnel_info["pid"] instance.public_url = tunnel_info["url"] instance.url = tunnel_info["url"] logger.info("Created tunnel for instance %s: %s", instance.id, tunnel_info["url"]) except Exception as exc: logger.error("Failed to create tunnel for instance %s: %s", instance.id, exc) instance.status = "error" instance.url = None async def _run_readiness_probe( instance: ToolInstance, tool_type: ToolType ) -> tuple[bool, list[str]]: """Run readiness probe if configured.""" if not tool_type.readiness_probe or not instance.container_id: return True, [] probe = tool_type.readiness_probe command = probe.get("command", "") if not command: return True, [] return await execute_probe( container_id=instance.container_id, command=command, timeout=probe.get("timeout", 30), interval=probe.get("interval", 2), )