Developer
5ed5e1c84b
- Fix ProjectsPage tests by wrapping renders in MemoryRouter (9 passing) - Improve session auto-naming to 'project / repo / tool' format - Add missing /users/me/sessions endpoint for sidebar session loading - Handle git history 500s: catch RuntimeError in endpoints, graceful empty repo handling - Add git status badge and discard-changes button to FileEditor toolbar Quality gates: tsc pass, build pass, Python syntax pass
421 lines
15 KiB
Python
421 lines
15 KiB
Python
"""High-level tool instance lifecycle orchestration.
|
|
|
|
Coordinates Docker compose, container, tunnel, and config staging services
|
|
to create, start, stop, restart, and delete tool instances.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from src.models.config_folder import ConfigFolder
|
|
from src.models.config_profile import ConfigProfile
|
|
from src.models.git_repository import GitRepository
|
|
from src.models.project import Project
|
|
from src.models.tool_config import ToolConfig
|
|
from src.models.tool_instance import ToolInstance
|
|
from src.models.tool_type import ToolType
|
|
from src.models.user import User
|
|
from src.services.docker import compose as compose_svc
|
|
from src.services.docker import config_staging
|
|
from src.services.docker import container as container_svc
|
|
from src.services.docker import tunnel as tunnel_svc
|
|
from src.services.docker_build import build_image
|
|
from src.services.readiness_probe import execute_probe
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def create_new_instance(
|
|
session: AsyncSession,
|
|
project: Project,
|
|
repo: GitRepository,
|
|
tool_type: ToolType,
|
|
user: User,
|
|
display_name: str | None,
|
|
selected_profile: ConfigProfile | None,
|
|
) -> ToolInstance:
|
|
"""Create a new tool instance record and its compose file."""
|
|
instance_name = await compose_svc._generate_instance_name(
|
|
session, project.name, tool_type.name
|
|
)
|
|
instance_dir = compose_svc.ensure_instance_directory(instance_name)
|
|
tool_port = container_svc.find_free_port()
|
|
|
|
compose_path = await _build_or_render_compose(
|
|
tool_type, instance_name, instance_dir, repo, user, project.id, tool_port
|
|
)
|
|
|
|
instance = ToolInstance(
|
|
name=instance_name,
|
|
display_name=display_name or f"{project.name} / {repo.name} / {tool_type.display_name}",
|
|
tool_type_id=tool_type.id,
|
|
repository_id=repo.id,
|
|
project_id=project.id,
|
|
owner_id=user.id,
|
|
status="pending",
|
|
compose_path=compose_path,
|
|
port=tool_port,
|
|
selected_profile_id=selected_profile.id if selected_profile else None,
|
|
)
|
|
session.add(instance)
|
|
await session.commit()
|
|
await session.refresh(instance)
|
|
return instance
|
|
|
|
|
|
async def start_existing_instance(
|
|
session: AsyncSession,
|
|
instance: ToolInstance,
|
|
user: User,
|
|
project_id: Any,
|
|
) -> dict:
|
|
"""Start an existing instance: stage configs, compose up, probe, tunnel."""
|
|
if not instance.compose_path or not os.path.exists(instance.compose_path):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST, detail="compose file not found"
|
|
)
|
|
|
|
instance.status = "building"
|
|
await session.commit()
|
|
|
|
env_vars, config_files, port_override, start_command, working_directory, _extra_env, extra_volumes = await _fetch_tool_configs(
|
|
session, user.id, instance.tool_type_id, project_id
|
|
)
|
|
|
|
selected_profile = None
|
|
if instance.selected_profile_id:
|
|
selected_profile = await session.get(ConfigProfile, instance.selected_profile_id)
|
|
if selected_profile and selected_profile.user_id == user.id:
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
env_vars, port_override, start_command, working_directory, extra_volumes = await compose_svc._apply_resolved_profile(
|
|
selected_profile,
|
|
instance_dir,
|
|
env_vars,
|
|
port_override,
|
|
start_command,
|
|
working_directory,
|
|
extra_volumes,
|
|
)
|
|
|
|
env_file_path, extra_volumes = await _stage_configs_and_folders(
|
|
session, user.id, project_id, os.path.dirname(instance.compose_path),
|
|
env_vars, config_files, extra_volumes
|
|
)
|
|
|
|
if port_override or start_command or working_directory or extra_volumes:
|
|
compose_svc._modify_compose_file(
|
|
instance.compose_path, port_override, start_command, working_directory, extra_volumes
|
|
)
|
|
|
|
returncode, _stdout, stderr = compose_svc.execute_compose_command(
|
|
instance.compose_path, "up", env_file=env_file_path
|
|
)
|
|
if returncode != 0:
|
|
instance.status = "error"
|
|
await session.commit()
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"failed to start instance: {stderr}",
|
|
)
|
|
|
|
container_id = container_svc.get_container_id(instance.name)
|
|
if container_id:
|
|
instance.container_id = container_id
|
|
container_name = container_svc.get_container_name(instance.name)
|
|
if container_name:
|
|
instance.container_name = container_name
|
|
container_svc.connect_container_to_network(container_name, "backend")
|
|
|
|
instance.status = "starting"
|
|
instance.last_started_at = datetime.now()
|
|
await session.commit()
|
|
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
success, probe_logs = await _run_readiness_probe(instance, tool_type)
|
|
if not success:
|
|
instance.status = "failed"
|
|
instance.url = None
|
|
instance.public_url = None
|
|
await session.commit()
|
|
return {
|
|
"status": "failed",
|
|
"error": f"Readiness probe failed: {' '.join(probe_logs)}",
|
|
}
|
|
|
|
instance.status = "running"
|
|
await session.commit()
|
|
await _start_tunnel_if_web(instance, tool_type)
|
|
await session.commit()
|
|
|
|
return {"status": instance.status, "url": instance.url}
|
|
|
|
|
|
async def restart_existing_instance(
|
|
session: AsyncSession,
|
|
instance: ToolInstance,
|
|
user: User,
|
|
project_id: Any,
|
|
) -> dict:
|
|
"""Restart an instance: re-stage configs, compose restart, tunnel."""
|
|
if instance.tunnel_id:
|
|
try:
|
|
tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id)
|
|
except Exception as exc:
|
|
logger.warning("Failed to stop old tunnel: %s", exc)
|
|
|
|
if not instance.compose_path or not os.path.exists(instance.compose_path):
|
|
instance.status = "error"
|
|
await session.commit()
|
|
return {"status": instance.status}
|
|
|
|
env_vars, config_files, port_override, start_command, working_directory, _extra_env, extra_volumes = await _fetch_tool_configs(
|
|
session, user.id, instance.tool_type_id, project_id
|
|
)
|
|
|
|
stored_profile = None
|
|
if instance.selected_profile_id:
|
|
stored_profile = await session.get(ConfigProfile, instance.selected_profile_id)
|
|
if stored_profile and stored_profile.user_id == user.id:
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
env_vars, port_override, start_command, working_directory, extra_volumes = await compose_svc._apply_resolved_profile(
|
|
stored_profile, instance_dir, env_vars, port_override, start_command, working_directory, extra_volumes
|
|
)
|
|
|
|
env_file_path, extra_volumes = await _stage_configs_and_folders(
|
|
session, user.id, project_id, os.path.dirname(instance.compose_path),
|
|
env_vars, config_files, extra_volumes
|
|
)
|
|
|
|
if port_override or start_command or working_directory or extra_volumes:
|
|
compose_svc._modify_compose_file(
|
|
instance.compose_path, port_override, start_command, working_directory, extra_volumes
|
|
)
|
|
|
|
returncode, _stdout, _stderr = compose_svc.execute_compose_command(
|
|
instance.compose_path, "restart", env_file=env_file_path
|
|
)
|
|
if returncode != 0:
|
|
instance.status = "error"
|
|
await session.commit()
|
|
return {"status": instance.status}
|
|
|
|
instance.status = "running"
|
|
instance.last_started_at = datetime.now()
|
|
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
await _start_tunnel_if_web(instance, tool_type)
|
|
await session.commit()
|
|
|
|
return {"status": instance.status, "url": instance.url}
|
|
|
|
|
|
async def stop_existing_instance(session: AsyncSession, instance: ToolInstance) -> None:
|
|
"""Stop an instance and its tunnel."""
|
|
if instance.tunnel_id:
|
|
try:
|
|
tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id)
|
|
except Exception as exc:
|
|
logger.warning("Failed to stop tunnel: %s", exc)
|
|
|
|
if instance.compose_path and os.path.exists(instance.compose_path):
|
|
compose_svc.execute_compose_command(instance.compose_path, "stop")
|
|
|
|
instance.status = "stopped"
|
|
instance.last_stopped_at = datetime.now()
|
|
instance.url = None
|
|
instance.public_url = None
|
|
instance.tunnel_id = None
|
|
await session.commit()
|
|
|
|
|
|
async def delete_existing_instance(session: AsyncSession, instance: ToolInstance) -> None:
|
|
"""Delete an instance, its containers, and its directory."""
|
|
if instance.tunnel_id:
|
|
try:
|
|
tunnel_svc.stop_cloudflared_tunnel(instance.tunnel_id)
|
|
except Exception as exc:
|
|
logger.warning("Failed to stop tunnel: %s", exc)
|
|
|
|
if instance.compose_path and os.path.exists(instance.compose_path):
|
|
compose_svc.execute_compose_command(instance.compose_path, "down")
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
if os.path.exists(instance_dir):
|
|
shutil.rmtree(instance_dir)
|
|
|
|
await session.delete(instance)
|
|
await session.commit()
|
|
|
|
|
|
# ── Internal helpers ───────────────────────────────────────────────────────
|
|
|
|
async def _build_or_render_compose(
|
|
tool_type: ToolType,
|
|
instance_name: str,
|
|
instance_dir: str,
|
|
repo: GitRepository,
|
|
user: User,
|
|
project_id: Any,
|
|
tool_port: int,
|
|
) -> str:
|
|
"""Build Dockerfile or render compose template."""
|
|
if tool_type.definition_type == "dockerfile":
|
|
image_tag = f"headquarter/{instance_name}:latest"
|
|
if tool_type.dockerfile_template:
|
|
returncode, _stdout, stderr = build_image(
|
|
instance_dir=instance_dir,
|
|
dockerfile=tool_type.dockerfile_template,
|
|
tag=image_tag,
|
|
build_context=tool_type.build_context,
|
|
)
|
|
if returncode != 0:
|
|
logger.error("Build failed for %s: %s", instance_name, stderr)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Failed to build Docker image: {stderr[:500]}",
|
|
)
|
|
|
|
compose_content = (
|
|
f'version: "3.8"\nservices:\n app:\n'
|
|
f' image: {image_tag}\n'
|
|
f' container_name: {instance_name}\n'
|
|
f' ports:\n - "{tool_port}:{tool_type.default_port}"\n'
|
|
f' volumes:\n - {repo.path}:/workspace\n'
|
|
f' restart: unless-stopped\n'
|
|
)
|
|
else:
|
|
variables = {
|
|
"REPO_PATH": repo.path,
|
|
"INSTANCE_NAME": instance_name,
|
|
"INSTANCE_ID": instance_name,
|
|
"TOOL_NAME": instance_name,
|
|
"TOOL_PORT": tool_port,
|
|
"USER_ID": str(user.id),
|
|
"PROJECT_ID": str(project_id),
|
|
}
|
|
compose_content = compose_svc.render_compose_template(
|
|
tool_type.compose_template, variables
|
|
)
|
|
|
|
compose_svc.write_compose_file(instance_dir, compose_content)
|
|
return os.path.join(instance_dir, "docker-compose.yml")
|
|
|
|
|
|
async def _fetch_tool_configs(
|
|
session: AsyncSession,
|
|
user_id: Any,
|
|
tool_type_id: Any,
|
|
project_id: Any,
|
|
) -> tuple[dict, dict, Any, Any, Any, dict, list]:
|
|
"""Fetch tool configs and return parsed values."""
|
|
env_vars: dict[str, str] = {}
|
|
config_files: dict[str, str] = {}
|
|
port_override = None
|
|
start_command = None
|
|
working_directory = None
|
|
extra_env_vars: dict[str, str] = {}
|
|
extra_volumes: list[dict] = []
|
|
|
|
query = (
|
|
select(ToolConfig)
|
|
.where(ToolConfig.user_id == user_id, ToolConfig.tool_type_id == tool_type_id)
|
|
.where((ToolConfig.project_id == project_id) | (ToolConfig.project_id.is_(None)))
|
|
)
|
|
configs = (await session.execute(query)).scalars().all()
|
|
|
|
for cfg in configs:
|
|
if cfg.config_type == "env":
|
|
env_vars[cfg.key] = cfg.value
|
|
elif cfg.config_type == "file" and cfg.file_path:
|
|
config_files[cfg.file_path] = cfg.value
|
|
if cfg.port_override:
|
|
port_override = cfg.port_override
|
|
if cfg.start_command:
|
|
start_command = cfg.start_command
|
|
if cfg.working_directory:
|
|
working_directory = cfg.working_directory
|
|
if cfg.environment_variables:
|
|
extra_env_vars.update(cfg.environment_variables)
|
|
if cfg.volumes:
|
|
extra_volumes.extend(cfg.volumes)
|
|
|
|
env_vars.update(extra_env_vars)
|
|
return env_vars, config_files, port_override, start_command, working_directory, extra_env_vars, extra_volumes
|
|
|
|
|
|
async def _stage_configs_and_folders(
|
|
session: AsyncSession,
|
|
user_id: Any,
|
|
project_id: Any,
|
|
instance_dir: str,
|
|
env_vars: dict[str, str],
|
|
config_files: dict[str, str],
|
|
extra_volumes: list[dict],
|
|
) -> tuple[str | None, list[dict]]:
|
|
"""Write env/config files and config folders."""
|
|
env_file_path: str | None = None
|
|
if env_vars:
|
|
env_file_path = compose_svc.write_env_file(instance_dir, env_vars)
|
|
if config_files:
|
|
config_staging.write_config_files(instance_dir, config_files)
|
|
|
|
folder_query = select(ConfigFolder).where(
|
|
ConfigFolder.user_id == user_id, ConfigFolder.is_active.is_(True)
|
|
)
|
|
folders = (await session.execute(folder_query)).scalars().all()
|
|
if folders:
|
|
folder_volumes = config_staging.write_config_folder_files(
|
|
instance_dir, folders, str(project_id)
|
|
)
|
|
extra_volumes.extend(folder_volumes)
|
|
|
|
return env_file_path, extra_volumes
|
|
|
|
|
|
async def _start_tunnel_if_web(instance: ToolInstance, tool_type: ToolType) -> None:
|
|
"""Create Cloudflare tunnel for web-enabled tools."""
|
|
if "web" not in tool_type.interfaces or not tool_type.default_port:
|
|
instance.url = None
|
|
instance.public_url = None
|
|
return
|
|
|
|
try:
|
|
tunnel_info = tunnel_svc.start_cloudflared_tunnel(
|
|
container_name=instance.container_name or instance.name,
|
|
port=tool_type.default_port,
|
|
)
|
|
instance.tunnel_id = tunnel_info["pid"]
|
|
instance.public_url = tunnel_info["url"]
|
|
instance.url = tunnel_info["url"]
|
|
logger.info("Created tunnel for instance %s: %s", instance.id, tunnel_info["url"])
|
|
except Exception as exc:
|
|
logger.error("Failed to create tunnel for instance %s: %s", instance.id, exc)
|
|
instance.status = "error"
|
|
instance.url = None
|
|
|
|
|
|
async def _run_readiness_probe(
|
|
instance: ToolInstance, tool_type: ToolType
|
|
) -> tuple[bool, list[str]]:
|
|
"""Run readiness probe if configured."""
|
|
if not tool_type.readiness_probe or not instance.container_id:
|
|
return True, []
|
|
|
|
probe = tool_type.readiness_probe
|
|
command = probe.get("command", "")
|
|
if not command:
|
|
return True, []
|
|
|
|
return await execute_probe(
|
|
container_id=instance.container_id,
|
|
command=command,
|
|
timeout=probe.get("timeout", 30),
|
|
interval=probe.get("interval", 2),
|
|
)
|