alex
2247ec47c9
Use canonical profile files and writable Git working copies so editor and container changes share one source. Require confirmation before destructive Git refreshes and overlay profile files without composite snapshots.
2537 lines
88 KiB
Python
2537 lines
88 KiB
Python
"""Tool instance service functions."""
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import fcntl
|
|
import glob as glob_module
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import uuid
|
|
from datetime import datetime
|
|
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from src.models import (
|
|
ConfigProfile,
|
|
GitRepository,
|
|
Project,
|
|
SSHKey,
|
|
ToolInstance,
|
|
ToolType,
|
|
Workspace,
|
|
)
|
|
from src.schemas.tool import CreateInstanceRequest, StartInstanceRequest
|
|
from src.services.git.clone import check_dirty_state, clone_repository
|
|
from src.services.config.config_profile_resolver import (
|
|
ConfigProfileCycleError,
|
|
ResolvedProfile,
|
|
apply_resolved_profile,
|
|
expand_container_path,
|
|
resolve_profile,
|
|
)
|
|
from src.services.docker import (
|
|
connect_container_to_network,
|
|
ensure_instance_directory,
|
|
execute_compose_command,
|
|
find_free_port,
|
|
get_backend_network_name,
|
|
get_container_id,
|
|
get_container_ip_on_network,
|
|
get_container_logs,
|
|
get_container_status,
|
|
is_container_on_network,
|
|
render_compose_template,
|
|
sort_volumes_by_specificity,
|
|
wait_for_container_running,
|
|
write_compose_file,
|
|
write_config_files,
|
|
write_env_file,
|
|
)
|
|
from src.services.shared.tunnel import (
|
|
check_tunnel_health,
|
|
recreate_tunnel,
|
|
start_tunnel,
|
|
stop_tunnel,
|
|
)
|
|
from src.services.build.docker_build import build_image
|
|
from src.services.build.manifest_compiler import (
|
|
compile_compose,
|
|
compile_dockerfile,
|
|
compile_entrypoint,
|
|
compute_image_tag,
|
|
deep_merge,
|
|
get_manifest_home_dir,
|
|
merge_with_config,
|
|
resolve_base,
|
|
)
|
|
from src.services.shared.permission_fixer import (
|
|
apply_mount_permissions,
|
|
apply_ssh_permissions,
|
|
)
|
|
from src.services.shared.readiness_probe import execute_probe
|
|
from src.services.shared.ssh_keys import prepare_ssh_key_files
|
|
from src.services.instance.event_bus import InstanceEventBus
|
|
from src.services.instance.lifecycle_hooks import publish_lifecycle_event
|
|
|
|
logger = logging.getLogger(__name__)
|
|
_event_bus = InstanceEventBus()
|
|
|
|
|
|
def _slugify_directory_name(name: str) -> str:
|
|
"""Return a filesystem-safe, lowercase directory slug."""
|
|
slug = name.lower().strip()
|
|
slug = re.sub(r"[^a-z0-9_-]+", "-", slug)
|
|
slug = re.sub(r"-+", "-", slug).strip("-")
|
|
return slug or "project"
|
|
|
|
|
|
def _get_repository_mount_name(
|
|
project: Project,
|
|
repo: GitRepository,
|
|
workspace: "Workspace | None" = None,
|
|
) -> str:
|
|
"""Return the directory name the repository should appear under in the container.
|
|
|
|
The in-container layout is now ``/home/user/{project_name}/`` so that the
|
|
terminal starts directly in the project directory. The workspace path is
|
|
ignored for naming purposes; it only provides the source directory to
|
|
mount.
|
|
"""
|
|
return _slugify_directory_name(project.name)
|
|
|
|
|
|
def _chown_path(path: str, uid: int, gid: int) -> None:
|
|
"""Recursively chown a path, suppressing permission errors."""
|
|
try:
|
|
if os.path.isdir(path):
|
|
for root, dirs, files in os.walk(path):
|
|
for name in dirs + files:
|
|
full = os.path.join(root, name)
|
|
with contextlib.suppress(OSError):
|
|
os.chown(full, uid, gid)
|
|
with contextlib.suppress(OSError):
|
|
os.chown(path, uid, gid)
|
|
except Exception as exc:
|
|
logger.warning("Failed to chown %s to %s:%s: %s", path, uid, gid, exc)
|
|
|
|
|
|
def _chown_staged_mounts(
|
|
extra_volumes: list[dict],
|
|
instance_dir: str,
|
|
uid: int,
|
|
gid: int,
|
|
) -> None:
|
|
"""Recursively chown writable profile and instance mount sources.
|
|
|
|
Canonical non-Git profile sources are shared by compatible instances, so
|
|
they must be writable by the container user rather than copied per
|
|
instance. Instance-local composites and SSH mounts remain supported.
|
|
"""
|
|
canonical_profile_root = os.path.join(
|
|
os.path.dirname(instance_dir), "config-profiles"
|
|
)
|
|
for vol in extra_volumes:
|
|
source = vol.get("source", "")
|
|
if not source or not (
|
|
source.startswith(instance_dir) or source.startswith(canonical_profile_root)
|
|
):
|
|
continue
|
|
_chown_path(source, uid, gid)
|
|
|
|
|
|
def _relative_under(parent: str, child: str) -> str | None:
|
|
"""Return the relative path of ``child`` under ``parent`` if it is inside.
|
|
|
|
Returns ``""`` when the paths are equal. Returns ``None`` when ``child``
|
|
is not under ``parent``.
|
|
"""
|
|
parent = os.path.normpath(parent)
|
|
child = os.path.normpath(child)
|
|
if child == parent:
|
|
return ""
|
|
prefix = parent + os.sep
|
|
if child.startswith(prefix):
|
|
return child[len(prefix) :]
|
|
return None
|
|
|
|
|
|
def _stage_profile_mounts(profile_mounts: list[dict], instance_dir: str) -> list[dict]:
|
|
"""Copy profile bind sources into an instance-local, writable staging area."""
|
|
staged_mounts: list[dict] = []
|
|
staging_root = os.path.join(instance_dir, "mounts", "profiles")
|
|
|
|
for mount in profile_mounts:
|
|
source = mount.get("source", "")
|
|
target = mount.get("target", "")
|
|
if not source or not target:
|
|
continue
|
|
if not os.path.exists(source):
|
|
logger.warning("Skipping missing config profile mount source: %s", source)
|
|
continue
|
|
|
|
digest = hashlib.sha256(f"{source}\0{target}".encode()).hexdigest()[:16]
|
|
staged_source = os.path.join(staging_root, digest)
|
|
try:
|
|
if os.path.lexists(staged_source):
|
|
if os.path.isdir(staged_source):
|
|
shutil.rmtree(staged_source)
|
|
else:
|
|
os.unlink(staged_source)
|
|
os.makedirs(os.path.dirname(staged_source), exist_ok=True)
|
|
|
|
if os.path.isdir(source):
|
|
shutil.copytree(source, staged_source, symlinks=True)
|
|
else:
|
|
shutil.copy2(source, staged_source, follow_symlinks=False)
|
|
except OSError as exc:
|
|
logger.error("Failed to stage config profile mount %s: %s", source, exc)
|
|
continue
|
|
|
|
staged_mount = dict(mount)
|
|
staged_mount["source"] = staged_source
|
|
staged_mounts.append(staged_mount)
|
|
|
|
return staged_mounts
|
|
|
|
|
|
def _mounts_overlap(first_target: str, second_target: str) -> bool:
|
|
"""Return whether two normalized container mount targets intersect."""
|
|
return (
|
|
_relative_under(first_target, second_target) is not None
|
|
or _relative_under(second_target, first_target) is not None
|
|
)
|
|
|
|
|
|
def _copy_mount_source(source: str, destination: str) -> None:
|
|
"""Copy a bind-mount source into its destination in a composite tree."""
|
|
try:
|
|
if os.path.isdir(source):
|
|
os.makedirs(destination, exist_ok=True)
|
|
for entry in os.listdir(source):
|
|
source_entry = os.path.join(source, entry)
|
|
destination_entry = os.path.join(destination, entry)
|
|
if os.path.isdir(source_entry):
|
|
shutil.copytree(
|
|
source_entry,
|
|
destination_entry,
|
|
dirs_exist_ok=True,
|
|
symlinks=True,
|
|
)
|
|
else:
|
|
os.makedirs(os.path.dirname(destination_entry), exist_ok=True)
|
|
shutil.copy2(source_entry, destination_entry, follow_symlinks=False)
|
|
return
|
|
|
|
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
shutil.copy2(source, destination, follow_symlinks=False)
|
|
except OSError as exc:
|
|
raise RuntimeError(f"Unable to compose mount source {source}: {exc}") from exc
|
|
|
|
|
|
def _stack_profile_mounts_with_git_mounts(
|
|
profile_mounts: list[dict],
|
|
git_mount_volumes: list[dict],
|
|
instance_dir: str,
|
|
) -> list[dict]:
|
|
"""Overlay canonical profile files on Git directories without snapshots.
|
|
|
|
An overlapping profile directory is expanded into individual child-file
|
|
mounts. Docker then mounts the Git working directory first and the more
|
|
specific canonical profile files last, preserving shared writable sources
|
|
instead of constructing an instance-local composite copy.
|
|
"""
|
|
del instance_dir
|
|
result: list[dict] = list(git_mount_volumes)
|
|
|
|
for profile_mount in profile_mounts:
|
|
source = profile_mount.get("source", "")
|
|
target = profile_mount.get("target", "")
|
|
overlaps_git = any(
|
|
_mounts_overlap(target, git_mount.get("target", ""))
|
|
for git_mount in git_mount_volumes
|
|
)
|
|
if not overlaps_git or not os.path.isdir(source):
|
|
result.append(profile_mount)
|
|
continue
|
|
|
|
for root, _dirs, files in os.walk(source):
|
|
for filename in files:
|
|
file_source = os.path.join(root, filename)
|
|
relative_path = os.path.relpath(file_source, source)
|
|
result.append(
|
|
{
|
|
**profile_mount,
|
|
"source": file_source,
|
|
"target": os.path.join(target, relative_path),
|
|
}
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
async def resolve_git_mounts(
|
|
session: AsyncSession,
|
|
resolved: ResolvedProfile,
|
|
instance_dir: str | None = None,
|
|
working_directory: str | None = None,
|
|
home_dir: str = "/root",
|
|
) -> list[dict]:
|
|
"""Convert git mounts from resolved profile to Docker volume mounts.
|
|
|
|
Looks up repository paths, auto-clones if needed, handles branch checkout,
|
|
expands glob patterns, and prepares bind mount entries.
|
|
Logs warnings for missing repos or invalid paths (non-blocking).
|
|
"""
|
|
if not resolved.git_mounts:
|
|
return []
|
|
|
|
# Git mount sources are profile-scoped, not instance-scoped, so compatible
|
|
# instances bind the same canonical checkout.
|
|
clone_parent = os.path.join(
|
|
os.path.dirname(instance_dir or ""), "config-profiles", str(resolved.profile_id)
|
|
)
|
|
|
|
# Process all git mounts concurrently
|
|
tasks = []
|
|
for git_mount in resolved.git_mounts:
|
|
tasks.append(
|
|
resolve_single_git_mount(
|
|
session, git_mount, clone_parent, working_directory, home_dir
|
|
)
|
|
)
|
|
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
volume_mounts = []
|
|
for result in results:
|
|
if isinstance(result, Exception):
|
|
logger.warning("Git mount failed: %s", result)
|
|
continue
|
|
if isinstance(result, list):
|
|
volume_mounts.extend(result)
|
|
|
|
return volume_mounts
|
|
|
|
|
|
def normalize_git_mount(entry: dict) -> dict:
|
|
"""Normalize a git mount entry to the unified mappings format.
|
|
|
|
Converts legacy source_path + target_path into a single-entry mappings array.
|
|
"""
|
|
entry = dict(entry)
|
|
if "mappings" not in entry or not entry.get("mappings"):
|
|
source = entry.get("source_path", ".")
|
|
target = entry.get("target_path")
|
|
if target is not None:
|
|
entry["mappings"] = [{"source_path": source, "target_path": target}]
|
|
entry.pop("source_path", None)
|
|
entry.pop("target_path", None)
|
|
return entry
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def _git_mount_lock(clone_parent: str, remote_url: str, branch: str | None):
|
|
"""Serialize clone and refresh operations for one canonical Git source."""
|
|
lock_dir = os.path.join(clone_parent, "git-mounts")
|
|
identity = f"{remote_url}:{branch or 'default'}"
|
|
lock_path = os.path.join(
|
|
lock_dir, f".{hashlib.sha256(identity.encode()).hexdigest()}.lock"
|
|
)
|
|
try:
|
|
os.makedirs(lock_dir, exist_ok=True)
|
|
with open(lock_path, "a+", encoding="utf-8") as lock_file:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
except OSError as exc:
|
|
raise RuntimeError(f"Cannot lock Git mount source: {lock_path}") from exc
|
|
|
|
|
|
def clone_git_repo_locked(
|
|
remote_url: str,
|
|
branch: str | None,
|
|
clone_parent: str,
|
|
project_name: str | None = None,
|
|
) -> str:
|
|
"""Clone or refresh a canonical source while holding its process lock."""
|
|
with _git_mount_lock(clone_parent, remote_url, branch):
|
|
return clone_git_repo(remote_url, branch, clone_parent, project_name)
|
|
|
|
|
|
def clone_git_repo(
|
|
remote_url: str,
|
|
branch: str | None,
|
|
clone_parent: str,
|
|
project_name: str | None = None,
|
|
) -> str:
|
|
"""Clone or pull a git repository.
|
|
|
|
Returns the path to the cloned repo directory.
|
|
"""
|
|
import hashlib
|
|
|
|
# Include the branch in the hash so different branches of the same repo
|
|
# get separate clone directories and cannot race each other.
|
|
branch_segment = branch or "default"
|
|
url_hash = hashlib.md5(f"{remote_url}:{branch_segment}".encode()).hexdigest()[:12]
|
|
repo_name = remote_url.split("/")[-1].replace(".git", "") or "repo"
|
|
clone_dir = os.path.join(clone_parent, "git-mounts", f"{repo_name}-{url_hash}")
|
|
clone_name = _slugify_directory_name(project_name) if project_name else "repo-clone"
|
|
repo_path = os.path.join(clone_dir, clone_name)
|
|
|
|
if os.path.isdir(os.path.join(repo_path, ".git")):
|
|
# Reuse the deterministic per-repository cache on repeated starts.
|
|
try:
|
|
pull_repository_updates(repo_path, remote_url)
|
|
logger.debug("Pulled updates for git mount %s", remote_url)
|
|
except Exception as exc:
|
|
logger.warning("Failed to pull updates for %s: %s", remote_url, exc)
|
|
else:
|
|
try:
|
|
# A failed clone can leave its destination behind. Remove only the
|
|
# computed clone path so the next start can retry cleanly.
|
|
if os.path.lexists(repo_path):
|
|
logger.warning("Removing incomplete git mount clone at %s", repo_path)
|
|
if os.path.isdir(repo_path) and not os.path.islink(repo_path):
|
|
shutil.rmtree(repo_path)
|
|
else:
|
|
os.unlink(repo_path)
|
|
|
|
os.makedirs(clone_dir, exist_ok=True)
|
|
repo_path = clone_repository(
|
|
remote_url,
|
|
None, # No SSH key for now - can be added later
|
|
clone_dir,
|
|
branch or "main",
|
|
project_name=project_name,
|
|
)
|
|
logger.debug("Cloned git mount repository %s to %s", remote_url, repo_path)
|
|
except Exception as exc:
|
|
logger.warning("Clone failed for git mount %s: %s", remote_url, exc)
|
|
raise
|
|
|
|
# Handle branch checkout if specified
|
|
if branch and repo_path:
|
|
success = checkout_branch(repo_path, branch)
|
|
if success:
|
|
logger.debug("Checked out branch %s for %s", branch, remote_url)
|
|
else:
|
|
logger.warning(
|
|
"Branch %s not found in %s, using current branch", branch, remote_url
|
|
)
|
|
|
|
return repo_path
|
|
|
|
|
|
def resolve_git_mount_mappings(
|
|
repo_path: str,
|
|
mappings: list[dict],
|
|
working_directory: str | None,
|
|
home_dir: str = "/root",
|
|
) -> list[dict]:
|
|
"""Resolve mappings from an already-cloned repo to volume mount entries.
|
|
|
|
Returns a flat list of volume mount dicts.
|
|
"""
|
|
volume_mounts = []
|
|
|
|
for mapping in mappings:
|
|
source_path = mapping.get("source_path", ".")
|
|
target_path = mapping.get("target_path")
|
|
|
|
if not target_path:
|
|
logger.warning("Invalid mapping skipped: missing target_path")
|
|
continue
|
|
|
|
# Expand ~ and $HOME in target path
|
|
target_path = expand_container_path(target_path, home_dir)
|
|
|
|
# Resolve relative target paths against working directory
|
|
final_target = target_path
|
|
if not target_path.startswith("/"):
|
|
if not working_directory:
|
|
logger.warning(
|
|
"Git mount skipped: target_path '%s' is relative but no working_directory is configured. "
|
|
"Set working_directory in the tool config or use an absolute path.",
|
|
target_path,
|
|
)
|
|
continue
|
|
final_target = os.path.join(working_directory, target_path)
|
|
|
|
# Build source path and expand globs
|
|
if source_path and source_path != ".":
|
|
source_full = os.path.join(repo_path, source_path)
|
|
else:
|
|
source_full = repo_path
|
|
|
|
# Expand glob patterns
|
|
matched_paths = expand_glob_source(source_full, repo_path)
|
|
|
|
if not matched_paths:
|
|
logger.warning(
|
|
"Git mount skipped: no files matched source path %s in repo",
|
|
source_path,
|
|
)
|
|
continue
|
|
|
|
for matched_path in matched_paths:
|
|
if not os.path.exists(matched_path):
|
|
continue
|
|
|
|
# Determine target path for this match
|
|
if len(matched_paths) == 1:
|
|
# Single match: mount directly to target_path
|
|
mount_target = final_target
|
|
else:
|
|
# Multiple matches: append the path relative to the glob's base
|
|
# directory so `packages/*` → `/app/packages` yields
|
|
# `/app/packages/api` instead of `/app/packages/packages/api`.
|
|
first_glob_idx = min(
|
|
(source_path.find(c) for c in "*?[" if c in source_path),
|
|
default=len(source_path),
|
|
)
|
|
base_relative = os.path.dirname(source_path[: first_glob_idx + 1])
|
|
base_full = (
|
|
os.path.join(repo_path, base_relative)
|
|
if base_relative
|
|
else repo_path
|
|
)
|
|
rel_path = os.path.relpath(matched_path, base_full)
|
|
mount_target = os.path.join(final_target, rel_path)
|
|
|
|
volume_mounts.append(
|
|
{
|
|
"source": matched_path,
|
|
"target": mount_target,
|
|
"type": "bind",
|
|
}
|
|
)
|
|
logger.debug(
|
|
"Added git mount: %s -> %s",
|
|
matched_path,
|
|
mount_target,
|
|
)
|
|
|
|
return volume_mounts
|
|
|
|
|
|
async def resolve_single_git_mount(
|
|
session: AsyncSession,
|
|
git_mount: dict,
|
|
clone_parent: str | None = None,
|
|
working_directory: str | None = None,
|
|
home_dir: str = "/root",
|
|
) -> list[dict]:
|
|
"""Resolve a single git mount to volume mount entries.
|
|
|
|
Clones directly from remote_url, no database lookup needed.
|
|
Returns a list of volume mounts (one for each matched file/directory).
|
|
"""
|
|
git_mount = normalize_git_mount(git_mount)
|
|
remote_url = git_mount.get("remote_url")
|
|
branch = git_mount.get("branch")
|
|
mappings = git_mount.get("mappings", [])
|
|
|
|
if not remote_url:
|
|
logger.warning("Invalid git mount skipped: missing remote_url")
|
|
return []
|
|
|
|
if not mappings:
|
|
logger.warning("Invalid git mount skipped: no mappings")
|
|
return []
|
|
|
|
if not clone_parent:
|
|
logger.warning("Git mount skipped: no canonical profile directory provided")
|
|
return []
|
|
|
|
# Clone or pull the repository. Git mounts are auxiliary, so they keep
|
|
# using the repository URL basename rather than the project name.
|
|
try:
|
|
repo_path = await asyncio.to_thread(
|
|
clone_git_repo_locked, remote_url, branch, clone_parent
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Git mount clone failed for %s (branch=%s): %s",
|
|
remote_url,
|
|
branch,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
return []
|
|
|
|
# Resolve all mappings from the cloned repo
|
|
volumes = resolve_git_mount_mappings(
|
|
repo_path, mappings, working_directory, home_dir
|
|
)
|
|
# Profile-scoped Git working copies are writable and shared by compatible
|
|
# containers. Explicit refresh replaces local edits with the remote ref.
|
|
for volume in volumes:
|
|
volume["readonly"] = False
|
|
return volumes
|
|
|
|
|
|
def checkout_branch(repo_path: str, branch: str) -> bool:
|
|
"""Checkout a specific branch in a git repository.
|
|
|
|
Returns True if checkout succeeded, False if it failed.
|
|
On failure, the repository remains on its current branch.
|
|
"""
|
|
import subprocess
|
|
|
|
# First try to checkout existing branch
|
|
result = subprocess.run(
|
|
["git", "-C", repo_path, "checkout", branch],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
# Try fetching and checking out
|
|
subprocess.run(
|
|
["git", "-C", repo_path, "fetch", "origin", branch],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
result = subprocess.run(
|
|
["git", "-C", repo_path, "checkout", "-b", branch, f"origin/{branch}"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
logger.warning(
|
|
"Failed to checkout branch %s in %s: %s",
|
|
branch,
|
|
repo_path,
|
|
result.stderr.strip(),
|
|
)
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def pull_repository_updates(repo_path: str, remote_url: str) -> None:
|
|
"""Replace a profile working copy with its current remote branch.
|
|
|
|
Git profile mounts are writable shared working copies. Refresh discards
|
|
local container/editor edits after fetching the remote baseline.
|
|
"""
|
|
import subprocess
|
|
|
|
# Fetch latest changes
|
|
result = subprocess.run(
|
|
["git", "-C", repo_path, "fetch", "origin"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"Failed to fetch updates: {result.stderr}")
|
|
|
|
branch_result = subprocess.run(
|
|
["git", "-C", repo_path, "branch", "--show-current"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
branch = branch_result.stdout.strip() if branch_result.returncode == 0 else ""
|
|
if not branch:
|
|
raise RuntimeError("Unable to determine Git working-copy branch")
|
|
|
|
result = subprocess.run(
|
|
["git", "-C", repo_path, "reset", "--hard", f"origin/{branch}"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"Failed to reset working copy: {result.stderr}")
|
|
|
|
|
|
def expand_glob_source(source_path: str, repo_path: str) -> list[str]:
|
|
"""Expand glob patterns in source path.
|
|
|
|
Returns a list of matched absolute paths.
|
|
Limits results to prevent abuse.
|
|
"""
|
|
MAX_GLOB_MATCHES = 100
|
|
|
|
# Check if path contains glob characters
|
|
if not any(c in source_path for c in "*?["):
|
|
# No glob pattern: return single path if it exists
|
|
return [source_path] if os.path.exists(source_path) else []
|
|
|
|
# Expand glob pattern
|
|
matched = glob_module.glob(source_path, recursive=True)
|
|
total_matched = len(matched)
|
|
|
|
# Filter to only paths within the repo and limit count
|
|
results = []
|
|
for path in matched:
|
|
abs_path = os.path.abspath(path)
|
|
if abs_path.startswith(os.path.abspath(repo_path)):
|
|
results.append(abs_path)
|
|
if len(results) >= MAX_GLOB_MATCHES:
|
|
logger.warning(
|
|
"Glob pattern matched %d files, limited to %d",
|
|
total_matched,
|
|
MAX_GLOB_MATCHES,
|
|
)
|
|
break
|
|
|
|
return results
|
|
|
|
|
|
async def validate_config_profile(
|
|
session: AsyncSession,
|
|
profile_id: str | None,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
tool_type_id: uuid.UUID,
|
|
) -> uuid.UUID | None:
|
|
"""Validate a config profile selection.
|
|
|
|
Args:
|
|
session: Database session.
|
|
profile_id: Profile ID string or None.
|
|
user_id: Authenticated user ID.
|
|
project_id: Project ID for compatibility check.
|
|
tool_type_id: Tool type ID for compatibility check.
|
|
|
|
Returns:
|
|
Validated UUID or None.
|
|
|
|
Raises:
|
|
HTTPException: If profile is not found, not owned, or incompatible.
|
|
"""
|
|
if not profile_id:
|
|
return None
|
|
|
|
try:
|
|
profile_uuid = uuid.UUID(profile_id)
|
|
except ValueError:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"Invalid config profile ID: {profile_id}",
|
|
)
|
|
|
|
profile = await session.get(ConfigProfile, profile_uuid)
|
|
if profile is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Config profile not found: {profile_id}",
|
|
)
|
|
|
|
if profile.user_id != user_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Not authorized to use this config profile",
|
|
)
|
|
|
|
# Check compatibility: profile must be portable or match project/tool
|
|
is_compatible = (
|
|
(profile.project_id is None and profile.tool_type_id is None)
|
|
or (profile.project_id == project_id)
|
|
or (profile.tool_type_id == tool_type_id)
|
|
or (profile.project_id == project_id and profile.tool_type_id == tool_type_id)
|
|
)
|
|
|
|
if not is_compatible:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Selected config profile is not compatible with this project and tool type",
|
|
)
|
|
|
|
return profile_uuid
|
|
|
|
|
|
def sanitize_compose_file(compose_path: str) -> None:
|
|
"""Remove invalid port mappings (target port 0) from compose file."""
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
compose_file = Path(compose_path)
|
|
if not compose_file.exists():
|
|
return
|
|
|
|
content = compose_file.read_text()
|
|
compose_data = yaml.safe_load(content)
|
|
|
|
if not compose_data or "services" not in compose_data:
|
|
return
|
|
|
|
modified = False
|
|
for service_name, service_config in compose_data["services"].items():
|
|
if "ports" in service_config:
|
|
valid_ports = []
|
|
for port_mapping in service_config["ports"]:
|
|
if isinstance(port_mapping, str) and ":" in port_mapping:
|
|
parts = port_mapping.split(":")
|
|
if len(parts) == 2:
|
|
host_port, container_port = parts
|
|
# Skip invalid mappings (target port 0 or empty)
|
|
if container_port == "0" or not container_port:
|
|
modified = True
|
|
continue
|
|
valid_ports.append(port_mapping)
|
|
|
|
if valid_ports:
|
|
service_config["ports"] = valid_ports
|
|
else:
|
|
del service_config["ports"]
|
|
modified = True
|
|
break # Only check first service
|
|
|
|
if modified:
|
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
|
|
|
|
|
def modify_compose_file(
|
|
compose_path: str,
|
|
port_override: int | None = None,
|
|
start_command: str | None = None,
|
|
working_directory: str | None = None,
|
|
extra_volumes: list[dict] | None = None,
|
|
home_dir: str = "/root",
|
|
) -> None:
|
|
"""Modify compose file with runtime overrides."""
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
compose_file = Path(compose_path)
|
|
content = compose_file.read_text()
|
|
compose_data = yaml.safe_load(content)
|
|
|
|
if not compose_data or "services" not in compose_data:
|
|
return
|
|
|
|
# Apply modifications to the first service
|
|
for service_name, service_config in compose_data["services"].items():
|
|
if port_override and "ports" in service_config:
|
|
# Update port mapping
|
|
for i, port_mapping in enumerate(service_config["ports"]):
|
|
if isinstance(port_mapping, str) and ":" in port_mapping:
|
|
host_port, container_port = port_mapping.split(":", 1)
|
|
service_config["ports"][i] = f"{port_override}:{container_port}"
|
|
break
|
|
|
|
if start_command:
|
|
service_config["command"] = start_command
|
|
|
|
if working_directory:
|
|
service_config["working_dir"] = expand_container_path(
|
|
working_directory, home_dir
|
|
)
|
|
|
|
if extra_volumes:
|
|
if "volumes" not in service_config:
|
|
service_config["volumes"] = []
|
|
for vol in extra_volumes:
|
|
source = vol.get("source", "")
|
|
target = expand_container_path(vol.get("target", ""), home_dir)
|
|
vol_type = vol.get("type", "bind")
|
|
readonly = ":ro" if vol.get("readonly") else ""
|
|
if vol_type == "bind":
|
|
service_config["volumes"].append(f"{source}:{target}{readonly}")
|
|
else:
|
|
service_config["volumes"].append(
|
|
f"{source}:{target}:{vol_type}{readonly}"
|
|
)
|
|
|
|
# Sort volumes so parent paths come before child paths
|
|
if service_config.get("volumes"):
|
|
service_config["volumes"] = sort_volumes_by_specificity(
|
|
service_config["volumes"]
|
|
)
|
|
|
|
break # Only modify the first service
|
|
|
|
# Write back
|
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
|
|
|
|
|
def ensure_container_name_in_compose(compose_path: str, container_name: str) -> None:
|
|
"""Ensure compose file has explicit container_name for predictable naming.
|
|
|
|
Docker Compose auto-generates container names from the project directory
|
|
when container_name is absent. This breaks tunnel connectivity because
|
|
get_container_name(instance.name) cannot find the container. We inject
|
|
container_name into every service so the container has a predictable name.
|
|
"""
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
compose_file = Path(compose_path)
|
|
if not compose_file.exists():
|
|
return
|
|
|
|
content = compose_file.read_text()
|
|
compose_data = yaml.safe_load(content)
|
|
|
|
if not compose_data or "services" not in compose_data:
|
|
return
|
|
|
|
modified = False
|
|
for svc_name, svc_config in compose_data["services"].items():
|
|
if "container_name" not in svc_config:
|
|
svc_config["container_name"] = container_name.lower()
|
|
modified = True
|
|
|
|
if modified:
|
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
|
logger.info(
|
|
"Injected container_name '%s' into compose file",
|
|
container_name.lower(),
|
|
)
|
|
|
|
|
|
def ensure_web_bind_address(
|
|
compose_path: str, tool_type_name: str, default_port: int
|
|
) -> None:
|
|
"""Auto-inject bind address for known web tools that default to 127.0.0.1.
|
|
|
|
Many web tools (code-server, jupyter) bind to localhost by default,
|
|
making them inaccessible from the Docker network. This function detects
|
|
known tool images and injects the correct --bind-addr or --ip flag.
|
|
"""
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
if default_port <= 0:
|
|
return
|
|
|
|
KNOWN_BIND_FIXES: dict[str, str] = {
|
|
"code-server": f"--bind-addr 0.0.0.0:{default_port}",
|
|
"jupyter-notebook": f"start-notebook.sh --ip=0.0.0.0 --port={default_port} --no-browser",
|
|
}
|
|
|
|
bind_command = KNOWN_BIND_FIXES.get(tool_type_name)
|
|
if not bind_command:
|
|
return
|
|
|
|
compose_file = Path(compose_path)
|
|
if not compose_file.exists():
|
|
return
|
|
|
|
content = compose_file.read_text()
|
|
compose_data = yaml.safe_load(content)
|
|
|
|
if not compose_data or "services" not in compose_data:
|
|
return
|
|
|
|
for service_config in compose_data["services"].values():
|
|
image = service_config.get("image", "")
|
|
if not image:
|
|
continue
|
|
|
|
# LSIO images already bind to 0.0.0.0 — command override breaks s6 init
|
|
if "linuxserver" in image:
|
|
existing_command = service_config.get("command", "")
|
|
if "--bind-addr" in existing_command or "--host" in existing_command:
|
|
del service_config["command"]
|
|
compose_file.write_text(
|
|
yaml.dump(compose_data, default_flow_style=False)
|
|
)
|
|
logger.warning(
|
|
"Removed broken command override from LSIO image: %s",
|
|
existing_command,
|
|
)
|
|
return
|
|
return
|
|
|
|
# Check if the image matches a known tool
|
|
is_code_server = tool_type_name == "code-server" and (
|
|
"code-server" in image or "coder" in image
|
|
)
|
|
is_jupyter = tool_type_name == "jupyter-notebook" and (
|
|
"jupyter" in image or "notebook" in image
|
|
)
|
|
if not is_code_server and not is_jupyter:
|
|
continue
|
|
|
|
existing_command = service_config.get("command", "")
|
|
if existing_command:
|
|
# Already correct — nothing to do
|
|
if bind_command in existing_command:
|
|
return
|
|
# Fix broken or outdated bind flags
|
|
if (
|
|
"--bind-addr" in existing_command
|
|
or "--host" in existing_command
|
|
or "--ip=" in existing_command
|
|
):
|
|
service_config["command"] = bind_command
|
|
compose_file.write_text(
|
|
yaml.dump(compose_data, default_flow_style=False)
|
|
)
|
|
logger.warning(
|
|
"Replaced broken bind address for %s: %s → %s",
|
|
tool_type_name,
|
|
existing_command,
|
|
bind_command,
|
|
)
|
|
return
|
|
# Some other command override exists — don't touch it
|
|
return
|
|
|
|
# No command yet — inject the correct bind address
|
|
service_config["command"] = bind_command
|
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
|
logger.info("Injected bind address for %s: %s", tool_type_name, bind_command)
|
|
return
|
|
|
|
|
|
def ensure_backend_network_in_compose(compose_path: str) -> None:
|
|
"""Inject the backend network into the compose file so compose up attaches it.
|
|
|
|
Instead of running 'docker network connect' after container creation (which
|
|
is prone to race conditions and silent failures), we declare the network in
|
|
the compose file itself. Docker Compose then connects the container to the
|
|
network atomically during 'docker compose up'.
|
|
"""
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
compose_file = Path(compose_path)
|
|
if not compose_file.exists():
|
|
return
|
|
|
|
content = compose_file.read_text()
|
|
compose_data = yaml.safe_load(content)
|
|
|
|
if not compose_data or "services" not in compose_data:
|
|
return
|
|
|
|
network_name = get_backend_network_name()
|
|
modified = False
|
|
|
|
for svc_config in compose_data["services"].values():
|
|
existing = svc_config.get("networks", [])
|
|
if network_name not in existing:
|
|
svc_config["networks"] = existing + [network_name]
|
|
modified = True
|
|
break # Only modify first service
|
|
|
|
# Declare the network as external at the top level
|
|
if "networks" not in compose_data:
|
|
compose_data["networks"] = {}
|
|
if network_name not in compose_data["networks"]:
|
|
compose_data["networks"][network_name] = {"external": True}
|
|
modified = True
|
|
|
|
if modified:
|
|
compose_file.write_text(yaml.dump(compose_data, default_flow_style=False))
|
|
logger.info("Injected backend network '%s' into compose file", network_name)
|
|
|
|
|
|
async def prepare_manifest_instance(
|
|
session: AsyncSession,
|
|
instance: ToolInstance,
|
|
instance_dir: str,
|
|
repo_path: str,
|
|
env_vars: dict,
|
|
extra_volumes: list,
|
|
working_directory: str | None,
|
|
) -> tuple[str, str, dict, str]:
|
|
"""Build image and generate compose from a manifest-based tool type.
|
|
|
|
Returns:
|
|
Tuple of (image_tag, compose_content, resolved_manifest, home_dir)
|
|
"""
|
|
from src.models import ToolDefinitionManifest
|
|
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if not tool_type:
|
|
raise RuntimeError(f"Tool type not found for instance {instance.id}")
|
|
|
|
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
|
|
if not manifest_def:
|
|
raise RuntimeError(f"Manifest not found for tool type {tool_type.id}")
|
|
|
|
manifest = dict(manifest_def.manifest)
|
|
|
|
# Resolve base if referenced
|
|
if manifest_def.base_definition_id:
|
|
base_def = await session.get(
|
|
ToolDefinitionManifest, manifest_def.base_definition_id
|
|
)
|
|
if base_def:
|
|
base_manifest = dict(base_def.manifest)
|
|
manifest = resolve_base(deep_merge(base_manifest, manifest))
|
|
else:
|
|
logger.warning(
|
|
"Base definition %s not found for manifest %s",
|
|
manifest_def.base_definition_id,
|
|
manifest_def.id,
|
|
)
|
|
|
|
manifest = merge_with_config(manifest)
|
|
|
|
# Resolve extra env and volumes from merge_with_config
|
|
extra_env = manifest.pop("_extra_env", {})
|
|
extra_cfg_volumes = manifest.pop("_extra_volumes", [])
|
|
env_vars.update(extra_env)
|
|
extra_volumes.extend(extra_cfg_volumes)
|
|
|
|
# Compute image tag
|
|
image_tag = compute_image_tag(tool_type.name, manifest)
|
|
|
|
# Check if image already exists
|
|
check = subprocess.run(
|
|
["docker", "images", "-q", image_tag],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
image_exists = check.returncode == 0 and check.stdout.strip()
|
|
|
|
if not image_exists:
|
|
# Compile and build
|
|
dockerfile = compile_dockerfile(manifest)
|
|
entrypoint = compile_entrypoint(manifest)
|
|
|
|
logger.debug(
|
|
"Compiled Dockerfile for instance %s (%d chars)",
|
|
instance.id,
|
|
len(dockerfile),
|
|
)
|
|
|
|
build_ctx = {
|
|
"Dockerfile": dockerfile,
|
|
".headquarter/entrypoint.sh": entrypoint,
|
|
}
|
|
|
|
returncode, stdout, stderr = await asyncio.to_thread(
|
|
build_image,
|
|
instance_dir=instance_dir,
|
|
dockerfile=dockerfile,
|
|
tag=image_tag,
|
|
build_context=build_ctx,
|
|
)
|
|
|
|
if returncode != 0:
|
|
raise RuntimeError(f"Docker build failed: {stderr}")
|
|
|
|
logger.info("Built image %s for instance %s", image_tag, instance.id)
|
|
else:
|
|
logger.info("Reusing existing image %s for instance %s", image_tag, instance.id)
|
|
|
|
# Prepare SSH path for mount resolution
|
|
ssh_path = ""
|
|
|
|
# Resolve git mount variables from config profile
|
|
git_mount_vars = {}
|
|
if instance.selected_config_profile_id:
|
|
resolved_profile = await resolve_profile(
|
|
session, instance.selected_config_profile_id
|
|
)
|
|
for gm in resolved_profile.git_mounts or []:
|
|
ref = gm.get("git_mount_ref", "default")
|
|
# The actual resolution happens in resolve_git_mounts; we store placeholder
|
|
git_mount_vars[f"GIT_MOUNT_{ref}"] = ""
|
|
|
|
# Use the repository name for the workspace/repo mount target, not the
|
|
# directory name of a workspace/clone path (which may be "main" or similar).
|
|
# Prefer the actual on-disk workspace directory name when a workspace is
|
|
# mounted, otherwise fall back to parsing the remote URL like git clone.
|
|
repo = await session.get(GitRepository, instance.repository_id)
|
|
workspace: Workspace | None = None
|
|
if instance.workspace_id:
|
|
workspace = await session.get(Workspace, instance.workspace_id)
|
|
project = (
|
|
await session.get(Project, instance.project_id) if instance.project_id else None
|
|
)
|
|
repo_name = (
|
|
_get_repository_mount_name(project, repo, workspace)
|
|
if project and repo
|
|
else os.path.basename(os.path.normpath(repo_path))
|
|
)
|
|
variables = {
|
|
"IMAGE_TAG": image_tag,
|
|
"INSTANCE_NAME": instance.name.lower(),
|
|
"INSTANCE_DIR": instance_dir,
|
|
"WORKSPACE_PATH": repo_path,
|
|
"REPO_PATH": repo_path,
|
|
"REPO_NAME": repo_name,
|
|
"WORKSPACE_NAME": repo_name,
|
|
"SSH_PATH": ssh_path,
|
|
"TOOL_PORT": instance.port or 0,
|
|
"EXTRA_ENV": env_vars,
|
|
"EXTRA_VOLUMES": extra_volumes,
|
|
**git_mount_vars,
|
|
}
|
|
|
|
compose_content = compile_compose(manifest, variables)
|
|
|
|
logger.debug(
|
|
"_prepare_manifest_instance for %s: repo_path=%s compose_volumes=%s",
|
|
instance.id,
|
|
repo_path or "<empty>",
|
|
manifest.get("mounts", []),
|
|
)
|
|
logger.debug(
|
|
"Generated compose for %s:\n%s",
|
|
instance.id,
|
|
compose_content,
|
|
)
|
|
|
|
# Cache
|
|
instance.image_tag = image_tag
|
|
instance.manifest_compiled_at = datetime.now()
|
|
|
|
home_dir = get_manifest_home_dir(manifest)
|
|
return image_tag, compose_content, manifest, home_dir
|
|
|
|
|
|
async def create_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
data: "CreateInstanceRequest",
|
|
) -> ToolInstance:
|
|
"""Create a new tool instance for a repository.
|
|
|
|
Returns the created ToolInstance.
|
|
Raises ValueError for invalid input, RuntimeError for internal failures.
|
|
"""
|
|
repo = await session.get(GitRepository, repo_id)
|
|
if repo is None or repo.project_id != project_id:
|
|
raise ValueError("repository not found")
|
|
|
|
# Resolve the project up front: it is needed both for auto-generated display
|
|
# names and for the in-container mount name (/home/user/{project_name}),
|
|
# regardless of the tool definition type below.
|
|
project = await session.get(Project, project_id)
|
|
if project is None:
|
|
raise ValueError("project not found")
|
|
|
|
tool_type_id = uuid.UUID(data.tool_type_id)
|
|
tool_type = await session.get(ToolType, tool_type_id)
|
|
if tool_type is None:
|
|
raise ValueError("tool type not found")
|
|
|
|
# Validate config profile if provided
|
|
selected_profile_id = await validate_config_profile(
|
|
session, data.config_profile_id, user_id, project_id, tool_type_id
|
|
)
|
|
|
|
# Resolve workspace if provided
|
|
workspace = None
|
|
workspace_id = None
|
|
if data.workspace_id:
|
|
from src.models import Workspace as WorkspaceModel
|
|
|
|
try:
|
|
workspace_id = uuid.UUID(data.workspace_id)
|
|
except ValueError:
|
|
raise ValueError("Invalid workspace_id format")
|
|
workspace = await session.get(WorkspaceModel, workspace_id)
|
|
if workspace is None:
|
|
raise ValueError("workspace not found")
|
|
if workspace.repo_id != repo_id:
|
|
raise ValueError("workspace does not belong to this repository")
|
|
|
|
# Generate unique name
|
|
instance_name = f"{tool_type.name}-{repo.name}-{uuid.uuid4().hex[:8]}"
|
|
|
|
# Auto-generate display name with scoped numbering.
|
|
if data.display_name:
|
|
instance_display = data.display_name
|
|
else:
|
|
project_name = project.name if project else "Unknown"
|
|
scope_name = workspace.name if workspace else repo.name
|
|
auto_name = f"{project_name} / {scope_name} / {tool_type.display_name}"
|
|
|
|
if workspace:
|
|
count_query = (
|
|
select(ToolInstance)
|
|
.where(ToolInstance.workspace_id == workspace_id)
|
|
.where(ToolInstance.tool_type_id == tool_type_id)
|
|
.where(ToolInstance.owner_id == user_id)
|
|
)
|
|
else:
|
|
count_query = (
|
|
select(ToolInstance)
|
|
.where(ToolInstance.repository_id == repo_id)
|
|
.where(ToolInstance.tool_type_id == tool_type_id)
|
|
.where(ToolInstance.owner_id == user_id)
|
|
)
|
|
|
|
result = await session.execute(count_query)
|
|
existing_count = len(result.scalars().all())
|
|
if existing_count > 0:
|
|
instance_display = f"{auto_name} #{existing_count + 1}"
|
|
else:
|
|
instance_display = auto_name
|
|
|
|
# Create instance directory
|
|
instance_dir = ensure_instance_directory(instance_name)
|
|
compose_path = os.path.join(instance_dir, "docker-compose.yml")
|
|
|
|
# Find free port
|
|
tool_port = find_free_port()
|
|
|
|
# Determine the source path to mount. Workspaces are the canonical path;
|
|
# legacy instances without a workspace fall back to the repository path.
|
|
if workspace:
|
|
repo_path = workspace.path
|
|
else:
|
|
repo_path = repo.path
|
|
|
|
# Handle based on definition type
|
|
if tool_type.definition_type == "dockerfile":
|
|
image_tag = f"headquarter/{instance_name}:latest".lower()
|
|
|
|
if tool_type.dockerfile_template:
|
|
returncode, stdout, stderr = await asyncio.to_thread(
|
|
build_image,
|
|
instance_dir=instance_dir,
|
|
dockerfile=tool_type.dockerfile_template,
|
|
tag=image_tag,
|
|
build_context=tool_type.build_context,
|
|
)
|
|
|
|
if returncode != 0:
|
|
logger.error(
|
|
"Failed to build image for instance %s: %s",
|
|
instance_name,
|
|
stderr,
|
|
)
|
|
raise RuntimeError(f"Failed to build Docker image: {stderr[:500]}")
|
|
|
|
logger.info(
|
|
"Successfully built image %s for instance %s",
|
|
image_tag,
|
|
instance_name,
|
|
)
|
|
|
|
ports_section = (
|
|
f""" ports:\n - "{tool_port}:{tool_type.default_port}"\n"""
|
|
if tool_type.default_port and tool_type.default_port > 0
|
|
else ""
|
|
)
|
|
|
|
home_dir = tool_type.home_directory or "/home/user"
|
|
mount_name = _get_repository_mount_name(project, repo, workspace)
|
|
workspace_target = f"{home_dir}/{mount_name}"
|
|
|
|
compose_content = f"""version: "3.8"\nservices:
|
|
app:
|
|
image: {image_tag}
|
|
container_name: {instance_name.lower()}
|
|
stdin_open: true
|
|
tty: true
|
|
{ports_section} environment:
|
|
- HOME={home_dir}
|
|
volumes:
|
|
- {repo_path}:{workspace_target}
|
|
working_dir: {workspace_target}
|
|
restart: "no"
|
|
"""
|
|
write_compose_file(instance_dir, compose_content)
|
|
|
|
elif tool_type.definition_type == "manifest":
|
|
from src.models import ToolDefinitionManifest
|
|
|
|
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
|
|
if not manifest_def:
|
|
raise RuntimeError("Manifest definition not found for this tool type")
|
|
|
|
manifest = dict(manifest_def.manifest)
|
|
if manifest_def.base_definition_id:
|
|
base_def = await session.get(
|
|
ToolDefinitionManifest, manifest_def.base_definition_id
|
|
)
|
|
if base_def:
|
|
manifest = resolve_base(deep_merge(dict(base_def.manifest), manifest))
|
|
|
|
image_tag = compute_image_tag(tool_type.name, manifest)
|
|
|
|
# Manifest templates use WORKSPACE_PATH; REPO_PATH is retained as a
|
|
# deprecated alias for backward compatibility with older templates.
|
|
mount_name = _get_repository_mount_name(project, repo, workspace)
|
|
variables = {
|
|
"IMAGE_TAG": image_tag,
|
|
"INSTANCE_NAME": instance_name.lower(),
|
|
"INSTANCE_DIR": instance_dir,
|
|
"WORKSPACE_PATH": repo_path,
|
|
"REPO_PATH": repo_path,
|
|
"REPO_NAME": mount_name,
|
|
"WORKSPACE_NAME": mount_name,
|
|
"SSH_PATH": "",
|
|
"TOOL_PORT": tool_port,
|
|
"EXTRA_ENV": {},
|
|
"EXTRA_VOLUMES": [],
|
|
}
|
|
compose_content = compile_compose(manifest, variables)
|
|
write_compose_file(instance_dir, compose_content)
|
|
|
|
else:
|
|
if not tool_type.compose_template:
|
|
raise ValueError("Tool type has no compose template configured")
|
|
variables = {
|
|
"WORKSPACE_PATH": repo_path,
|
|
"REPO_PATH": repo_path,
|
|
"INSTANCE_NAME": instance_name,
|
|
"INSTANCE_ID": instance_name,
|
|
"TOOL_NAME": instance_name,
|
|
"TOOL_PORT": tool_port,
|
|
"USER_ID": str(user_id),
|
|
"PROJECT_ID": str(project_id),
|
|
"WORKSPACE_NAME": _get_repository_mount_name(project, repo, workspace),
|
|
"HOME_DIRECTORY": tool_type.home_directory or "/home/user",
|
|
}
|
|
compose_content = render_compose_template(tool_type.compose_template, variables)
|
|
|
|
write_compose_file(instance_dir, compose_content)
|
|
|
|
# Create database record
|
|
instance = ToolInstance(
|
|
name=instance_name,
|
|
display_name=instance_display,
|
|
tool_type_id=tool_type_id,
|
|
repository_id=repo_id,
|
|
project_id=project_id,
|
|
owner_id=user_id,
|
|
status="pending",
|
|
compose_path=compose_path,
|
|
port=tool_port,
|
|
workspace_id=workspace_id,
|
|
clone_mode=None,
|
|
branch=None,
|
|
selected_config_profile_id=selected_profile_id,
|
|
ssh_key_ids=data.ssh_key_ids or None,
|
|
)
|
|
session.add(instance)
|
|
await session.commit()
|
|
await session.refresh(instance)
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.created",
|
|
created_by=user_id,
|
|
status="pending",
|
|
message="Instance created",
|
|
)
|
|
|
|
return instance
|
|
|
|
|
|
async def start_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
data: "StartInstanceRequest | None",
|
|
) -> dict:
|
|
"""Start a tool instance.
|
|
|
|
Returns a dict with status and url.
|
|
Raises ValueError for invalid input, RuntimeError for internal failures.
|
|
"""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
|
|
# Validate and store config profile selection
|
|
if data and data.config_profile_id is not None:
|
|
selected_profile_id = await validate_config_profile(
|
|
session, data.config_profile_id, user_id, project_id, instance.tool_type_id
|
|
)
|
|
instance.selected_config_profile_id = selected_profile_id
|
|
await session.commit()
|
|
|
|
# Store SSH key selection if provided
|
|
if data and data.ssh_key_ids is not None:
|
|
instance.ssh_key_ids = data.ssh_key_ids or None
|
|
await session.commit()
|
|
|
|
if not instance.compose_path or not os.path.exists(instance.compose_path):
|
|
raise ValueError("compose file not found")
|
|
|
|
instance.status = "building"
|
|
await session.commit()
|
|
logger.info("Starting instance %s (name=%s)", instance.id, instance.name)
|
|
|
|
# Runtime overrides populated by config profiles
|
|
env_vars = {}
|
|
config_files = {}
|
|
port_override = None
|
|
start_command = None
|
|
working_directory = None
|
|
extra_volumes = []
|
|
|
|
# Fetch tool type early to determine home directory and container user
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
home_dir = (
|
|
tool_type.home_directory if tool_type and tool_type.home_directory else "/root"
|
|
)
|
|
container_uid = 0
|
|
container_gid = 0
|
|
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
|
from src.models import ToolDefinitionManifest
|
|
|
|
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
|
|
if manifest_def:
|
|
manifest = dict(manifest_def.manifest)
|
|
if manifest_def.base_definition_id:
|
|
base_def = await session.get(
|
|
ToolDefinitionManifest, manifest_def.base_definition_id
|
|
)
|
|
if base_def:
|
|
manifest = resolve_base(
|
|
deep_merge(dict(base_def.manifest), manifest)
|
|
)
|
|
home_dir = get_manifest_home_dir(manifest)
|
|
runtime = manifest.get("runtime", {})
|
|
if runtime.get("working_dir"):
|
|
working_directory = expand_container_path(
|
|
runtime["working_dir"], home_dir
|
|
)
|
|
user_cfg = manifest.get("user")
|
|
if user_cfg:
|
|
container_uid = user_cfg.get("uid", 0)
|
|
container_gid = user_cfg.get("gid", 0)
|
|
logger.debug(
|
|
"Manifest user resolved for instance %s: uid=%s, gid=%s, home=%s, working_directory=%s",
|
|
instance.id,
|
|
container_uid,
|
|
container_gid,
|
|
home_dir,
|
|
working_directory,
|
|
)
|
|
|
|
# Apply selected config profile if any
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
if instance.selected_config_profile_id is not None:
|
|
try:
|
|
resolved = await resolve_profile(
|
|
session, instance.selected_config_profile_id
|
|
)
|
|
# Profile working-directory hints determine where individual
|
|
# canonical profile files are bind-mounted at container creation.
|
|
profile_hints = resolved.runtime_hints
|
|
if profile_hints.get("working_directory"):
|
|
working_directory = expand_container_path(
|
|
profile_hints["working_directory"], home_dir
|
|
)
|
|
profile_env, profile_files, profile_mounts, profile_hints = (
|
|
apply_resolved_profile(
|
|
instance_dir, resolved, home_dir, working_directory
|
|
)
|
|
)
|
|
# Profile hints override the manifest/tool defaults, and git mounts
|
|
# need the final working directory to resolve relative target paths.
|
|
if profile_hints.get("start_command"):
|
|
start_command = profile_hints["start_command"]
|
|
if profile_hints.get("port_override"):
|
|
port_override = profile_hints["port_override"]
|
|
env_vars.update(profile_env)
|
|
config_files.update(profile_files)
|
|
git_mount_volumes = await resolve_git_mounts(
|
|
session, resolved, instance_dir, working_directory, home_dir
|
|
)
|
|
# Non-Git profile mounts bind directly to canonical profile
|
|
# storage so edits made by one compatible container are visible to
|
|
# every other container and the profile editor readback path.
|
|
composed_mounts = _stack_profile_mounts_with_git_mounts(
|
|
profile_mounts,
|
|
git_mount_volumes,
|
|
instance_dir,
|
|
)
|
|
extra_volumes.extend(composed_mounts)
|
|
logger.debug(
|
|
"Applied config profile %s to instance %s (env=%d, files=%d, profile_mounts=%d, git_mounts=%d, composed_mounts=%d)",
|
|
resolved.profile_name,
|
|
instance.id,
|
|
len(profile_env),
|
|
len(profile_files),
|
|
len(profile_mounts),
|
|
len(git_mount_volumes),
|
|
len(composed_mounts),
|
|
)
|
|
except ConfigProfileCycleError as exc:
|
|
logger.error(
|
|
"Cycle detected in config profile for instance %s: %s", instance.id, exc
|
|
)
|
|
raise ValueError(f"Config profile cycle detected: {exc}")
|
|
else:
|
|
logger.debug("No config profile selected for instance %s", instance.id)
|
|
|
|
# Write env file and config files
|
|
env_file_path = None
|
|
|
|
if env_vars:
|
|
env_file_path = write_env_file(instance_dir, env_vars)
|
|
logger.debug("Wrote env file for instance %s: %s", instance.id, env_file_path)
|
|
|
|
if config_files:
|
|
write_config_files(instance_dir, config_files)
|
|
logger.debug(
|
|
"Wrote %d config files for instance %s", len(config_files), instance.id
|
|
)
|
|
|
|
# Mount selected SSH keys into container home dir
|
|
if instance.ssh_key_ids:
|
|
from src.services.shared.ssh_keys import write_ssh_config, _sanitize_filename
|
|
|
|
ssh_keys_to_mount = []
|
|
for key_id in instance.ssh_key_ids:
|
|
ssh_key = await session.get(SSHKey, uuid.UUID(key_id))
|
|
if ssh_key and ssh_key.user_id == user_id:
|
|
ssh_keys_to_mount.append(ssh_key)
|
|
else:
|
|
logger.warning(
|
|
"SSH key %s not found or not authorized for user %s",
|
|
key_id,
|
|
user_id,
|
|
)
|
|
|
|
if ssh_keys_to_mount:
|
|
ssh_dir = os.path.join(instance_dir, "mounts", "ssh", ".ssh")
|
|
try:
|
|
os.makedirs(ssh_dir, exist_ok=True)
|
|
except OSError as exc:
|
|
logger.error(
|
|
"Failed to create SSH mount directory for instance %s: %s",
|
|
instance.id,
|
|
exc,
|
|
)
|
|
ssh_keys_to_mount = []
|
|
|
|
key_filenames = []
|
|
for ssh_key in ssh_keys_to_mount:
|
|
key_name = _sanitize_filename(ssh_key.name)
|
|
base_filename = f"id_ed25519_{key_name}"
|
|
filename = base_filename
|
|
counter = 1
|
|
while filename in key_filenames:
|
|
filename = f"{base_filename}_{counter}"
|
|
counter += 1
|
|
key_filenames.append(filename)
|
|
|
|
try:
|
|
prepare_ssh_key_files(
|
|
instance_dir,
|
|
ssh_key,
|
|
subdir="mounts/ssh/.ssh",
|
|
uid=container_uid,
|
|
gid=container_gid,
|
|
key_filename=filename,
|
|
write_config=False,
|
|
)
|
|
logger.debug(
|
|
"Prepared SSH key %s as %s for instance %s",
|
|
ssh_key.name,
|
|
filename,
|
|
instance.id,
|
|
)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"Failed to prepare SSH key %s for instance %s: %s",
|
|
ssh_key.id,
|
|
instance.id,
|
|
exc,
|
|
)
|
|
|
|
try:
|
|
write_ssh_config(
|
|
ssh_dir,
|
|
key_filenames,
|
|
uid=container_uid,
|
|
gid=container_gid,
|
|
)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"Failed to write SSH config for instance %s: %s",
|
|
instance.id,
|
|
exc,
|
|
)
|
|
|
|
ssh_target = os.path.join(home_dir, ".ssh")
|
|
extra_volumes.append(
|
|
{
|
|
"source": ssh_dir,
|
|
"target": ssh_target,
|
|
"type": "bind",
|
|
}
|
|
)
|
|
logger.debug(
|
|
"Mounted %d SSH key(s) for instance %s to %s",
|
|
len(ssh_keys_to_mount),
|
|
instance.id,
|
|
ssh_target,
|
|
)
|
|
|
|
# Ensure staged bind-mount sources are owned by the container user so
|
|
# directories like ~/.config remain writable inside the container.
|
|
_chown_staged_mounts(extra_volumes, instance_dir, container_uid, container_gid)
|
|
|
|
# ── MANIFEST-BASED FLOW ──────────────────────────────────────
|
|
resolved_manifest = None
|
|
|
|
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
|
logger.info("Using manifest-based startup for instance %s", instance.id)
|
|
|
|
repo_path = ""
|
|
if instance.workspace_id:
|
|
from src.models import Workspace as WorkspaceModel
|
|
|
|
workspace = await session.get(WorkspaceModel, instance.workspace_id)
|
|
if workspace:
|
|
repo_path = workspace.path
|
|
else:
|
|
repo = await session.get(GitRepository, instance.repository_id)
|
|
repo_path = repo.path if repo else ""
|
|
if instance.clone_mode == "clone":
|
|
repo_path = os.path.join(instance_dir, "repo-clone")
|
|
|
|
try:
|
|
(
|
|
image_tag,
|
|
compose_content,
|
|
resolved_manifest,
|
|
_home_dir,
|
|
) = await prepare_manifest_instance(
|
|
session=session,
|
|
instance=instance,
|
|
instance_dir=instance_dir,
|
|
repo_path=repo_path,
|
|
env_vars=env_vars,
|
|
extra_volumes=extra_volumes,
|
|
working_directory=working_directory,
|
|
)
|
|
write_compose_file(instance_dir, compose_content)
|
|
logger.debug(
|
|
"Generated manifest-based compose for instance %s", instance.id
|
|
)
|
|
except Exception as exc:
|
|
logger.exception(
|
|
"Manifest compilation failed for instance %s: %s", instance.id, exc
|
|
)
|
|
instance.status = "error"
|
|
await session.commit()
|
|
raise RuntimeError(f"Manifest compilation failed: {exc}")
|
|
else:
|
|
# ── LEGACY FLOW ──────────────────────────────────────────
|
|
if instance.clone_mode == "clone" and not instance.workspace_id:
|
|
clone_project = (
|
|
await session.get(Project, instance.project_id)
|
|
if instance.project_id
|
|
else None
|
|
)
|
|
clone_name = (
|
|
_slugify_directory_name(clone_project.name)
|
|
if clone_project
|
|
else "repo-clone"
|
|
)
|
|
repo_path = os.path.join(instance_dir, clone_name)
|
|
|
|
repo = await session.get(GitRepository, instance.repository_id)
|
|
if repo and repo.ssh_key_id:
|
|
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
|
|
if ssh_key:
|
|
try:
|
|
ssh_dir = prepare_ssh_key_files(
|
|
instance_dir, ssh_key, uid=0, gid=0
|
|
)
|
|
extra_volumes.append(
|
|
{
|
|
"source": ssh_dir,
|
|
"target": "/root/.ssh",
|
|
"type": "bind",
|
|
}
|
|
)
|
|
logger.debug(
|
|
"Mounted SSH key for clone-mode instance %s", instance.id
|
|
)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"Failed to prepare SSH key for instance %s: %s",
|
|
instance.id,
|
|
exc,
|
|
)
|
|
|
|
# Ensure the legacy compose mounts the project-named clone directory.
|
|
extra_volumes.append(
|
|
{
|
|
"source": repo_path,
|
|
"target": f"{home_dir}/{clone_name}",
|
|
"type": "bind",
|
|
}
|
|
)
|
|
|
|
if port_override or start_command or working_directory or extra_volumes:
|
|
modify_compose_file(
|
|
instance.compose_path,
|
|
port_override,
|
|
start_command,
|
|
working_directory,
|
|
extra_volumes,
|
|
home_dir,
|
|
)
|
|
logger.debug("Modified compose file for instance %s", instance.id)
|
|
|
|
# Sanitize compose file
|
|
sanitize_compose_file(instance.compose_path)
|
|
|
|
# Auto-fix bind address for known web tools
|
|
if tool_type and tool_type.interface_type == "web":
|
|
ensure_web_bind_address(
|
|
instance.compose_path, tool_type.name, tool_type.default_port
|
|
)
|
|
|
|
# Ensure predictable container name
|
|
ensure_container_name_in_compose(instance.compose_path, instance.name)
|
|
ensure_backend_network_in_compose(instance.compose_path)
|
|
|
|
# Execute docker compose up
|
|
logger.debug(
|
|
"Running docker compose up for instance %s (compose_path=%s)",
|
|
instance.id,
|
|
instance.compose_path,
|
|
)
|
|
returncode, stdout, stderr = execute_compose_command(
|
|
instance.compose_path, "up", env_file=env_file_path
|
|
)
|
|
logger.debug(
|
|
"Docker compose up completed for instance %s: returncode=%d, stdout=%s, stderr=%s",
|
|
instance.id,
|
|
returncode,
|
|
stdout[:200] if stdout else "",
|
|
stderr[:500] if stderr else "",
|
|
)
|
|
|
|
if returncode != 0:
|
|
instance.status = "error"
|
|
await session.commit()
|
|
logger.error("Failed to start instance %s: %s", instance.id, stderr)
|
|
raise RuntimeError(f"failed to start instance: {stderr}")
|
|
|
|
# Get container ID and name
|
|
expected_container_name = instance.name.lower()
|
|
container_id = get_container_id(expected_container_name)
|
|
if container_id:
|
|
instance.container_id = container_id
|
|
logger.debug("Container ID for instance %s: %s", instance.id, container_id)
|
|
|
|
instance.container_name = expected_container_name
|
|
logger.debug(
|
|
"Container name for instance %s: %s", instance.id, expected_container_name
|
|
)
|
|
|
|
# Verify container reached running state
|
|
if instance.container_id:
|
|
instance.status = "starting"
|
|
instance.last_started_at = datetime.now()
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.started",
|
|
created_by=user_id,
|
|
status="starting",
|
|
message="Container starting...",
|
|
)
|
|
logger.debug("Instance %s: verifying container startup...", instance.id)
|
|
|
|
startup_result = wait_for_container_running(
|
|
instance.container_id, timeout=30, interval=2.0
|
|
)
|
|
|
|
if not startup_result["success"]:
|
|
error_msg = f"Container failed to start: status={startup_result['status']}"
|
|
if startup_result["exit_code"] is not None:
|
|
error_msg += f", exit_code={startup_result['exit_code']}"
|
|
|
|
logs = get_container_logs(instance.container_id, tail=50)
|
|
|
|
instance.status = "error"
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.error",
|
|
created_by=user_id,
|
|
status="error",
|
|
message=error_msg,
|
|
metadata={
|
|
"exit_code": startup_result["exit_code"],
|
|
"error_type": "container",
|
|
},
|
|
)
|
|
logger.error(
|
|
"Instance %s container startup failed after %.1fs: %s\nLogs:\n%s",
|
|
instance.id,
|
|
startup_result["waited_seconds"],
|
|
error_msg,
|
|
logs,
|
|
)
|
|
return {
|
|
"status": "error",
|
|
"error": error_msg,
|
|
"logs": logs,
|
|
}
|
|
|
|
logger.debug(
|
|
"Instance %s container started successfully after %.1fs",
|
|
instance.id,
|
|
startup_result["waited_seconds"],
|
|
)
|
|
|
|
# Apply mount permission fixes for manifest-based instances
|
|
if resolved_manifest and instance.container_id:
|
|
mounts = resolved_manifest.get("mounts", [])
|
|
if mounts:
|
|
logger.debug(
|
|
"Applying permission fixes for instance %s (%d mounts)",
|
|
instance.id,
|
|
len(mounts),
|
|
)
|
|
permission_results = apply_mount_permissions(
|
|
instance.container_id,
|
|
mounts,
|
|
)
|
|
for result in permission_results:
|
|
if not result["success"]:
|
|
logger.warning(
|
|
"Permission fix failed for mount %s on instance %s: %s",
|
|
result["mount_name"],
|
|
instance.id,
|
|
result["error"],
|
|
)
|
|
|
|
# Fix SSH key ownership/permissions inside the container
|
|
if instance.ssh_key_ids and instance.container_id:
|
|
container_user = (
|
|
"root"
|
|
if home_dir == "/root"
|
|
else home_dir[6:]
|
|
if home_dir.startswith("/home/")
|
|
else "root"
|
|
)
|
|
ssh_target = os.path.join(home_dir, ".ssh")
|
|
logger.debug(
|
|
"Applying SSH permissions for user %s on %s in instance %s",
|
|
container_user,
|
|
ssh_target,
|
|
instance.id,
|
|
)
|
|
ssh_perm_result = apply_ssh_permissions(
|
|
instance.container_id,
|
|
ssh_target,
|
|
container_user,
|
|
)
|
|
if not ssh_perm_result["success"]:
|
|
logger.warning(
|
|
"SSH permission fix failed for instance %s: %s",
|
|
instance.id,
|
|
ssh_perm_result["error"],
|
|
)
|
|
|
|
# Execute readiness probe if configured
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if tool_type and instance.container_id:
|
|
probe_command = None
|
|
probe_timeout = 30
|
|
probe_interval = 2
|
|
|
|
if tool_type.readiness_probe:
|
|
probe_config = tool_type.readiness_probe
|
|
probe_command = probe_config.get("command", "")
|
|
probe_timeout = probe_config.get("timeout", 30)
|
|
probe_interval = probe_config.get("interval", 2)
|
|
elif tool_type.interface_type == "web":
|
|
probe_command = f"curl -f http://localhost:{tool_type.default_port or 8080}"
|
|
probe_timeout = 30
|
|
probe_interval = 2
|
|
|
|
if probe_command:
|
|
instance.status = "probing"
|
|
await session.commit()
|
|
logger.debug(
|
|
"Executing readiness probe for instance %s: command='%s', timeout=%d, interval=%d",
|
|
instance.id,
|
|
probe_command,
|
|
probe_timeout,
|
|
probe_interval,
|
|
)
|
|
|
|
success, probe_logs = await execute_probe(
|
|
container_id=instance.container_id,
|
|
command=probe_command,
|
|
timeout=probe_timeout,
|
|
interval=probe_interval,
|
|
)
|
|
|
|
instance.probe_result = {
|
|
"success": success,
|
|
"command": probe_command,
|
|
"logs": probe_logs,
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
|
|
if not success:
|
|
instance.status = "unhealthy"
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.health_changed",
|
|
created_by=user_id,
|
|
status="unhealthy",
|
|
message="Readiness probe failed",
|
|
metadata={"probe_output": "\n".join(probe_logs)},
|
|
)
|
|
logger.error(
|
|
"Readiness probe failed for instance %s after %ds: %s",
|
|
instance.id,
|
|
probe_timeout,
|
|
"\n".join(probe_logs),
|
|
)
|
|
return {
|
|
"status": "unhealthy",
|
|
"error": f"Readiness probe failed after {probe_timeout}s",
|
|
"probe_logs": probe_logs,
|
|
}
|
|
|
|
logger.info("Readiness probe succeeded for instance %s", instance.id)
|
|
|
|
# Final stability check: the container must still be running after all
|
|
# post-start setup. If it has already exited/restarted, mark it failed now
|
|
# instead of optimistically reporting "running".
|
|
if instance.container_id:
|
|
final_check = get_container_status(instance.container_id)
|
|
if final_check["status"] != "running":
|
|
error_msg = (
|
|
f"Container stopped during startup: status={final_check['status']}"
|
|
)
|
|
if final_check["exit_code"] is not None:
|
|
error_msg += f", exit_code={final_check['exit_code']}"
|
|
logs = get_container_logs(instance.container_id, tail=50)
|
|
instance.status = "error"
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.error",
|
|
created_by=user_id,
|
|
status="error",
|
|
message=error_msg,
|
|
metadata={
|
|
"exit_code": final_check["exit_code"],
|
|
"error_type": "container",
|
|
},
|
|
)
|
|
logger.error(
|
|
"Instance %s container stopped during startup: %s\nLogs:\n%s",
|
|
instance.id,
|
|
error_msg,
|
|
logs,
|
|
)
|
|
return {"status": "error", "error": error_msg, "logs": logs}
|
|
|
|
instance.status = "running"
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.health_changed",
|
|
created_by=user_id,
|
|
status="running",
|
|
message="Container running",
|
|
metadata={"previous_status": "starting"},
|
|
)
|
|
logger.info("Instance %s is now running", instance.id)
|
|
|
|
# Get tool type for default port
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if not tool_type:
|
|
logger.error("Tool type %s not found", instance.tool_type_id)
|
|
instance.status = "error"
|
|
await session.commit()
|
|
raise RuntimeError(f"Tool type '{instance.tool_type_id}' not found")
|
|
|
|
logger.debug(
|
|
"Tool type for instance %s: name=%s, container_port=%s, interface_type=%s",
|
|
instance.id,
|
|
tool_type.name,
|
|
tool_type.default_port or 0,
|
|
tool_type.interface_type,
|
|
)
|
|
|
|
# Only create Cloudflare tunnel for web-enabled tools
|
|
if tool_type.interface_type == "web":
|
|
try:
|
|
logger.debug(
|
|
"Creating tunnel for instance %s (container_port=%d)",
|
|
instance.id,
|
|
tool_type.default_port or 0,
|
|
)
|
|
tunnel_info = start_tunnel(
|
|
instance_name=instance.name,
|
|
container_port=tool_type.default_port or 0,
|
|
)
|
|
instance.tunnel_id = tunnel_info["container_name"]
|
|
instance.public_url = tunnel_info["url"]
|
|
instance.url = tunnel_info["url"]
|
|
await session.commit()
|
|
logger.debug(
|
|
"Created tunnel for instance %s: container=%s, url=%s",
|
|
instance.id,
|
|
tunnel_info["container_name"],
|
|
tunnel_info["url"],
|
|
)
|
|
except Exception as exc:
|
|
import traceback
|
|
|
|
error_msg = str(exc)
|
|
error_trace = traceback.format_exc()
|
|
logger.error(
|
|
"Failed to create tunnel for instance %s: %s\nTraceback:\n%s",
|
|
instance.id,
|
|
error_msg,
|
|
error_trace,
|
|
)
|
|
instance.status = "error"
|
|
instance.url = None
|
|
await session.commit()
|
|
raise RuntimeError(f"Failed to create tunnel: {error_msg}")
|
|
else:
|
|
logger.info(
|
|
"Instance %s is terminal-only (no web interface), skipping tunnel creation",
|
|
instance.id,
|
|
)
|
|
instance.url = None
|
|
instance.public_url = None
|
|
await session.commit()
|
|
|
|
return {"status": instance.status, "url": instance.url}
|
|
|
|
|
|
async def restart_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
) -> dict:
|
|
"""Restart a tool instance.
|
|
|
|
Returns a dict with status and url.
|
|
Raises ValueError for invalid input, RuntimeError for internal failures.
|
|
"""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
|
|
# Stop old tunnel if exists
|
|
if instance.tunnel_id:
|
|
try:
|
|
stop_tunnel(instance.name)
|
|
logger.debug(
|
|
"Stopped old tunnel for instance %s (container=%s)",
|
|
instance.id,
|
|
instance.tunnel_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Failed to stop old tunnel for instance %s: %s", instance.id, exc
|
|
)
|
|
|
|
# Re-apply stored config profile on restart
|
|
if instance.compose_path and os.path.exists(instance.compose_path):
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
if instance.selected_config_profile_id is not None:
|
|
try:
|
|
resolved = await resolve_profile(
|
|
session, instance.selected_config_profile_id
|
|
)
|
|
profile_env, profile_files, profile_mounts, profile_hints = (
|
|
apply_resolved_profile(instance_dir, resolved)
|
|
)
|
|
if profile_env:
|
|
write_env_file(instance_dir, profile_env)
|
|
logger.debug(
|
|
"Re-applied config profile %s on restart for instance %s",
|
|
resolved.profile_name,
|
|
instance.id,
|
|
)
|
|
except ConfigProfileCycleError as exc:
|
|
logger.error(
|
|
"Cycle detected in stored config profile for instance %s: %s",
|
|
instance.id,
|
|
exc,
|
|
)
|
|
|
|
# Re-apply compose fixes
|
|
sanitize_compose_file(instance.compose_path)
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if tool_type and tool_type.interface_type == "web":
|
|
ensure_web_bind_address(
|
|
instance.compose_path, tool_type.name, tool_type.default_port
|
|
)
|
|
ensure_container_name_in_compose(instance.compose_path, instance.name)
|
|
ensure_backend_network_in_compose(instance.compose_path)
|
|
|
|
returncode, stdout, stderr = execute_compose_command(
|
|
instance.compose_path, "restart"
|
|
)
|
|
|
|
if returncode == 0:
|
|
instance.status = "running"
|
|
instance.last_started_at = datetime.now()
|
|
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if not tool_type or not tool_type.default_port:
|
|
logger.error(
|
|
"Tool type %s has no default_port configured. Cannot create tunnel.",
|
|
instance.tool_type_id,
|
|
)
|
|
instance.status = "error"
|
|
await session.commit()
|
|
raise RuntimeError(
|
|
f"Tool type '{tool_type.name if tool_type else 'unknown'}' has no port configured"
|
|
)
|
|
|
|
if tool_type.interface_type == "web":
|
|
try:
|
|
tunnel_info = start_tunnel(
|
|
instance_name=instance.name,
|
|
container_port=tool_type.default_port or 0,
|
|
)
|
|
instance.tunnel_id = tunnel_info["container_name"]
|
|
instance.public_url = tunnel_info["url"]
|
|
instance.url = tunnel_info["url"]
|
|
logger.debug(
|
|
"Created new tunnel for instance %s: %s",
|
|
instance.id,
|
|
tunnel_info["url"],
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Failed to create tunnel for instance %s: %s",
|
|
instance.id,
|
|
exc,
|
|
)
|
|
instance.status = "error"
|
|
instance.url = None
|
|
await session.commit()
|
|
raise RuntimeError(f"Failed to create tunnel: {exc}")
|
|
else:
|
|
instance.url = None
|
|
instance.public_url = None
|
|
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.restarted",
|
|
created_by=user_id,
|
|
status="running",
|
|
message="Instance restarted",
|
|
)
|
|
return {"status": instance.status, "url": instance.url}
|
|
|
|
instance.status = "error"
|
|
await session.commit()
|
|
return {"status": instance.status}
|
|
|
|
|
|
async def delete_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
force: bool = False,
|
|
) -> None:
|
|
"""Delete a tool instance.
|
|
|
|
Raises ValueError for invalid input.
|
|
"""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
|
|
# Check dirty state for clone-mode instances
|
|
if instance.clone_mode == "clone" and not force:
|
|
instance_dir = (
|
|
os.path.dirname(instance.compose_path) if instance.compose_path else None
|
|
)
|
|
if instance_dir:
|
|
clone_project = (
|
|
await session.get(Project, instance.project_id)
|
|
if instance.project_id
|
|
else None
|
|
)
|
|
clone_name = (
|
|
_slugify_directory_name(clone_project.name)
|
|
if clone_project
|
|
else "repo-clone"
|
|
)
|
|
clone_path = os.path.join(instance_dir, clone_name)
|
|
if not os.path.exists(clone_path):
|
|
clone_path = os.path.join(instance_dir, "repo-clone")
|
|
if os.path.exists(clone_path):
|
|
is_dirty, changed_files = check_dirty_state(clone_path)
|
|
if is_dirty:
|
|
raise RuntimeError(
|
|
f"Repository has uncommitted changes: {changed_files}"
|
|
)
|
|
|
|
# Stop Cloudflare tunnel if exists
|
|
if instance.tunnel_id:
|
|
try:
|
|
stop_tunnel(instance.name)
|
|
logger.debug(
|
|
"Stopped tunnel for instance %s (container=%s)",
|
|
instance.id,
|
|
instance.tunnel_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Failed to stop tunnel for instance %s: %s", instance.id, exc
|
|
)
|
|
|
|
# Stop and remove container
|
|
if instance.compose_path and os.path.exists(instance.compose_path):
|
|
execute_compose_command(instance.compose_path, "down")
|
|
|
|
# Remove instance directory
|
|
if instance.compose_path:
|
|
instance_dir = os.path.dirname(instance.compose_path)
|
|
if os.path.exists(instance_dir):
|
|
import shutil
|
|
|
|
try:
|
|
shutil.rmtree(instance_dir)
|
|
except OSError as exc:
|
|
logger.error(
|
|
"Failed to remove instance directory %s: %s", instance_dir, exc
|
|
)
|
|
raise RuntimeError("Failed to remove instance files") from exc
|
|
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.deleted",
|
|
created_by=user_id,
|
|
status="deleted",
|
|
message="Instance deleted",
|
|
)
|
|
await session.delete(instance)
|
|
await session.commit()
|
|
|
|
|
|
async def recreate_instance_tunnel(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
) -> dict:
|
|
"""Recreate the temporary tunnel for an instance.
|
|
|
|
Returns a dict with status and url.
|
|
Raises ValueError for invalid input, RuntimeError for internal failures.
|
|
"""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
|
|
if instance.status != "running":
|
|
raise ValueError("instance must be running to recreate tunnel")
|
|
|
|
tool_type = await session.get(ToolType, instance.tool_type_id)
|
|
if not tool_type:
|
|
raise ValueError("Tool type not found for this instance")
|
|
|
|
expected_name = instance.name.lower()
|
|
logger.info(
|
|
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
|
|
instance.id,
|
|
expected_name,
|
|
tool_type.default_port,
|
|
)
|
|
|
|
# Find the tool container
|
|
tool_container_id = instance.container_id
|
|
if tool_container_id:
|
|
logger.info("Using stored container_id: %s", tool_container_id)
|
|
else:
|
|
tool_container_id = get_container_id(expected_name)
|
|
if tool_container_id:
|
|
logger.info("Found container by name: %s", tool_container_id)
|
|
else:
|
|
logger.error("Container %s not found", expected_name)
|
|
raise ValueError("Could not find running container for this instance")
|
|
|
|
# Ensure the tool container is on the backend network
|
|
network_name = get_backend_network_name()
|
|
on_network = is_container_on_network(tool_container_id, network_name)
|
|
logger.info(
|
|
"Container %s on network %s: %s",
|
|
tool_container_id,
|
|
network_name,
|
|
on_network,
|
|
)
|
|
if not on_network:
|
|
logger.info(
|
|
"Connecting container %s to network %s",
|
|
tool_container_id,
|
|
network_name,
|
|
)
|
|
connected = connect_container_to_network(tool_container_id, network_name)
|
|
logger.info("Network connect result: %s", connected)
|
|
|
|
# Get the container's IP on the backend network
|
|
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
|
if target_ip:
|
|
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
|
logger.info(
|
|
"Tunnel target for instance %s: %s (IP %s on %s)",
|
|
instance.id,
|
|
target_url,
|
|
target_ip,
|
|
network_name,
|
|
)
|
|
else:
|
|
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
|
|
logger.warning(
|
|
"Could not get container IP, falling back to name-based target: %s",
|
|
target_url,
|
|
)
|
|
|
|
try:
|
|
tunnel_info = recreate_tunnel(
|
|
instance_name=instance.name,
|
|
container_port=tool_type.default_port or 0,
|
|
target_url=target_url,
|
|
)
|
|
logger.info(
|
|
"Tunnel recreated: container=%s, url=%s",
|
|
tunnel_info["container_name"],
|
|
tunnel_info["url"],
|
|
)
|
|
|
|
# Verify the tunnel can actually reach the origin
|
|
health = check_tunnel_health(tunnel_info["url"], timeout=10)
|
|
logger.info(
|
|
"Tunnel health check: status=%s, code=%s, error=%s",
|
|
health.get("tunnel_status"),
|
|
health.get("status_code"),
|
|
health.get("error"),
|
|
)
|
|
|
|
# Also probe from inside the API container directly to the target
|
|
probe = subprocess.run(
|
|
[
|
|
"curl",
|
|
"-s",
|
|
"-o",
|
|
"/dev/null",
|
|
"-w",
|
|
"%{http_code}",
|
|
"--max-time",
|
|
"5",
|
|
target_url,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
logger.info(
|
|
"Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip()
|
|
)
|
|
|
|
instance.tunnel_id = tunnel_info["container_name"]
|
|
instance.public_url = tunnel_info["url"]
|
|
instance.url = tunnel_info["url"]
|
|
await session.commit()
|
|
return {"status": "healthy", "url": instance.url}
|
|
except Exception as exc:
|
|
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
|
raise RuntimeError(f"Failed to recreate tunnel: {str(exc)}")
|
|
|
|
|
|
async def stop_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
) -> dict:
|
|
"""Stop a tool instance.
|
|
|
|
Returns a dict with the stopped status.
|
|
Raises ValueError for invalid input.
|
|
"""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
|
|
if instance.tunnel_id:
|
|
try:
|
|
stop_tunnel(instance.name)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Failed to stop tunnel for instance %s: %s", instance.id, exc
|
|
)
|
|
|
|
if instance.compose_path and os.path.exists(instance.compose_path):
|
|
execute_compose_command(instance.compose_path, "stop")
|
|
|
|
instance.status = "stopped"
|
|
instance.last_stopped_at = datetime.now()
|
|
instance.url = None
|
|
instance.public_url = None
|
|
instance.tunnel_id = None
|
|
await session.commit()
|
|
await publish_lifecycle_event(
|
|
event_bus=_event_bus,
|
|
session=session,
|
|
instance=instance,
|
|
event_type="instance.stopped",
|
|
created_by=user_id,
|
|
status="stopped",
|
|
message="Instance stopped",
|
|
)
|
|
return {"status": instance.status}
|
|
|
|
|
|
async def rename_tool_instance(
|
|
session: AsyncSession,
|
|
user_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
repo_id: uuid.UUID,
|
|
instance_id: uuid.UUID,
|
|
display_name: str,
|
|
) -> ToolInstance:
|
|
"""Rename a tool instance (update display_name only)."""
|
|
instance = await session.get(ToolInstance, instance_id)
|
|
if instance is None or instance.repository_id != repo_id:
|
|
raise ValueError("instance not found")
|
|
if instance.owner_id != user_id:
|
|
raise ValueError("not authorized")
|
|
|
|
instance.display_name = display_name.strip()
|
|
await session.commit()
|
|
await session.refresh(instance)
|
|
return instance
|