refactor: slim backend routers to ≤500 lines
- tool_instances.py: 2108 → 496 lines - git_repositories.py: 1422 → 500 lines - config_profiles.py: 474 → 300 lines (already committed) Extract business logic into services: - services/tool/instance_service.py - services/git/operations.py - services/config/crud_service.py Quality gates: py_compile pass on all files
This commit is contained in:
Developer
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -45,3 +45,91 @@ class GitRepositoryResponse(BaseModel):
|
||||
|
||||
class UpdateSSHKeyRequest(BaseModel):
|
||||
ssh_key_id: str | None = None
|
||||
|
||||
|
||||
class FileListResponse(BaseModel):
|
||||
path: str
|
||||
branch: str
|
||||
entries: list[dict]
|
||||
|
||||
|
||||
class FileContentResponse(BaseModel):
|
||||
path: str
|
||||
branch: str
|
||||
content: str
|
||||
size: int
|
||||
encoding: str
|
||||
language: str | None
|
||||
is_binary: bool
|
||||
last_commit: dict | None
|
||||
|
||||
|
||||
class BranchesResponse(BaseModel):
|
||||
branches: list[dict]
|
||||
default_branch: str
|
||||
|
||||
|
||||
class FileUpdateRequest(BaseModel):
|
||||
path: str
|
||||
branch: str
|
||||
content: str
|
||||
commit_message: str
|
||||
|
||||
|
||||
class FileUpdateResponse(BaseModel):
|
||||
commit_hash: str
|
||||
message: str
|
||||
branch: str
|
||||
|
||||
|
||||
class StatusResponse(BaseModel):
|
||||
branch: str
|
||||
modified: list[str]
|
||||
added: list[str]
|
||||
deleted: list[str]
|
||||
untracked: list[str]
|
||||
renamed: list[str]
|
||||
ahead: int
|
||||
behind: int
|
||||
|
||||
|
||||
class BranchCreateRequest(BaseModel):
|
||||
name: str
|
||||
base_branch: str = "HEAD"
|
||||
|
||||
|
||||
class CheckoutRequest(BaseModel):
|
||||
branch: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
message: str
|
||||
files: list[str] | None = None
|
||||
|
||||
|
||||
class CommitResponse(BaseModel):
|
||||
commit_hash: str
|
||||
message: str
|
||||
|
||||
|
||||
class FetchResponse(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class PullResponse(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class PushResponse(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class MergeRequest(BaseModel):
|
||||
source_branch: str
|
||||
target_branch: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
class MergeResponse(BaseModel):
|
||||
commit_hash: str
|
||||
message: str
|
||||
|
||||
@@ -167,3 +167,47 @@ def init_working_repository(repo_path: str) -> None:
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"failed to set initial branch: {ref_result.stderr}",
|
||||
)
|
||||
|
||||
|
||||
def list_remote_branches(remote_url: str, ssh_key: SSHKey | None = None) -> tuple[list[str], str]:
|
||||
"""List branches from a remote repository via ls-remote.
|
||||
|
||||
Returns:
|
||||
Tuple of (branch_names, default_branch).
|
||||
"""
|
||||
ssh_result = prepare_ssh_env(ssh_key)
|
||||
env, key_path = ssh_result if ssh_result else (None, None)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "ls-remote", "--heads", remote_url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
env={**os.environ, **env} if env else None,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("ls-remote returned %d: %s", result.returncode, result.stderr)
|
||||
raise RuntimeError(f"ls-remote failed: {result.stderr}")
|
||||
branches = []
|
||||
default_branch = "main"
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
if len(parts) == 2:
|
||||
ref = parts[1]
|
||||
if ref.startswith("refs/heads/"):
|
||||
branch_name = ref[len("refs/heads/"):]
|
||||
branches.append(branch_name)
|
||||
if branch_name in ("main", "master"):
|
||||
default_branch = branch_name
|
||||
return branches, default_branch
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("ls-remote timed out for %s", remote_url)
|
||||
raise RuntimeError("ls-remote timed out")
|
||||
except Exception as e:
|
||||
logger.warning("ls-remote failed for %s: %s", remote_url, str(e))
|
||||
raise RuntimeError(f"ls-remote failed: {e}")
|
||||
finally:
|
||||
if key_path and os.path.exists(key_path):
|
||||
os.unlink(key_path)
|
||||
|
||||
@@ -1182,3 +1182,980 @@ async def create_tool_instance(
|
||||
)
|
||||
|
||||
return instance
|
||||
|
||||
|
||||
async def start_tool_instance(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
data: "StartInstanceRequest | None",
|
||||
) -> dict:
|
||||
"""Start a tool instance.
|
||||
|
||||
Returns a dict with status and url.
|
||||
Raises ValueError for invalid input, RuntimeError for internal failures.
|
||||
"""
|
||||
instance = await session.get(ToolInstance, instance_id)
|
||||
if instance is None or instance.repository_id != repo_id:
|
||||
raise ValueError("instance not found")
|
||||
|
||||
# Validate and store config profile selection
|
||||
if data and data.config_profile_id is not None:
|
||||
selected_profile_id = await validate_config_profile(
|
||||
session, data.config_profile_id, user_id, project_id, instance.tool_type_id
|
||||
)
|
||||
instance.selected_config_profile_id = selected_profile_id
|
||||
await session.commit()
|
||||
|
||||
# Store SSH key selection if provided
|
||||
if data and data.ssh_key_ids is not None:
|
||||
instance.ssh_key_ids = data.ssh_key_ids or None
|
||||
await session.commit()
|
||||
|
||||
if not instance.compose_path or not os.path.exists(instance.compose_path):
|
||||
raise ValueError("compose file not found")
|
||||
|
||||
instance.status = "building"
|
||||
await session.commit()
|
||||
logger.info("Starting instance %s (name=%s)", instance.id, instance.name)
|
||||
|
||||
# Runtime overrides populated by config profiles
|
||||
env_vars = {}
|
||||
config_files = {}
|
||||
port_override = None
|
||||
start_command = None
|
||||
working_directory = None
|
||||
extra_volumes = []
|
||||
|
||||
# Fetch tool type early to determine home directory and container user
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
home_dir = "/root"
|
||||
container_uid = 0
|
||||
container_gid = 0
|
||||
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
||||
from src.models import ToolDefinitionManifest
|
||||
|
||||
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
|
||||
if manifest_def:
|
||||
manifest = dict(manifest_def.manifest)
|
||||
if manifest_def.base_definition_id:
|
||||
base_def = await session.get(
|
||||
ToolDefinitionManifest, manifest_def.base_definition_id
|
||||
)
|
||||
if base_def:
|
||||
manifest = resolve_base(
|
||||
deep_merge(dict(base_def.manifest), manifest)
|
||||
)
|
||||
home_dir = get_manifest_home_dir(manifest)
|
||||
user_cfg = manifest.get("user")
|
||||
if user_cfg:
|
||||
container_uid = user_cfg.get("uid", 0)
|
||||
container_gid = user_cfg.get("gid", 0)
|
||||
logger.debug(
|
||||
"Manifest user resolved for instance %s: uid=%s, gid=%s, home=%s",
|
||||
instance.id,
|
||||
container_uid,
|
||||
container_gid,
|
||||
home_dir,
|
||||
)
|
||||
|
||||
# Apply selected config profile if any
|
||||
instance_dir = os.path.dirname(instance.compose_path)
|
||||
if instance.selected_config_profile_id is not None:
|
||||
try:
|
||||
resolved = await resolve_profile(
|
||||
session, instance.selected_config_profile_id
|
||||
)
|
||||
profile_env, profile_files, profile_mounts, profile_hints = (
|
||||
apply_resolved_profile(instance_dir, resolved, home_dir)
|
||||
)
|
||||
env_vars.update(profile_env)
|
||||
config_files.update(profile_files)
|
||||
extra_volumes.extend(profile_mounts)
|
||||
git_mount_volumes = await resolve_git_mounts(
|
||||
session, resolved, instance_dir, working_directory, home_dir
|
||||
)
|
||||
extra_volumes.extend(git_mount_volumes)
|
||||
if profile_hints.get("start_command"):
|
||||
start_command = profile_hints["start_command"]
|
||||
if profile_hints.get("working_directory"):
|
||||
working_directory = profile_hints["working_directory"]
|
||||
if profile_hints.get("port_override"):
|
||||
port_override = profile_hints["port_override"]
|
||||
logger.debug(
|
||||
"Applied config profile %s to instance %s (env=%d, files=%d, mounts=%d, git_mounts=%d)",
|
||||
resolved.profile_name,
|
||||
instance.id,
|
||||
len(profile_env),
|
||||
len(profile_files),
|
||||
len(profile_mounts),
|
||||
len(git_mount_volumes),
|
||||
)
|
||||
except ConfigProfileCycleError as exc:
|
||||
logger.error(
|
||||
"Cycle detected in config profile for instance %s: %s", instance.id, exc
|
||||
)
|
||||
raise ValueError(f"Config profile cycle detected: {exc}")
|
||||
else:
|
||||
logger.debug("No config profile selected for instance %s", instance.id)
|
||||
|
||||
# Write env file and config files
|
||||
env_file_path = None
|
||||
|
||||
if env_vars:
|
||||
env_file_path = write_env_file(instance_dir, env_vars)
|
||||
logger.debug("Wrote env file for instance %s: %s", instance.id, env_file_path)
|
||||
|
||||
if config_files:
|
||||
write_config_files(instance_dir, config_files)
|
||||
logger.debug(
|
||||
"Wrote %d config files for instance %s", len(config_files), instance.id
|
||||
)
|
||||
|
||||
# Mount selected SSH keys into container home dir
|
||||
if instance.ssh_key_ids:
|
||||
from src.services.shared.ssh_keys import write_ssh_config, _sanitize_filename
|
||||
|
||||
ssh_keys_to_mount = []
|
||||
for key_id in instance.ssh_key_ids:
|
||||
ssh_key = await session.get(SSHKey, uuid.UUID(key_id))
|
||||
if ssh_key and ssh_key.user_id == user_id:
|
||||
ssh_keys_to_mount.append(ssh_key)
|
||||
else:
|
||||
logger.warning(
|
||||
"SSH key %s not found or not authorized for user %s",
|
||||
key_id,
|
||||
user_id,
|
||||
)
|
||||
|
||||
if ssh_keys_to_mount:
|
||||
ssh_dir = os.path.join(instance_dir, "mounts", "ssh", ".ssh")
|
||||
os.makedirs(ssh_dir, exist_ok=True)
|
||||
|
||||
key_filenames = []
|
||||
for ssh_key in ssh_keys_to_mount:
|
||||
key_name = _sanitize_filename(ssh_key.name)
|
||||
base_filename = f"id_ed25519_{key_name}"
|
||||
filename = base_filename
|
||||
counter = 1
|
||||
while filename in key_filenames:
|
||||
filename = f"{base_filename}_{counter}"
|
||||
counter += 1
|
||||
key_filenames.append(filename)
|
||||
|
||||
try:
|
||||
prepare_ssh_key_files(
|
||||
instance_dir,
|
||||
ssh_key,
|
||||
subdir="mounts/ssh/.ssh",
|
||||
uid=container_uid,
|
||||
gid=container_gid,
|
||||
key_filename=filename,
|
||||
write_config=False,
|
||||
)
|
||||
logger.debug(
|
||||
"Prepared SSH key %s as %s for instance %s",
|
||||
ssh_key.name,
|
||||
filename,
|
||||
instance.id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to prepare SSH key %s for instance %s: %s",
|
||||
ssh_key.id,
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
try:
|
||||
write_ssh_config(
|
||||
ssh_dir,
|
||||
key_filenames,
|
||||
uid=container_uid,
|
||||
gid=container_gid,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to write SSH config for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
ssh_target = os.path.join(home_dir, ".ssh")
|
||||
extra_volumes.append(
|
||||
{
|
||||
"source": ssh_dir,
|
||||
"target": ssh_target,
|
||||
"type": "bind",
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"Mounted %d SSH key(s) for instance %s to %s",
|
||||
len(ssh_keys_to_mount),
|
||||
instance.id,
|
||||
ssh_target,
|
||||
)
|
||||
|
||||
# ── MANIFEST-BASED FLOW ──────────────────────────────────────
|
||||
resolved_manifest = None
|
||||
|
||||
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
||||
logger.info("Using manifest-based startup for instance %s", instance.id)
|
||||
|
||||
repo_path = ""
|
||||
if instance.workspace_id:
|
||||
from src.models import Workspace as WorkspaceModel
|
||||
|
||||
workspace = await session.get(WorkspaceModel, instance.workspace_id)
|
||||
if workspace:
|
||||
repo_path = workspace.path
|
||||
else:
|
||||
repo = await session.get(GitRepository, instance.repository_id)
|
||||
repo_path = repo.path if repo else ""
|
||||
if instance.clone_mode == "clone":
|
||||
repo_path = os.path.join(instance_dir, "repo-clone")
|
||||
|
||||
try:
|
||||
(
|
||||
image_tag,
|
||||
compose_content,
|
||||
resolved_manifest,
|
||||
_home_dir,
|
||||
) = await prepare_manifest_instance(
|
||||
session=session,
|
||||
instance=instance,
|
||||
instance_dir=instance_dir,
|
||||
repo_path=repo_path,
|
||||
env_vars=env_vars,
|
||||
extra_volumes=extra_volumes,
|
||||
working_directory=working_directory,
|
||||
)
|
||||
write_compose_file(instance_dir, compose_content)
|
||||
logger.debug(
|
||||
"Generated manifest-based compose for instance %s", instance.id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Manifest compilation failed for instance %s: %s", instance.id, exc
|
||||
)
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
raise RuntimeError(f"Manifest compilation failed: {exc}")
|
||||
else:
|
||||
# ── LEGACY FLOW ──────────────────────────────────────────
|
||||
if instance.clone_mode == "clone" and not instance.workspace_id:
|
||||
repo = await session.get(GitRepository, instance.repository_id)
|
||||
if repo and repo.ssh_key_id:
|
||||
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
|
||||
if ssh_key:
|
||||
try:
|
||||
ssh_dir = prepare_ssh_key_files(
|
||||
instance_dir, ssh_key, uid=0, gid=0
|
||||
)
|
||||
extra_volumes.append(
|
||||
{
|
||||
"source": ssh_dir,
|
||||
"target": "/root/.ssh",
|
||||
"type": "bind",
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"Mounted SSH key for clone-mode instance %s", instance.id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to prepare SSH key for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if port_override or start_command or working_directory or extra_volumes:
|
||||
modify_compose_file(
|
||||
instance.compose_path,
|
||||
port_override,
|
||||
start_command,
|
||||
working_directory,
|
||||
extra_volumes,
|
||||
home_dir,
|
||||
)
|
||||
logger.debug("Modified compose file for instance %s", instance.id)
|
||||
|
||||
# Sanitize compose file
|
||||
sanitize_compose_file(instance.compose_path)
|
||||
|
||||
# Auto-fix bind address for known web tools
|
||||
if tool_type and tool_type.interface_type == "web":
|
||||
ensure_web_bind_address(
|
||||
instance.compose_path, tool_type.name, tool_type.default_port
|
||||
)
|
||||
|
||||
# Ensure predictable container name
|
||||
ensure_container_name_in_compose(instance.compose_path, instance.name)
|
||||
ensure_backend_network_in_compose(instance.compose_path)
|
||||
|
||||
# Execute docker compose up
|
||||
logger.debug(
|
||||
"Running docker compose up for instance %s (compose_path=%s)",
|
||||
instance.id,
|
||||
instance.compose_path,
|
||||
)
|
||||
returncode, stdout, stderr = execute_compose_command(
|
||||
instance.compose_path, "up", env_file=env_file_path
|
||||
)
|
||||
logger.debug(
|
||||
"Docker compose up completed for instance %s: returncode=%d, stdout=%s, stderr=%s",
|
||||
instance.id,
|
||||
returncode,
|
||||
stdout[:200] if stdout else "",
|
||||
stderr[:500] if stderr else "",
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
logger.error("Failed to start instance %s: %s", instance.id, stderr)
|
||||
raise RuntimeError(f"failed to start instance: {stderr}")
|
||||
|
||||
# Get container ID and name
|
||||
expected_container_name = instance.name.lower()
|
||||
container_id = get_container_id(expected_container_name)
|
||||
if container_id:
|
||||
instance.container_id = container_id
|
||||
logger.debug("Container ID for instance %s: %s", instance.id, container_id)
|
||||
|
||||
instance.container_name = expected_container_name
|
||||
logger.debug(
|
||||
"Container name for instance %s: %s", instance.id, expected_container_name
|
||||
)
|
||||
|
||||
# Verify container reached running state
|
||||
if instance.container_id:
|
||||
instance.status = "starting"
|
||||
instance.last_started_at = datetime.now()
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.started",
|
||||
created_by=user_id,
|
||||
status="starting",
|
||||
message="Container starting...",
|
||||
)
|
||||
logger.debug("Instance %s: verifying container startup...", instance.id)
|
||||
|
||||
startup_result = wait_for_container_running(
|
||||
instance.container_id, timeout=30, interval=2.0
|
||||
)
|
||||
|
||||
if not startup_result["success"]:
|
||||
error_msg = f"Container failed to start: status={startup_result['status']}"
|
||||
if startup_result["exit_code"] is not None:
|
||||
error_msg += f", exit_code={startup_result['exit_code']}"
|
||||
|
||||
logs = get_container_logs(instance.container_id, tail=50)
|
||||
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.error",
|
||||
created_by=user_id,
|
||||
status="error",
|
||||
message=error_msg,
|
||||
metadata={
|
||||
"exit_code": startup_result["exit_code"],
|
||||
"error_type": "container",
|
||||
},
|
||||
)
|
||||
logger.error(
|
||||
"Instance %s container startup failed after %.1fs: %s\nLogs:\n%s",
|
||||
instance.id,
|
||||
startup_result["waited_seconds"],
|
||||
error_msg,
|
||||
logs,
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"error": error_msg,
|
||||
"logs": logs,
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
"Instance %s container started successfully after %.1fs",
|
||||
instance.id,
|
||||
startup_result["waited_seconds"],
|
||||
)
|
||||
|
||||
# Apply mount permission fixes for manifest-based instances
|
||||
if resolved_manifest and instance.container_id:
|
||||
mounts = resolved_manifest.get("mounts", [])
|
||||
if mounts:
|
||||
logger.debug(
|
||||
"Applying permission fixes for instance %s (%d mounts)",
|
||||
instance.id,
|
||||
len(mounts),
|
||||
)
|
||||
permission_results = apply_mount_permissions(
|
||||
instance.container_id,
|
||||
mounts,
|
||||
)
|
||||
for result in permission_results:
|
||||
if not result["success"]:
|
||||
logger.warning(
|
||||
"Permission fix failed for mount %s on instance %s: %s",
|
||||
result["mount_name"],
|
||||
instance.id,
|
||||
result["error"],
|
||||
)
|
||||
|
||||
# Fix SSH key ownership/permissions inside the container
|
||||
if instance.ssh_key_ids and instance.container_id:
|
||||
container_user = (
|
||||
"root"
|
||||
if home_dir == "/root"
|
||||
else home_dir[6:]
|
||||
if home_dir.startswith("/home/")
|
||||
else "root"
|
||||
)
|
||||
ssh_target = os.path.join(home_dir, ".ssh")
|
||||
logger.debug(
|
||||
"Applying SSH permissions for user %s on %s in instance %s",
|
||||
container_user,
|
||||
ssh_target,
|
||||
instance.id,
|
||||
)
|
||||
ssh_perm_result = apply_ssh_permissions(
|
||||
instance.container_id,
|
||||
ssh_target,
|
||||
container_user,
|
||||
)
|
||||
if not ssh_perm_result["success"]:
|
||||
logger.warning(
|
||||
"SSH permission fix failed for instance %s: %s",
|
||||
instance.id,
|
||||
ssh_perm_result["error"],
|
||||
)
|
||||
|
||||
# Execute readiness probe if configured
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if tool_type and instance.container_id:
|
||||
probe_command = None
|
||||
probe_timeout = 30
|
||||
probe_interval = 2
|
||||
|
||||
if tool_type.readiness_probe:
|
||||
probe_config = tool_type.readiness_probe
|
||||
probe_command = probe_config.get("command", "")
|
||||
probe_timeout = probe_config.get("timeout", 30)
|
||||
probe_interval = probe_config.get("interval", 2)
|
||||
elif tool_type.interface_type == "web":
|
||||
probe_command = f"curl -f http://localhost:{tool_type.default_port or 8080}"
|
||||
probe_timeout = 30
|
||||
probe_interval = 2
|
||||
|
||||
if probe_command:
|
||||
instance.status = "probing"
|
||||
await session.commit()
|
||||
logger.debug(
|
||||
"Executing readiness probe for instance %s: command='%s', timeout=%d, interval=%d",
|
||||
instance.id,
|
||||
probe_command,
|
||||
probe_timeout,
|
||||
probe_interval,
|
||||
)
|
||||
|
||||
success, probe_logs = await execute_probe(
|
||||
container_id=instance.container_id,
|
||||
command=probe_command,
|
||||
timeout=probe_timeout,
|
||||
interval=probe_interval,
|
||||
)
|
||||
|
||||
instance.probe_result = {
|
||||
"success": success,
|
||||
"command": probe_command,
|
||||
"logs": probe_logs,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
if not success:
|
||||
instance.status = "unhealthy"
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.health_changed",
|
||||
created_by=user_id,
|
||||
status="unhealthy",
|
||||
message="Readiness probe failed",
|
||||
metadata={"probe_output": "\n".join(probe_logs)},
|
||||
)
|
||||
logger.error(
|
||||
"Readiness probe failed for instance %s after %ds: %s",
|
||||
instance.id,
|
||||
probe_timeout,
|
||||
"\n".join(probe_logs),
|
||||
)
|
||||
return {
|
||||
"status": "unhealthy",
|
||||
"error": f"Readiness probe failed after {probe_timeout}s",
|
||||
"probe_logs": probe_logs,
|
||||
}
|
||||
|
||||
logger.info("Readiness probe succeeded for instance %s", instance.id)
|
||||
|
||||
instance.status = "running"
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.health_changed",
|
||||
created_by=user_id,
|
||||
status="running",
|
||||
message="Container running",
|
||||
metadata={"previous_status": "starting"},
|
||||
)
|
||||
logger.info("Instance %s is now running", instance.id)
|
||||
|
||||
# Get tool type for default port
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if not tool_type:
|
||||
logger.error("Tool type %s not found", instance.tool_type_id)
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
raise RuntimeError(f"Tool type '{instance.tool_type_id}' not found")
|
||||
|
||||
logger.debug(
|
||||
"Tool type for instance %s: name=%s, container_port=%s, interface_type=%s",
|
||||
instance.id,
|
||||
tool_type.name,
|
||||
tool_type.default_port or 0,
|
||||
tool_type.interface_type,
|
||||
)
|
||||
|
||||
# Only create Cloudflare tunnel for web-enabled tools
|
||||
if tool_type.interface_type == "web":
|
||||
try:
|
||||
logger.debug(
|
||||
"Creating tunnel for instance %s (container_port=%d)",
|
||||
instance.id,
|
||||
tool_type.default_port or 0,
|
||||
)
|
||||
tunnel_info = start_tunnel(
|
||||
instance_name=instance.name,
|
||||
container_port=tool_type.default_port or 0,
|
||||
)
|
||||
instance.tunnel_id = tunnel_info["container_name"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
instance.url = tunnel_info["url"]
|
||||
await session.commit()
|
||||
logger.debug(
|
||||
"Created tunnel for instance %s: container=%s, url=%s",
|
||||
instance.id,
|
||||
tunnel_info["container_name"],
|
||||
tunnel_info["url"],
|
||||
)
|
||||
except Exception as exc:
|
||||
import traceback
|
||||
|
||||
error_msg = str(exc)
|
||||
error_trace = traceback.format_exc()
|
||||
logger.error(
|
||||
"Failed to create tunnel for instance %s: %s\nTraceback:\n%s",
|
||||
instance.id,
|
||||
error_msg,
|
||||
error_trace,
|
||||
)
|
||||
instance.status = "error"
|
||||
instance.url = None
|
||||
await session.commit()
|
||||
raise RuntimeError(f"Failed to create tunnel: {error_msg}")
|
||||
else:
|
||||
logger.info(
|
||||
"Instance %s is terminal-only (no web interface), skipping tunnel creation",
|
||||
instance.id,
|
||||
)
|
||||
instance.url = None
|
||||
instance.public_url = None
|
||||
await session.commit()
|
||||
|
||||
return {"status": instance.status, "url": instance.url}
|
||||
|
||||
|
||||
async def restart_tool_instance(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
) -> dict:
|
||||
"""Restart a tool instance.
|
||||
|
||||
Returns a dict with status and url.
|
||||
Raises ValueError for invalid input, RuntimeError for internal failures.
|
||||
"""
|
||||
instance = await session.get(ToolInstance, instance_id)
|
||||
if instance is None or instance.repository_id != repo_id:
|
||||
raise ValueError("instance not found")
|
||||
|
||||
# Stop old tunnel if exists
|
||||
if instance.tunnel_id:
|
||||
try:
|
||||
stop_tunnel(instance.name)
|
||||
logger.debug(
|
||||
"Stopped old tunnel for instance %s (container=%s)",
|
||||
instance.id,
|
||||
instance.tunnel_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to stop old tunnel for instance %s: %s", instance.id, exc
|
||||
)
|
||||
|
||||
# Re-apply stored config profile on restart
|
||||
if instance.compose_path and os.path.exists(instance.compose_path):
|
||||
instance_dir = os.path.dirname(instance.compose_path)
|
||||
if instance.selected_config_profile_id is not None:
|
||||
try:
|
||||
resolved = await resolve_profile(
|
||||
session, instance.selected_config_profile_id
|
||||
)
|
||||
profile_env, profile_files, profile_mounts, profile_hints = (
|
||||
apply_resolved_profile(instance_dir, resolved)
|
||||
)
|
||||
if profile_env:
|
||||
write_env_file(instance_dir, profile_env)
|
||||
logger.debug(
|
||||
"Re-applied config profile %s on restart for instance %s",
|
||||
resolved.profile_name,
|
||||
instance.id,
|
||||
)
|
||||
except ConfigProfileCycleError as exc:
|
||||
logger.error(
|
||||
"Cycle detected in stored config profile for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
# Re-apply compose fixes
|
||||
sanitize_compose_file(instance.compose_path)
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if tool_type and tool_type.interface_type == "web":
|
||||
ensure_web_bind_address(
|
||||
instance.compose_path, tool_type.name, tool_type.default_port
|
||||
)
|
||||
ensure_container_name_in_compose(instance.compose_path, instance.name)
|
||||
ensure_backend_network_in_compose(instance.compose_path)
|
||||
|
||||
returncode, stdout, stderr = execute_compose_command(
|
||||
instance.compose_path, "restart"
|
||||
)
|
||||
|
||||
if returncode == 0:
|
||||
instance.status = "running"
|
||||
instance.last_started_at = datetime.now()
|
||||
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if not tool_type or not tool_type.default_port:
|
||||
logger.error(
|
||||
"Tool type %s has no default_port configured. Cannot create tunnel.",
|
||||
instance.tool_type_id,
|
||||
)
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
raise RuntimeError(
|
||||
f"Tool type '{tool_type.name if tool_type else 'unknown'}' has no port configured"
|
||||
)
|
||||
|
||||
if tool_type.interface_type == "web":
|
||||
try:
|
||||
tunnel_info = start_tunnel(
|
||||
instance_name=instance.name,
|
||||
container_port=tool_type.default_port or 0,
|
||||
)
|
||||
instance.tunnel_id = tunnel_info["container_name"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
instance.url = tunnel_info["url"]
|
||||
logger.debug(
|
||||
"Created new tunnel for instance %s: %s",
|
||||
instance.id,
|
||||
tunnel_info["url"],
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to create tunnel for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
instance.status = "error"
|
||||
instance.url = None
|
||||
await session.commit()
|
||||
raise RuntimeError(f"Failed to create tunnel: {exc}")
|
||||
else:
|
||||
instance.url = None
|
||||
instance.public_url = None
|
||||
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.restarted",
|
||||
created_by=user_id,
|
||||
status="running",
|
||||
message="Instance restarted",
|
||||
)
|
||||
return {"status": instance.status, "url": instance.url}
|
||||
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
return {"status": instance.status}
|
||||
|
||||
|
||||
async def delete_tool_instance(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
"""Delete a tool instance.
|
||||
|
||||
Raises ValueError for invalid input.
|
||||
"""
|
||||
instance = await session.get(ToolInstance, instance_id)
|
||||
if instance is None or instance.repository_id != repo_id:
|
||||
raise ValueError("instance not found")
|
||||
|
||||
# Check dirty state for clone-mode instances
|
||||
if instance.clone_mode == "clone" and not force:
|
||||
instance_dir = (
|
||||
os.path.dirname(instance.compose_path) if instance.compose_path else None
|
||||
)
|
||||
if instance_dir:
|
||||
clone_path = os.path.join(instance_dir, "repo-clone")
|
||||
if os.path.exists(clone_path):
|
||||
is_dirty, changed_files = check_dirty_state(clone_path)
|
||||
if is_dirty:
|
||||
raise RuntimeError(
|
||||
f"Repository has uncommitted changes: {changed_files}"
|
||||
)
|
||||
|
||||
# Stop Cloudflare tunnel if exists
|
||||
if instance.tunnel_id:
|
||||
try:
|
||||
stop_tunnel(instance.name)
|
||||
logger.debug(
|
||||
"Stopped tunnel for instance %s (container=%s)",
|
||||
instance.id,
|
||||
instance.tunnel_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to stop tunnel for instance %s: %s", instance.id, exc
|
||||
)
|
||||
|
||||
# Stop and remove container
|
||||
if instance.compose_path and os.path.exists(instance.compose_path):
|
||||
execute_compose_command(instance.compose_path, "down")
|
||||
|
||||
# Remove instance directory
|
||||
if instance.compose_path:
|
||||
instance_dir = os.path.dirname(instance.compose_path)
|
||||
if os.path.exists(instance_dir):
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(instance_dir)
|
||||
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.deleted",
|
||||
created_by=user_id,
|
||||
status="deleted",
|
||||
message="Instance deleted",
|
||||
)
|
||||
await session.delete(instance)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def recreate_instance_tunnel(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
) -> dict:
|
||||
"""Recreate the temporary tunnel for an instance.
|
||||
|
||||
Returns a dict with status and url.
|
||||
Raises ValueError for invalid input, RuntimeError for internal failures.
|
||||
"""
|
||||
instance = await session.get(ToolInstance, instance_id)
|
||||
if instance is None or instance.repository_id != repo_id:
|
||||
raise ValueError("instance not found")
|
||||
|
||||
if instance.status != "running":
|
||||
raise ValueError("instance must be running to recreate tunnel")
|
||||
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if not tool_type:
|
||||
raise ValueError("Tool type not found for this instance")
|
||||
|
||||
expected_name = instance.name.lower()
|
||||
logger.info(
|
||||
"Recreate tunnel for instance %s (expected container name: %s, default_port: %s)",
|
||||
instance.id,
|
||||
expected_name,
|
||||
tool_type.default_port,
|
||||
)
|
||||
|
||||
# Find the tool container
|
||||
tool_container_id = instance.container_id
|
||||
if tool_container_id:
|
||||
logger.info("Using stored container_id: %s", tool_container_id)
|
||||
else:
|
||||
tool_container_id = get_container_id(expected_name)
|
||||
if tool_container_id:
|
||||
logger.info("Found container by name: %s", tool_container_id)
|
||||
else:
|
||||
logger.error("Container %s not found", expected_name)
|
||||
raise ValueError("Could not find running container for this instance")
|
||||
|
||||
# Ensure the tool container is on the backend network
|
||||
network_name = get_backend_network_name()
|
||||
on_network = is_container_on_network(tool_container_id, network_name)
|
||||
logger.info(
|
||||
"Container %s on network %s: %s",
|
||||
tool_container_id,
|
||||
network_name,
|
||||
on_network,
|
||||
)
|
||||
if not on_network:
|
||||
logger.info(
|
||||
"Connecting container %s to network %s",
|
||||
tool_container_id,
|
||||
network_name,
|
||||
)
|
||||
connected = connect_container_to_network(tool_container_id, network_name)
|
||||
logger.info("Network connect result: %s", connected)
|
||||
|
||||
# Get the container's IP on the backend network
|
||||
target_ip = get_container_ip_on_network(tool_container_id, network_name)
|
||||
if target_ip:
|
||||
target_url = f"http://{target_ip}:{tool_type.default_port or 0}"
|
||||
logger.info(
|
||||
"Tunnel target for instance %s: %s (IP %s on %s)",
|
||||
instance.id,
|
||||
target_url,
|
||||
target_ip,
|
||||
network_name,
|
||||
)
|
||||
else:
|
||||
target_url = f"http://{expected_name}:{tool_type.default_port or 0}"
|
||||
logger.warning(
|
||||
"Could not get container IP, falling back to name-based target: %s",
|
||||
target_url,
|
||||
)
|
||||
|
||||
try:
|
||||
tunnel_info = recreate_tunnel(
|
||||
instance_name=instance.name,
|
||||
container_port=tool_type.default_port or 0,
|
||||
target_url=target_url,
|
||||
)
|
||||
logger.info(
|
||||
"Tunnel recreated: container=%s, url=%s",
|
||||
tunnel_info["container_name"],
|
||||
tunnel_info["url"],
|
||||
)
|
||||
|
||||
# Verify the tunnel can actually reach the origin
|
||||
health = check_tunnel_health(tunnel_info["url"], timeout=10)
|
||||
logger.info(
|
||||
"Tunnel health check: status=%s, code=%s, error=%s",
|
||||
health.get("tunnel_status"),
|
||||
health.get("status_code"),
|
||||
health.get("error"),
|
||||
)
|
||||
|
||||
# Also probe from inside the API container directly to the target
|
||||
probe = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-s",
|
||||
"-o",
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
"--max-time",
|
||||
"5",
|
||||
target_url,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
logger.info(
|
||||
"Direct probe from API to %s: HTTP %s", target_url, probe.stdout.strip()
|
||||
)
|
||||
|
||||
instance.tunnel_id = tunnel_info["container_name"]
|
||||
instance.public_url = tunnel_info["url"]
|
||||
instance.url = tunnel_info["url"]
|
||||
await session.commit()
|
||||
return {"status": "healthy", "url": instance.url}
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to recreate tunnel for instance %s", instance.id)
|
||||
raise RuntimeError(f"Failed to recreate tunnel: {str(exc)}")
|
||||
|
||||
|
||||
async def stop_tool_instance(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
) -> dict:
|
||||
"""Stop a tool instance.
|
||||
|
||||
Returns a dict with the stopped status.
|
||||
Raises ValueError for invalid input.
|
||||
"""
|
||||
instance = await session.get(ToolInstance, instance_id)
|
||||
if instance is None or instance.repository_id != repo_id:
|
||||
raise ValueError("instance not found")
|
||||
|
||||
if instance.tunnel_id:
|
||||
try:
|
||||
stop_tunnel(instance.name)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to stop tunnel for instance %s: %s", instance.id, exc)
|
||||
|
||||
if instance.compose_path and os.path.exists(instance.compose_path):
|
||||
execute_compose_command(instance.compose_path, "stop")
|
||||
|
||||
instance.status = "stopped"
|
||||
instance.last_stopped_at = datetime.now()
|
||||
instance.url = None
|
||||
instance.public_url = None
|
||||
instance.tunnel_id = None
|
||||
await session.commit()
|
||||
await publish_lifecycle_event(
|
||||
event_bus=_event_bus,
|
||||
session=session,
|
||||
instance=instance,
|
||||
event_type="instance.stopped",
|
||||
created_by=user_id,
|
||||
status="stopped",
|
||||
message="Instance stopped",
|
||||
)
|
||||
return {"status": instance.status}
|
||||
|
||||
Reference in New Issue
Block a user