alex/headquarter
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent failed containers from showing as running on dashboard

Browse Source 
- Add final get_container_status check in start_tool_instance before
  writing status=running; mark as error and return logs if container stopped
- Treat restarting as error in HealthMonitor when DB status was already
  running, so crash loops are surfaced instead of preserved
- Disable auto-restart (restart: unless-stopped -> restart: no) for tool
  instances in manifest compiler, legacy dockerfile path, and built-in seeds

Quality gates:
- pytest tests/unit: 210 passed
- ruff: clean on changed files
- mypy: clean on changed files
This commit is contained in:
Developer
2026-06-14 21:52:02 +00:00
parent a4e6c46a47
commit 089d802f1d
25 changed files with 185 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/tool/.pi-map.index.md
+1 -1
View File
@@ -2,7 +2,7 @@
dir: apps/api/src/services/tool
## role
Provides containerized execution environment for tools by managing Docker instances, git repositories, and compose orchestration.
Provides Docker container lifecycle management for tool instances with git repository mounting, configuration resolution, and SSH tunnel connectivity.
## parent
index: apps/api/src/services/.pi-map.index.md
map: apps/api/src/services/.pi-map.md
apps/api/src/services/tool/.pi-map.md
+3 -3
View File
File diff suppressed because one or more lines are too long
apps/api/src/services/tool/instance_service.py
+63 -16
View File
@@ -13,7 +13,14 @@ from fastapi import HTTPException, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.models import ConfigProfile, GitRepository, Project, SSHKey, ToolInstance, ToolType
from src.models import (
ConfigProfile,
GitRepository,
Project,
SSHKey,
ToolInstance,
ToolType,
)
from src.schemas.tool import CreateInstanceRequest, StartInstanceRequest
from src.services.git.clone import check_dirty_state, clone_repository
from src.services.config.config_profile_resolver import (
@@ -32,6 +39,7 @@ from src.services.docker import (
get_container_id,
get_container_ip_on_network,
get_container_logs,
get_container_status,
is_container_on_network,
render_compose_template,
sort_volumes_by_specificity,
@@ -774,7 +782,6 @@ def ensure_backend_network_in_compose(compose_path: str) -> None:
logger.info("Injected backend network '%s' into compose file", network_name)
async def prepare_manifest_instance(
session: AsyncSession,
instance: ToolInstance,
@@ -918,8 +925,6 @@ async def prepare_manifest_instance(
return image_tag, compose_content, manifest, home_dir
async def create_tool_instance(
session: AsyncSession,
user_id: uuid.UUID,
@@ -1047,15 +1052,25 @@ async def create_tool_instance(
repo_name = os.path.basename(os.path.normpath(repo_path))
workspace_target = f"{home_dir}/{repo_name}"
compose_content = f"""version: "3.8"\nservices:\n app:\n image: {image_tag}\n container_name: {instance_name.lower()}\n stdin_open: true\n tty: true\n{ports_section} environment:\n - HOME={home_dir}\n volumes:\n - {repo_path}:{workspace_target}\n working_dir: {workspace_target}\n restart: unless-stopped\n"""
compose_content = f"""version: "3.8"\nservices:
app:
image: {image_tag}
container_name: {instance_name.lower()}
stdin_open: true
tty: true
{ports_section} environment:
- HOME={home_dir}
volumes:
- {repo_path}:{workspace_target}
working_dir: {workspace_target}
restart: "no"
"""
write_compose_file(instance_dir, compose_content)
elif tool_type.definition_type == "manifest":
from src.models import ToolDefinitionManifest
manifest_def = await session.get(
ToolDefinitionManifest, tool_type.manifest_id
)
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
if not manifest_def:
raise RuntimeError("Manifest definition not found for this tool type")
@@ -1065,9 +1080,7 @@ async def create_tool_instance(
ToolDefinitionManifest, manifest_def.base_definition_id
)
if base_def:
manifest = resolve_base(
deep_merge(dict(base_def.manifest), manifest)
)
manifest = resolve_base(deep_merge(dict(base_def.manifest), manifest))
image_tag = compute_image_tag(tool_type.name, manifest)
@@ -1105,9 +1118,7 @@ async def create_tool_instance(
"WORKSPACE_NAME": os.path.basename(os.path.normpath(repo_path)),
"HOME_DIRECTORY": tool_type.home_directory or "/home/user",
}
compose_content = render_compose_template(
tool_type.compose_template, variables
)
compose_content = render_compose_template(tool_type.compose_template, variables)
write_compose_file(instance_dir, compose_content)
@@ -1674,6 +1685,41 @@ async def start_tool_instance(
logger.info("Readiness probe succeeded for instance %s", instance.id)
# Final stability check: the container must still be running after all
# post-start setup. If it has already exited/restarted, mark it failed now
# instead of optimistically reporting "running".
if instance.container_id:
final_check = get_container_status(instance.container_id)
if final_check["status"] != "running":
error_msg = (
f"Container stopped during startup: status={final_check['status']}"
)
if final_check["exit_code"] is not None:
error_msg += f", exit_code={final_check['exit_code']}"
logs = get_container_logs(instance.container_id, tail=50)
instance.status = "error"
await session.commit()
await publish_lifecycle_event(
event_bus=_event_bus,
session=session,
instance=instance,
event_type="instance.error",
created_by=user_id,
status="error",
message=error_msg,
metadata={
"exit_code": final_check["exit_code"],
"error_type": "container",
},
)
logger.error(
"Instance %s container stopped during startup: %s\nLogs:\n%s",
instance.id,
error_msg,
logs,
)
return {"status": "error", "error": error_msg, "logs": logs}
instance.status = "running"
await session.commit()
await publish_lifecycle_event(
@@ -2102,7 +2148,9 @@ async def stop_tool_instance(
try:
stop_tunnel(instance.name)
except Exception as exc:
logger.warning("Failed to stop tunnel for instance %s: %s", instance.id, exc)
logger.warning(
"Failed to stop tunnel for instance %s: %s", instance.id, exc
)
if instance.compose_path and os.path.exists(instance.compose_path):
execute_compose_command(instance.compose_path, "stop")
@@ -2144,4 +2192,3 @@ async def rename_tool_instance(
await session.commit()
await session.refresh(instance)
return instance