alex/headquarter
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent failed containers from showing as running on dashboard

Browse Source 
- Add final get_container_status check in start_tool_instance before
  writing status=running; mark as error and return logs if container stopped
- Treat restarting as error in HealthMonitor when DB status was already
  running, so crash loops are surfaced instead of preserved
- Disable auto-restart (restart: unless-stopped -> restart: no) for tool
  instances in manifest compiler, legacy dockerfile path, and built-in seeds

Quality gates:
- pytest tests/unit: 210 passed
- ruff: clean on changed files
- mypy: clean on changed files
This commit is contained in:
Developer
2026-06-14 21:52:02 +00:00
parent a4e6c46a47
commit 089d802f1d
25 changed files with 185 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/tool/instance_service.py
+63 -16
View File
@@ -13,7 +13,14 @@ from fastapi import HTTPException, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.models import ConfigProfile, GitRepository, Project, SSHKey, ToolInstance, ToolType
from src.models import (
ConfigProfile,
GitRepository,
Project,
SSHKey,
ToolInstance,
ToolType,
)
from src.schemas.tool import CreateInstanceRequest, StartInstanceRequest
from src.services.git.clone import check_dirty_state, clone_repository
from src.services.config.config_profile_resolver import (
@@ -32,6 +39,7 @@ from src.services.docker import (
get_container_id,
get_container_ip_on_network,
get_container_logs,
get_container_status,
is_container_on_network,
render_compose_template,
sort_volumes_by_specificity,
@@ -774,7 +782,6 @@ def ensure_backend_network_in_compose(compose_path: str) -> None:
logger.info("Injected backend network '%s' into compose file", network_name)
async def prepare_manifest_instance(
session: AsyncSession,
instance: ToolInstance,
@@ -918,8 +925,6 @@ async def prepare_manifest_instance(
return image_tag, compose_content, manifest, home_dir
async def create_tool_instance(
session: AsyncSession,
user_id: uuid.UUID,
@@ -1047,15 +1052,25 @@ async def create_tool_instance(
repo_name = os.path.basename(os.path.normpath(repo_path))
workspace_target = f"{home_dir}/{repo_name}"
compose_content = f"""version: "3.8"\nservices:\n app:\n image: {image_tag}\n container_name: {instance_name.lower()}\n stdin_open: true\n tty: true\n{ports_section} environment:\n - HOME={home_dir}\n volumes:\n - {repo_path}:{workspace_target}\n working_dir: {workspace_target}\n restart: unless-stopped\n"""
compose_content = f"""version: "3.8"\nservices:
app:
image: {image_tag}
container_name: {instance_name.lower()}
stdin_open: true
tty: true
{ports_section} environment:
- HOME={home_dir}
volumes:
- {repo_path}:{workspace_target}
working_dir: {workspace_target}
restart: "no"
"""
write_compose_file(instance_dir, compose_content)
elif tool_type.definition_type == "manifest":
from src.models import ToolDefinitionManifest
manifest_def = await session.get(
ToolDefinitionManifest, tool_type.manifest_id
)
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
if not manifest_def:
raise RuntimeError("Manifest definition not found for this tool type")
@@ -1065,9 +1080,7 @@ async def create_tool_instance(
ToolDefinitionManifest, manifest_def.base_definition_id
)
if base_def:
manifest = resolve_base(
deep_merge(dict(base_def.manifest), manifest)
)
manifest = resolve_base(deep_merge(dict(base_def.manifest), manifest))
image_tag = compute_image_tag(tool_type.name, manifest)
@@ -1105,9 +1118,7 @@ async def create_tool_instance(
"WORKSPACE_NAME": os.path.basename(os.path.normpath(repo_path)),
"HOME_DIRECTORY": tool_type.home_directory or "/home/user",
}
compose_content = render_compose_template(
tool_type.compose_template, variables
)
compose_content = render_compose_template(tool_type.compose_template, variables)
write_compose_file(instance_dir, compose_content)
@@ -1674,6 +1685,41 @@ async def start_tool_instance(
logger.info("Readiness probe succeeded for instance %s", instance.id)
# Final stability check: the container must still be running after all
# post-start setup. If it has already exited/restarted, mark it failed now
# instead of optimistically reporting "running".
if instance.container_id:
final_check = get_container_status(instance.container_id)
if final_check["status"] != "running":
error_msg = (
f"Container stopped during startup: status={final_check['status']}"
)
if final_check["exit_code"] is not None:
error_msg += f", exit_code={final_check['exit_code']}"
logs = get_container_logs(instance.container_id, tail=50)
instance.status = "error"
await session.commit()
await publish_lifecycle_event(
event_bus=_event_bus,
session=session,
instance=instance,
event_type="instance.error",
created_by=user_id,
status="error",
message=error_msg,
metadata={
"exit_code": final_check["exit_code"],
"error_type": "container",
},
)
logger.error(
"Instance %s container stopped during startup: %s\nLogs:\n%s",
instance.id,
error_msg,
logs,
)
return {"status": "error", "error": error_msg, "logs": logs}
instance.status = "running"
await session.commit()
await publish_lifecycle_event(
@@ -2102,7 +2148,9 @@ async def stop_tool_instance(
try:
stop_tunnel(instance.name)
except Exception as exc:
logger.warning("Failed to stop tunnel for instance %s: %s", instance.id, exc)
logger.warning(
"Failed to stop tunnel for instance %s: %s", instance.id, exc
)
if instance.compose_path and os.path.exists(instance.compose_path):
execute_compose_command(instance.compose_path, "stop")
@@ -2144,4 +2192,3 @@ async def rename_tool_instance(
await session.commit()
await session.refresh(instance)
return instance