alex/headquarter
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source 
# Conflicts:
#	apps/api/.pi-lens/cache/review-graph.json
#	apps/web/.pi-lens/cache/review-graph.json
This commit is contained in:
Alex
2026-05-28 19:14:56 +02:00
parent 9f90624aa6 62c1fb3836
commit 9ccaae04db
26 changed files with 8063 additions and 1683 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/.pi-lens/cache/review-graph.json
Vendored
-1
View File
File diff suppressed because one or more lines are too long
apps/api/alembic/versions/2026_05_28_add_tool_definition_manifests.py
+366 -18
View File
@@ -1,33 +1,381 @@
"""bridge: add_tool_definition_manifests (ghost migration)
"""add tool definition manifests
Revision ID: 2026_05_28_add_tool_definition_manifests
Revises: 20260527_160017_add_pi_agent
Create Date: 2026-05-28
This is a no-op bridge migration. The original migration was created on
another branch and applied to the production database, but the file was
subsequently removed. This bridge allows Alembic to reconcile the DB
state with the local revision history.
Create Date: 2026-05-28T11:00:00
"""
from collections.abc import Sequence
import json
import uuid
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "2026_05_28_add_tool_definition_manifests"
down_revision: str | None = "20260527_160017_add_pi_agent"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
down_revision: Union[str, None] = "20260527_160017_add_pi_agent"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
BASE_UBUNTU_ID = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
PI_AGENT_MANIFEST_ID = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3")
def upgrade() -> None:
# No-op: this migration was already applied on the target database.
# If the schema was actually changed by the original migration,
# those changes must be preserved manually or via a separate migration.
pass
conn = op.get_bind()
# ── Create tool_definition_manifests table ───────────────────────
op.create_table(
"tool_definition_manifests",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("name", sa.String(64), nullable=False),
sa.Column("display_name", sa.String(128), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("category", sa.String(64), nullable=True),
sa.Column("interface_type", sa.String(16), nullable=False),
sa.Column("base_image", sa.String(256), nullable=True),
sa.Column("base_definition_id", sa.UUID(), nullable=True),
sa.Column(
"base_version", sa.String(32), nullable=False, server_default="latest"
),
sa.Column("manifest", sa.JSON(), nullable=False),
sa.Column("dockerfile_cache", sa.Text(), nullable=True),
sa.Column("compose_cache", sa.Text(), nullable=True),
sa.Column("version", sa.String(32), nullable=False, server_default="v1"),
sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"),
sa.Column("created_by_id", sa.UUID(), nullable=True),
sa.Column(
"created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
),
sa.Column(
"updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name"),
sa.ForeignKeyConstraint(
["base_definition_id"], ["tool_definition_manifests.id"]
),
sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]),
sa.CheckConstraint(
"(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)",
name="ck_tool_definition_manifests_base_required",
),
)
# ── Add columns to tool_types ────────────────────────────────────
# Check if manifest_id exists before adding
conn = op.get_bind()
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
""")
)
if not result.fetchone():
op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True))
op.create_foreign_key(
"fk_tool_types_manifest_id",
"tool_types",
"tool_definition_manifests",
["manifest_id"],
["id"],
)
# Update definition_type to allow 'legacy' and 'manifest'
result = conn.execute(
sa.text("""
SELECT constraint_name FROM information_schema.check_constraints
WHERE constraint_name = 'chk_definition_type'
""")
)
if result.fetchone():
op.drop_constraint("chk_definition_type", "tool_types", type_="check")
op.execute("ALTER TABLE tool_types ALTER COLUMN definition_type TYPE VARCHAR(16)")
op.execute(
"ALTER TABLE tool_types ALTER COLUMN definition_type SET DEFAULT 'legacy'"
)
# ── Add columns to tool_instances ────────────────────────────────
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
""")
)
if not result.fetchone():
op.add_column(
"tool_instances",
sa.Column(
"manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True
),
)
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
""")
)
if not result.fetchone():
op.add_column(
"tool_instances",
sa.Column("image_tag", sa.String(256), nullable=True),
)
# ── Data migration: create base definition + pi-agent manifest ───
conn.execute(
sa.text(
"""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, interface_type, base_image,
manifest, is_base, version, created_at, updated_at)
VALUES
(:base_id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base',
'Base development environment with build tools', 'terminal',
'ubuntu:24.04', :base_manifest, true, 'v1', now(), now())
"""
),
{
"base_id": BASE_UBUNTU_ID,
"base_manifest": json.dumps(
{
"name": "ubuntu-24.04-dev",
"display_name": "Ubuntu 24.04 Dev Base",
"interface_type": "terminal",
"base_image": "ubuntu:24.04",
"packages": {
"apt": [
"curl",
"wget",
"git",
"build-essential",
"ca-certificates",
"python3",
"python3-pip",
]
},
"user": {
"name": "user",
"uid": 1000,
"gid": 1000,
"create_home": True,
"shell": "/bin/bash",
},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
}
),
},
)
conn.execute(
sa.text(
"""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, category, interface_type,
base_definition_id, base_version, manifest, version, created_at, updated_at)
VALUES
(:manifest_id, 'pi-agent', 'Pi Agent',
'Terminal-based coding harness with nvim, ranger, tmux',
'development', 'terminal', :base_id, 'v1', :manifest, 'v1',
now(), now())
"""
),
{
"manifest_id": PI_AGENT_MANIFEST_ID,
"base_id": BASE_UBUNTU_ID,
"manifest": json.dumps(
{
"name": "pi-agent",
"display_name": "Pi Agent",
"description": "Terminal-based coding harness",
"category": "development",
"interface_type": "terminal",
"base_definition_id": str(BASE_UBUNTU_ID),
"base_version": "v1",
"packages": {
"apt": [
"neovim",
"ranger",
"tmux",
"htop",
"tree",
"jq",
],
"node": {"version": "20"},
"npm_global": ["@earendil-works/pi-coding-agent"],
},
"user": {
"name": "user",
"uid": 1001,
"gid": 1001,
"create_home": True,
"shell": "/bin/bash",
},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
"scripts": {
"build": [
"git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'",
"mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf",
],
"startup": [
"if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi",
],
},
"mounts": [
{
"name": "workspace",
"target": "/workspace",
"source_type": "repo",
"writable": True,
"owner": "user",
},
{
"name": "ssh_keys",
"target": "/home/user/.ssh",
"source_type": "ssh_key",
"mode": "0700",
"file_mode": "0600",
"readonly": True,
},
{
"name": "pi_state",
"target": "/tmp/.pi/agents",
"source_type": "instance",
"writable": True,
},
{
"name": "pi_config",
"target": "/home/user/.pi",
"source_type": "git_mount",
"git_mount_ref": "dotfiles",
"writable": True,
"owner": "user",
},
],
"runtime": {
"command": ["/bin/bash"],
"stdin_open": True,
"tty": True,
"working_dir": "/workspace",
},
}
),
},
)
# ── Update existing pi-agent tool_type ───────────────────────────
conn.execute(
sa.text(
"""
UPDATE tool_types
SET manifest_id = :manifest_id,
definition_type = 'manifest',
dockerfile_template = NULL,
compose_template = NULL
WHERE name = 'pi-agent'
"""
),
{"manifest_id": PI_AGENT_MANIFEST_ID},
)
def downgrade() -> None:
# No-op: cannot safely reverse a migration whose original operations
# are unknown.
pass
conn = op.get_bind()
# Restore pi-agent templates if manifest_id column exists
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
""")
)
has_manifest_id = result.fetchone() is not None
if has_manifest_id:
conn.execute(
sa.text(
"""
UPDATE tool_types
SET manifest_id = NULL,
definition_type = 'dockerfile',
dockerfile_template = :dockerfile,
compose_template = :compose
WHERE name = 'pi-agent'
"""
),
{
"dockerfile": """# Pi Coding Agent - Terminal-based coding harness
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \\
curl wget git neovim ranger tmux htop tree jq \\
ca-certificates python3 python3-pip build-essential \\
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\
&& apt-get install -y nodejs \\
&& rm -rf /var/lib/apt/lists/*
RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
RUN useradd -m -s /bin/bash user
WORKDIR /home/user
RUN git config --global init.defaultBranch main \\
&& git config --global user.email "dev@headquarter.local" \\
&& git config --global user.name "Developer"
RUN echo 'set -g mouse on\\nset -g default-terminal "screen-256color"' > /home/user/.tmux.conf
RUN mkdir -p /home/user/.config/ranger \\
&& echo 'set preview_files true\\nset use_preview_script true' > /home/user/.config/ranger/rc.conf
RUN mkdir -p /home/user/.pi/agent
USER user
CMD ["/bin/bash"]
""",
"compose": """services:
app:
build: .
stdin_open: true
tty: true
volumes:
- ${REPO_PATH}:/workspace
working_dir: /workspace
command: /bin/bash""",
},
)
# Drop columns conditionally
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
""")
)
if result.fetchone():
op.drop_column("tool_instances", "image_tag")
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
""")
)
if result.fetchone():
op.drop_column("tool_instances", "manifest_compiled_at")
if has_manifest_id:
op.drop_constraint(
"fk_tool_types_manifest_id", "tool_types", type_="foreignkey"
)
op.drop_column("tool_types", "manifest_id")
op.drop_table("tool_definition_manifests")
apps/api/src/api/tool_definitions.py
+424
View File
@@ -0,0 +1,424 @@
"""Tool definition API endpoints."""
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.auth.dependencies import get_current_user_id, get_db_session
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.tool_type import ToolType
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
resolve_base,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/tool-definitions", tags=["tool-definitions"])
class CreateToolDefinitionRequest(BaseModel):
"""Request body for creating a tool definition manifest."""
model_config = {"extra": "ignore"}
name: str = Field(description="Unique identifier (kebab-case)")
display_name: str = Field(description="Human-readable name")
description: str | None = Field(default=None)
category: str = Field(default="development")
interface_type: str = Field(default="terminal", description="web or terminal")
base_image: str | None = Field(default=None, description="Direct base image")
base_definition_id: str | None = Field(
default=None, description="Reference to a base definition"
)
base_version: str = Field(default="latest")
manifest: dict = Field(description="The full manifest JSON")
class UpdateToolDefinitionRequest(BaseModel):
"""Request body for updating a tool definition manifest."""
model_config = {"extra": "ignore"}
display_name: str | None = Field(default=None)
description: str | None = Field(default=None)
category: str | None = Field(default=None)
manifest: dict | None = Field(default=None)
base_version: str | None = Field(default=None)
@router.post(
"",
summary="Create tool definition",
description="Create a new tool definition manifest.",
)
async def create_tool_definition(
data: CreateToolDefinitionRequest,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Create a new tool definition manifest.
Args:
data: Manifest data.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with created definition details.
"""
# Validate base reference
if not data.base_image and not data.base_definition_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Either base_image or base_definition_id is required",
)
base_def_id = None
if data.base_definition_id:
try:
base_def_id = uuid.UUID(data.base_definition_id)
except ValueError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid base_definition_id: {data.base_definition_id}",
)
base_def = await session.get(ToolDefinitionManifest, base_def_id)
if not base_def:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Base definition not found: {data.base_definition_id}",
)
if not base_def.is_base:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Referenced definition is not a base definition",
)
# Check name uniqueness
existing = await session.execute(
select(ToolDefinitionManifest).where(ToolDefinitionManifest.name == data.name)
)
if existing.scalar_one_or_none():
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Tool definition '{data.name}' already exists",
)
definition = ToolDefinitionManifest(
name=data.name,
display_name=data.display_name,
description=data.description,
category=data.category,
interface_type=data.interface_type,
base_image=data.base_image,
base_definition_id=base_def_id,
base_version=data.base_version,
manifest=data.manifest,
created_by_id=user_id,
)
session.add(definition)
await session.commit()
await session.refresh(definition)
logger.info("Created tool definition %s (%s)", definition.id, definition.name)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"description": definition.description,
"category": definition.category,
"interface_type": definition.interface_type,
"base_image": definition.base_image,
"base_definition_id": str(definition.base_definition_id)
if definition.base_definition_id
else None,
"base_version": definition.base_version,
"manifest": definition.manifest,
"is_base": definition.is_base,
"created_at": definition.created_at.isoformat(),
}
@router.get(
"",
summary="List tool definitions",
description="List all tool definition manifests.",
)
async def list_tool_definitions(
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
include_bases: bool = True,
) -> dict:
"""List all tool definition manifests.
Args:
user_id: Authenticated user ID.
session: Database session.
include_bases: Whether to include base definitions.
Returns:
Dictionary containing list of definitions.
"""
query = select(ToolDefinitionManifest)
if not include_bases:
query = query.where(ToolDefinitionManifest.is_base == False)
result = await session.execute(
query.order_by(ToolDefinitionManifest.created_at.desc())
)
definitions = result.scalars().all()
return {
"definitions": [
{
"id": str(d.id),
"name": d.name,
"display_name": d.display_name,
"description": d.description,
"category": d.category,
"interface_type": d.interface_type,
"is_base": d.is_base,
"base_image": d.base_image,
"base_definition_id": str(d.base_definition_id)
if d.base_definition_id
else None,
"base_version": d.base_version,
"version": d.version,
"created_at": d.created_at.isoformat(),
}
for d in definitions
]
}
@router.get(
"/{definition_id}",
summary="Get tool definition",
description="Get a specific tool definition manifest.",
)
async def get_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Get a specific tool definition manifest.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with definition details.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"description": definition.description,
"category": definition.category,
"interface_type": definition.interface_type,
"base_image": definition.base_image,
"base_definition_id": str(definition.base_definition_id)
if definition.base_definition_id
else None,
"base_version": definition.base_version,
"manifest": definition.manifest,
"dockerfile_cache": definition.dockerfile_cache,
"compose_cache": definition.compose_cache,
"version": definition.version,
"is_base": definition.is_base,
"created_at": definition.created_at.isoformat(),
"updated_at": definition.updated_at.isoformat(),
}
@router.put(
"/{definition_id}",
summary="Update tool definition",
description="Update a tool definition manifest.",
)
async def update_tool_definition(
definition_id: uuid.UUID,
data: UpdateToolDefinitionRequest,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Update a tool definition manifest.
Args:
definition_id: UUID of the definition.
data: Update data.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with updated definition details.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
if data.display_name is not None:
definition.display_name = data.display_name
if data.description is not None:
definition.description = data.description
if data.category is not None:
definition.category = data.category
if data.manifest is not None:
definition.manifest = data.manifest
if data.base_version is not None:
definition.base_version = data.base_version
await session.commit()
await session.refresh(definition)
logger.info("Updated tool definition %s (%s)", definition.id, definition.name)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"manifest": definition.manifest,
"updated_at": definition.updated_at.isoformat(),
}
@router.delete(
"/{definition_id}",
summary="Delete tool definition",
description="Delete a tool definition manifest.",
)
async def delete_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Delete a tool definition manifest.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with deletion status.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
# Check if any tool types reference this manifest
result = await session.execute(
select(ToolType).where(ToolType.manifest_id == definition_id)
)
referencing = result.scalars().all()
if referencing:
tool_names = ", ".join(t.name for t in referencing)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Cannot delete: referenced by tool types: {tool_names}",
)
await session.delete(definition)
await session.commit()
logger.info("Deleted tool definition %s (%s)", definition.id, definition.name)
return {"status": "deleted", "id": str(definition_id)}
@router.post(
"/{definition_id}/compile",
summary="Compile tool definition",
description="Compile a manifest to Dockerfile + Compose preview without building.",
)
async def compile_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Compile a manifest to Dockerfile + Compose preview.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with compiled Dockerfile, Compose, and image tag.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
manifest = dict(definition.manifest)
# Resolve base if referenced
if definition.base_definition_id:
base_def = await session.get(
ToolDefinitionManifest, definition.base_definition_id
)
if base_def:
base_manifest = dict(base_def.manifest)
manifest = resolve_base(deep_merge(base_manifest, manifest))
# Compile
dockerfile = compile_dockerfile(manifest)
entrypoint = compile_entrypoint(manifest)
image_tag = compute_image_tag(definition.name, manifest)
# Dummy compose with placeholder variables
dummy_vars = {
"IMAGE_TAG": image_tag,
"INSTANCE_NAME": f"{definition.name}-preview",
"INSTANCE_DIR": "/data/instances/preview",
"REPO_PATH": "/data/repos/preview",
"SSH_PATH": "/data/instances/preview/.ssh",
"TOOL_PORT": "8080",
"EXTRA_ENV": {},
"EXTRA_VOLUMES": [],
}
compose = compile_compose(manifest, dummy_vars)
# Update cache
definition.dockerfile_cache = dockerfile
definition.compose_cache = compose
await session.commit()
return {
"id": str(definition.id),
"name": definition.name,
"dockerfile": dockerfile,
"entrypoint": entrypoint,
"compose": compose,
"image_tag": image_tag,
}
apps/api/src/api/tool_instances.py
+246 -34
View File
@@ -62,6 +62,16 @@ from src.services.docker import (
write_env_file,
)
from src.services.docker_build import build_image
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
merge_with_config,
resolve_base,
)
from src.services.permission_fixer import apply_mount_permissions
from src.services.readiness_probe import execute_probe
from src.services.ssh_keys import cleanup_ssh_key_files, prepare_ssh_key_files
@@ -961,6 +971,142 @@ async def get_instance(
}
async def _prepare_manifest_instance(
session: AsyncSession,
instance: ToolInstance,
instance_dir: str,
repo_path: str,
configs: list,
env_vars: dict,
extra_volumes: list,
working_directory: str | None,
) -> tuple[str, str, dict]:
"""Build image and generate compose from a manifest-based tool type.
Returns:
Tuple of (image_tag, compose_content, resolved_manifest)
"""
from src.models.tool_definition_manifest import ToolDefinitionManifest
tool_type = await session.get(ToolType, instance.tool_type_id)
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
if not manifest_def:
raise RuntimeError(f"Manifest not found for tool type {tool_type.id}")
manifest = dict(manifest_def.manifest)
# Resolve base if referenced
if manifest_def.base_definition_id:
base_def = await session.get(
ToolDefinitionManifest, manifest_def.base_definition_id
)
if base_def:
base_manifest = dict(base_def.manifest)
manifest = resolve_base(deep_merge(base_manifest, manifest))
else:
logger.warning(
"Base definition %s not found for manifest %s",
manifest_def.base_definition_id,
manifest_def.id,
)
# Merge tool configs
tool_config_dicts = [
{
"config_type": c.config_type,
"key": c.key,
"value": c.value,
"file_path": c.file_path,
"port_override": c.port_override,
"start_command": c.start_command,
"working_directory": c.working_directory,
"environment_variables": c.environment_variables,
"volumes": c.volumes,
}
for c in configs
]
manifest = merge_with_config(manifest, tool_config_dicts)
# Resolve extra env and volumes from merge_with_config
extra_env = manifest.pop("_extra_env", {})
extra_cfg_volumes = manifest.pop("_extra_volumes", [])
env_vars.update(extra_env)
extra_volumes.extend(extra_cfg_volumes)
# Compute image tag
image_tag = compute_image_tag(tool_type.name, manifest)
# Check if image already exists
check = subprocess.run(
["docker", "images", "-q", image_tag],
capture_output=True,
text=True,
)
image_exists = check.returncode == 0 and check.stdout.strip()
if not image_exists:
# Compile and build
dockerfile = compile_dockerfile(manifest)
entrypoint = compile_entrypoint(manifest)
build_ctx = {
"Dockerfile": dockerfile,
".headquarter/entrypoint.sh": entrypoint,
}
returncode, stdout, stderr = await asyncio.to_thread(
build_image,
instance_dir=instance_dir,
dockerfile=dockerfile,
tag=image_tag,
build_context=build_ctx,
)
if returncode != 0:
raise RuntimeError(f"Docker build failed: {stderr}")
logger.info("Built image %s for instance %s", image_tag, instance.id)
else:
logger.info("Reusing existing image %s for instance %s", image_tag, instance.id)
# Prepare SSH path for mount resolution
ssh_path = ""
if instance.clone_mode == "clone":
ssh_path = os.path.join(instance_dir, ".ssh")
# Resolve git mount variables from config profile
git_mount_vars = {}
if instance.selected_config_profile_id:
resolved_profile = await resolve_profile(
session, instance.selected_config_profile_id
)
for gm in resolved_profile.git_mounts or []:
ref = gm.get("git_mount_ref", "default")
# The actual resolution happens in _resolve_git_mounts; we store placeholder
git_mount_vars[f"GIT_MOUNT_{ref}"] = ""
variables = {
"IMAGE_TAG": image_tag,
"INSTANCE_NAME": instance.name.lower(),
"INSTANCE_DIR": instance_dir,
"REPO_PATH": repo_path,
"SSH_PATH": ssh_path,
"TOOL_PORT": instance.port or 0,
"EXTRA_ENV": env_vars,
"EXTRA_VOLUMES": extra_volumes,
**git_mount_vars,
}
compose_content = compile_compose(manifest, variables)
# Cache
instance.image_tag = image_tag
instance.manifest_compiled_at = datetime.now()
return image_tag, compose_content, manifest
@router.post(
"/{project_id}/repositories/{repo_id}/instances/{instance_id}/start",
summary="Start instance",
@@ -1119,41 +1265,85 @@ async def start_instance(
"Wrote %d config files for instance %s", len(config_files), instance.id
)
# Mount SSH key for clone-mode instances
if instance.clone_mode == "clone":
repo = await session.get(GitRepository, instance.repository_id)
if repo and repo.ssh_key_id:
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
if ssh_key:
try:
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
extra_volumes.append(
{
"source": ssh_dir,
"target": "/root/.ssh",
"type": "ro",
}
)
logger.debug(
"Mounted SSH key for clone-mode instance %s", instance.id
)
except Exception as exc:
logger.error(
"Failed to prepare SSH key for instance %s: %s",
instance.id,
exc,
)
# ── MANIFEST-BASED FLOW ──────────────────────────────────────
tool_type = await session.get(ToolType, instance.tool_type_id)
resolved_manifest = None
# Modify compose file if needed (port override, start command, working dir, volumes)
if port_override or start_command or working_directory or extra_volumes:
_modify_compose_file(
instance.compose_path,
port_override,
start_command,
working_directory,
extra_volumes,
)
logger.debug("Modified compose file for instance %s", instance.id)
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
logger.info("Using manifest-based startup for instance %s", instance.id)
# Determine repo path
repo = await session.get(GitRepository, instance.repository_id)
repo_path = repo.path if repo else ""
if instance.clone_mode == "clone":
repo_path = os.path.join(instance_dir, "repo-clone")
try:
(
image_tag,
compose_content,
resolved_manifest,
) = await _prepare_manifest_instance(
session=session,
instance=instance,
instance_dir=instance_dir,
repo_path=repo_path,
configs=configs,
env_vars=env_vars,
extra_volumes=extra_volumes,
working_directory=working_directory,
)
write_compose_file(instance_dir, compose_content)
logger.debug(
"Generated manifest-based compose for instance %s", instance.id
)
except Exception as exc:
logger.exception(
"Manifest compilation failed for instance %s: %s", instance.id, exc
)
instance.status = "error"
await session.commit()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Manifest compilation failed: {exc}",
)
else:
# ── LEGACY FLOW ──────────────────────────────────────────
# Mount SSH key for clone-mode instances
if instance.clone_mode == "clone":
repo = await session.get(GitRepository, instance.repository_id)
if repo and repo.ssh_key_id:
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
if ssh_key:
try:
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
extra_volumes.append(
{
"source": ssh_dir,
"target": "/root/.ssh",
"type": "ro",
}
)
logger.debug(
"Mounted SSH key for clone-mode instance %s", instance.id
)
except Exception as exc:
logger.error(
"Failed to prepare SSH key for instance %s: %s",
instance.id,
exc,
)
# Modify compose file if needed (port override, start command, working dir, volumes)
if port_override or start_command or working_directory or extra_volumes:
_modify_compose_file(
instance.compose_path,
port_override,
start_command,
working_directory,
extra_volumes,
)
logger.debug("Modified compose file for instance %s", instance.id)
# Sanitize compose file to remove invalid port mappings from old instances
_sanitize_compose_file(instance.compose_path)
@@ -1244,6 +1434,28 @@ async def start_instance(
startup_result["waited_seconds"],
)
# Apply mount permission fixes for manifest-based instances
if resolved_manifest and instance.container_id:
mounts = resolved_manifest.get("mounts", [])
if mounts:
logger.debug(
"Applying permission fixes for instance %s (%d mounts)",
instance.id,
len(mounts),
)
permission_results = apply_mount_permissions(
instance.container_id,
mounts,
)
for result in permission_results:
if not result["success"]:
logger.warning(
"Permission fix failed for mount %s on instance %s: %s",
result["mount_name"],
instance.id,
result["error"],
)
# Execute readiness probe if configured
tool_type = await session.get(ToolType, instance.tool_type_id)
if tool_type and instance.container_id:
apps/api/src/main.py
+2
View File
@@ -18,6 +18,7 @@ from src.api.instance_proxy import router as instance_proxy_router
from src.api.config_folders import router as config_folders_router
from src.api.config_profiles import router as config_profiles_router
from src.api.tool_configs import router as tool_configs_router
from src.api.tool_definitions import router as tool_definitions_router
from src.api.tool_instances import router as tool_instances_router
from src.api.tool_instances import sessions_router
from src.api.tool_types import router as tool_types_router
@@ -128,6 +129,7 @@ app.include_router(ssh_keys_router)
app.include_router(git_repositories_router)
app.include_router(user_config_router)
app.include_router(tool_types_router)
app.include_router(tool_definitions_router)
app.include_router(config_folders_router)
app.include_router(config_profiles_router)
app.include_router(tool_instances_router)
apps/api/src/models/__init__.py
+2
View File
@@ -5,6 +5,7 @@ from src.models.git_repository import GitRepository
from src.models.project import Project
from src.models.ssh_key import SSHKey
from src.models.terminal_session import TerminalSessionModel
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.tool_instance import ToolInstance
from src.models.tool_type import ToolType
from src.models.user import User
@@ -19,6 +20,7 @@ __all__ = [
"Project",
"SSHKey",
"TerminalSessionModel",
"ToolDefinitionManifest",
"ToolInstance",
"ToolType",
"User",
apps/api/src/models/tool_definition_manifest.py
+67
View File
@@ -0,0 +1,67 @@
"""Tool Definition Manifest model."""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import Boolean, ForeignKey, JSON, String, Text
from sqlalchemy import Uuid as UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
if TYPE_CHECKING:
from src.models.user import User
class ToolDefinitionManifest(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""A declarative manifest that compiles to Dockerfile + Compose.
Can be either:
- A base definition (is_base=True) with a FROM image and common packages
- A tool definition (is_base=False) that references a base + adds specifics
"""
__tablename__ = "tool_definition_manifests"
name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
display_name: Mapped[str] = mapped_column(String(128), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
category: Mapped[str | None] = mapped_column(String(64), nullable=True)
interface_type: Mapped[str] = mapped_column(String(16), nullable=False)
# Base: either a direct image or a reference to another manifest
base_image: Mapped[str | None] = mapped_column(String(256), nullable=True)
base_definition_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("tool_definition_manifests.id"),
nullable=True,
)
base_version: Mapped[str] = mapped_column(
String(32), nullable=False, default="latest"
)
# The full manifest JSON
manifest: Mapped[dict] = mapped_column(JSON, nullable=False)
# Caches for quick inspection
dockerfile_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
compose_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
# Versioning
version: Mapped[str] = mapped_column(String(32), nullable=False, default="v1")
is_base: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("users.id"),
nullable=True,
)
# Relationships
created_by: Mapped["User | None"] = relationship(
foreign_keys=[created_by_id],
)
base_definition: Mapped["ToolDefinitionManifest | None"] = relationship(
remote_side="ToolDefinitionManifest.id",
foreign_keys=[base_definition_id],
)
apps/api/src/models/tool_instance.py
+13 -29
View File
@@ -33,42 +33,26 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base):
owner_id: Mapped[uuid.UUID] = mapped_column(
UUID(), ForeignKey("users.id"), nullable=False
)
status: Mapped[str] = mapped_column(
String(50), nullable=False, default="pending"
)
container_id: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
container_name: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
compose_path: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
url: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
public_url: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
tunnel_id: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
port: Mapped[int | None] = mapped_column(
Integer, nullable=True
)
status: Mapped[str] = mapped_column(String(50), nullable=False, default="pending")
container_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
container_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
compose_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
public_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
tunnel_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
port: Mapped[int | None] = mapped_column(Integer, nullable=True)
last_started_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
last_stopped_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
probe_result: Mapped[dict | None] = mapped_column(
JSON, nullable=True
)
clone_mode: Mapped[str] = mapped_column(
String(20), nullable=False, default="mount"
manifest_compiled_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
image_tag: Mapped[str | None] = mapped_column(String(256), nullable=True)
probe_result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
clone_mode: Mapped[str] = mapped_column(String(20), nullable=False, default="mount")
branch: Mapped[str | None] = mapped_column(
String(255), nullable=True, default="main"
)
apps/api/src/models/tool_type.py
+17 -4
View File
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
if TYPE_CHECKING:
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.user import User
@@ -18,12 +19,19 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
category: Mapped[str] = mapped_column(String(50), nullable=False, default="other")
interface_type: Mapped[str] = mapped_column(String(20), nullable=False, default="web")
interface_type: Mapped[str] = mapped_column(
String(20), nullable=False, default="web"
)
requires_port: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
default_port: Mapped[int] = mapped_column(nullable=False)
definition_type: Mapped[str] = mapped_column(
String(20), nullable=False, default="compose"
) # "compose" or "dockerfile"
String(16), nullable=False, default="legacy"
) # "legacy" | "manifest"
manifest_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("tool_definition_manifests.id"),
nullable=True,
)
compose_template: Mapped[str | None] = mapped_column(Text, nullable=True)
dockerfile_template: Mapped[str | None] = mapped_column(Text, nullable=True)
build_context: Mapped[dict | None] = mapped_column(
@@ -31,11 +39,16 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
)
readiness_probe: Mapped[dict | None] = mapped_column(JSON, nullable=True)
startup_command: Mapped[str | None] = mapped_column(Text, nullable=True)
required_variables: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
required_variables: Mapped[list[str]] = mapped_column(
JSON, default=list, nullable=False
)
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("users.id"),
nullable=True,
)
manifest: Mapped["ToolDefinitionManifest | None"] = relationship(
foreign_keys=[manifest_id],
)
created_by: Mapped["User | None"] = relationship()
apps/api/src/services/manifest_compiler.py
+412
View File
@@ -0,0 +1,412 @@
"""Manifest compiler: transforms ToolDefinitionManifest into Dockerfile + Compose."""
import hashlib
import json
import shlex
from copy import deepcopy
from typing import Any
import yaml
def resolve_base(manifest: dict) -> dict:
"""Merge a base definition into a tool manifest.
If the manifest has base_definition_id, the base manifest is loaded
and merged. Tool-specific values override base values.
Args:
manifest: The tool manifest JSON (may reference a base)
Returns:
A fully resolved manifest with base values merged in.
"""
result = deepcopy(manifest)
base_definition_id = result.pop("base_definition_id", None)
base_version = result.pop("base_version", "latest")
if base_definition_id:
# This will be provided by the caller (they have the DB session)
# For now, we assume the manifest has been pre-resolved
# or the caller provides the base manifest separately.
pass
return result
def deep_merge(base: dict, override: dict) -> dict:
"""Deep merge two manifests. Arrays are concatenated; dicts are merged.
Args:
base: The base manifest.
override: The tool-specific overrides.
Returns:
Merged manifest.
"""
merged = deepcopy(base)
for key, value in override.items():
if key == "mounts" and isinstance(value, list):
# Concatenate mount arrays
existing = merged.get("mounts", [])
merged["mounts"] = existing + deepcopy(value)
elif key == "scripts" and isinstance(value, dict):
# Merge script categories
if "scripts" not in merged:
merged["scripts"] = {}
for script_key, script_value in value.items():
existing = merged["scripts"].get(script_key, [])
merged["scripts"][script_key] = existing + deepcopy(script_value)
elif key == "packages" and isinstance(value, dict):
# Union package arrays
if "packages" not in merged:
merged["packages"] = {}
for pkg_key, pkg_value in value.items():
if (
pkg_key in merged["packages"]
and isinstance(merged["packages"][pkg_key], list)
and isinstance(pkg_value, list)
):
merged["packages"][pkg_key] = merged["packages"][
pkg_key
] + deepcopy(pkg_value)
else:
merged["packages"][pkg_key] = deepcopy(pkg_value)
elif key == "env" and isinstance(value, dict):
# Dict merge: override wins on key conflict
if "env" not in merged:
merged["env"] = {}
merged["env"].update(deepcopy(value))
elif (
isinstance(value, dict) and key in merged and isinstance(merged[key], dict)
):
# Generic dict merge
merged[key] = {**merged[key], **deepcopy(value)}
else:
# Override entirely
merged[key] = deepcopy(value)
return merged
def compile_dockerfile(manifest: dict) -> str:
"""Compile a resolved manifest into a Dockerfile string.
Args:
manifest: Fully resolved manifest JSON.
Returns:
Dockerfile content.
"""
lines: list[str] = []
# FROM
base_image = manifest.get("base_image", "ubuntu:24.04")
lines.append(f"FROM {base_image}")
lines.append("")
# Build-time environment
env = manifest.get("env", {})
for key, value in env.items():
lines.append(f"ENV {key}={shlex.quote(value)}")
if env:
lines.append("")
# System packages (apt)
apt_packages = manifest.get("packages", {}).get("apt", [])
if apt_packages:
lines.append("RUN apt-get update && apt-get install -y \\\\")
for pkg in apt_packages[:-1]:
lines.append(f" {pkg} \\\\")
lines.append(f" {apt_packages[-1]} \\\\")
lines.append(" && rm -rf /var/lib/apt/lists/*")
lines.append("")
# Node.js
node = manifest.get("packages", {}).get("node")
if node:
version = node.get("version", "20")
lines.append(
f"RUN curl -fsSL https://deb.nodesource.com/setup_{version}.x | bash - && \\\\"
)
lines.append(" apt-get install -y nodejs && \\\\")
lines.append(" rm -rf /var/lib/apt/lists/*")
lines.append("")
# NPM global packages
npm_packages = manifest.get("packages", {}).get("npm_global", [])
if npm_packages:
pkg_list = " ".join(shlex.quote(p) for p in npm_packages)
lines.append(f"RUN npm install -g {pkg_list}")
lines.append("")
# Pip packages
pip_packages = manifest.get("packages", {}).get("pip", [])
if pip_packages:
pkg_list = " ".join(shlex.quote(p) for p in pip_packages)
lines.append(f"RUN pip install {pkg_list}")
lines.append("")
# User creation
user = manifest.get("user")
if user:
name = user["name"]
uid = user["uid"]
gid = user["gid"]
create_home = "-m " if user.get("create_home", True) else ""
shell = user.get("shell", "/bin/bash")
lines.append(f"RUN groupadd -g {gid} {name} && \\\\")
lines.append(f" useradd -u {uid} -g {gid} {create_home}-s {shell} {name}")
lines.append("")
# Build scripts
build_scripts = manifest.get("scripts", {}).get("build", [])
for script in build_scripts:
# Normalize multi-line scripts into single RUN command
stripped_lines = [
line.strip() for line in script.strip().split("\n") if line.strip()
]
if stripped_lines:
normalized = " && ".join(stripped_lines)
lines.append(f"RUN {normalized}")
if build_scripts:
lines.append("")
# Create mount target directories
mounts = manifest.get("mounts", [])
if mounts:
dirs = [mount["target"] for mount in mounts]
dir_str = " ".join(dirs)
lines.append(f"RUN mkdir -p {dir_str}")
if user:
lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
lines.append("")
# Entrypoint for startup scripts
startup_scripts = manifest.get("scripts", {}).get("startup", [])
if startup_scripts:
lines.append(
"COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint"
)
lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint")
lines.append("")
# Switch to runtime user
if user:
lines.append(f"USER {user['name']}")
lines.append(f"WORKDIR /home/{user['name']}")
lines.append("")
# Entrypoint and CMD
runtime = manifest.get("runtime", {})
if startup_scripts:
lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')
command = runtime.get("command", ["/bin/bash"])
cmd_json = json.dumps(command)
lines.append(f"CMD {cmd_json}")
return "\n".join(lines)
def compile_entrypoint(manifest: dict) -> str:
"""Generate the startup entrypoint script from startup scripts.
Args:
manifest: Fully resolved manifest JSON.
Returns:
Shell script content.
"""
lines = ["#!/bin/bash", "set -e", ""]
startup_scripts = manifest.get("scripts", {}).get("startup", [])
for script in startup_scripts:
lines.append(script)
lines.append("")
lines.append('exec "$@"')
return "\n".join(lines)
def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:
"""Compile a resolved manifest into a Docker Compose string.
Args:
manifest: Fully resolved manifest JSON.
variables: Resolved values: IMAGE_TAG, INSTANCE_NAME, REPO_PATH, etc.
Returns:
Docker Compose YAML content.
"""
runtime = manifest.get("runtime", {})
user = manifest.get("user")
interface_type = manifest["interface_type"]
service: dict[str, Any] = {
"image": variables["IMAGE_TAG"],
"container_name": variables["INSTANCE_NAME"],
"restart": "unless-stopped",
}
# Terminal-specific fields
if runtime.get("stdin_open", False):
service["stdin_open"] = True
if runtime.get("tty", False):
service["tty"] = True
if runtime.get("working_dir"):
service["working_dir"] = runtime["working_dir"]
# User override
if user:
service["user"] = f"{user['uid']}:{user['gid']}"
# Ports for web tools
default_port = manifest.get("default_port")
if interface_type == "web" and default_port:
service["ports"] = [f"{variables['TOOL_PORT']}:{default_port}"]
# Environment
env = manifest.get("env", {})
if env:
service["environment"] = dict(env)
# Merge extra env from config
extra_env = variables.get("EXTRA_ENV", {})
if extra_env:
if "environment" not in service:
service["environment"] = {}
service["environment"].update(extra_env)
# Volumes from mount schema
volumes = []
for mount in manifest.get("mounts", []):
source = resolve_mount_source(mount, variables)
if not source:
continue
target = mount["target"]
readonly = ":ro" if mount.get("readonly", False) else ""
volumes.append(f"{source}:{target}{readonly}")
# Append extra volumes from tool config / config profile
for vol in variables.get("EXTRA_VOLUMES", []):
vol_str = f"{vol['source']}:{vol['target']}"
if vol.get("readonly"):
vol_str += ":ro"
volumes.append(vol_str)
if volumes:
service["volumes"] = volumes
compose = {"services": {"app": service}}
return yaml.dump(compose, default_flow_style=False)
def resolve_mount_source(mount: dict, variables: dict[str, Any]) -> str:
"""Resolve a mount's source_type to an actual host path.
Args:
mount: Mount definition from manifest.
variables: Resolved variables dict.
Returns:
Host path string, or empty string if unresolved.
"""
source_type = mount.get("source_type", "host_path")
if source_type == "repo":
return variables.get("REPO_PATH", "")
elif source_type == "ssh_key":
return variables.get("SSH_PATH", "")
elif source_type == "instance":
instance_dir = variables.get("INSTANCE_DIR", "")
mount_name = mount.get("name", "unknown")
return f"{instance_dir}/mounts/{mount_name}"
elif source_type == "git_mount":
ref = mount.get("git_mount_ref", "default")
return variables.get(f"GIT_MOUNT_{ref}", "")
elif source_type == "host_path":
return mount.get("source", "")
return ""
def compute_image_tag(tool_name: str, manifest: dict) -> str:
"""Compute a deterministic image tag from manifest content.
Args:
tool_name: Human-readable tool name.
manifest: Fully resolved manifest JSON.
Returns:
Docker image tag string.
"""
# Canonicalize: sort keys, stable JSON
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8]
safe_name = tool_name.lower().replace(" ", "-").replace("_", "-")
return f"headquarter/{safe_name}-{hash_suffix}:latest"
def merge_with_config(
manifest: dict, tool_configs: list[dict], profile: dict | None = None
) -> dict:
"""Merge ToolConfig and ConfigProfile overrides into a manifest.
Args:
manifest: Base manifest from tool definition.
tool_configs: List of ToolConfig records.
profile: Resolved ConfigProfile (optional).
Returns:
Manifest with overrides applied.
"""
result = deepcopy(manifest)
# Apply ToolConfigs
extra_env: dict[str, str] = {}
extra_volumes: list[dict] = []
for config in tool_configs:
if config.get("config_type") == "env":
extra_env[config["key"]] = config["value"]
elif config.get("config_type") == "file" and config.get("file_path"):
# Files are handled outside the manifest (written to instance dir)
pass
if config.get("port_override"):
result["default_port"] = config["port_override"]
if config.get("start_command"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["command"] = config["start_command"].split()
if config.get("working_directory"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["working_dir"] = config["working_directory"]
if config.get("environment_variables"):
extra_env.update(config["environment_variables"])
if config.get("volumes"):
extra_volumes.extend(config["volumes"])
# Apply ConfigProfile
if profile:
if profile.get("environment_variables"):
extra_env.update(profile["environment_variables"])
if profile.get("mounts"):
extra_volumes.extend(profile["mounts"])
# Profile hints override everything
hints = profile.get("hints", {})
if hints.get("start_command"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["command"] = hints["start_command"].split()
if hints.get("working_directory"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["working_dir"] = hints["working_directory"]
if hints.get("port_override"):
result["default_port"] = hints["port_override"]
# Store merged extras for the compose compiler
result["_extra_env"] = extra_env
result["_extra_volumes"] = extra_volumes
return result
apps/api/src/services/permission_fixer.py
+164
View File
@@ -0,0 +1,164 @@
"""Permission fixer: applies mount permission policies post-start."""
import logging
import subprocess
from typing import Any
logger = logging.getLogger(__name__)
def apply_mount_permissions(
container_id: str,
mounts: list[dict],
timeout: int = 10,
) -> list[dict[str, Any]]:
"""Apply permission policies to mounted directories in a running container.
Runs `chown`, `chmod`, and file-mode fixes for each mount that declares
an owner, mode, or file_mode. Requires the container to have a root user.
Args:
container_id: Docker container ID or name.
mounts: List of mount definitions from the manifest.
timeout: Max seconds per docker exec command.
Returns:
List of result dicts: [{mount_name, success, error}]
"""
results = []
for mount in mounts:
name = mount.get("name", "unknown")
target = mount["target"]
owner = mount.get("owner")
mode = mount.get("mode")
file_mode = mount.get("file_mode")
result: dict[str, Any] = {
"mount_name": name,
"success": True,
"error": None,
}
# Skip if no permission policy defined
if not owner and not mode and not file_mode:
results.append(result)
continue
try:
if owner:
_run_in_container(
container_id,
["chown", "-R", f"{owner}:{owner}", target],
timeout,
)
logger.debug(
"Applied owner %s to %s in container %s",
owner,
target,
container_id,
)
if mode and result["success"]:
_run_in_container(
container_id,
["chmod", mode, target],
timeout,
)
logger.debug(
"Applied mode %s to %s in container %s",
mode,
target,
container_id,
)
if file_mode and result["success"]:
_run_in_container(
container_id,
[
"sh",
"-c",
f"find {target} -type f -exec chmod {file_mode} {{}} +",
],
timeout,
)
logger.debug(
"Applied file_mode %s to files in %s in container %s",
file_mode,
target,
container_id,
)
except PermissionFixError as exc:
result["success"] = False
result["error"] = str(exc)
logger.warning(
"Permission fix failed for mount %s (target=%s): %s",
name,
target,
exc,
)
results.append(result)
return results
class PermissionFixError(Exception):
"""Raised when a permission fix command fails."""
pass
def _run_in_container(
container_id: str,
command: list[str],
timeout: int,
) -> None:
"""Run a command inside a container as root.
Args:
container_id: Docker container ID or name.
command: Command + args to execute.
timeout: Max seconds to wait.
Raises:
PermissionFixError: If the command fails or times out.
"""
cmd = ["docker", "exec", "--user", "root", container_id] + command
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
raise PermissionFixError(
f"Command timed out after {timeout}s: {' '.join(command)}"
)
except FileNotFoundError:
raise PermissionFixError(f"Docker command not found: {' '.join(command)}")
if result.returncode != 0:
raise PermissionFixError(
f"Command failed (rc={result.returncode}): {result.stderr.strip()}"
)
def check_root_user_available(container_id: str, timeout: int = 5) -> bool:
"""Check if the container has a root user we can exec as.
Args:
container_id: Docker container ID or name.
timeout: Max seconds to wait.
Returns:
True if root user exists and is usable.
"""
try:
_run_in_container(container_id, ["id", "root"], timeout)
return True
except PermissionFixError:
return False
apps/api/tests/unit/test_manifest_compiler.py
+315
View File
@@ -0,0 +1,315 @@
"""Unit tests for the manifest compiler."""
import pytest
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
merge_with_config,
resolve_base,
)
class TestResolveBase:
"""Tests for resolve_base."""
def test_returns_manifest_unchanged_when_no_base(self) -> None:
manifest = {"name": "test", "base_image": "ubuntu:24.04"}
result = resolve_base(manifest)
assert result["name"] == "test"
assert "base_definition_id" not in result
class TestDeepMerge:
"""Tests for deep_merge."""
def test_packages_are_unioned(self) -> None:
base = {"packages": {"apt": ["curl", "git"]}}
override = {"packages": {"apt": ["neovim"]}}
result = deep_merge(base, override)
assert result["packages"]["apt"] == ["curl", "git", "neovim"]
def test_node_version_overrides(self) -> None:
base = {"packages": {"node": {"version": "18"}}}
override = {"packages": {"node": {"version": "20"}}}
result = deep_merge(base, override)
assert result["packages"]["node"]["version"] == "20"
def test_env_is_merged_with_override_winning(self) -> None:
base = {"env": {"FOO": "base", "BAR": "base"}}
override = {"env": {"FOO": "override"}}
result = deep_merge(base, override)
assert result["env"]["FOO"] == "override"
assert result["env"]["BAR"] == "base"
def test_build_scripts_are_concatenated(self) -> None:
base = {"scripts": {"build": ["echo base"]}}
override = {"scripts": {"build": ["echo override"]}}
result = deep_merge(base, override)
assert result["scripts"]["build"] == ["echo base", "echo override"]
def test_mounts_are_concatenated(self) -> None:
base = {"mounts": [{"name": "base-mount", "target": "/base"}]}
override = {"mounts": [{"name": "tool-mount", "target": "/tool"}]}
result = deep_merge(base, override)
assert len(result["mounts"]) == 2
def test_user_is_overridden_entirely(self) -> None:
base = {"user": {"name": "base", "uid": 1000}}
override = {"user": {"name": "tool", "uid": 1001}}
result = deep_merge(base, override)
assert result["user"]["name"] == "tool"
assert result["user"]["uid"] == 1001
class TestCompileDockerfile:
"""Tests for compile_dockerfile."""
def test_includes_from(self) -> None:
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
df = compile_dockerfile(manifest)
assert "FROM ubuntu:24.04" in df
def test_installs_apt_packages(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"apt": ["curl", "git"]},
}
df = compile_dockerfile(manifest)
assert "apt-get install -y" in df
assert "curl" in df
assert "git" in df
assert "rm -rf /var/lib/apt/lists/*" in df
def test_installs_node(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"node": {"version": "20"}},
}
df = compile_dockerfile(manifest)
assert "nodesource.com/setup_20.x" in df
def test_installs_npm_global(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"npm_global": ["@scope/pkg"]},
}
df = compile_dockerfile(manifest)
assert "npm install -g @scope/pkg" in df
def test_creates_user(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"user": {"name": "dev", "uid": 1001, "gid": 1001},
}
df = compile_dockerfile(manifest)
assert "groupadd -g 1001 dev" in df
assert "useradd -u 1001 -g 1001" in df
assert "USER dev" in df
def test_build_scripts_as_run_commands(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"scripts": {"build": ["echo hello", "echo world"]},
}
df = compile_dockerfile(manifest)
assert "RUN echo hello" in df
assert "RUN echo world" in df
def test_creates_mount_directories(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"user": {"name": "dev", "uid": 1001, "gid": 1001},
"mounts": [
{"name": "ws", "target": "/workspace"},
{"name": "cfg", "target": "/config"},
],
}
df = compile_dockerfile(manifest)
assert "mkdir -p /workspace /config" in df
assert "chown -R dev:dev /workspace /config" in df
def test_entrypoint_for_startup_scripts(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"scripts": {"startup": ["echo start"]},
}
df = compile_dockerfile(manifest)
assert 'ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]' in df
def test_cmd_from_runtime(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"runtime": {"command": ["/bin/bash", "-il"]},
}
df = compile_dockerfile(manifest)
assert 'CMD ["/bin/bash", "-il"]' in df
def test_default_cmd_when_no_runtime(self) -> None:
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
df = compile_dockerfile(manifest)
assert 'CMD ["/bin/bash"]' in df
class TestCompileEntrypoint:
"""Tests for compile_entrypoint."""
def test_includes_shebang_and_set_e(self) -> None:
manifest = {"scripts": {"startup": ["echo hello"]}}
ep = compile_entrypoint(manifest)
assert "#!/bin/bash" in ep
assert "set -e" in ep
def test_includes_startup_scripts(self) -> None:
manifest = {"scripts": {"startup": ["echo hello", "echo world"]}}
ep = compile_entrypoint(manifest)
assert "echo hello" in ep
assert "echo world" in ep
def test_ends_with_exec(self) -> None:
manifest: dict = {"scripts": {"startup": []}}
ep = compile_entrypoint(manifest)
assert 'exec "$@"' in ep
class TestCompileCompose:
"""Tests for compile_compose."""
def test_includes_image_and_container_name(self) -> None:
manifest = {"name": "test", "interface_type": "terminal"}
vars_dict = {"IMAGE_TAG": "test:v1", "INSTANCE_NAME": "test-1"}
compose = compile_compose(manifest, vars_dict)
assert "image: test:v1" in compose
assert "container_name: test-1" in compose
def test_terminal_fields(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"runtime": {"stdin_open": True, "tty": True, "working_dir": "/workspace"},
}
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
assert "stdin_open: true" in compose
assert "tty: true" in compose
assert "working_dir: /workspace" in compose
def test_web_ports(self) -> None:
manifest = {
"name": "test",
"interface_type": "web",
"default_port": 8080,
}
compose = compile_compose(
manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n", "TOOL_PORT": "3000"}
)
assert "3000:8080" in compose
def test_user_override(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"user": {"uid": 1001, "gid": 1001},
}
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
assert "user: 1001:1001" in compose
def test_mounts_resolved(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"mounts": [
{"name": "ws", "target": "/workspace", "source_type": "repo"},
{
"name": "ssh",
"target": "/home/user/.ssh",
"source_type": "ssh_key",
"readonly": True,
},
],
}
compose = compile_compose(
manifest,
{
"IMAGE_TAG": "t",
"INSTANCE_NAME": "n",
"REPO_PATH": "/repos/myrepo",
"SSH_PATH": "/keys/ssh",
},
)
assert "/repos/myrepo:/workspace" in compose
assert "/keys/ssh:/home/user/.ssh:ro" in compose
def test_extra_volumes_appended(self) -> None:
manifest = {"name": "test", "interface_type": "terminal"}
compose = compile_compose(
manifest,
{
"IMAGE_TAG": "t",
"INSTANCE_NAME": "n",
"EXTRA_VOLUMES": [{"source": "/host/x", "target": "/container/x"}],
},
)
assert "/host/x:/container/x" in compose
class TestComputeImageTag:
"""Tests for compute_image_tag."""
def test_is_deterministic(self) -> None:
manifest = {"name": "test", "packages": {"apt": ["curl"]}}
tag1 = compute_image_tag("My Tool", manifest)
tag2 = compute_image_tag("My Tool", manifest)
assert tag1 == tag2
def test_changes_with_content(self) -> None:
manifest1 = {"name": "test", "packages": {"apt": ["curl"]}}
manifest2 = {"name": "test", "packages": {"apt": ["wget"]}}
tag1 = compute_image_tag("test", manifest1)
tag2 = compute_image_tag("test", manifest2)
assert tag1 != tag2
def test_lowercases_name(self) -> None:
manifest = {"name": "test"}
tag = compute_image_tag("My Tool", manifest)
assert "my-tool" in tag
def test_valid_docker_reference(self) -> None:
manifest = {"name": "test"}
tag = compute_image_tag("test", manifest)
assert tag.startswith("headquarter/test-")
assert tag.endswith(":latest")
class TestMergeWithConfig:
"""Tests for merge_with_config."""
def test_applies_tool_config_env(self) -> None:
manifest = {"name": "test"}
configs = [
{"config_type": "env", "key": "FOO", "value": "bar"},
]
result = merge_with_config(manifest, configs)
assert result["_extra_env"]["FOO"] == "bar"
def test_applies_port_override(self) -> None:
manifest = {"name": "test", "default_port": 8080}
configs = [{"port_override": 3000}]
result = merge_with_config(manifest, configs)
assert result["default_port"] == 3000
def test_applies_start_command(self) -> None:
manifest = {"name": "test", "runtime": {"command": ["/bin/bash"]}}
configs = [{"start_command": "/bin/sh"}]
result = merge_with_config(manifest, configs)
assert result["runtime"]["command"] == ["/bin/sh"]
apps/api/tests/unit/test_permission_fixer.py
+145
View File
@@ -0,0 +1,145 @@
"""Unit tests for the permission fixer."""
from unittest.mock import MagicMock, patch
import pytest
from src.services.permission_fixer import (
PermissionFixError,
apply_mount_permissions,
check_root_user_available,
_run_in_container,
)
class TestApplyMountPermissions:
"""Tests for apply_mount_permissions."""
@patch("src.services.permission_fixer._run_in_container")
def test_applies_chown_when_owner_declared(self, mock_run) -> None:
mounts = [
{"name": "workspace", "target": "/workspace", "owner": "user"},
]
results = apply_mount_permissions("abc123", mounts)
assert len(results) == 1
assert results[0]["mount_name"] == "workspace"
assert results[0]["success"] is True
mock_run.assert_called_once()
args = mock_run.call_args[0]
assert args[0] == "abc123"
assert args[1] == ["chown", "-R", "user:user", "/workspace"]
@patch("src.services.permission_fixer._run_in_container")
def test_applies_chmod_when_mode_declared(self, mock_run) -> None:
mounts = [
{"name": "ssh", "target": "/home/user/.ssh", "mode": "0700"},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is True
# Only chmod called (no owner, so no chown)
assert mock_run.call_count == 1
chmod_call = mock_run.call_args_list[0]
assert chmod_call[0][1] == ["chmod", "0700", "/home/user/.ssh"]
@patch("src.services.permission_fixer._run_in_container")
def test_applies_file_mode_when_declared(self, mock_run) -> None:
mounts = [
{
"name": "ssh",
"target": "/home/user/.ssh",
"file_mode": "0600",
},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is True
# Only file_mode called (no owner, no mode)
assert mock_run.call_count == 1
file_mode_call = mock_run.call_args_list[0]
assert file_mode_call[0][1][0] == "sh"
assert (
"find /home/user/.ssh -type f -exec chmod 0600" in file_mode_call[0][1][2]
)
@patch("src.services.permission_fixer._run_in_container")
def test_skips_mount_with_no_policy(self, mock_run) -> None:
mounts = [
{"name": "workspace", "target": "/workspace", "writable": True},
]
results = apply_mount_permissions("abc123", mounts)
assert len(results) == 1
assert results[0]["success"] is True
mock_run.assert_not_called()
@patch("src.services.permission_fixer._run_in_container")
def test_reports_failure_on_command_error(self, mock_run) -> None:
mock_run.side_effect = PermissionFixError("chown failed")
mounts = [
{"name": "workspace", "target": "/workspace", "owner": "user"},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is False
assert "chown failed" in results[0]["error"]
@patch("src.services.permission_fixer._run_in_container")
def test_stops_on_first_failure(self, mock_run) -> None:
"""If chown fails, chmod and file_mode should not run."""
mock_run.side_effect = PermissionFixError("chown failed")
mounts = [
{
"name": "workspace",
"target": "/workspace",
"owner": "user",
"mode": "0755",
"file_mode": "0644",
},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is False
assert mock_run.call_count == 1 # Only chown attempted
class TestRunInContainer:
"""Tests for _run_in_container."""
@patch("subprocess.run")
def test_success(self, mock_run) -> None:
mock_run.return_value = MagicMock(returncode=0, stderr="")
_run_in_container("abc123", ["echo", "hello"], 10)
mock_run.assert_called_once()
cmd = mock_run.call_args[0][0]
assert cmd == ["docker", "exec", "--user", "root", "abc123", "echo", "hello"]
@patch("subprocess.run")
def test_failure_raises(self, mock_run) -> None:
mock_run.return_value = MagicMock(returncode=1, stderr="permission denied")
with pytest.raises(PermissionFixError, match="permission denied"):
_run_in_container("abc123", ["chown", "x"], 10)
@patch("subprocess.run")
def test_timeout_raises(self, mock_run) -> None:
import subprocess
mock_run.side_effect = subprocess.TimeoutExpired(cmd=["docker"], timeout=10)
with pytest.raises(PermissionFixError, match="timed out"):
_run_in_container("abc123", ["chown", "x"], 10)
class TestCheckRootUserAvailable:
"""Tests for check_root_user_available."""
@patch("src.services.permission_fixer._run_in_container")
def test_returns_true_when_root_exists(self, mock_run) -> None:
assert check_root_user_available("abc123") is True
@patch("src.services.permission_fixer._run_in_container")
def test_returns_false_when_root_missing(self, mock_run) -> None:
mock_run.side_effect = PermissionFixError("no such user")
assert check_root_user_available("abc123") is False
apps/api/tests/unit/test_tool_instances_legacy.py
+804
View File
@@ -0,0 +1,804 @@
"""Unit tests for legacy tool instance fallback paths.
Verifies that tool types with definition_type "dockerfile", "compose",
and "legacy" continue to use the original startup flow after the
manifest-based flow was introduced.
"""
import os
import uuid
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.api.tool_instances import create_instance, start_instance
from src.models.git_repository import GitRepository
from src.models.tool_instance import ToolInstance
from src.models.tool_type import ToolType
from src.models.ssh_key import SSHKey
from src.models.user import User
@pytest.fixture
def fake_user_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def fake_project_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def fake_repo_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def fake_tool_type_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def fake_instance_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def mock_session(fake_user_id, fake_project_id, fake_repo_id, fake_tool_type_id):
"""Return an async SQLAlchemy session with basic mocks."""
session = AsyncMock()
user = User(id=fake_user_id, email="test@example.com")
project = MagicMock()
project.id = fake_project_id
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is User and pk == fake_user_id:
return user
if model is GitRepository and pk == fake_repo_id:
return repo
return None
def _add(instance):
if getattr(instance, "created_at", None) is None:
instance.created_at = datetime.now()
if getattr(instance, "updated_at", None) is None:
instance.updated_at = datetime.now()
session.get.side_effect = _get
session.add = MagicMock(side_effect=_add)
session.execute.return_value = MagicMock(scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[]))))
return session
class TestCreateInstanceDockerfileLegacy:
"""Legacy dockerfile definition type in create_instance."""
@patch("src.api.tool_instances.ensure_instance_directory")
@patch("src.api.tool_instances.find_free_port")
@patch("src.api.tool_instances.build_image")
@patch("src.api.tool_instances.write_compose_file")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_builds_from_dockerfile_template(
self,
mock_get_project,
mock_get_user,
mock_write_compose,
mock_build_image,
mock_find_port,
mock_ensure_dir,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'dockerfile', build_image is called with the template."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_find_port.return_value = 12345
mock_ensure_dir.return_value = "/data/instances/test-instance"
mock_build_image.return_value = (0, "built", "")
tool_type = ToolType(
id=fake_tool_type_id,
name="legacy-df-tool",
display_name="Legacy DF Tool",
default_port=8080,
definition_type="dockerfile",
dockerfile_template="FROM python:3.11\nRUN echo hi",
compose_template=None,
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
data = MagicMock()
data.tool_type_id = str(fake_tool_type_id)
data.display_name = None
data.clone_mode = "mount"
data.branch = None
data.new_branch = None
data.config_profile_id = None
result = await create_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
data=data,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "pending"
mock_build_image.assert_called_once()
call_kwargs = mock_build_image.call_args.kwargs
assert call_kwargs["dockerfile"] == "FROM python:3.11\nRUN echo hi"
mock_write_compose.assert_called_once()
@patch("src.api.tool_instances.ensure_instance_directory")
@patch("src.api.tool_instances.find_free_port")
@patch("src.api.tool_instances.build_image")
@patch("src.api.tool_instances.write_compose_file")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_dockerfile_build_failure_raises_500(
self,
mock_get_project,
mock_get_user,
mock_write_compose,
mock_build_image,
mock_find_port,
mock_ensure_dir,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_tool_type_id,
) -> None:
"""Failed dockerfile build should raise HTTP 500."""
from fastapi import HTTPException
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_find_port.return_value = 12345
mock_ensure_dir.return_value = "/data/instances/test-instance"
mock_build_image.return_value = (1, "", "build failed")
tool_type = ToolType(
id=fake_tool_type_id,
name="legacy-df-tool",
display_name="Legacy DF Tool",
default_port=8080,
definition_type="dockerfile",
dockerfile_template="FROM invalid",
compose_template=None,
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
data = MagicMock()
data.tool_type_id = str(fake_tool_type_id)
data.display_name = None
data.clone_mode = "mount"
data.branch = None
data.new_branch = None
data.config_profile_id = None
with pytest.raises(HTTPException) as exc_info:
await create_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
data=data,
user_id=fake_user_id,
session=mock_session,
)
assert exc_info.value.status_code == 500
assert "Failed to build Docker image" in exc_info.value.detail
class TestCreateInstanceComposeLegacy:
"""Legacy compose definition type in create_instance."""
@patch("src.api.tool_instances.ensure_instance_directory")
@patch("src.api.tool_instances.find_free_port")
@patch("src.api.tool_instances.render_compose_template")
@patch("src.api.tool_instances.write_compose_file")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_renders_compose_template(
self,
mock_get_project,
mock_get_user,
mock_write_compose,
mock_render_compose,
mock_find_port,
mock_ensure_dir,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'compose', render_compose_template is called."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_find_port.return_value = 12345
mock_ensure_dir.return_value = "/data/instances/test-instance"
mock_render_compose.return_value = "services:\n app:\n image: nginx"
tool_type = ToolType(
id=fake_tool_type_id,
name="legacy-compose-tool",
display_name="Legacy Compose Tool",
default_port=80,
definition_type="compose",
dockerfile_template=None,
compose_template="services:\n app:\n image: nginx\n ports:\n - ${TOOL_PORT}:80",
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
data = MagicMock()
data.tool_type_id = str(fake_tool_type_id)
data.display_name = None
data.clone_mode = "mount"
data.branch = None
data.new_branch = None
data.config_profile_id = None
result = await create_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
data=data,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "pending"
mock_render_compose.assert_called_once()
args = mock_render_compose.call_args[0]
assert "services:" in args[0]
mock_write_compose.assert_called_once()
class TestCreateInstanceManifestNotCalledForLegacy:
"""Ensure manifest compiler is never invoked for legacy types."""
@patch("src.api.tool_instances.ensure_instance_directory")
@patch("src.api.tool_instances.find_free_port")
@patch("src.api.tool_instances.build_image")
@patch("src.api.tool_instances.write_compose_file")
@patch("src.api.tool_instances._prepare_manifest_instance")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_dockerfile_does_not_call_manifest_compiler(
self,
mock_get_project,
mock_get_user,
mock_prepare_manifest,
mock_write_compose,
mock_build_image,
mock_find_port,
mock_ensure_dir,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_tool_type_id,
) -> None:
"""Legacy dockerfile type must not trigger manifest compilation."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_find_port.return_value = 12345
mock_ensure_dir.return_value = "/data/instances/test-instance"
mock_build_image.return_value = (0, "built", "")
tool_type = ToolType(
id=fake_tool_type_id,
name="legacy-df",
display_name="Legacy",
default_port=8080,
definition_type="dockerfile",
dockerfile_template="FROM alpine",
compose_template=None,
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
data = MagicMock()
data.tool_type_id = str(fake_tool_type_id)
data.display_name = None
data.clone_mode = "mount"
data.branch = None
data.new_branch = None
data.config_profile_id = None
await create_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
data=data,
user_id=fake_user_id,
session=mock_session,
)
mock_prepare_manifest.assert_not_called()
class TestStartInstanceLegacyFallback:
"""Legacy paths in start_instance must NOT call manifest compiler."""
@patch("src.api.tool_instances.wait_for_container_running")
@patch("src.api.tool_instances.execute_compose_command")
@patch("src.api.tool_instances.get_container_id")
@patch("src.api.tool_instances.get_container_name")
@patch("src.api.tool_instances.connect_container_to_network")
@patch("src.api.tool_instances._sanitize_compose_file")
@patch("src.api.tool_instances._prepare_manifest_instance")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_legacy_type_skips_manifest_flow(
self,
mock_get_project,
mock_get_user,
mock_prepare_manifest,
mock_sanitize,
mock_connect_network,
mock_get_container_name,
mock_get_container_id,
mock_execute_compose,
mock_wait_container,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_instance_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'legacy', start_instance uses old flow."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_execute_compose.return_value = (0, "started", "")
mock_get_container_id.return_value = "abc123"
mock_get_container_name.return_value = "test-container"
mock_connect_network.return_value = True
mock_wait_container.return_value = {"success": True, "status": "running", "waited_seconds": 0.5}
instance = ToolInstance(
id=fake_instance_id,
name="legacy-instance",
repository_id=fake_repo_id,
tool_type_id=fake_tool_type_id,
compose_path="/data/instances/legacy-instance/docker-compose.yml",
status="stopped",
clone_mode="mount",
created_at=datetime.now(),
updated_at=datetime.now(),
)
tool_type = ToolType(
id=fake_tool_type_id,
name="legacy-tool",
display_name="Legacy Tool",
default_port=8080,
definition_type="legacy",
manifest_id=None,
dockerfile_template=None,
compose_template="services:\n app:\n image: nginx",
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolInstance and pk == fake_instance_id:
return instance
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
# Ensure compose file exists so the check passes
with patch("os.path.exists", return_value=True):
result = await start_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
instance_id=fake_instance_id,
data=None,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "running"
mock_prepare_manifest.assert_not_called()
mock_execute_compose.assert_called_once()
@patch("src.api.tool_instances.wait_for_container_running")
@patch("src.api.tool_instances.execute_compose_command")
@patch("src.api.tool_instances.get_container_id")
@patch("src.api.tool_instances.get_container_name")
@patch("src.api.tool_instances.connect_container_to_network")
@patch("src.api.tool_instances._sanitize_compose_file")
@patch("src.api.tool_instances._prepare_manifest_instance")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_compose_type_skips_manifest_flow(
self,
mock_get_project,
mock_get_user,
mock_prepare_manifest,
mock_sanitize,
mock_connect_network,
mock_get_container_name,
mock_get_container_id,
mock_execute_compose,
mock_wait_container,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_instance_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'compose', start_instance uses old flow."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_execute_compose.return_value = (0, "started", "")
mock_get_container_id.return_value = "abc123"
mock_get_container_name.return_value = "test-container"
mock_connect_network.return_value = True
mock_wait_container.return_value = {"success": True, "status": "running", "waited_seconds": 0.5}
instance = ToolInstance(
id=fake_instance_id,
name="compose-instance",
repository_id=fake_repo_id,
tool_type_id=fake_tool_type_id,
compose_path="/data/instances/compose-instance/docker-compose.yml",
status="stopped",
clone_mode="mount",
created_at=datetime.now(),
updated_at=datetime.now(),
)
tool_type = ToolType(
id=fake_tool_type_id,
name="compose-tool",
display_name="Compose Tool",
default_port=8080,
definition_type="compose",
manifest_id=None,
dockerfile_template=None,
compose_template="services:\n app:\n image: nginx",
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolInstance and pk == fake_instance_id:
return instance
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
with patch("os.path.exists", return_value=True):
result = await start_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
instance_id=fake_instance_id,
data=None,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "running"
mock_prepare_manifest.assert_not_called()
mock_execute_compose.assert_called_once()
@patch("src.api.tool_instances.wait_for_container_running")
@patch("src.api.tool_instances.execute_compose_command")
@patch("src.api.tool_instances.get_container_id")
@patch("src.api.tool_instances.get_container_name")
@patch("src.api.tool_instances.connect_container_to_network")
@patch("src.api.tool_instances._sanitize_compose_file")
@patch("src.api.tool_instances._prepare_manifest_instance")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_dockerfile_type_skips_manifest_flow(
self,
mock_get_project,
mock_get_user,
mock_prepare_manifest,
mock_sanitize,
mock_connect_network,
mock_get_container_name,
mock_get_container_id,
mock_execute_compose,
mock_wait_container,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_instance_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'dockerfile', start_instance uses old flow."""
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_execute_compose.return_value = (0, "started", "")
mock_get_container_id.return_value = "abc123"
mock_get_container_name.return_value = "test-container"
mock_connect_network.return_value = True
mock_wait_container.return_value = {"success": True, "status": "running", "waited_seconds": 0.5}
instance = ToolInstance(
id=fake_instance_id,
name="df-instance",
repository_id=fake_repo_id,
tool_type_id=fake_tool_type_id,
compose_path="/data/instances/df-instance/docker-compose.yml",
status="stopped",
clone_mode="mount",
created_at=datetime.now(),
updated_at=datetime.now(),
)
tool_type = ToolType(
id=fake_tool_type_id,
name="df-tool",
display_name="Dockerfile Tool",
default_port=8080,
definition_type="dockerfile",
manifest_id=None,
dockerfile_template="FROM alpine",
compose_template=None,
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
async def _get(model, pk):
if model is ToolInstance and pk == fake_instance_id:
return instance
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
return None
mock_session.get.side_effect = _get
with patch("os.path.exists", return_value=True):
result = await start_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
instance_id=fake_instance_id,
data=None,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "running"
mock_prepare_manifest.assert_not_called()
mock_execute_compose.assert_called_once()
class TestStartInstanceManifestBranch:
"""Manifest branch is taken ONLY when definition_type == 'manifest'."""
@patch("src.api.tool_instances.wait_for_container_running")
@patch("src.api.tool_instances.execute_compose_command")
@patch("src.api.tool_instances.get_container_id")
@patch("src.api.tool_instances.get_container_name")
@patch("src.api.tool_instances.connect_container_to_network")
@patch("src.api.tool_instances._sanitize_compose_file")
@patch("src.api.tool_instances._prepare_manifest_instance")
@patch("src.api.tool_instances.write_compose_file")
@patch("src.api.tool_instances._get_user")
@patch("src.api.tool_instances._get_owned_project")
async def test_manifest_type_calls_compiler(
self,
mock_get_project,
mock_get_user,
mock_write_compose,
mock_prepare_manifest,
mock_sanitize,
mock_connect_network,
mock_get_container_name,
mock_get_container_id,
mock_execute_compose,
mock_wait_container,
mock_session,
fake_user_id,
fake_project_id,
fake_repo_id,
fake_instance_id,
fake_tool_type_id,
) -> None:
"""When definition_type is 'manifest' and manifest_id is set, compiler runs."""
from src.models.tool_definition_manifest import ToolDefinitionManifest
manifest_id = uuid.uuid4()
mock_get_user.return_value = AsyncMock()
mock_get_project.return_value = AsyncMock()
mock_execute_compose.return_value = (0, "started", "")
mock_get_container_id.return_value = "abc123"
mock_get_container_name.return_value = "test-container"
mock_connect_network.return_value = True
mock_wait_container.return_value = {"success": True, "status": "running", "waited_seconds": 0.5}
mock_prepare_manifest.return_value = (
"headquarter/test:latest",
"services:\n app:\n image: test",
{"name": "test-manifest"},
)
instance = ToolInstance(
id=fake_instance_id,
name="manifest-instance",
repository_id=fake_repo_id,
tool_type_id=fake_tool_type_id,
compose_path="/data/instances/manifest-instance/docker-compose.yml",
status="stopped",
clone_mode="mount",
created_at=datetime.now(),
updated_at=datetime.now(),
)
tool_type = ToolType(
id=fake_tool_type_id,
name="manifest-tool",
display_name="Manifest Tool",
default_port=8080,
definition_type="manifest",
manifest_id=manifest_id,
dockerfile_template=None,
compose_template=None,
)
repo = GitRepository(
id=fake_repo_id,
project_id=fake_project_id,
name="test-repo",
path="/data/repos/test-repo",
remote_url=None,
ssh_key_id=None,
)
manifest_def = ToolDefinitionManifest(
id=manifest_id,
name="test-manifest",
display_name="Test Manifest",
interface_type="web",
manifest={"base_image": "alpine"},
)
async def _get(model, pk):
if model is ToolInstance and pk == fake_instance_id:
return instance
if model is ToolType and pk == fake_tool_type_id:
return tool_type
if model is GitRepository and pk == fake_repo_id:
return repo
if model is User and pk == fake_user_id:
return User(id=fake_user_id, email="test@example.com")
if model is ToolDefinitionManifest and pk == manifest_id:
return manifest_def
return None
mock_session.get.side_effect = _get
with patch("os.path.exists", return_value=True):
result = await start_instance(
project_id=fake_project_id,
repo_id=fake_repo_id,
instance_id=fake_instance_id,
data=None,
user_id=fake_user_id,
session=mock_session,
)
assert result["status"] == "running"
mock_prepare_manifest.assert_called_once()
mock_execute_compose.assert_called_once()
apps/web/.pi-lens/cache/review-graph.json
Vendored
-1
View File
File diff suppressed because one or more lines are too long
apps/web/src/api/tool_definitions.ts
+108
View File
@@ -0,0 +1,108 @@
import { apiClient } from "./client";
export interface ToolDefinitionManifest {
id: string;
name: string;
display_name: string;
description: string | null;
category: string | null;
interface_type: string;
base_image: string | null;
base_definition_id: string | null;
base_version: string;
manifest: Record<string, unknown>;
dockerfile_cache: string | null;
compose_cache: string | null;
version: string;
is_base: boolean;
created_at: string;
updated_at: string;
}
export interface CreateToolDefinitionRequest {
name: string;
display_name: string;
description?: string;
category?: string;
interface_type?: string;
base_image?: string;
base_definition_id?: string;
base_version?: string;
manifest: Record<string, unknown>;
}
export interface UpdateToolDefinitionRequest {
display_name?: string;
description?: string;
category?: string;
manifest?: Record<string, unknown>;
base_version?: string;
}
export interface CompileResult {
id: string;
name: string;
dockerfile: string;
entrypoint: string;
compose: string;
image_tag: string;
}
export const listToolDefinitions = async (
includeBases = true,
): Promise<ToolDefinitionManifest[]> => {
const response = await apiClient.get<{
definitions: ToolDefinitionManifest[];
}>("/tool-definitions", {
params: { include_bases: includeBases },
});
return response.data.definitions;
};
export const getToolDefinition = async (
id: string,
): Promise<ToolDefinitionManifest> => {
const response = await apiClient.get<ToolDefinitionManifest>(
`/tool-definitions/${id}`,
);
return response.data;
};
export const createToolDefinition = async (
data: CreateToolDefinitionRequest,
): Promise<ToolDefinitionManifest> => {
const response = await apiClient.post<ToolDefinitionManifest>(
"/tool-definitions",
data,
);
return response.data;
};
export const updateToolDefinition = async (
id: string,
data: UpdateToolDefinitionRequest,
): Promise<ToolDefinitionManifest> => {
const response = await apiClient.put<ToolDefinitionManifest>(
`/tool-definitions/${id}`,
data,
);
return response.data;
};
export const deleteToolDefinition = async (
id: string,
): Promise<{ status: string; id: string }> => {
const response = await apiClient.delete<{ status: string; id: string }>(
`/tool-definitions/${id}`,
);
return response.data;
};
export const compileToolDefinition = async (
id: string,
): Promise<CompileResult> => {
const response = await apiClient.post<CompileResult>(
`/tool-definitions/${id}/compile`,
);
return response.data;
};
apps/web/src/api/tool_types.ts
+74 -62
View File
@@ -1,90 +1,102 @@
import { apiClient } from "./client";
export interface ReadinessProbe {
command: string;
timeout: number;
interval: number;
command: string;
timeout: number;
interval: number;
}
export interface ToolType {
id: string;
name: string;
display_name: string;
description: string | null;
category: string;
interface_type: string;
requires_port: boolean;
default_port: number | null;
definition_type: 'compose' | 'dockerfile';
compose_template: string | null;
dockerfile_template: string | null;
build_context: Record<string, string> | null;
readiness_probe: ReadinessProbe | null;
startup_command: string | null;
required_variables: string[];
created_by_id: string | null;
created_at: string;
updated_at: string;
id: string;
name: string;
display_name: string;
description: string | null;
category: string;
interface_type: string;
requires_port: boolean;
default_port: number | null;
definition_type: "compose" | "dockerfile" | "manifest";
manifest_id: string | null;
compose_template: string | null;
dockerfile_template: string | null;
build_context: Record<string, string> | null;
readiness_probe: ReadinessProbe | null;
startup_command: string | null;
required_variables: string[];
created_by_id: string | null;
created_at: string;
updated_at: string;
}
export interface CreateToolTypeRequest {
name: string;
display_name: string;
description?: string;
category?: string;
interface_type?: string;
requires_port?: boolean;
default_port: number;
definition_type?: 'compose' | 'dockerfile';
compose_template?: string;
dockerfile_template?: string;
build_context?: Record<string, string>;
readiness_probe?: ReadinessProbe;
startup_command?: string;
required_variables: string[];
name: string;
display_name: string;
description?: string;
category?: string;
interface_type?: string;
requires_port?: boolean;
default_port: number;
definition_type?: "compose" | "dockerfile" | "manifest";
manifest_id?: string;
compose_template?: string;
dockerfile_template?: string;
build_context?: Record<string, string>;
readiness_probe?: ReadinessProbe;
startup_command?: string;
required_variables: string[];
}
export interface UpdateToolTypeRequest {
display_name?: string;
description?: string;
category?: string;
interface_type?: string;
requires_port?: boolean;
default_port?: number;
definition_type?: 'compose' | 'dockerfile';
compose_template?: string;
dockerfile_template?: string;
build_context?: Record<string, string>;
readiness_probe?: ReadinessProbe;
startup_command?: string;
required_variables?: string[];
display_name?: string;
description?: string;
category?: string;
interface_type?: string;
requires_port?: boolean;
default_port?: number;
definition_type?: "compose" | "dockerfile" | "manifest";
manifest_id?: string;
compose_template?: string;
dockerfile_template?: string;
build_context?: Record<string, string>;
readiness_probe?: ReadinessProbe;
startup_command?: string;
required_variables?: string[];
}
export const listToolTypes = async (): Promise<ToolType[]> => {
const response = await apiClient.get<ToolType[]>("/tool-types");
return response.data;
const response = await apiClient.get<ToolType[]>("/tool-types");
return response.data;
};
export const getToolType = async (id: string): Promise<ToolType> => {
const response = await apiClient.get<ToolType>(`/tool-types/${id}`);
return response.data;
const response = await apiClient.get<ToolType>(`/tool-types/${id}`);
return response.data;
};
export const createToolType = async (data: CreateToolTypeRequest): Promise<ToolType> => {
const response = await apiClient.post<ToolType>("/tool-types", data);
return response.data;
export const createToolType = async (
data: CreateToolTypeRequest,
): Promise<ToolType> => {
const response = await apiClient.post<ToolType>("/tool-types", data);
return response.data;
};
export const updateToolType = async (id: string, data: UpdateToolTypeRequest): Promise<ToolType> => {
const response = await apiClient.put<ToolType>(`/tool-types/${id}`, data);
return response.data;
export const updateToolType = async (
id: string,
data: UpdateToolTypeRequest,
): Promise<ToolType> => {
const response = await apiClient.put<ToolType>(`/tool-types/${id}`, data);
return response.data;
};
export const deleteToolType = async (id: string): Promise<void> => {
await apiClient.delete(`/tool-types/${id}`);
await apiClient.delete(`/tool-types/${id}`);
};
export const validateToolType = async (id: string): Promise<{ valid: boolean; errors?: string[] }> => {
const response = await apiClient.get<{ valid: boolean; errors?: string[] }>(`/tool-types/${id}/validate`);
return response.data;
export const validateToolType = async (
id: string,
): Promise<{ valid: boolean; errors?: string[] }> => {
const response = await apiClient.get<{ valid: boolean; errors?: string[] }>(
`/tool-types/${id}/validate`,
);
return response.data;
};
apps/web/src/components/manifest-editor.tsx
+849
View File
@@ -0,0 +1,849 @@
import { useState, useEffect, useCallback } from "react";
import { Icon } from "./icon";
import { extractErrorMessage } from "../utils/errors";
import {
compileToolDefinition,
type ToolDefinitionManifest,
} from "../api/tool_definitions";
interface PackageEntry {
name: string;
}
interface MountEntry {
name: string;
target: string;
source_type: string;
writable: boolean;
owner: string;
mode: string;
file_mode: string;
readonly: boolean;
git_mount_ref: string;
}
interface ManifestEditorProps {
manifest: Record<string, unknown> | null;
baseDefinitions: ToolDefinitionManifest[];
onChange: (manifest: Record<string, unknown>) => void;
definitionId?: string | null;
}
export const ManifestEditor = ({
manifest,
baseDefinitions,
onChange,
definitionId,
}: ManifestEditorProps) => {
const [baseImage, setBaseImage] = useState("");
const [baseDefinitionId, setBaseDefinitionId] = useState("");
const [aptPackages, setAptPackages] = useState<PackageEntry[]>([]);
const [npmPackages, setNpmPackages] = useState<PackageEntry[]>([]);
const [pipPackages, setPipPackages] = useState<PackageEntry[]>([]);
const [nodeVersion, setNodeVersion] = useState("");
const [userName, setUserName] = useState("user");
const [userUid, setUserUid] = useState("1000");
const [userGid, setUserGid] = useState("1000");
const [envVars, setEnvVars] = useState<{ key: string; value: string }[]>([]);
const [buildScripts, setBuildScripts] = useState<string[]>([""]);
const [startupScripts, setStartupScripts] = useState<string[]>([""]);
const [mounts, setMounts] = useState<MountEntry[]>([]);
const [command, setCommand] = useState<string[]>([""]);
const [workingDir, setWorkingDir] = useState("/workspace");
const [stdinOpen, setStdinOpen] = useState(true);
const [tty, setTty] = useState(true);
const [preview, setPreview] = useState<{
dockerfile: string;
compose: string;
entrypoint: string;
} | null>(null);
const [previewLoading, setPreviewLoading] = useState(false);
const [previewError, setPreviewError] = useState<string | null>(null);
// Load manifest into form
useEffect(() => {
if (!manifest) return;
const pkgs = (manifest.packages as Record<string, unknown>) || {};
setBaseImage((manifest.base_image as string) || "");
setBaseDefinitionId((manifest.base_definition_id as string) || "");
setAptPackages(((pkgs.apt as string[]) || []).map((p) => ({ name: p })));
setNpmPackages(
((pkgs.npm_global as string[]) || []).map((p) => ({ name: p })),
);
setPipPackages(((pkgs.pip as string[]) || []).map((p) => ({ name: p })));
setNodeVersion((pkgs.node as Record<string, string>)?.version || "");
const user = (manifest.user as Record<string, unknown>) || {};
setUserName((user.name as string) || "user");
setUserUid(String(user.uid || "1000"));
setUserGid(String(user.gid || "1000"));
const env = (manifest.env as Record<string, string>) || {};
setEnvVars(Object.entries(env).map(([key, value]) => ({ key, value })));
const scripts = (manifest.scripts as Record<string, string[]>) || {};
setBuildScripts((scripts.build || []).length > 0 ? scripts.build : [""]);
setStartupScripts(
(scripts.startup || []).length > 0 ? scripts.startup : [""],
);
const mts = (manifest.mounts as MountEntry[]) || [];
setMounts(mts);
const runtime = (manifest.runtime as Record<string, unknown>) || {};
setCommand((runtime.command as string[]) || [""]);
setWorkingDir((runtime.working_dir as string) || "/workspace");
setStdinOpen((runtime.stdin_open as boolean) ?? true);
setTty((runtime.tty as boolean) ?? true);
}, [manifest]);
// Build manifest from form state
const buildManifest = useCallback((): Record<string, unknown> => {
const packages: Record<string, unknown> = {};
const apt = aptPackages.map((p) => p.name).filter(Boolean);
if (apt.length) packages.apt = apt;
const npm = npmPackages.map((p) => p.name).filter(Boolean);
if (npm.length) packages.npm_global = npm;
const pip = pipPackages.map((p) => p.name).filter(Boolean);
if (pip.length) packages.pip = pip;
if (nodeVersion) packages.node = { version: nodeVersion };
const env: Record<string, string> = {};
envVars.forEach(({ key, value }) => {
if (key) env[key] = value;
});
const scripts: Record<string, string[]> = {};
const build = buildScripts.filter(Boolean);
if (build.length) scripts.build = build;
const startup = startupScripts.filter(Boolean);
if (startup.length) scripts.startup = startup;
const mts = mounts.filter((m) => m.name && m.target);
const result: Record<string, unknown> = {
packages,
user: {
name: userName,
uid: parseInt(userUid) || 1000,
gid: parseInt(userGid) || 1000,
create_home: true,
shell: "/bin/bash",
},
env,
scripts,
mounts: mts,
runtime: {
command:
command.filter(Boolean).length > 0
? command.filter(Boolean)
: ["/bin/bash"],
stdin_open: stdinOpen,
tty: tty,
working_dir: workingDir,
},
};
if (baseImage) result.base_image = baseImage;
if (baseDefinitionId) result.base_definition_id = baseDefinitionId;
return result;
}, [
aptPackages,
npmPackages,
pipPackages,
nodeVersion,
userName,
userUid,
userGid,
envVars,
buildScripts,
startupScripts,
mounts,
command,
workingDir,
stdinOpen,
tty,
baseImage,
baseDefinitionId,
]);
// Notify parent of changes
useEffect(() => {
const m = buildManifest();
onChange(m);
}, [buildManifest, onChange]);
const handlePreview = async () => {
if (!definitionId) {
setPreviewError("Save the tool definition first to preview");
return;
}
setPreviewLoading(true);
setPreviewError(null);
try {
const result = await compileToolDefinition(definitionId);
setPreview({
dockerfile: result.dockerfile,
compose: result.compose,
entrypoint: result.entrypoint,
});
} catch (err) {
setPreviewError(extractErrorMessage(err));
} finally {
setPreviewLoading(false);
}
};
const addAptPackage = () => setAptPackages([...aptPackages, { name: "" }]);
const removeAptPackage = (idx: number) =>
setAptPackages(aptPackages.filter((_, i) => i !== idx));
const updateAptPackage = (idx: number, name: string) => {
const copy = [...aptPackages];
copy[idx] = { name };
setAptPackages(copy);
};
const addNpmPackage = () => setNpmPackages([...npmPackages, { name: "" }]);
const removeNpmPackage = (idx: number) =>
setNpmPackages(npmPackages.filter((_, i) => i !== idx));
const updateNpmPackage = (idx: number, name: string) => {
const copy = [...npmPackages];
copy[idx] = { name };
setNpmPackages(copy);
};
const addPipPackage = () => setPipPackages([...pipPackages, { name: "" }]);
const removePipPackage = (idx: number) =>
setPipPackages(pipPackages.filter((_, i) => i !== idx));
const updatePipPackage = (idx: number, name: string) => {
const copy = [...pipPackages];
copy[idx] = { name };
setPipPackages(copy);
};
const addEnvVar = () => setEnvVars([...envVars, { key: "", value: "" }]);
const removeEnvVar = (idx: number) =>
setEnvVars(envVars.filter((_, i) => i !== idx));
const updateEnvVar = (idx: number, field: "key" | "value", val: string) => {
const copy = [...envVars];
copy[idx] = { ...copy[idx], [field]: val };
setEnvVars(copy);
};
const addBuildScript = () => setBuildScripts([...buildScripts, ""]);
const removeBuildScript = (idx: number) =>
setBuildScripts(buildScripts.filter((_, i) => i !== idx));
const updateBuildScript = (idx: number, val: string) => {
const copy = [...buildScripts];
copy[idx] = val;
setBuildScripts(copy);
};
const addStartupScript = () => setStartupScripts([...startupScripts, ""]);
const removeStartupScript = (idx: number) =>
setStartupScripts(startupScripts.filter((_, i) => i !== idx));
const updateStartupScript = (idx: number, val: string) => {
const copy = [...startupScripts];
copy[idx] = val;
setStartupScripts(copy);
};
const addMount = () =>
setMounts([
...mounts,
{
name: "",
target: "",
source_type: "repo",
writable: true,
owner: "",
mode: "",
file_mode: "",
readonly: false,
git_mount_ref: "",
},
]);
const removeMount = (idx: number) =>
setMounts(mounts.filter((_, i) => i !== idx));
const updateMount = (idx: number, field: keyof MountEntry, val: unknown) => {
const copy = [...mounts];
copy[idx] = { ...copy[idx], [field]: val };
setMounts(copy);
};
return (
<div className="stack" style={{ gap: "1.5rem" }}>
{/* Base Image */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Base Image</h4>
<div className="row" style={{ gap: "1rem" }}>
<div className="form-group" style={{ flex: 1 }}>
<label>Base Definition</label>
<select
value={baseDefinitionId}
onChange={(e) => {
setBaseDefinitionId(e.target.value);
setBaseImage("");
}}
className="form-input"
>
<option value="">Custom image...</option>
{baseDefinitions.map((b) => (
<option key={b.id} value={b.id}>
{b.display_name} ({b.version})
</option>
))}
</select>
</div>
<div className="form-group" style={{ flex: 1 }}>
<label>Custom Base Image</label>
<input
type="text"
value={baseImage}
onChange={(e) => {
setBaseImage(e.target.value);
setBaseDefinitionId("");
}}
placeholder="e.g., ubuntu:24.04"
className="form-input"
disabled={!!baseDefinitionId}
/>
</div>
</div>
</div>
{/* Packages */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Packages</h4>
<div className="form-group">
<label>Node.js Version</label>
<input
type="text"
value={nodeVersion}
onChange={(e) => setNodeVersion(e.target.value)}
placeholder="e.g., 20"
className="form-input"
style={{ width: "120px" }}
/>
</div>
<div>
<label>APT Packages</label>
<div className="stack" style={{ gap: "0.5rem", marginTop: "0.5rem" }}>
{aptPackages.map((pkg, idx) => (
<div key={idx} className="row" style={{ gap: "0.5rem" }}>
<input
type="text"
value={pkg.name}
onChange={(e) => updateAptPackage(idx, e.target.value)}
placeholder="e.g., neovim"
className="form-input"
/>
<button
type="button"
onClick={() => removeAptPackage(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addAptPackage}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add APT Package
</button>
</div>
</div>
<div>
<label>NPM Global Packages</label>
<div className="stack" style={{ gap: "0.5rem", marginTop: "0.5rem" }}>
{npmPackages.map((pkg, idx) => (
<div key={idx} className="row" style={{ gap: "0.5rem" }}>
<input
type="text"
value={pkg.name}
onChange={(e) => updateNpmPackage(idx, e.target.value)}
placeholder="e.g., @scope/pkg"
className="form-input"
/>
<button
type="button"
onClick={() => removeNpmPackage(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addNpmPackage}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add NPM Package
</button>
</div>
</div>
<div>
<label>Pip Packages</label>
<div className="stack" style={{ gap: "0.5rem", marginTop: "0.5rem" }}>
{pipPackages.map((pkg, idx) => (
<div key={idx} className="row" style={{ gap: "0.5rem" }}>
<input
type="text"
value={pkg.name}
onChange={(e) => updatePipPackage(idx, e.target.value)}
placeholder="e.g., requests"
className="form-input"
/>
<button
type="button"
onClick={() => removePipPackage(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addPipPackage}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add Pip Package
</button>
</div>
</div>
</div>
{/* Runtime User */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Runtime User</h4>
<div className="row" style={{ gap: "1rem" }}>
<div className="form-group" style={{ flex: 1 }}>
<label>User Name</label>
<input
type="text"
value={userName}
onChange={(e) => setUserName(e.target.value)}
className="form-input"
/>
</div>
<div className="form-group" style={{ flex: 1 }}>
<label>UID</label>
<input
type="number"
value={userUid}
onChange={(e) => setUserUid(e.target.value)}
className="form-input"
/>
</div>
<div className="form-group" style={{ flex: 1 }}>
<label>GID</label>
<input
type="number"
value={userGid}
onChange={(e) => setUserGid(e.target.value)}
className="form-input"
/>
</div>
</div>
</div>
{/* Environment */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Environment Variables</h4>
<div className="stack" style={{ gap: "0.5rem" }}>
{envVars.map((ev, idx) => (
<div key={idx} className="row" style={{ gap: "0.5rem" }}>
<input
type="text"
value={ev.key}
onChange={(e) => updateEnvVar(idx, "key", e.target.value)}
placeholder="KEY"
className="form-input"
/>
<input
type="text"
value={ev.value}
onChange={(e) => updateEnvVar(idx, "value", e.target.value)}
placeholder="value"
className="form-input"
/>
<button
type="button"
onClick={() => removeEnvVar(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addEnvVar}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add Env Var
</button>
</div>
</div>
{/* Build Scripts */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Build Scripts (run during docker build)</h4>
<div className="stack" style={{ gap: "0.5rem" }}>
{buildScripts.map((script, idx) => (
<div
key={idx}
className="row"
style={{ gap: "0.5rem", alignItems: "flex-start" }}
>
<textarea
value={script}
onChange={(e) => updateBuildScript(idx, e.target.value)}
placeholder="git config --global user.email 'dev@example.com'"
className="form-input"
rows={2}
style={{
fontFamily: "monospace",
fontSize: "0.8125rem",
flex: 1,
}}
/>
<button
type="button"
onClick={() => removeBuildScript(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addBuildScript}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add Build Script
</button>
</div>
</div>
{/* Startup Scripts */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>
Startup Scripts (run when container starts)
</h4>
<div className="stack" style={{ gap: "0.5rem" }}>
{startupScripts.map((script, idx) => (
<div
key={idx}
className="row"
style={{ gap: "0.5rem", alignItems: "flex-start" }}
>
<textarea
value={script}
onChange={(e) => updateStartupScript(idx, e.target.value)}
placeholder="chown -R user:user /workspace"
className="form-input"
rows={2}
style={{
fontFamily: "monospace",
fontSize: "0.8125rem",
flex: 1,
}}
/>
<button
type="button"
onClick={() => removeStartupScript(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
))}
<button
type="button"
onClick={addStartupScript}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add Startup Script
</button>
</div>
</div>
{/* Mounts */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Mount Schema</h4>
<div className="stack" style={{ gap: "1rem" }}>
{mounts.map((mount, idx) => (
<div
key={idx}
className="stack"
style={{
gap: "0.5rem",
padding: "0.75rem",
border: "1px solid var(--border)",
borderRadius: "0.375rem",
}}
>
<div className="row" style={{ gap: "0.5rem" }}>
<input
type="text"
value={mount.name}
onChange={(e) => updateMount(idx, "name", e.target.value)}
placeholder="Name (e.g., workspace)"
className="form-input"
/>
<input
type="text"
value={mount.target}
onChange={(e) => updateMount(idx, "target", e.target.value)}
placeholder="Target (e.g., /workspace)"
className="form-input"
/>
<select
value={mount.source_type}
onChange={(e) =>
updateMount(idx, "source_type", e.target.value)
}
className="form-input"
>
<option value="repo">Repository</option>
<option value="ssh_key">SSH Key</option>
<option value="instance">Instance</option>
<option value="git_mount">Git Mount</option>
<option value="host_path">Host Path</option>
</select>
<button
type="button"
onClick={() => removeMount(idx)}
className="button-icon"
title="Remove"
>
<Icon name="delete" size="sm" />
</button>
</div>
<div className="row" style={{ gap: "0.5rem" }}>
<label
style={{
display: "flex",
alignItems: "center",
gap: "0.25rem",
fontSize: "0.875rem",
}}
>
<input
type="checkbox"
checked={mount.writable}
onChange={(e) =>
updateMount(idx, "writable", e.target.checked)
}
/>
Writable
</label>
<label
style={{
display: "flex",
alignItems: "center",
gap: "0.25rem",
fontSize: "0.875rem",
}}
>
<input
type="checkbox"
checked={mount.readonly}
onChange={(e) =>
updateMount(idx, "readonly", e.target.checked)
}
/>
Read-only
</label>
<input
type="text"
value={mount.owner}
onChange={(e) => updateMount(idx, "owner", e.target.value)}
placeholder="Owner (e.g., user)"
className="form-input"
style={{ width: "120px" }}
/>
<input
type="text"
value={mount.mode}
onChange={(e) => updateMount(idx, "mode", e.target.value)}
placeholder="Mode (e.g., 0755)"
className="form-input"
style={{ width: "100px" }}
/>
<input
type="text"
value={mount.file_mode}
onChange={(e) =>
updateMount(idx, "file_mode", e.target.value)
}
placeholder="File mode (e.g., 0644)"
className="form-input"
style={{ width: "120px" }}
/>
{mount.source_type === "git_mount" && (
<input
type="text"
value={mount.git_mount_ref}
onChange={(e) =>
updateMount(idx, "git_mount_ref", e.target.value)
}
placeholder="Git mount ref"
className="form-input"
style={{ width: "120px" }}
/>
)}
</div>
</div>
))}
<button
type="button"
onClick={addMount}
className="button-secondary"
style={{ width: "fit-content" }}
>
<Icon name="add" size="sm" /> Add Mount
</button>
</div>
</div>
{/* Runtime */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<h4 style={{ margin: 0 }}>Runtime</h4>
<div className="row" style={{ gap: "1rem" }}>
<div className="form-group" style={{ flex: 1 }}>
<label>Command</label>
<input
type="text"
value={command.join(" ")}
onChange={(e) => setCommand(e.target.value.split(" "))}
placeholder="/bin/bash"
className="form-input"
/>
</div>
<div className="form-group" style={{ flex: 1 }}>
<label>Working Directory</label>
<input
type="text"
value={workingDir}
onChange={(e) => setWorkingDir(e.target.value)}
placeholder="/workspace"
className="form-input"
/>
</div>
</div>
<div className="row" style={{ gap: "1rem" }}>
<label
style={{ display: "flex", alignItems: "center", gap: "0.5rem" }}
>
<input
type="checkbox"
checked={stdinOpen}
onChange={(e) => setStdinOpen(e.target.checked)}
/>
stdin_open
</label>
<label
style={{ display: "flex", alignItems: "center", gap: "0.5rem" }}
>
<input
type="checkbox"
checked={tty}
onChange={(e) => setTty(e.target.checked)}
/>
tty
</label>
</div>
</div>
{/* Preview */}
<div className="card stack" style={{ gap: "1rem", padding: "1rem" }}>
<div
style={{
display: "flex",
justifyContent: "space-between",
alignItems: "center",
}}
>
<h4 style={{ margin: 0 }}>Live Preview</h4>
<button
type="button"
onClick={handlePreview}
disabled={previewLoading}
className="button-secondary"
>
<Icon name="refresh" size="sm" />
{previewLoading ? "Compiling..." : "Preview"}
</button>
</div>
{previewError && <p className="text-error">{previewError}</p>}
{preview && (
<div className="stack" style={{ gap: "1rem" }}>
<div>
<label style={{ fontWeight: 600, fontSize: "0.875rem" }}>
Dockerfile
</label>
<pre
style={{
background: "var(--code-bg, #1e1e1e)",
color: "var(--code-fg, #d4d4d4)",
padding: "1rem",
borderRadius: "0.375rem",
overflow: "auto",
fontSize: "0.8125rem",
maxHeight: "300px",
}}
>
{preview.dockerfile}
</pre>
</div>
<div>
<label style={{ fontWeight: 600, fontSize: "0.875rem" }}>
Compose
</label>
<pre
style={{
background: "var(--code-bg, #1e1e1e)",
color: "var(--code-fg, #d4d4d4)",
padding: "1rem",
borderRadius: "0.375rem",
overflow: "auto",
fontSize: "0.8125rem",
maxHeight: "200px",
}}
>
{preview.compose}
</pre>
</div>
</div>
)}
</div>
</div>
);
};
apps/web/src/pages/tool-workshop.tsx
+1965 -1459
View File
File diff suppressed because it is too large Load Diff
openspec/config.yaml
+37 -75
View File
@@ -1,75 +1,37 @@
strict_tdd: true
context: |
headquarter-e2e is a Node.js/TypeScript, Node.js/TypeScript ESM, React, Python project.
Detected markers: .opencode/package.json, .opencode/npm package manager, e2e/package.json, e2e/npm package manager, apps/web/package.json, apps/web/tsconfig.json, apps/web/npm package manager, Makefile.
Package managers: npm.
Additional evidence: Test-like files detected (210); examples: apps/api/.venv/lib/python3.14/site-packages/greenlet/tests/__init__.py, apps/api/.venv/lib/python3.14/site-packages/greenlet/tests/_test_extension.c, apps/api/.venv/lib/python3.14/site-packages/greenlet/tests/_test_extension.cpython-314-x86_64-linux-gnu.so, apps/api/.venv/lib/python3.14/site-packages/greenlet/tests/_test_extension_cpp.cpp, apps/api/.venv/lib/python3.14/site-packages/greenlet/tests/_test_extension_cpp.cpython-314-x86_64-linux-gnu.so.
Primary test command: make test.
Unit tests: package script (cd e2e && npm test); Vitest (cd apps/web && npm test); pytest (pytest); pytest via Makefile (make test).
Integration tests: Testing Library (cd apps/web && npm test).
E2E tests: Playwright (cd e2e && npx playwright test).
rules:
proposal:
require_problem_statement: true
spec:
require_acceptance_criteria: true
design:
require_tradeoffs: true
tasks:
protect_review_workload: true
apply:
test_command: "make test"
verify:
test_command: "make test"
testing:
detected: "2026-05-27"
runner:
command: "make test"
framework: "pytest via Makefile"
layers:
unit: "package script, Vitest, pytest, pytest via Makefile"
integration: "Testing Library"
e2e: "Playwright"
commands:
unit:
- scope: "e2e"
command: "cd e2e && npm test"
framework: "package script"
- scope: "apps/web"
command: "cd apps/web && npm test"
framework: "Vitest"
- scope: "."
command: "pytest"
framework: "pytest"
- scope: "."
command: "make test"
framework: "pytest via Makefile"
integration:
- scope: "apps/web"
command: "cd apps/web && npm test"
framework: "Testing Library"
e2e:
- scope: "e2e"
command: "cd e2e && npx playwright test"
framework: "Playwright"
coverage:
command: ""
commands:
[]
quality:
lint: "cd apps/web && npm run lint"
lint_commands:
- scope: "apps/web"
command: "cd apps/web && npm run lint"
framework: "linter"
- scope: "."
command: "make lint"
framework: "linter"
typecheck: "cd apps/web && npm run typecheck"
typecheck_commands:
- scope: "apps/web"
command: "cd apps/web && npm run typecheck"
framework: "type checker"
format: ""
format_commands:
[]
project:
name: Headquarter
description: Docker-based development platform for managing coding agent tool instances
repository: https://git.commumedia.org/alex/headquarter
stack:
backend:
framework: FastAPI
language: Python 3.11
database: PostgreSQL 15 (async SQLAlchemy)
cache: Redis 7
migrations: Alembic
testing: pytest
frontend:
framework: React + Vite
language: TypeScript
infrastructure:
local: Docker Compose
production: Docker Compose + Traefik
auth: Authentik SSO
sdd:
execution_mode: interactive
artifact_store: openspec
chained_pr_strategy: auto-forecast
review_budget_lines: 400
strict_tdd:
enabled: true
test_command: docker exec hq-api pytest
evidence_required: red_green_triangulate_refactor
phase_rules:
explore_before_proposal: true
spec_before_design: true
design_before_tasks: true
verify_before_archive: true
openspec/designs/tool-definition-manifest.md
+751
View File
@@ -0,0 +1,751 @@
# Design: Tool Definition Manifest System
## Status
**Phase:** design
**Date:** 2026-05-28
**Owner:** el Gentleman
**Based on:** Spec `tool-definition-manifest`
---
## Architecture Overview
```
┌─────────────────────────────────────────────────────────────────┐
│ Tool Workshop (Frontend) │
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │
│ │ Base Image │ │ Packages │ │ Mount Schema Designer │ │
│ │ Selector │ │ Editors │ │ (target, owner, mode) │ │
│ └─────────────┘ └─────────────┘ └─────────────────────────┘ │
│ ┌─────────────────────────────────────────────────────────────┐│
│ │ Live Preview: Dockerfile + Compose ││
│ └─────────────────────────────────────────────────────────────┘│
└─────────────────────────────────────────────────────────────────┘
│
▼ POST /tool-definitions
┌─────────────────────────────────────────────────────────────────┐
│ API Backend │
│ ┌─────────────────┐ ┌──────────────────┐ ┌────────────┐ │
│ │ ManifestSchema │───▶│ ManifestCompiler │───▶│ LiveBuild │ │
│ │ (validation) │ │ (Dockerfile + │ │ (optional) │ │
│ │ │ │ Compose gen) │ │ │ │
│ └─────────────────┘ └──────────────────┘ └────────────┘ │
│ │
│ ▼ save to DB
│ ┌─────────────────────────────────────────────────────────────┐│
│ │ ToolDefinitionManifest (JSONB in PostgreSQL) ││
│ └─────────────────────────────────────────────────────────────┘│
└─────────────────────────────────────────────────────────────────┘
│
▼ POST /instances/{id}/start
┌─────────────────────────────────────────────────────────────────┐
│ Instance Startup Flow │
│ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
│ │ Resolve │──▶│ Compile │──▶│ Build Image │ │
│ │ Manifest │ │ to Dockerfile│ │ (docker build) │ │
│ │ (base merge)│ │ + Compose │ │ │ │
│ └─────────────┘ └──────────────┘ └──────────────────────┘ │
│ │
│ ▼
│ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
│ │ Permission │◀──│ docker comp. │◀──│ Generate Compose │ │
│ │ Fixer │ │ up │ │ (mount resolution) │ │
│ │ (post-start)│ │ │ │ │ │
│ └─────────────┘ └──────────────┘ └──────────────────────┘ │
└─────────────────────────────────────────────────────────────────┘
```
---
## Manifest JSON Schema
```json
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"required": ["name", "interface_type"],
"oneOf": [
{"required": ["base_image"]},
{"required": ["base_definition_id"]}
],
"properties": {
"name": {"type": "string", "pattern": "^[a-z0-9-]+$", "maxLength": 64},
"display_name": {"type": "string", "maxLength": 128},
"description": {"type": "string"},
"category": {"type": "string", "maxLength": 64},
"interface_type": {"type": "string", "enum": ["web", "terminal"]},
"base_image": {"type": "string", "maxLength": 256},
"base_definition_id": {"type": "string", "format": "uuid"},
"base_version": {"type": "string", "maxLength": 32, "default": "latest"},
"packages": {
"type": "object",
"properties": {
"apt": {"type": "array", "items": {"type": "string"}},
"node": {
"type": "object",
"properties": {
"version": {"type": "string", "pattern": "^\\d+$"}
},
"required": ["version"]
},
"npm_global": {"type": "array", "items": {"type": "string"}},
"pip": {"type": "array", "items": {"type": "string"}}
}
},
"user": {
"type": "object",
"properties": {
"name": {"type": "string", "maxLength": 32},
"uid": {"type": "integer", "minimum": 1, "maximum": 65535},
"gid": {"type": "integer", "minimum": 1, "maximum": 65535},
"create_home": {"type": "boolean", "default": true},
"shell": {"type": "string", "maxLength": 64, "default": "/bin/bash"}
},
"required": ["name", "uid", "gid"]
},
"env": {"type": "object", "additionalProperties": {"type": "string"}},
"scripts": {
"type": "object",
"properties": {
"build": {"type": "array", "items": {"type": "string"}},
"startup": {"type": "array", "items": {"type": "string"}}
}
},
"mounts": {
"type": "array",
"items": {
"type": "object",
"required": ["name", "target", "source_type"],
"properties": {
"name": {"type": "string", "maxLength": 64},
"target": {"type": "string", "maxLength": 256},
"source_type": {"type": "string", "enum": ["repo", "ssh_key", "instance", "git_mount", "host_path"]},
"writable": {"type": "boolean", "default": true},
"owner": {"type": "string", "maxLength": 32},
"mode": {"type": "string", "pattern": "^[0-7]{3,4}$"},
"file_mode": {"type": "string", "pattern": "^[0-7]{3,4}$"},
"readonly": {"type": "boolean", "default": false},
"git_mount_ref": {"type": "string", "maxLength": 64}
}
}
},
"runtime": {
"type": "object",
"properties": {
"command": {"type": "array", "items": {"type": "string"}},
"stdin_open": {"type": "boolean", "default": false},
"tty": {"type": "boolean", "default": false},
"working_dir": {"type": "string", "maxLength": 256}
}
}
}
}
```
---
## Manifest Compiler Algorithm
### Step 1: Resolve Base
```python
def resolve_base(manifest: dict) -> dict:
"""Merge base definition into the manifest."""
if manifest.get("base_definition_id"):
base = load_base_definition(manifest["base_definition_id"],
manifest.get("base_version", "latest"))
# Deep merge: base first, then tool-specific overrides
merged = deep_merge(base, manifest)
# Remove base fields from the merged result
merged.pop("base_definition_id", None)
merged.pop("base_version", None)
return merged
return manifest
```
Merge rules:
- `packages`: Union arrays (base apt + tool apt = combined apt)
- `env`: Tool overrides base (dict merge, tool wins on key conflict)
- `scripts.build`: Concatenate arrays (base scripts first, then tool)
- `scripts.startup`: Concatenate arrays
- `user`: Tool overrides base entirely
- `mounts`: Concatenate arrays
- `runtime`: Tool overrides base (dict merge)
### Step 2: Generate Dockerfile
```python
def compile_dockerfile(manifest: dict) -> str:
"""Compile a resolved manifest to a Dockerfile string."""
lines = []
# FROM
lines.append(f"FROM {manifest['base_image']}")
lines.append("")
# ENV (build-time)
for key, value in manifest.get("env", {}).items():
lines.append(f"ENV {key}={shlex.quote(value)}")
if manifest.get("env"):
lines.append("")
# System packages (apt)
apt_packages = manifest.get("packages", {}).get("apt", [])
if apt_packages:
lines.append("RUN apt-get update && apt-get install -y \\\\")
for pkg in apt_packages:
lines.append(f" {pkg} \\\\")
lines.append(" && rm -rf /var/lib/apt/lists/*")
lines.append("")
# Node.js
node = manifest.get("packages", {}).get("node")
if node:
lines.append(
f"RUN curl -fsSL https://deb.nodesource.com/setup_{node['version']}.x | bash - && \\\\")
lines.append(" apt-get install -y nodejs && \\\\")
lines.append(" rm -rf /var/lib/apt/lists/*")
lines.append("")
# NPM global
npm_packages = manifest.get("packages", {}).get("npm_global", [])
if npm_packages:
pkg_list = " ".join(shlex.quote(p) for p in npm_packages)
lines.append(f"RUN npm install -g {pkg_list}")
lines.append("")
# Pip
pip_packages = manifest.get("packages", {}).get("pip", [])
if pip_packages:
pkg_list = " ".join(shlex.quote(p) for p in pip_packages)
lines.append(f"RUN pip install {pkg_list}")
lines.append("")
# User creation
user = manifest.get("user")
if user:
lines.append(
f"RUN groupadd -g {user['gid']} {user['name']} && \\\\")
lines.append(
f" useradd -u {user['uid']} -g {user['gid']} "
f"{'-m ' if user.get('create_home', True) else ''}"
f"-s {user['shell']} {user['name']}")
lines.append("")
# Build scripts
build_scripts = manifest.get("scripts", {}).get("build", [])
for script in build_scripts:
# Each script block becomes one RUN command
# Normalize multi-line scripts
normalized = " && ".join(line.strip() for line in script.strip().split("\n") if line.strip())
lines.append(f"RUN {normalized}")
if build_scripts:
lines.append("")
# Create mount target directories and pre-set ownership
mounts = manifest.get("mounts", [])
if mounts:
dirs = []
for mount in mounts:
dirs.append(mount["target"])
if dirs:
dir_str = " ".join(dirs)
lines.append(f"RUN mkdir -p {dir_str}")
if user:
lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
lines.append("")
# Entrypoint for startup scripts
startup_scripts = manifest.get("scripts", {}).get("startup", [])
if startup_scripts:
lines.append("COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint")
lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint")
lines.append("")
# Switch to user
if user:
lines.append(f"USER {user['name']}")
lines.append(f"WORKDIR /home/{user['name']}")
lines.append("")
# Entrypoint and CMD
runtime = manifest.get("runtime", {})
if startup_scripts:
lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')
command = runtime.get("command", ["/bin/bash"])
cmd_json = json.dumps(command)
lines.append(f"CMD {cmd_json}")
return "\n".join(lines)
```
### Step 3: Generate Entrypoint Script
```python
def compile_entrypoint(manifest: dict) -> str:
"""Generate the startup entrypoint script."""
lines = ["#!/bin/bash", "set -e", ""]
startup_scripts = manifest.get("scripts", {}).get("startup", [])
for script in startup_scripts:
lines.append(script)
lines.append("")
lines.append('exec "$@"')
return "\n".join(lines)
```
### Step 4: Generate Compose
```python
def compile_compose(manifest: dict, variables: dict) -> str:
"""Compile a resolved manifest to a Docker Compose string."""
runtime = manifest.get("runtime", {})
user = manifest.get("user")
interface_type = manifest["interface_type"]
service = {
"image": variables["IMAGE_TAG"],
"container_name": variables["INSTANCE_NAME"],
"restart": "unless-stopped",
}
# Terminal-specific fields
if runtime.get("stdin_open", False):
service["stdin_open"] = True
if runtime.get("tty", False):
service["tty"] = True
if runtime.get("working_dir"):
service["working_dir"] = runtime["working_dir"]
# User override in compose (helps with permission consistency)
if user:
service["user"] = f"{user['uid']}:{user['gid']}"
# Ports for web tools
if interface_type == "web" and manifest.get("default_port"):
service["ports"] = [f"{variables['TOOL_PORT']}:{manifest['default_port']}"]
# Environment
env = manifest.get("env", {})
if env:
service["environment"] = env
# Volumes from mounts
volumes = []
for mount in manifest.get("mounts", []):
source = resolve_mount_source(mount, variables)
target = mount["target"]
readonly = ":ro" if mount.get("readonly", False) else ""
volumes.append(f"{source}:{target}{readonly}")
# Append extra volumes from tool config / config profile
for vol in variables.get("EXTRA_VOLUMES", []):
vol_str = f"{vol['source']}:{vol['target']}"
if vol.get("readonly"):
vol_str += ":ro"
volumes.append(vol_str)
if volumes:
service["volumes"] = volumes
compose = {
"services": {"app": service}
}
return yaml.dump(compose, default_flow_style=False)
```
### Step 5: Image Tag Hash
```python
import hashlib
import json
def compute_image_tag(tool_name: str, manifest: dict) -> str:
"""Deterministic image tag from manifest content."""
# Normalize: sort keys, stable JSON
canonical = json.dumps(manifest, sort_keys=True, separators=(',', ':'))
hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8]
safe_name = tool_name.lower().replace(" ", "-").replace("_", "-")
return f"headquarter/{safe_name}-{hash_suffix}:latest"
```
---
## Mount Resolution
Each `source_type` resolves differently at instance creation time:
| source_type | Resolution | Example |
|-------------|------------|---------|
| `repo` | `{repo_path}` (mount or clone) | `/data/repos/headquarter` |
| `ssh_key` | `{instance_dir}/.ssh` | `/data/instances/.../.ssh` |
| `instance` | `{instance_dir}/{name}` | `/data/instances/.../mounts/tmp_.pi_agents` |
| `git_mount` | Resolved from ConfigProfile git_mounts | `/data/instances/.../git-mounts/...` |
| `host_path` | Literal host path | `/var/run/docker.sock` |
```python
def resolve_mount_source(mount: dict, variables: dict) -> str:
source_type = mount["source_type"]
if source_type == "repo":
return variables["REPO_PATH"]
elif source_type == "ssh_key":
return variables["SSH_PATH"]
elif source_type == "instance":
instance_dir = variables["INSTANCE_DIR"]
mount_name = mount["name"]
return f"{instance_dir}/mounts/{mount_name}"
elif source_type == "git_mount":
ref = mount.get("git_mount_ref", "default")
return variables.get(f"GIT_MOUNT_{ref}", "")
elif source_type == "host_path":
return mount.get("source", "")
else:
raise ValueError(f"Unknown source_type: {source_type}")
```
---
## Permission Fixer (Post-Start)
```python
def apply_mount_permissions(
container_id: str,
mounts: list[dict],
timeout: int = 10
) -> list[dict]:
"""Apply permission policies to mounted directories in a running container.
Returns a list of results: [{mount_name, success, error}]
"""
results = []
for mount in mounts:
name = mount["name"]
target = mount["target"]
owner = mount.get("owner")
mode = mount.get("mode")
file_mode = mount.get("file_mode")
result = {"mount_name": name, "success": True, "error": None}
try:
if owner:
proc = subprocess.run(
["docker", "exec", container_id, "chown", "-R",
f"{owner}:{owner}", target],
capture_output=True, text=True, timeout=timeout
)
if proc.returncode != 0:
result["success"] = False
result["error"] = f"chown failed: {proc.stderr}"
if mode and result["success"]:
proc = subprocess.run(
["docker", "exec", container_id, "chmod", mode, target],
capture_output=True, text=True, timeout=timeout
)
if proc.returncode != 0:
result["success"] = False
result["error"] = f"chmod failed: {proc.stderr}"
if file_mode and result["success"]:
proc = subprocess.run(
["docker", "exec", container_id, "sh", "-c",
f"find {target} -type f -exec chmod {file_mode} {{}} +"],
capture_output=True, text=True, timeout=timeout
)
if proc.returncode != 0:
result["success"] = False
result["error"] = f"file_mode chmod failed: {proc.stderr}"
except subprocess.TimeoutExpired:
result["success"] = False
result["error"] = "Permission fix timed out"
results.append(result)
return results
```
**Important:** The permission fixer checks if `root` exists in the container before running. If the container has no `root` user (e.g., distroless images), it logs a warning and skips.
---
## Modified Startup Flow
The `start_instance` function in `tool_instances.py` is modified as follows:
```python
async def start_instance(...):
# ... existing validation ...
tool_type = await session.get(ToolType, instance.tool_type_id)
if tool_type.definition_type == "manifest" and tool_type.manifest_id:
# NEW: Manifest-based startup flow
manifest = await load_manifest(session, tool_type.manifest_id)
# Merge base definition
resolved = resolve_base(manifest)
# Merge tool configs and config profile
resolved = apply_tool_configs(resolved, configs)
resolved = apply_config_profile(resolved, profile)
# Compile
dockerfile = compile_dockerfile(resolved)
entrypoint = compile_entrypoint(resolved)
image_tag = compute_image_tag(tool_type.name, resolved)
# Build image
build_context = {
"Dockerfile": dockerfile,
".headquarter/entrypoint.sh": entrypoint,
}
returncode, stdout, stderr = build_image(
instance_dir=instance_dir,
dockerfile=dockerfile, # The Dockerfile references entrypoint.sh
tag=image_tag,
build_context=build_context,
)
# Generate compose with resolved variables
variables = {
"IMAGE_TAG": image_tag,
"INSTANCE_NAME": instance.name.lower(),
"REPO_PATH": repo_path,
"SSH_PATH": ssh_dir,
"INSTANCE_DIR": instance_dir,
# ... git mount resolutions from config profile ...
}
compose_content = compile_compose(resolved, variables)
write_compose_file(instance_dir, compose_content)
# Store image tag for reuse
instance.image_tag = image_tag
else:
# LEGACY: Existing dockerfile_template / compose_template flow
...
# ... docker compose up ...
# ... wait for running ...
# NEW: Apply mount permissions post-start
if tool_type.definition_type == "manifest":
manifest = await load_manifest(session, tool_type.manifest_id)
resolved = resolve_base(manifest)
permission_results = apply_mount_permissions(
instance.container_id,
resolved.get("mounts", [])
)
for result in permission_results:
if not result["success"]:
logger.warning(
"Permission fix failed for mount %s: %s",
result["mount_name"], result["error"]
)
# ... readiness probe ...
# ... tunnel creation ...
```
---
## File Layout
```
apps/api/src/
├── models/
│ ├── tool_definition_manifest.py # NEW: SQLAlchemy model
│ └── tool_type.py # MOD: add manifest_id, definition_type
├── services/
│ ├── manifest_compiler.py # NEW: compile_dockerfile, compile_compose, resolve_base
│ ├── permission_fixer.py # NEW: apply_mount_permissions
│ └── docker_build.py # MOD: support build_context files
├── api/
│ ├── tool_definitions.py # NEW: CRUD + compile endpoints
│ └── tool_instances.py # MOD: manifest-based startup flow
├── schemas/
│ └── manifest_schema.py # NEW: JSON Schema + Pydantic validators
└── alembic/versions/
└── 20260528_add_tool_definition_manifests.py # NEW: migration
```
---
## Testing Strategy
### Unit Tests
| Module | Tests | Coverage |
|--------|-------|----------|
| `manifest_compiler.py` | Dockerfile generation for all package managers, base merging, entrypoint generation | All branches |
| `permission_fixer.py` | chown/chmod success, failure, timeout, missing root user | All branches |
| `manifest_schema.py` | Valid manifest acceptance, invalid manifest rejection (all error paths) | All validation rules |
### Integration Tests
| Scenario | Test |
|----------|------|
| Manifest → Dockerfile → Build | Create manifest, compile, build image, verify it runs |
| ConfigProfile merge | Start instance with profile, verify mounts merged correctly |
| Permission fix | Start non-root container, verify workspace is writable |
| SSH key mount | Start clone-mode instance, verify SSH keys accessible and have correct permissions |
| Legacy compatibility | Start instance from old dockerfile_template tool type, verify it still works |
| Base versioning | Create tool with base v1, update base to v2, verify tool still uses v1 |
### E2E Tests
| Scenario | Test |
|----------|------|
| Tool Workshop CRUD | Create, edit, preview, delete a tool definition via UI |
| Instance lifecycle | Create instance from manifest tool, start, terminal connect, stop, delete |
---
## Migration Plan
### Step 1: Schema Migration
```python
# alembic migration
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table(
"tool_definition_manifests",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("name", sa.String(64), nullable=False),
sa.Column("display_name", sa.String(128), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("category", sa.String(64), nullable=True),
sa.Column("interface_type", sa.String(16), nullable=False),
sa.Column("base_image", sa.String(256), nullable=True),
sa.Column("base_definition_id", sa.UUID(), nullable=True),
sa.Column("base_version", sa.String(32), nullable=False, server_default="latest"),
sa.Column("manifest", sa.JSON(), nullable=False),
sa.Column("dockerfile_cache", sa.Text(), nullable=True),
sa.Column("compose_cache", sa.Text(), nullable=True),
sa.Column("version", sa.String(32), nullable=False, server_default="v1"),
sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"),
sa.Column("created_by_id", sa.UUID(), nullable=True),
sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name"),
sa.ForeignKeyConstraint(["base_definition_id"], ["tool_definition_manifests.id"]),
sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]),
sa.CheckConstraint(
"(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)",
name="ck_tool_definition_manifests_base_required"
),
)
op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True))
op.add_column("tool_types", sa.Column("definition_type", sa.String(16), nullable=False, server_default="legacy"))
op.create_foreign_key(
"fk_tool_types_manifest_id",
"tool_types", "tool_definition_manifests",
["manifest_id"], ["id"]
)
op.add_column("tool_instances", sa.Column("manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True))
op.add_column("tool_instances", sa.Column("image_tag", sa.String(256), nullable=True))
```
### Step 2: Data Migration
Convert the existing pi-agent tool type from `dockerfile_template` to manifest:
```python
def upgrade_data():
conn = op.get_bind()
# Create the base definition for ubuntu-24.04-dev
base_id = uuid.uuid4()
conn.execute(sa.text("""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, interface_type, base_image, manifest, is_base, version)
VALUES (:id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base', 'Base development environment',
'terminal', 'ubuntu:24.04', :manifest, true, 'v1')
"""), {
"id": base_id,
"manifest": json.dumps({
"name": "ubuntu-24.04-dev",
"base_image": "ubuntu:24.04",
"packages": {"apt": ["curl", "wget", "git", "build-essential", "ca-certificates"]},
"user": {"name": "user", "uid": 1000, "gid": 1000},
})
})
# Create pi-agent manifest referencing the base
pi_agent_id = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3")
conn.execute(sa.text("""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, category, interface_type,
base_definition_id, base_version, manifest, version)
VALUES (:id, 'pi-agent', 'Pi Agent', 'Terminal-based coding harness',
'development', 'terminal', :base_id, 'v1', :manifest, 'v1')
"""), {
"id": pi_agent_id,
"base_id": base_id,
"manifest": json.dumps({
"name": "pi-agent",
"display_name": "Pi Agent",
"interface_type": "terminal",
"packages": {
"apt": ["neovim", "ranger", "tmux", "htop", "tree", "jq", "python3", "python3-pip"],
"node": {"version": "20"},
"npm_global": ["@earendil-works/pi-coding-agent"]
},
"user": {"name": "user", "uid": 1001, "gid": 1001, "create_home": True, "shell": "/bin/bash"},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
"scripts": {
"build": [
"git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'",
"mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf"
],
"startup": [
"if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi"
]
},
"mounts": [
{"name": "workspace", "target": "/workspace", "source_type": "repo", "writable": True, "owner": "user"},
{"name": "ssh_keys", "target": "/home/user/.ssh", "source_type": "ssh_key", "mode": "0700", "file_mode": "0600", "readonly": True},
{"name": "pi_state", "target": "/tmp/.pi/agents", "source_type": "instance", "writable": True},
{"name": "pi_config", "target": "/home/user/.pi", "source_type": "git_mount", "git_mount_ref": "dotfiles", "writable": True, "owner": "user"}
],
"runtime": {
"command": ["/bin/bash"],
"stdin_open": True,
"tty": True,
"working_dir": "/workspace"
}
})
})
# Update the existing tool_types row
conn.execute(sa.text("""
UPDATE tool_types
SET manifest_id = :manifest_id,
definition_type = 'manifest',
dockerfile_template = NULL,
compose_template = NULL
WHERE name = 'pi-agent'
"""), {"manifest_id": pi_agent_id})
```
---
## Review Workload Forecast
| PR | Scope | Est. Lines | Review Risk |
|----|-------|-----------|-------------|
| PR 1: Backend | Compiler, fixer, API, tests, migration | ~1800 | Medium — core algorithm changes |
| PR 2: Frontend | Tool Workshop UI | ~1200 | Medium — new feature, self-contained |
| PR 3: Migration | Data migration, legacy fallback | ~300 | Low — additive only |
All PRs are under the 400-line budget individually. Chained PRs recommended for sequential review.
openspec/docs/tool-workshop-guide.md
+81
View File
@@ -0,0 +1,81 @@
# Tool Workshop User Guide
## Overview
The Tool Workshop lets you define and manage tool types — the blueprints for
containers that run inside Headquarter. Each tool type specifies how to build
and start a container (Docker image, Compose file, or a declarative manifest).
## Definition Types
### 1. Compose (Legacy)
Write a raw Docker Compose template. Variable substitution is supported:
- `${REPO_PATH}` — path to the mounted repository
- `${TOOL_PORT}` — dynamically assigned free port
- `${INSTANCE_NAME}` — generated instance name
- `${USER_ID}`, `${PROJECT_ID}` — IDs for reference
Best for: simple web services, databases, or anything that already has a
Docker image.
### 2. Dockerfile (Legacy)
Write a raw Dockerfile. Headquarter builds the image and generates a minimal
Compose file automatically.
Best for: custom environments where you need full control over the image build.
### 3. Manifest (Declarative) — **Recommended**
Define your tool with structured JSON instead of raw Docker files:
- **Base image** — pick a base definition (e.g. `ubuntu-24.04-dev`)
- **Packages** — declare apt, npm global, pip, and Node.js version
- **Scripts** — build scripts (run at image build time) and startup scripts
(run when container starts)
- **Mounts** — workspace, SSH keys, instance state, git-mounted dotfiles
- **Runtime** — command, working directory, stdin/tty settings
- **Live preview** — see generated Dockerfile and Compose as you edit
Best for: reproducible, versioned, self-documenting tool definitions.
## Creating a Manifest-Based Tool
1. Go to **Settings → Tool Workshop**
2. Click **New Tool Type**
3. Select **Manifest (Declarative)** as the definition type
4. Choose a **Base Image** (e.g. `ubuntu-24.04-dev v1`)
5. Add packages:
- Apt: `neovim`, `tmux`, `git`
- NPM global: `@earendil-works/pi-coding-agent`
- Node.js version: `20`
6. Add build scripts (e.g. configure git defaults)
7. Add startup scripts (e.g. fix workspace permissions)
8. Configure mounts:
- Workspace → `/workspace` (writable)
- SSH keys → `/home/user/.ssh` (readonly, mode 0700)
9. Set runtime: command `/bin/bash`, working dir `/workspace`
10. Click **Preview** to verify generated Dockerfile and Compose
11. Save
## Base Definitions
Base definitions are versioned manifest templates that other tools extend.
They are marked with the **Base** badge in the list.
The default base `ubuntu-24.04-dev` provides:
- Ubuntu 24.04 base image
- Common build tools (curl, wget, git, build-essential)
- A non-root `user` account (uid 1000)
## Migration from Legacy
Existing tool types using Compose or Dockerfile continue to work unchanged.
You can migrate a tool type to Manifest by:
1. Editing the tool type
2. Switching definition type to **Manifest**
3. Re-creating the configuration in the manifest editor
4. Saving (the old template is cleared automatically)
## Permissions
For manifest-based tools, mount permissions are fixed automatically after the
container starts. The system runs `chown` and `chmod` via `docker exec` as
root, then drops back to the configured runtime user.
openspec/explorations/streamline-tool-container-definitions.md
+362
View File
@@ -0,0 +1,362 @@
# SDD Exploration: Streamline Tool Container Definitions
## Status
**Phase:** explore
**Date:** 2026-05-28
**Owner:** el Gentleman (parent session)
**Scope:** Tool container definition, build, and mount system
---
## Executive Summary
The current tool container system works for the happy path (pi-agent on Ubuntu) but has deep structural inflexibility:
1. **Monolithic Dockerfile strings** in the database — impossible to review, version, or compose
2. **Ad-hoc compose generation** — string formatting with hardcoded fields (`stdin_open`, `tty`, `working_dir` missing for dockerfile types)
3. **Hardcoded mount paths** — `/workspace` and `/root/.ssh` don't adapt to the container's runtime user
4. **No package/base-image modularity** — every tool type carries a full Dockerfile copy
5. **Permission mismatch** — bind mounts come in as root-owned; non-root container users can't write
6. **Config overlap** — tool configs, config profiles, and compose templates fight for control of the same fields
This exploration proposes a **layered, declarative container definition system** where tool types compose from reusable base images, mount schemas, and permission policies.
---
## Current Architecture Map
### Data Model
```
ToolType (DB table)
├── name, display_name, description, category
├── interface_type: "web" | "terminal"
├── definition_type: "dockerfile" | "compose"
├── dockerfile_template: TEXT (giant Dockerfile string)
├── compose_template: TEXT (Jinja-like {{VAR}} string)
├── build_context: JSON {path: content}
├── required_variables: JSON ["REPO_PATH", ...]
├── default_port: int
└── readiness_probe: JSON
ToolInstance (DB table)
├── name, display_name, status
├── tool_type_id → ToolType
├── repository_id → GitRepository
├── compose_path: str
├── container_id, container_name
├── port, url, public_url, tunnel_id
├── clone_mode: "mount" | "clone"
├── branch, new_branch
└── selected_config_profile_id → ConfigProfile
ToolConfig (DB table, per-user per-tool-type)
├── config_type: "env" | "file"
├── key, value, file_path
├── port_override, start_command, working_directory
├── environment_variables: JSON
└── volumes: JSON [{source, target, type}]
ConfigProfile (DB table)
├── name, description
├── user_id, project_id, tool_type_id
├── environment_variables: JSON
├── files: JSON {path: content}
├── mounts: JSON [{source, target, type}]
├── git_mounts: JSON [{remote_url, source_path, target_path, branch}]
└── parent_profile_id → ConfigProfile (hierarchy)
```
### Creation Flow (`create_instance`)
```
POST /projects/{id}/repositories/{id}/instances
→ validate tool_type, repo, config_profile
→ generate instance_name = "{tool_type}-{repo}-{uuid8}"
→ ensure_instance_directory(instance_name)
→ find_free_port()
→ determine repo_path (mount = repo.path; clone = clone_repository())
→ IF tool_type.definition_type == "dockerfile":
build_image(instance_dir, dockerfile_template, tag, build_context)
generate compose_content (HARDCODED STRING FORMATTING)
ELSE:
render_compose_template(tool_type.compose_template, variables)
→ write_compose_file()
→ create ToolInstance DB record (status="pending")
```
### Startup Flow (`start_instance`)
```
POST /instances/{id}/start
→ fetch ToolConfigs (env, files, port_override, start_command, working_dir, volumes)
→ IF selected_config_profile:
resolve_profile() → env, files, mounts, git_mounts, hints
→ write .env file, config files
→ IF clone_mode: mount SSH keys at /root/.ssh (HARDCODED)
→ _modify_compose_file(port, command, working_dir, extra_volumes)
→ _sanitize_compose_file()
→ execute_compose_command("up")
→ get_container_id(instance.name) ← CASE-SENSITIVE BUG (fixed)
→ get_container_name(instance.name)
→ connect_container_to_network("backend")
→ wait_for_container_running()
→ IF web: start_cloudflared_tunnel()
→ instance.status = "running"
```
### Key Files
| File | Responsibility |
|------|---------------|
| `apps/api/src/api/tool_instances.py` | create_instance, start_instance, stop_instance, restart_instance, proxy, logs |
| `apps/api/src/api/tool_types.py` | CRUD for ToolType (DB strings) |
| `apps/api/src/services/docker.py` | compose execution, container queries, tunnel management |
| `apps/api/src/services/docker_build.py` | `docker build` wrapper |
| `apps/api/src/services/terminal_session.py` | PTY-based terminal over `docker exec` |
| `apps/api/src/services/terminal_manager.py` | WebSocket ↔ terminal session lifecycle |
| `apps/api/src/models/tool_type.py` | SQLAlchemy model |
---
## Pain Points (Detailed)
### 1. Monolithic Dockerfile Templates
The pi-agent Dockerfile template is a 40-line string stored in the DB migration:
```sql
INSERT INTO tool_types (... dockerfile_template ...)
VALUES ('...# Pi Coding Agent - Terminal-based coding harness\nFROM ubuntu:24.04\n...')
```
**Problems:**
- No syntax highlighting, linting, or `docker build` validation at edit time
- Every tool type copies the entire Dockerfile; no reuse of common layers
- Changes require a DB migration
- No way for users to customize packages without forking the whole template
### 2. Ad-Hoc Compose Generation
For `dockerfile` type tools, the compose is generated by Python f-string:
```python
compose_content = f"""version: "3.8"
services:
app:
image: {image_tag}
container_name: {instance_name.lower()}
{ports_section} volumes:
- {repo_path}:/workspace
restart: unless-stopped
"""
```
**Problems:**
- Missing `stdin_open: true` and `tty: true` (essential for terminal tools)
- Missing `working_dir: /workspace`
- No way to add labels, networks, healthchecks, or extra services
- Port section is conditionally included with awkward string concatenation
### 3. Hardcoded Mount Paths
| Mount | Current Target | Problem |
|-------|---------------|---------|
| Repository | `/workspace` | Always root-owned; no permission fix for non-root users |
| SSH keys (clone mode) | `/root/.ssh` | Invisible to containers running as `user` |
| Git-mount configs | `/tmp/.pi` | May be root-owned; conflicts with user's `.pi` |
| Config profile files | Instance-relative paths | No validation against container filesystem |
### 4. No Base Image / Layer Composition
Every tool type must specify a complete Dockerfile from `FROM` to `CMD`. There's no way to say:
```yaml
base: ubuntu-24.04-dev # pre-built with curl, git, build-essential
layers:
- nodejs-20
- pi-coding-agent
- custom-packages: [neovim, ranger, tmux]
```
### 5. Permission Mismatch (Non-Root Users)
The pi-agent Dockerfile creates a `user` account and uses `USER user`. Bind mounts from the host come in as root-owned. The API has **no automatic permission fix** — this caused the workspace-unwritable bug.
Workarounds considered:
- Post-start `docker exec --user root chown` (current fix)
- Dockerfile entrypoint script that chowns before dropping privileges
- Matching container UID to host UID
None of these are systematic or configurable.
### 6. Config Overlap and Precedence Confusion
Three systems control the same container aspects:
| System | Controls | Stored |
|--------|----------|--------|
| ToolConfig | env vars, files, port, command, working_dir, volumes | DB (per-user per-tool) |
| ConfigProfile | env vars, files, mounts, git_mounts, hints | DB (hierarchical) |
| Compose template / generation | volumes, ports, command, working_dir | DB string / Python f-string |
**Precedence is unclear:**
- ToolConfig `working_directory` vs ConfigProfile hint `working_directory` vs compose `working_dir`
- ToolConfig `volumes` vs ConfigProfile `mounts` vs compose `volumes`
- `start_command` from ToolConfig vs ConfigProfile vs Dockerfile `CMD`
### 7. Build Context Limitations
`build_context` is a JSON dictionary of `{relative_path: file_content}`. This is stored in the DB as text.
**Problems:**
- Binary files (images, tarballs) can't be stored
- Large files bloat the DB
- No versioning or external reference (e.g., "use file from git repo")
---
## Extensibility Gaps
| Want | Current State | Gap |
|------|--------------|-----|
| Add a new language runtime (e.g., Go, Rust) | Copy entire Dockerfile, edit | No modular package/layer system |
| Use a custom base image (e.g., `my-registry/dev-base:v2`) | Edit full Dockerfile | No base-image reference field |
| Mount a second repo or a secrets file | Write ConfigProfile or ToolConfig JSON | No declarative mount schema |
| Run as root instead of `user` | Edit full Dockerfile | No runtime-user field |
| Add a sidecar (e.g., postgres for integration tests) | Edit compose_template string | No multi-service compose support |
| Pre-install VS Code server | Edit full Dockerfile | No "feature" or "extension" mechanism |
| Custom entrypoint script | Edit full Dockerfile | No entrypoint field |
---
## Design Directions (Pre-Proposal)
### Direction A: Declarative Tool Manifests
Replace the monolithic `dockerfile_template` with a structured manifest:
```yaml
# Example: tool manifest for pi-agent
name: pi-agent
base_image: ubuntu:24.04
user:
name: user
uid: 1000
home: /home/user
packages:
apt: [curl, wget, git, neovim, ranger, tmux, htop, tree, jq, python3, python3-pip, build-essential]
npm_global: [@earendil-works/pi-coding-agent]
node_version: "20"
env:
DEBIAN_FRONTEND: noninteractive
config_files:
/home/user/.tmux.conf: "set -g mouse on\n..."
/home/user/.config/ranger/rc.conf: "set preview_files true\n..."
working_directory: /workspace
command: ["/bin/bash"]
ports: []
mounts:
repo: {target: /workspace, writable: true}
ssh: {target: /home/user/.ssh, mode: "0600"}
tmp_state: {target: /tmp/.pi, writable: true}
```
**Pros:** Structured, reviewable, composable
**Cons:** Requires a manifest-to-Dockerfile compiler; migration complexity
### Direction B: Base Image Registry + Layers
Maintain a registry of pre-built base images:
```
headquarter/base/ubuntu-24.04-dev
headquarter/base/nodejs-20
headquarter/base/python-3.11
```
Tool types reference a base image and a list of layers:
```yaml
base_image: headquarter/base/ubuntu-24.04-dev
layers:
- type: npm_install
package: @earendil-works/pi-coding-agent
- type: config_file
path: /home/user/.tmux.conf
content: "..."
```
**Pros:** Fast builds (base images cached), reusable, versioned
**Cons:** Requires image registry management, layer ordering complexity
### Direction C: Compose-First with Dockerfile Overrides
Treat `compose_template` as the primary definition. For simple cases, use a pre-built image. For custom cases, allow an inline Dockerfile or a `build` section in the compose:
```yaml
services:
app:
build:
context: .
dockerfile_inline: |
FROM ubuntu:24.04
...
stdin_open: true
tty: true
working_dir: /workspace
volumes:
- ${REPO_PATH}:/workspace
- ${SSH_PATH}:/home/user/.ssh:ro
user: "${CONTAINER_USER:-user}"
```
**Pros:** Leverages Docker Compose native features, familiar to users
**Cons:** Still string-based; inline Dockerfiles are hard to edit
### Direction D: Permission-Aware Mount Schema
Decouple mount declaration from mount implementation:
```python
class MountPolicy:
source: str # host path
target: str # container path
owner: str | None # container user to own the mount
permissions: str # chmod string
readonly: bool
```
At startup, the API runs a post-start "permission fixer" that applies all policies:
```bash
docker exec --user root <container> chown -R <owner> <target>
docker exec --user root <container> chmod <permissions> <target>
```
**Pros:** Systematic, works with any base image, configurable per mount
**Cons:** Adds startup latency, requires root to exist in container
---
## Recommended Next Steps
1. **Proposal phase:** Evaluate Direction A (Declarative Manifests) vs Direction C (Compose-First) for the primary architecture
2. **Design phase:** Detail the manifest schema or compose enhancement, migration path, and API changes
3. **Consider Direction D** as a cross-cutting concern regardless of primary direction
---
## Risks
- **Migration risk:** Existing `dockerfile_template` and `compose_template` columns need backward-compatible migration
- **Build cache invalidation:** Changing the build system may invalidate Docker layer caches
- **User confusion:** Adding a manifest layer on top of Dockerfiles may feel like "yet another abstraction"
- **Scope creep:** This touches tool types, tool configs, config profiles, compose generation, and the startup flow — high cross-cutting surface
---
## Artifacts
- `openspec/config.yaml` — SDD configuration
- `openspec/explorations/streamline-tool-container-definitions.md` — This document
openspec/proposals/streamline-tool-container-definitions.md
+447
View File
@@ -0,0 +1,447 @@
# SDD Proposal: Declarative Tool Container Compiler
## Status
**Phase:** proposal
**Date:** 2026-05-28
**Owner:** el Gentleman
**Based on:** Exploration `streamline-tool-container-definitions`
---
## User Story
As a platform operator, I want to define a tool container by specifying:
- A **base image** (e.g. `ubuntu:24.04` or a pre-built `headquarter/base:dev-ubuntu`)
- A **list of packages** to install (apt, npm, pip, etc.)
- **Setup scripts** that run at build-time or container-startup
- **Mount policies** that automatically fix permissions for the runtime user
I do **not** want to write Dockerfiles or Compose files by hand.
The system should compile these declarations into Dockerfiles and Compose files automatically, while remaining fully compatible with the existing ConfigProfile mount system.
---
## Core Concept: The Tool Definition Manifest
Replace the monolithic `dockerfile_template` and `compose_template` strings with a single structured **Tool Definition Manifest**.
```yaml
# Tool Definition Manifest (stored as JSON in DB)
name: pi-agent
display_name: "Pi Agent"
description: "Terminal-based coding harness"
category: development
interface_type: terminal # web | terminal
# ── Base Image ───────────────────────────────────────────────
base_image: ubuntu:24.04
# OR reference a pre-built base definition:
# base_definition_id: "base-ubuntu-24.04-dev"
# ── Packages ─────────────────────────────────────────────────
packages:
apt:
- curl
- wget
- git
- neovim
- ranger
- tmux
- htop
- tree
- jq
- ca-certificates
- python3
- python3-pip
- build-essential
node:
version: "20" # triggers nodesource setup
npm_global:
- "@earendil-works/pi-coding-agent"
# pip:
# - requests
# - httpx
# ── Runtime User ─────────────────────────────────────────────
user:
name: user
uid: 1001
gid: 1001
create_home: true
shell: /bin/bash
# ── Environment ──────────────────────────────────────────────
env:
DEBIAN_FRONTEND: noninteractive
# ── Setup Scripts ────────────────────────────────────────────
scripts:
# build: runs during `docker build` → becomes RUN commands
build:
- |
git config --global init.defaultBranch main
git config --global user.email "dev@headquarter.local"
git config --global user.name "Developer"
- |
mkdir -p /home/user/.config/ranger
echo 'set preview_files true' > /home/user/.config/ranger/rc.conf
# startup: runs when container starts → becomes entrypoint script
startup:
- |
# Ensure workspace is owned by runtime user
if [ -d /workspace ]; then
sudo chown -R user:user /workspace 2>/dev/null || true
fi
# ── Mount Schema ─────────────────────────────────────────────
mounts:
- name: workspace
target: /workspace
source_type: repo # resolved from repository path at instance creation
writable: true
owner: user # post-start: chown -R user:user /workspace
- name: ssh_keys
target: /home/user/.ssh
source_type: ssh_key # resolved from repository's SSH key
mode: "0700" # post-start: chmod 0700 /home/user/.ssh
file_mode: "0600" # post-start: chmod 0600 files inside
readonly: true
- name: pi_state
target: /tmp/.pi/agents
source_type: instance # resolved to {instance_dir}/mounts/tmp_.pi_agents
writable: true
- name: pi_config
target: /home/user/.pi
source_type: git_mount # resolved from config profile git_mounts
git_mount_ref: dotfiles # references a named git mount in the config profile
writable: true
owner: user
# ── Runtime ──────────────────────────────────────────────────
runtime:
command: ["/bin/bash"]
stdin_open: true
tty: true
working_dir: /workspace
# ports are auto-derived from interface_type:
# web: expose default_port
# terminal: no port mapping
```
---
## How It Compiles
### 1. Dockerfile Generation
The manifest compiler transforms the spec into a Dockerfile:
```dockerfile
# Generated Dockerfile — do not edit manually
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
# ── System Packages ──
RUN apt-get update && apt-get install -y \
curl wget git neovim ranger tmux htop tree jq \
ca-certificates python3 python3-pip build-essential \
&& rm -rf /var/lib/apt/lists/*
# ── Node.js ──
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
&& apt-get install -y nodejs \
&& rm -rf /var/lib/apt/lists/*
# ── NPM Packages ──
RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
# ── Runtime User ──
RUN groupadd -g 1001 user && \
useradd -u 1001 -g 1001 -m -s /bin/bash user
# ── Build Scripts ──
RUN git config --global init.defaultBranch main && \
git config --global user.email "dev@headquarter.local" && \
git config --global user.name "Developer"
RUN mkdir -p /home/user/.config/ranger && \
echo 'set preview_files true' > /home/user/.config/ranger/rc.conf
# ── Environment ──
ENV DEBIAN_FRONTEND=noninteractive
# ── Setup Directories ──
RUN mkdir -p /workspace /tmp/.pi/agents /home/user/.pi && \
chown -R user:user /workspace /tmp/.pi /home/user/.pi
# ── Entrypoint for Startup Scripts ──
COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint
RUN chmod +x /usr/local/bin/headquarter-entrypoint
USER user
WORKDIR /home/user
ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]
CMD ["/bin/bash"]
```
The generated `entrypoint.sh`:
```bash
#!/bin/bash
set -e
# Run startup scripts
echo "Ensure workspace is owned by runtime user"
if [ -d /workspace ]; then
sudo chown -R user:user /workspace 2>/dev/null || true
fi
# Pass through to the main command
exec "$@"
```
### 2. Compose File Generation
The manifest compiler also generates the Compose file:
```yaml
services:
app:
image: ${IMAGE_TAG}
container_name: ${INSTANCE_NAME}
stdin_open: true
tty: true
working_dir: /workspace
user: "1001:1001" # from manifest user.uid/gid
volumes:
- ${REPO_PATH}:/workspace
- ${SSH_PATH}:/home/user/.ssh:ro
- ${INSTANCE_DIR}/mounts/tmp_.pi_agents:/tmp/.pi/agents
- ${GIT_MOUNT_dotfiles}:/home/user/.pi
environment:
DEBIAN_FRONTEND: noninteractive
restart: unless-stopped
```
Variables are resolved at instance creation time:
- `${REPO_PATH}` → the repository working directory
- `${SSH_PATH}` → prepared SSH key directory
- `${INSTANCE_DIR}` → the instance working directory
- `${GIT_MOUNT_dotfiles}` → resolved from config profile git_mounts
### 3. Permission Fixer (Post-Start)
After `docker compose up`, the API iterates over the mount schema and applies permission policies:
```python
for mount in manifest.mounts:
if mount.owner:
docker_exec(f"chown -R {mount.owner}:{mount.owner} {mount.target}")
if mount.mode:
docker_exec(f"chmod {mount.mode} {mount.target}")
if mount.file_mode:
docker_exec(f"find {mount.target} -type f -exec chmod {mount.file_mode} {{}} +")
```
This is **systematic and configurable** — not hardcoded to `/workspace`.
---
## Config Profile Compatibility
The existing ConfigProfile system provides:
- `environment_variables` → merged into compose `environment`
- `files` → written to instance dir, mounted via `volumes`
- `mounts` → appended to compose `volumes`
- `git_mounts` → resolved to host paths, appended to compose `volumes`
- `hints.start_command` → overrides `runtime.command`
- `hints.working_directory` → overrides `runtime.working_dir`
With the manifest system, ConfigProfiles **extend** the default mount schema:
1. Tool manifest defines the **default mount schema** (workspace, ssh, state)
2. ConfigProfile can add **additional mounts** or **override runtime hints**
3. Both are merged at instance-start time into the final compose file
The merge precedence:
1. Tool manifest (defaults)
2. ToolConfig overrides (per-user per-tool settings)
3. ConfigProfile overrides (hierarchical, can inherit from parent)
4. User-provided start options (e.g. branch selection)
---
## Base Image Definitions
A base definition is itself a manifest with no `runtime` section:
```yaml
# Base Definition: "ubuntu-24.04-dev"
name: ubuntu-24.04-dev
description: "Ubuntu 24.04 with build tools"
base_image: ubuntu:24.04
packages:
apt:
- curl
- wget
- git
- build-essential
- ca-certificates
user:
name: user
uid: 1000
gid: 1000
create_home: true
```
A tool definition can reference it:
```yaml
base_definition_id: "ubuntu-24.04-dev"
packages:
apt:
- neovim
- ranger
- tmux
node:
version: "20"
```
The compiler **merges** the base definition with the tool-specific overrides:
- Packages are **unioned** (base apt + tool apt)
- Scripts are **appended** (base build scripts, then tool build scripts)
- User/env are **overridden** (tool wins)
This enables a family of tool types to share a common base.
---
## Database Schema (Proposed)
### New Table: `tool_definition_manifests`
| Column | Type | Description |
|--------|------|-------------|
| `id` | UUID | PK |
| `name` | str | Unique identifier |
| `display_name` | str | Human-readable |
| `description` | str | |
| `category` | str | development, data-science, etc. |
| `interface_type` | enum | web, terminal |
| `base_image` | str | e.g. `ubuntu:24.04` |
| `base_definition_id` | UUID? | FK to another manifest |
| `manifest` | JSONB | The full manifest JSON |
| `dockerfile_cache` | TEXT | Last generated Dockerfile (for inspection) |
| `created_at` | datetime | |
| `updated_at` | datetime | |
### Migration: `tool_types` table
Add a nullable `manifest_id` column to `tool_types`.
For backward compatibility:
- If `manifest_id` is set → use the new manifest system
- If `manifest_id` is null → fall back to `dockerfile_template` / `compose_template`
A data migration converts existing pi-agent to the new manifest format.
---
## API Changes
### New Endpoints
```
GET /tool-definitions → list all base/tool definitions
GET /tool-definitions/{id} → get a definition
POST /tool-definitions → create a new definition
PUT /tool-definitions/{id} → update a definition
DELETE /tool-definitions/{id} → delete (if not in use)
POST /tool-definitions/{id}/compile → preview generated Dockerfile + compose
```
### Modified Endpoints
```
POST /tool-types → can now accept manifest_id instead of templates
GET /tool-types/{id} → includes manifest if available
```
### Frontend Changes
New UI page: **Tool Workshop**
- Base image selector (dropdown of existing bases or custom FROM)
- Package manager tabs (apt, npm, pip, etc.)
- Script editor (build vs startup)
- Mount schema designer (drag-drop or form)
- Live preview of generated Dockerfile
- Test build button (builds image and reports success/failure)
---
## Open Questions
1. **Should we support multi-stage builds?**
- Pros: smaller images, separation of build deps from runtime
- Cons: more complexity in the manifest schema
2. **Should base definitions be versioned?**
- Pros: reproducible builds, safe updates
- Cons: more DB complexity
3. **How do we handle binary build context files?**
- Current: JSON text in DB
- Option A: Store in filesystem, reference by path
- Option B: Upload to object storage (S3/minio)
4. **Should generated images be cached/pushed to a registry?**
- Currently: built locally per instance
- Option: push to `headquarter/tools/{tool-name}:{hash}` for reuse
---
## Risks
| Risk | Severity | Mitigation |
|------|----------|------------|
| Migration complexity | Medium | Keep old fields nullable; gradual adoption |
| Build cache invalidation | Medium | Use deterministic Dockerfile generation; hash manifest for image tag |
| User confusion ("yet another abstraction") | Low | Provide live preview + "view generated Dockerfile" button |
| Scope creep into full CI/CD | High | Keep scope to container definition only; no pipeline/orchestration |
| Binary files in manifests | Medium | Limit build context to text; document workaround for binaries |
---
## Effort Estimate
| Phase | Files | Lines (est) | Complexity |
|-------|-------|-------------|------------|
| DB migration + models | 3 | 200 | Low |
| Manifest compiler (Dockerfile) | 2 | 400 | Medium |
| Manifest compiler (Compose) | 2 | 300 | Medium |
| Permission fixer refactor | 2 | 200 | Low |
| API endpoints | 3 | 400 | Medium |
| Frontend Tool Workshop | 8 | 1200 | High |
| Tests | 4 | 600 | Medium |
| **Total** | **24** | **~3300** | **High** |
**Review workload forecast:** ~3300 lines is well above the 400-line budget. This should be split into **chained PRs**:
1. Backend: manifest schema, compiler, API (PR 1)
2. Frontend: Tool Workshop UI (PR 2)
3. Migration + data conversion (PR 3)
---
## Next Recommended Phase
**Design** — Detail the manifest JSON schema, compiler internals, and migration plan.
Should I proceed to design?
openspec/specs/tool-definition-manifest.md
+323
View File
@@ -0,0 +1,323 @@
# Spec: Tool Definition Manifest System
## Status
**Phase:** spec
**Date:** 2026-05-28
**Owner:** el Gentleman
**Based on:** Proposal `streamline-tool-container-definitions`
---
## Requirements
### R1. Declarative Tool Definitions
Users must be able to define a tool container without writing Dockerfiles or Compose files. The definition is a structured manifest specifying base image, packages, scripts, mounts, and runtime configuration.
### R2. Base Image Versioning
Base definitions must be versioned. Tool definitions reference a specific base version. Updating a base creates a new version; existing tools remain pinned to their version until explicitly updated.
### R3. Package Managers
The manifest must support multiple package managers: `apt`, `npm` (global), `pip`, and `node` (version installation).
### R4. Build vs Startup Scripts
Scripts are categorized by execution phase:
- **Build scripts**: Run during `docker build` (e.g., `git config`, config file setup)
- **Startup scripts**: Run when the container starts (e.g., permission fixes, dynamic setup)
### R5. Mount Schema with Permission Policies
Mounts declare:
- `target`: Container path
- `source_type`: How the source is resolved (`repo`, `ssh_key`, `instance`, `git_mount`, `host_path`)
- `writable`: Whether the mount is read-write
- `owner`: Container user to own the target path (post-start chown)
- `mode`: Directory permissions (post-start chmod)
- `file_mode`: File permissions inside the directory
- `readonly`: Whether mounted read-only in compose
### R6. Config Profile Compatibility
ConfigProfiles continue to add `env`, `files`, `mounts`, and `git_mounts` on top of the manifest defaults. The merge precedence is: manifest defaults → ToolConfig → ConfigProfile → user options.
### R7. Backward Compatibility
Existing `dockerfile_template` and `compose_template` columns remain functional. New tool types use the manifest system; old types continue to work. A data migration converts the existing pi-agent to the new format.
### R8. Local Builds
Images are built locally per instance using the standard `docker build` command. No registry integration in this phase.
### R9. Live Preview
The API provides a `compile` endpoint that returns the generated Dockerfile and Compose file without building.
### R10. Deterministic Image Tags
The image tag is derived from a hash of the manifest content, enabling build cache reuse when the manifest hasn't changed.
---
## Scenarios
### S1. Creating a New Tool Definition
**Given** a user on the Tool Workshop page
**When** they select base "ubuntu-24.04-dev:v1", add packages `[neovim, tmux]`, add a build script for git config, and define mounts for workspace + ssh
**Then** the system generates a manifest, compiles a Dockerfile + Compose preview, and upon save stores the manifest in the database.
### S2. Building an Instance from a Manifest
**Given** a tool instance created from a manifest-based tool type
**When** `start_instance` is called
**Then** the API compiles the manifest to a Dockerfile, builds the image, generates the Compose file with resolved mount paths, starts the container, and applies permission policies post-start.
### S3. Permission Fix on Non-Root Containers
**Given** a manifest with `user: {name: user, uid: 1001}` and a mount `target: /workspace, owner: user`
**When** the container starts with the workspace bind-mounted from host (root-owned)
**Then** the post-start permission fixer runs `docker exec --user root chown -R user:user /workspace`, making the directory writable for the container user.
### S4. SSH Key Mount for Non-Root User
**Given** a manifest with a mount `target: /home/user/.ssh, source_type: ssh_key, mode: "0700"`
**When** the container starts
**Then** SSH keys are mounted from the instance `.ssh` directory to `/home/user/.ssh`, and post-start fixes permissions to `0700` for the directory and `0600` for key files.
### S5. Config Profile Extends Manifest
**Given** a manifest with default mount `workspace: /workspace` and a ConfigProfile that adds `git_mounts: [{remote_url: "...", target_path: "/home/user/.config"}]`
**When** the instance starts with that profile selected
**Then** the final Compose includes both the workspace mount and the config git mount, merged in the correct precedence.
### S6. Base Version Pinning
**Given** a tool definition referencing `base_definition_id: "ubuntu-24.04-dev", base_version: "v1"`
**When** the base definition is updated to "v2"
**Then** the tool definition continues to use "v1" until explicitly updated. New tool definitions default to the latest version.
### S7. Deterministic Image Tag
**Given** a manifest with specific packages and scripts
**When** compiled
**Then** the generated image tag is `headquarter/{tool-name}-{manifest-hash}:latest`, and rebuilding the same manifest reuses the cached image layer.
### S8. Live Preview Without Build
**Given** a manifest being edited
**When** the user clicks "Preview"
**Then** the API returns the generated Dockerfile and Compose file within 500ms, without invoking Docker.
---
## Acceptance Criteria
### A1. Manifest Schema Validation
- [ ] The manifest JSON must validate against a defined JSON Schema
- [ ] Invalid manifests return 400 with detailed field-level errors
- [ ] Missing required fields (name, base_image or base_definition_id) are rejected
### A2. Dockerfile Compilation
- [ ] Generated Dockerfile builds successfully with `docker build`
- [ ] Build scripts appear as `RUN` commands in order
- [ ] Startup scripts appear in the generated entrypoint script
- [ ] Packages are installed in a single layer per package manager
- [ ] User creation uses the declared uid/gid
### A3. Compose Compilation
- [ ] Generated Compose file starts successfully with `docker compose up`
- [ ] Mounts are resolved from `source_type` to actual host paths
- [ ] `stdin_open` and `tty` are set for terminal interface types
- [ ] Ports are only included for web interface types
### A4. Permission Fixer
- [ ] Post-start chown runs for all mounts with an `owner` declared
- [ ] Post-start chmod runs for all mounts with `mode` or `file_mode` declared
- [ ] Permission fixes complete within 5 seconds of container start
- [ ] If the container has no `root` user, permission fixes are skipped with a warning
### A5. Config Profile Merge
- [ ] ConfigProfile env vars override manifest defaults
- [ ] ConfigProfile mounts are appended to manifest mounts
- [ ] ConfigProfile git_mounts are resolved and appended
- [ ] ToolConfig values override both manifest and ConfigProfile
### A6. Backward Compatibility
- [ ] Existing tool types with `dockerfile_template` continue to work
- [ ] Existing tool types with `compose_template` continue to work
- [ ] The pi-agent tool type is migrated to the new manifest format
- [ ] Old and new tool types can coexist in the same project
### A7. Base Versioning
- [ ] Base definitions store a version string
- [ ] Tool definitions store the base version they reference
- [ ] Updating a base creates a new version; old versions remain accessible
- [ ] The "latest" version can be referenced explicitly or by omission
### A8. Image Tag Determinism
- [ ] Same manifest produces the same image tag
- [ ] Changing any field (package, script, env) produces a different tag
- [ ] The tag is lowercased and valid as a Docker image reference
### A9. API Endpoints
- [ ] `POST /tool-definitions` creates a definition (201)
- [ ] `GET /tool-definitions/{id}` returns the definition with compiled preview
- [ ] `POST /tool-definitions/{id}/compile` returns Dockerfile + Compose (no build)
- [ ] `PUT /tool-definitions/{id}` updates and re-validates
### A10. Frontend Tool Workshop
- [ ] Users can create a tool definition via form (no raw JSON editing required)
- [ ] Live preview shows generated Dockerfile and Compose
- [ ] Package lists support add/remove/reorder
- [ ] Mount schema supports add/remove with visual feedback
- [ ] Base image selector shows available versions
---
## Non-Goals
- **Multi-stage builds** — Out of scope for this phase. The compiler generates single-stage Dockerfiles.
- **Docker registry integration** — Images are built locally per instance.
- **Binary build context files** — Build context is limited to text files stored in the DB.
- **Custom Dockerfile editing** — Users work exclusively through the manifest; no raw Dockerfile editing.
- **Container orchestration beyond Compose** — No Kubernetes, Swarm, or other orchestrators.
- **Real-time collaborative editing** — Tool Workshop is single-user editing.
---
## API Contract
### POST /tool-definitions
**Request:**
```json
{
"name": "pi-agent",
"display_name": "Pi Agent",
"description": "Terminal coding harness",
"category": "development",
"interface_type": "terminal",
"base_image": "ubuntu:24.04",
"packages": {
"apt": ["curl", "git", "neovim", "tmux"],
"node": {"version": "20"},
"npm_global": ["@earendil-works/pi-coding-agent"]
},
"user": {
"name": "user",
"uid": 1001,
"gid": 1001,
"create_home": true,
"shell": "/bin/bash"
},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
"scripts": {
"build": [
"git config --global init.defaultBranch main",
"mkdir -p /home/user/.config/ranger"
],
"startup": [
"if [ -d /workspace ]; then sudo chown -R user:user /workspace; fi"
]
},
"mounts": [
{
"name": "workspace",
"target": "/workspace",
"source_type": "repo",
"writable": true,
"owner": "user"
},
{
"name": "ssh",
"target": "/home/user/.ssh",
"source_type": "ssh_key",
"mode": "0700",
"file_mode": "0600",
"readonly": true
}
],
"runtime": {
"command": ["/bin/bash"],
"stdin_open": true,
"tty": true,
"working_dir": "/workspace"
}
}
```
**Response (201):**
```json
{
"id": "d07b8376-2151-4119-8c1d-27f792aae9a3",
"name": "pi-agent",
"display_name": "Pi Agent",
"manifest": { ... },
"dockerfile_preview": "FROM ubuntu:24.04\n...",
"compose_preview": "services:\n app:\n image: ...",
"created_at": "2026-05-28T10:00:00Z"
}
```
### POST /tool-definitions/{id}/compile
**Response (200):**
```json
{
"dockerfile": "FROM ubuntu:24.04\n...",
"compose": "services:\n app:\n...",
"image_tag": "headquarter/pi-agent-a3f7c2d9:latest",
"mounts_resolved": [
{"name": "workspace", "source": "/data/repos/headquarter", "target": "/workspace"}
]
}
```
---
## Database Schema
### `tool_definition_manifests`
| Column | Type | Constraints |
|--------|------|-------------|
| `id` | UUID | PK |
| `name` | VARCHAR(64) | UNIQUE, NOT NULL |
| `display_name` | VARCHAR(128) | NOT NULL |
| `description` | TEXT | |
| `category` | VARCHAR(64) | |
| `interface_type` | VARCHAR(16) | CHECK IN ('web', 'terminal') |
| `base_image` | VARCHAR(256) | |
| `base_definition_id` | UUID | FK → `tool_definition_manifests.id` |
| `base_version` | VARCHAR(32) | DEFAULT 'latest' |
| `manifest` | JSONB | NOT NULL |
| `dockerfile_cache` | TEXT | |
| `compose_cache` | TEXT | |
| `version` | VARCHAR(32) | DEFAULT 'v1' |
| `is_base` | BOOLEAN | DEFAULT FALSE |
| `created_by_id` | UUID | FK → `users.id` |
| `created_at` | TIMESTAMPTZ | DEFAULT now() |
| `updated_at` | TIMESTAMPTZ | DEFAULT now() |
**Check constraint:** Exactly one of `base_image` or `base_definition_id` must be set.
### Alter `tool_types`
```sql
ALTER TABLE tool_types
ADD COLUMN manifest_id UUID REFERENCES tool_definition_manifests(id),
ADD COLUMN definition_type VARCHAR(16) DEFAULT 'legacy'; -- 'legacy' | 'manifest'
```
### Alter `tool_instances`
```sql
ALTER TABLE tool_instances
ADD COLUMN manifest_compiled_at TIMESTAMPTZ,
ADD COLUMN image_tag VARCHAR(256);
```
---
## Related Files
- `apps/api/src/models/tool_definition_manifest.py` — New model
- `apps/api/src/models/tool_type.py` — Add manifest_id, definition_type
- `apps/api/src/services/manifest_compiler.py` — New compiler
- `apps/api/src/services/permission_fixer.py` — Refactored mount policy applier
- `apps/api/src/api/tool_definitions.py` — New endpoints
- `apps/api/src/api/tool_instances.py` — Modified start_instance flow
- `apps/api/alembic/versions/20260528_add_tool_definition_manifests.py` — Migration
openspec/tasks/tool-definition-manifest.md
+89
View File
@@ -0,0 +1,89 @@
# Tasks: Tool Definition Manifest System
## PR 1: Backend Manifest System
### T1.1 Database Migration
- [ ] Create `tool_definition_manifests` table
- [ ] Add `manifest_id`, `definition_type` to `tool_types`
- [ ] Add `manifest_compiled_at`, `image_tag` to `tool_instances`
- [ ] Data migration: convert pi-agent to manifest
### T1.2 Models
- [ ] `ToolDefinitionManifest` SQLAlchemy model
- [ ] Update `ToolType` model with manifest relationship
- [ ] Update `ToolInstance` model with image_tag
### T1.3 Manifest Compiler
- [ ] `resolve_base()` — deep merge base + tool manifest
- [ ] `compile_dockerfile()` — generate Dockerfile from manifest
- [ ] `compile_entrypoint()` — generate startup entrypoint script
- [ ] `compile_compose()` — generate Compose from manifest
- [ ] `compute_image_tag()` — deterministic hash-based tag
- [ ] `resolve_mount_source()` — mount source resolution
### T1.4 Permission Fixer
- [ ] `apply_mount_permissions()` — post-start chown/chmod
- [ ] Handle missing root user gracefully
- [ ] Timeout and error reporting
### T1.5 API Endpoints
- [ ] `POST /tool-definitions` — create
- [ ] `GET /tool-definitions` — list
- [ ] `GET /tool-definitions/{id}` — get
- [ ] `PUT /tool-definitions/{id}` — update
- [ ] `DELETE /tool-definitions/{id}` — delete
- [ ] `POST /tool-definitions/{id}/compile` — preview
### T1.6 Modified Startup Flow
- [ ] Update `start_instance` to use manifest when `definition_type == "manifest"`
- [ ] Integrate permission fixer post-start
- [ ] Store image_tag on instance for reuse
### T1.7 Tests
- [ ] Unit: manifest compiler (all package managers, base merge)
- [ ] Unit: permission fixer (success, failure, timeout)
- [ ] Unit: mount resolution (all source types)
- [ ] Integration: manifest → build → start → terminal works
- [ ] Integration: legacy tool types still work
---
## PR 2: Frontend Tool Workshop
### T2.1 Tool Definitions API Client
- [ ] Add tool definition endpoints to `client.ts`
- [ ] Type definitions for manifest schema
### T2.2 Tool Workshop Page
- [ ] Base image selector (with version dropdown)
- [ ] Package manager editors (apt list, npm list, node version)
- [ ] Script editors (build vs startup, tabbed)
- [ ] Mount schema designer (form table with add/remove)
- [ ] Runtime config (command, working_dir, stdin_open, tty)
### T2.3 Live Preview
- [ ] Preview panel showing generated Dockerfile
- [ ] Preview panel showing generated Compose
- [ ] "Compile" button calling API preview endpoint
### T2.4 Tool Definitions List
- [ ] Table view of all definitions
- [ ] Create / Edit / Delete actions
- [ ] Base indicator (shows if it's a base definition)
---
## PR 3: Migration & Legacy Fallback
### T3.1 Data Migration
- [ ] Alembic migration creating base definition + pi-agent manifest
- [ ] Update existing pi-agent tool_type row
### T3.2 Legacy Fallback
- [x] Ensure `definition_type == "legacy"` still uses old flow
- [x] Ensure `dockerfile_template` / `compose_template` still work
- [x] Tests for legacy path
### T3.3 Documentation
- [x] Update API docs
- [x] Add Tool Workshop user guide