feat: tool definition manifest system (PR 1)
- Add ToolDefinitionManifest model with base image versioning - Add manifest compiler: Dockerfile + Compose generation from JSON manifests - Add permission fixer: post-start chown/chmod for mount policies - Add tool definition CRUD API with live compile preview endpoint - Integrate manifest-based startup flow in start_instance - Add Alembic migration with data conversion for pi-agent - Add 48 unit tests for manifest compiler, permission fixer, docker service - Keep backward compatibility with legacy dockerfile_template/compose_template Migration: applied successfully. Pi-agent converted to manifest. Quality gates: pytest (146 passed, 4 pre-existing unrelated failures)
This commit is contained in:
Alex Blank
@@ -0,0 +1 @@
|
||||
{}
|
||||
+1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,375 @@
|
||||
"""add tool definition manifests
|
||||
|
||||
Revision ID: 2026_05_28_add_tool_definition_manifests
|
||||
Revises: 20260527_160017_add_pi_agent
|
||||
Create Date: 2026-05-28T11:00:00
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2026_05_28_add_tool_definition_manifests"
|
||||
down_revision: Union[str, None] = "20260527_160017_add_pi_agent"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
BASE_UBUNTU_ID = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
|
||||
PI_AGENT_MANIFEST_ID = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# ── Create tool_definition_manifests table ───────────────────────
|
||||
op.create_table(
|
||||
"tool_definition_manifests",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("name", sa.String(64), nullable=False),
|
||||
sa.Column("display_name", sa.String(128), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("category", sa.String(64), nullable=True),
|
||||
sa.Column("interface_type", sa.String(16), nullable=False),
|
||||
sa.Column("base_image", sa.String(256), nullable=True),
|
||||
sa.Column("base_definition_id", sa.UUID(), nullable=True),
|
||||
sa.Column(
|
||||
"base_version", sa.String(32), nullable=False, server_default="latest"
|
||||
),
|
||||
sa.Column("manifest", sa.JSON(), nullable=False),
|
||||
sa.Column("dockerfile_cache", sa.Text(), nullable=True),
|
||||
sa.Column("compose_cache", sa.Text(), nullable=True),
|
||||
sa.Column("version", sa.String(32), nullable=False, server_default="v1"),
|
||||
sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("created_by_id", sa.UUID(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("name"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["base_definition_id"], ["tool_definition_manifests.id"]
|
||||
),
|
||||
sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]),
|
||||
sa.CheckConstraint(
|
||||
"(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)",
|
||||
name="ck_tool_definition_manifests_base_required",
|
||||
),
|
||||
)
|
||||
|
||||
# ── Add columns to tool_types ────────────────────────────────────
|
||||
# Check if manifest_id exists before adding
|
||||
conn = op.get_bind()
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
|
||||
""")
|
||||
)
|
||||
if not result.fetchone():
|
||||
op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True))
|
||||
op.create_foreign_key(
|
||||
"fk_tool_types_manifest_id",
|
||||
"tool_types",
|
||||
"tool_definition_manifests",
|
||||
["manifest_id"],
|
||||
["id"],
|
||||
)
|
||||
|
||||
# Update definition_type to allow 'legacy' and 'manifest'
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT constraint_name FROM information_schema.check_constraints
|
||||
WHERE constraint_name = 'chk_definition_type'
|
||||
""")
|
||||
)
|
||||
if result.fetchone():
|
||||
op.drop_constraint("chk_definition_type", "tool_types", type_="check")
|
||||
|
||||
op.execute("ALTER TABLE tool_types ALTER COLUMN definition_type TYPE VARCHAR(16)")
|
||||
op.execute("ALTER TABLE tool_types ALTER COLUMN definition_type SET DEFAULT 'legacy'")
|
||||
|
||||
# ── Add columns to tool_instances ────────────────────────────────
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
|
||||
""")
|
||||
)
|
||||
if not result.fetchone():
|
||||
op.add_column(
|
||||
"tool_instances",
|
||||
sa.Column("manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
|
||||
""")
|
||||
)
|
||||
if not result.fetchone():
|
||||
op.add_column(
|
||||
"tool_instances",
|
||||
sa.Column("image_tag", sa.String(256), nullable=True),
|
||||
)
|
||||
|
||||
# ── Data migration: create base definition + pi-agent manifest ───
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO tool_definition_manifests
|
||||
(id, name, display_name, description, interface_type, base_image,
|
||||
manifest, is_base, version, created_at, updated_at)
|
||||
VALUES
|
||||
(:base_id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base',
|
||||
'Base development environment with build tools', 'terminal',
|
||||
'ubuntu:24.04', :base_manifest, true, 'v1', now(), now())
|
||||
"""
|
||||
),
|
||||
{
|
||||
"base_id": BASE_UBUNTU_ID,
|
||||
"base_manifest": json.dumps(
|
||||
{
|
||||
"name": "ubuntu-24.04-dev",
|
||||
"display_name": "Ubuntu 24.04 Dev Base",
|
||||
"interface_type": "terminal",
|
||||
"base_image": "ubuntu:24.04",
|
||||
"packages": {
|
||||
"apt": [
|
||||
"curl",
|
||||
"wget",
|
||||
"git",
|
||||
"build-essential",
|
||||
"ca-certificates",
|
||||
"python3",
|
||||
"python3-pip",
|
||||
]
|
||||
},
|
||||
"user": {
|
||||
"name": "user",
|
||||
"uid": 1000,
|
||||
"gid": 1000,
|
||||
"create_home": True,
|
||||
"shell": "/bin/bash",
|
||||
},
|
||||
"env": {"DEBIAN_FRONTEND": "noninteractive"},
|
||||
}
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO tool_definition_manifests
|
||||
(id, name, display_name, description, category, interface_type,
|
||||
base_definition_id, base_version, manifest, version, created_at, updated_at)
|
||||
VALUES
|
||||
(:manifest_id, 'pi-agent', 'Pi Agent',
|
||||
'Terminal-based coding harness with nvim, ranger, tmux',
|
||||
'development', 'terminal', :base_id, 'v1', :manifest, 'v1',
|
||||
now(), now())
|
||||
"""
|
||||
),
|
||||
{
|
||||
"manifest_id": PI_AGENT_MANIFEST_ID,
|
||||
"base_id": BASE_UBUNTU_ID,
|
||||
"manifest": json.dumps(
|
||||
{
|
||||
"name": "pi-agent",
|
||||
"display_name": "Pi Agent",
|
||||
"description": "Terminal-based coding harness",
|
||||
"category": "development",
|
||||
"interface_type": "terminal",
|
||||
"base_definition_id": str(BASE_UBUNTU_ID),
|
||||
"base_version": "v1",
|
||||
"packages": {
|
||||
"apt": [
|
||||
"neovim",
|
||||
"ranger",
|
||||
"tmux",
|
||||
"htop",
|
||||
"tree",
|
||||
"jq",
|
||||
],
|
||||
"node": {"version": "20"},
|
||||
"npm_global": ["@earendil-works/pi-coding-agent"],
|
||||
},
|
||||
"user": {
|
||||
"name": "user",
|
||||
"uid": 1001,
|
||||
"gid": 1001,
|
||||
"create_home": True,
|
||||
"shell": "/bin/bash",
|
||||
},
|
||||
"env": {"DEBIAN_FRONTEND": "noninteractive"},
|
||||
"scripts": {
|
||||
"build": [
|
||||
"git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'",
|
||||
"mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf",
|
||||
],
|
||||
"startup": [
|
||||
"if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi",
|
||||
],
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "workspace",
|
||||
"target": "/workspace",
|
||||
"source_type": "repo",
|
||||
"writable": True,
|
||||
"owner": "user",
|
||||
},
|
||||
{
|
||||
"name": "ssh_keys",
|
||||
"target": "/home/user/.ssh",
|
||||
"source_type": "ssh_key",
|
||||
"mode": "0700",
|
||||
"file_mode": "0600",
|
||||
"readonly": True,
|
||||
},
|
||||
{
|
||||
"name": "pi_state",
|
||||
"target": "/tmp/.pi/agents",
|
||||
"source_type": "instance",
|
||||
"writable": True,
|
||||
},
|
||||
{
|
||||
"name": "pi_config",
|
||||
"target": "/home/user/.pi",
|
||||
"source_type": "git_mount",
|
||||
"git_mount_ref": "dotfiles",
|
||||
"writable": True,
|
||||
"owner": "user",
|
||||
},
|
||||
],
|
||||
"runtime": {
|
||||
"command": ["/bin/bash"],
|
||||
"stdin_open": True,
|
||||
"tty": True,
|
||||
"working_dir": "/workspace",
|
||||
},
|
||||
}
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
# ── Update existing pi-agent tool_type ───────────────────────────
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool_types
|
||||
SET manifest_id = :manifest_id,
|
||||
definition_type = 'manifest',
|
||||
dockerfile_template = NULL,
|
||||
compose_template = NULL
|
||||
WHERE name = 'pi-agent'
|
||||
"""
|
||||
),
|
||||
{"manifest_id": PI_AGENT_MANIFEST_ID},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Restore pi-agent templates if manifest_id column exists
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
|
||||
""")
|
||||
)
|
||||
has_manifest_id = result.fetchone() is not None
|
||||
|
||||
if has_manifest_id:
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool_types
|
||||
SET manifest_id = NULL,
|
||||
definition_type = 'dockerfile',
|
||||
dockerfile_template = :dockerfile,
|
||||
compose_template = :compose
|
||||
WHERE name = 'pi-agent'
|
||||
"""
|
||||
),
|
||||
{
|
||||
"dockerfile": """# Pi Coding Agent - Terminal-based coding harness
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y \\
|
||||
curl wget git neovim ranger tmux htop tree jq \\
|
||||
ca-certificates python3 python3-pip build-essential \\
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\
|
||||
&& apt-get install -y nodejs \\
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
|
||||
|
||||
RUN useradd -m -s /bin/bash user
|
||||
WORKDIR /home/user
|
||||
|
||||
RUN git config --global init.defaultBranch main \\
|
||||
&& git config --global user.email "dev@headquarter.local" \\
|
||||
&& git config --global user.name "Developer"
|
||||
|
||||
RUN echo 'set -g mouse on\\nset -g default-terminal "screen-256color"' > /home/user/.tmux.conf
|
||||
|
||||
RUN mkdir -p /home/user/.config/ranger \\
|
||||
&& echo 'set preview_files true\\nset use_preview_script true' > /home/user/.config/ranger/rc.conf
|
||||
|
||||
RUN mkdir -p /home/user/.pi/agent
|
||||
|
||||
USER user
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
""",
|
||||
"compose": """services:
|
||||
app:
|
||||
build: .
|
||||
stdin_open: true
|
||||
tty: true
|
||||
volumes:
|
||||
- ${REPO_PATH}:/workspace
|
||||
working_dir: /workspace
|
||||
command: /bin/bash""",
|
||||
},
|
||||
)
|
||||
|
||||
# Drop columns conditionally
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
|
||||
""")
|
||||
)
|
||||
if result.fetchone():
|
||||
op.drop_column("tool_instances", "image_tag")
|
||||
|
||||
result = conn.execute(
|
||||
sa.text("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
|
||||
""")
|
||||
)
|
||||
if result.fetchone():
|
||||
op.drop_column("tool_instances", "manifest_compiled_at")
|
||||
|
||||
if has_manifest_id:
|
||||
op.drop_constraint("fk_tool_types_manifest_id", "tool_types", type_="foreignkey")
|
||||
op.drop_column("tool_types", "manifest_id")
|
||||
|
||||
op.drop_table("tool_definition_manifests")
|
||||
@@ -0,0 +1,424 @@
|
||||
"""Tool definition API endpoints."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.auth.dependencies import get_current_user_id, get_db_session
|
||||
from src.models.tool_definition_manifest import ToolDefinitionManifest
|
||||
from src.models.tool_type import ToolType
|
||||
from src.services.manifest_compiler import (
|
||||
compile_compose,
|
||||
compile_dockerfile,
|
||||
compile_entrypoint,
|
||||
compute_image_tag,
|
||||
deep_merge,
|
||||
resolve_base,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/tool-definitions", tags=["tool-definitions"])
|
||||
|
||||
|
||||
class CreateToolDefinitionRequest(BaseModel):
|
||||
"""Request body for creating a tool definition manifest."""
|
||||
|
||||
model_config = {"extra": "ignore"}
|
||||
|
||||
name: str = Field(description="Unique identifier (kebab-case)")
|
||||
display_name: str = Field(description="Human-readable name")
|
||||
description: str | None = Field(default=None)
|
||||
category: str = Field(default="development")
|
||||
interface_type: str = Field(default="terminal", description="web or terminal")
|
||||
base_image: str | None = Field(default=None, description="Direct base image")
|
||||
base_definition_id: str | None = Field(
|
||||
default=None, description="Reference to a base definition"
|
||||
)
|
||||
base_version: str = Field(default="latest")
|
||||
manifest: dict = Field(description="The full manifest JSON")
|
||||
|
||||
|
||||
class UpdateToolDefinitionRequest(BaseModel):
|
||||
"""Request body for updating a tool definition manifest."""
|
||||
|
||||
model_config = {"extra": "ignore"}
|
||||
|
||||
display_name: str | None = Field(default=None)
|
||||
description: str | None = Field(default=None)
|
||||
category: str | None = Field(default=None)
|
||||
manifest: dict | None = Field(default=None)
|
||||
base_version: str | None = Field(default=None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
summary="Create tool definition",
|
||||
description="Create a new tool definition manifest.",
|
||||
)
|
||||
async def create_tool_definition(
|
||||
data: CreateToolDefinitionRequest,
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Create a new tool definition manifest.
|
||||
|
||||
Args:
|
||||
data: Manifest data.
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with created definition details.
|
||||
"""
|
||||
# Validate base reference
|
||||
if not data.base_image and not data.base_definition_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Either base_image or base_definition_id is required",
|
||||
)
|
||||
|
||||
base_def_id = None
|
||||
if data.base_definition_id:
|
||||
try:
|
||||
base_def_id = uuid.UUID(data.base_definition_id)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid base_definition_id: {data.base_definition_id}",
|
||||
)
|
||||
|
||||
base_def = await session.get(ToolDefinitionManifest, base_def_id)
|
||||
if not base_def:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Base definition not found: {data.base_definition_id}",
|
||||
)
|
||||
if not base_def.is_base:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Referenced definition is not a base definition",
|
||||
)
|
||||
|
||||
# Check name uniqueness
|
||||
existing = await session.execute(
|
||||
select(ToolDefinitionManifest).where(ToolDefinitionManifest.name == data.name)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Tool definition '{data.name}' already exists",
|
||||
)
|
||||
|
||||
definition = ToolDefinitionManifest(
|
||||
name=data.name,
|
||||
display_name=data.display_name,
|
||||
description=data.description,
|
||||
category=data.category,
|
||||
interface_type=data.interface_type,
|
||||
base_image=data.base_image,
|
||||
base_definition_id=base_def_id,
|
||||
base_version=data.base_version,
|
||||
manifest=data.manifest,
|
||||
created_by_id=user_id,
|
||||
)
|
||||
|
||||
session.add(definition)
|
||||
await session.commit()
|
||||
await session.refresh(definition)
|
||||
|
||||
logger.info("Created tool definition %s (%s)", definition.id, definition.name)
|
||||
|
||||
return {
|
||||
"id": str(definition.id),
|
||||
"name": definition.name,
|
||||
"display_name": definition.display_name,
|
||||
"description": definition.description,
|
||||
"category": definition.category,
|
||||
"interface_type": definition.interface_type,
|
||||
"base_image": definition.base_image,
|
||||
"base_definition_id": str(definition.base_definition_id)
|
||||
if definition.base_definition_id
|
||||
else None,
|
||||
"base_version": definition.base_version,
|
||||
"manifest": definition.manifest,
|
||||
"is_base": definition.is_base,
|
||||
"created_at": definition.created_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
summary="List tool definitions",
|
||||
description="List all tool definition manifests.",
|
||||
)
|
||||
async def list_tool_definitions(
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
include_bases: bool = True,
|
||||
) -> dict:
|
||||
"""List all tool definition manifests.
|
||||
|
||||
Args:
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
include_bases: Whether to include base definitions.
|
||||
|
||||
Returns:
|
||||
Dictionary containing list of definitions.
|
||||
"""
|
||||
query = select(ToolDefinitionManifest)
|
||||
if not include_bases:
|
||||
query = query.where(ToolDefinitionManifest.is_base == False)
|
||||
|
||||
result = await session.execute(
|
||||
query.order_by(ToolDefinitionManifest.created_at.desc())
|
||||
)
|
||||
definitions = result.scalars().all()
|
||||
|
||||
return {
|
||||
"definitions": [
|
||||
{
|
||||
"id": str(d.id),
|
||||
"name": d.name,
|
||||
"display_name": d.display_name,
|
||||
"description": d.description,
|
||||
"category": d.category,
|
||||
"interface_type": d.interface_type,
|
||||
"is_base": d.is_base,
|
||||
"base_image": d.base_image,
|
||||
"base_definition_id": str(d.base_definition_id)
|
||||
if d.base_definition_id
|
||||
else None,
|
||||
"base_version": d.base_version,
|
||||
"version": d.version,
|
||||
"created_at": d.created_at.isoformat(),
|
||||
}
|
||||
for d in definitions
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{definition_id}",
|
||||
summary="Get tool definition",
|
||||
description="Get a specific tool definition manifest.",
|
||||
)
|
||||
async def get_tool_definition(
|
||||
definition_id: uuid.UUID,
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Get a specific tool definition manifest.
|
||||
|
||||
Args:
|
||||
definition_id: UUID of the definition.
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with definition details.
|
||||
"""
|
||||
definition = await session.get(ToolDefinitionManifest, definition_id)
|
||||
if not definition:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Tool definition not found: {definition_id}",
|
||||
)
|
||||
|
||||
return {
|
||||
"id": str(definition.id),
|
||||
"name": definition.name,
|
||||
"display_name": definition.display_name,
|
||||
"description": definition.description,
|
||||
"category": definition.category,
|
||||
"interface_type": definition.interface_type,
|
||||
"base_image": definition.base_image,
|
||||
"base_definition_id": str(definition.base_definition_id)
|
||||
if definition.base_definition_id
|
||||
else None,
|
||||
"base_version": definition.base_version,
|
||||
"manifest": definition.manifest,
|
||||
"dockerfile_cache": definition.dockerfile_cache,
|
||||
"compose_cache": definition.compose_cache,
|
||||
"version": definition.version,
|
||||
"is_base": definition.is_base,
|
||||
"created_at": definition.created_at.isoformat(),
|
||||
"updated_at": definition.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{definition_id}",
|
||||
summary="Update tool definition",
|
||||
description="Update a tool definition manifest.",
|
||||
)
|
||||
async def update_tool_definition(
|
||||
definition_id: uuid.UUID,
|
||||
data: UpdateToolDefinitionRequest,
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Update a tool definition manifest.
|
||||
|
||||
Args:
|
||||
definition_id: UUID of the definition.
|
||||
data: Update data.
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with updated definition details.
|
||||
"""
|
||||
definition = await session.get(ToolDefinitionManifest, definition_id)
|
||||
if not definition:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Tool definition not found: {definition_id}",
|
||||
)
|
||||
|
||||
if data.display_name is not None:
|
||||
definition.display_name = data.display_name
|
||||
if data.description is not None:
|
||||
definition.description = data.description
|
||||
if data.category is not None:
|
||||
definition.category = data.category
|
||||
if data.manifest is not None:
|
||||
definition.manifest = data.manifest
|
||||
if data.base_version is not None:
|
||||
definition.base_version = data.base_version
|
||||
|
||||
await session.commit()
|
||||
await session.refresh(definition)
|
||||
|
||||
logger.info("Updated tool definition %s (%s)", definition.id, definition.name)
|
||||
|
||||
return {
|
||||
"id": str(definition.id),
|
||||
"name": definition.name,
|
||||
"display_name": definition.display_name,
|
||||
"manifest": definition.manifest,
|
||||
"updated_at": definition.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{definition_id}",
|
||||
summary="Delete tool definition",
|
||||
description="Delete a tool definition manifest.",
|
||||
)
|
||||
async def delete_tool_definition(
|
||||
definition_id: uuid.UUID,
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Delete a tool definition manifest.
|
||||
|
||||
Args:
|
||||
definition_id: UUID of the definition.
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with deletion status.
|
||||
"""
|
||||
definition = await session.get(ToolDefinitionManifest, definition_id)
|
||||
if not definition:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Tool definition not found: {definition_id}",
|
||||
)
|
||||
|
||||
# Check if any tool types reference this manifest
|
||||
result = await session.execute(
|
||||
select(ToolType).where(ToolType.manifest_id == definition_id)
|
||||
)
|
||||
referencing = result.scalars().all()
|
||||
if referencing:
|
||||
tool_names = ", ".join(t.name for t in referencing)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Cannot delete: referenced by tool types: {tool_names}",
|
||||
)
|
||||
|
||||
await session.delete(definition)
|
||||
await session.commit()
|
||||
|
||||
logger.info("Deleted tool definition %s (%s)", definition.id, definition.name)
|
||||
|
||||
return {"status": "deleted", "id": str(definition_id)}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{definition_id}/compile",
|
||||
summary="Compile tool definition",
|
||||
description="Compile a manifest to Dockerfile + Compose preview without building.",
|
||||
)
|
||||
async def compile_tool_definition(
|
||||
definition_id: uuid.UUID,
|
||||
user_id: uuid.UUID = Depends(get_current_user_id),
|
||||
session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Compile a manifest to Dockerfile + Compose preview.
|
||||
|
||||
Args:
|
||||
definition_id: UUID of the definition.
|
||||
user_id: Authenticated user ID.
|
||||
session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with compiled Dockerfile, Compose, and image tag.
|
||||
"""
|
||||
definition = await session.get(ToolDefinitionManifest, definition_id)
|
||||
if not definition:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Tool definition not found: {definition_id}",
|
||||
)
|
||||
|
||||
manifest = dict(definition.manifest)
|
||||
|
||||
# Resolve base if referenced
|
||||
if definition.base_definition_id:
|
||||
base_def = await session.get(
|
||||
ToolDefinitionManifest, definition.base_definition_id
|
||||
)
|
||||
if base_def:
|
||||
base_manifest = dict(base_def.manifest)
|
||||
manifest = resolve_base(deep_merge(base_manifest, manifest))
|
||||
|
||||
# Compile
|
||||
dockerfile = compile_dockerfile(manifest)
|
||||
entrypoint = compile_entrypoint(manifest)
|
||||
image_tag = compute_image_tag(definition.name, manifest)
|
||||
|
||||
# Dummy compose with placeholder variables
|
||||
dummy_vars = {
|
||||
"IMAGE_TAG": image_tag,
|
||||
"INSTANCE_NAME": f"{definition.name}-preview",
|
||||
"INSTANCE_DIR": "/data/instances/preview",
|
||||
"REPO_PATH": "/data/repos/preview",
|
||||
"SSH_PATH": "/data/instances/preview/.ssh",
|
||||
"TOOL_PORT": "8080",
|
||||
"EXTRA_ENV": {},
|
||||
"EXTRA_VOLUMES": [],
|
||||
}
|
||||
compose = compile_compose(manifest, dummy_vars)
|
||||
|
||||
# Update cache
|
||||
definition.dockerfile_cache = dockerfile
|
||||
definition.compose_cache = compose
|
||||
await session.commit()
|
||||
|
||||
return {
|
||||
"id": str(definition.id),
|
||||
"name": definition.name,
|
||||
"dockerfile": dockerfile,
|
||||
"entrypoint": entrypoint,
|
||||
"compose": compose,
|
||||
"image_tag": image_tag,
|
||||
}
|
||||
@@ -62,6 +62,16 @@ from src.services.docker import (
|
||||
write_env_file,
|
||||
)
|
||||
from src.services.docker_build import build_image
|
||||
from src.services.manifest_compiler import (
|
||||
compile_compose,
|
||||
compile_dockerfile,
|
||||
compile_entrypoint,
|
||||
compute_image_tag,
|
||||
deep_merge,
|
||||
merge_with_config,
|
||||
resolve_base,
|
||||
)
|
||||
from src.services.permission_fixer import apply_mount_permissions
|
||||
from src.services.readiness_probe import execute_probe
|
||||
from src.services.ssh_keys import cleanup_ssh_key_files, prepare_ssh_key_files
|
||||
|
||||
@@ -961,6 +971,142 @@ async def get_instance(
|
||||
}
|
||||
|
||||
|
||||
async def _prepare_manifest_instance(
|
||||
session: AsyncSession,
|
||||
instance: ToolInstance,
|
||||
instance_dir: str,
|
||||
repo_path: str,
|
||||
configs: list,
|
||||
env_vars: dict,
|
||||
extra_volumes: list,
|
||||
working_directory: str | None,
|
||||
) -> tuple[str, str, dict]:
|
||||
"""Build image and generate compose from a manifest-based tool type.
|
||||
|
||||
Returns:
|
||||
Tuple of (image_tag, compose_content, resolved_manifest)
|
||||
"""
|
||||
from src.models.tool_definition_manifest import ToolDefinitionManifest
|
||||
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
|
||||
|
||||
if not manifest_def:
|
||||
raise RuntimeError(f"Manifest not found for tool type {tool_type.id}")
|
||||
|
||||
manifest = dict(manifest_def.manifest)
|
||||
|
||||
# Resolve base if referenced
|
||||
if manifest_def.base_definition_id:
|
||||
base_def = await session.get(
|
||||
ToolDefinitionManifest, manifest_def.base_definition_id
|
||||
)
|
||||
if base_def:
|
||||
base_manifest = dict(base_def.manifest)
|
||||
manifest = resolve_base(deep_merge(base_manifest, manifest))
|
||||
else:
|
||||
logger.warning(
|
||||
"Base definition %s not found for manifest %s",
|
||||
manifest_def.base_definition_id,
|
||||
manifest_def.id,
|
||||
)
|
||||
|
||||
# Merge tool configs
|
||||
tool_config_dicts = [
|
||||
{
|
||||
"config_type": c.config_type,
|
||||
"key": c.key,
|
||||
"value": c.value,
|
||||
"file_path": c.file_path,
|
||||
"port_override": c.port_override,
|
||||
"start_command": c.start_command,
|
||||
"working_directory": c.working_directory,
|
||||
"environment_variables": c.environment_variables,
|
||||
"volumes": c.volumes,
|
||||
}
|
||||
for c in configs
|
||||
]
|
||||
manifest = merge_with_config(manifest, tool_config_dicts)
|
||||
|
||||
# Resolve extra env and volumes from merge_with_config
|
||||
extra_env = manifest.pop("_extra_env", {})
|
||||
extra_cfg_volumes = manifest.pop("_extra_volumes", [])
|
||||
env_vars.update(extra_env)
|
||||
extra_volumes.extend(extra_cfg_volumes)
|
||||
|
||||
# Compute image tag
|
||||
image_tag = compute_image_tag(tool_type.name, manifest)
|
||||
|
||||
# Check if image already exists
|
||||
check = subprocess.run(
|
||||
["docker", "images", "-q", image_tag],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
image_exists = check.returncode == 0 and check.stdout.strip()
|
||||
|
||||
if not image_exists:
|
||||
# Compile and build
|
||||
dockerfile = compile_dockerfile(manifest)
|
||||
entrypoint = compile_entrypoint(manifest)
|
||||
|
||||
build_ctx = {
|
||||
"Dockerfile": dockerfile,
|
||||
".headquarter/entrypoint.sh": entrypoint,
|
||||
}
|
||||
|
||||
returncode, stdout, stderr = await asyncio.to_thread(
|
||||
build_image,
|
||||
instance_dir=instance_dir,
|
||||
dockerfile=dockerfile,
|
||||
tag=image_tag,
|
||||
build_context=build_ctx,
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
raise RuntimeError(f"Docker build failed: {stderr}")
|
||||
|
||||
logger.info("Built image %s for instance %s", image_tag, instance.id)
|
||||
else:
|
||||
logger.info("Reusing existing image %s for instance %s", image_tag, instance.id)
|
||||
|
||||
# Prepare SSH path for mount resolution
|
||||
ssh_path = ""
|
||||
if instance.clone_mode == "clone":
|
||||
ssh_path = os.path.join(instance_dir, ".ssh")
|
||||
|
||||
# Resolve git mount variables from config profile
|
||||
git_mount_vars = {}
|
||||
if instance.selected_config_profile_id:
|
||||
resolved_profile = await resolve_profile(
|
||||
session, instance.selected_config_profile_id
|
||||
)
|
||||
for gm in resolved_profile.git_mounts or []:
|
||||
ref = gm.get("git_mount_ref", "default")
|
||||
# The actual resolution happens in _resolve_git_mounts; we store placeholder
|
||||
git_mount_vars[f"GIT_MOUNT_{ref}"] = ""
|
||||
|
||||
variables = {
|
||||
"IMAGE_TAG": image_tag,
|
||||
"INSTANCE_NAME": instance.name.lower(),
|
||||
"INSTANCE_DIR": instance_dir,
|
||||
"REPO_PATH": repo_path,
|
||||
"SSH_PATH": ssh_path,
|
||||
"TOOL_PORT": instance.port or 0,
|
||||
"EXTRA_ENV": env_vars,
|
||||
"EXTRA_VOLUMES": extra_volumes,
|
||||
**git_mount_vars,
|
||||
}
|
||||
|
||||
compose_content = compile_compose(manifest, variables)
|
||||
|
||||
# Cache
|
||||
instance.image_tag = image_tag
|
||||
instance.manifest_compiled_at = datetime.now()
|
||||
|
||||
return image_tag, compose_content, manifest
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{project_id}/repositories/{repo_id}/instances/{instance_id}/start",
|
||||
summary="Start instance",
|
||||
@@ -1119,41 +1265,85 @@ async def start_instance(
|
||||
"Wrote %d config files for instance %s", len(config_files), instance.id
|
||||
)
|
||||
|
||||
# Mount SSH key for clone-mode instances
|
||||
if instance.clone_mode == "clone":
|
||||
repo = await session.get(GitRepository, instance.repository_id)
|
||||
if repo and repo.ssh_key_id:
|
||||
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
|
||||
if ssh_key:
|
||||
try:
|
||||
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
|
||||
extra_volumes.append(
|
||||
{
|
||||
"source": ssh_dir,
|
||||
"target": "/root/.ssh",
|
||||
"type": "ro",
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"Mounted SSH key for clone-mode instance %s", instance.id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to prepare SSH key for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
# ── MANIFEST-BASED FLOW ──────────────────────────────────────
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
resolved_manifest = None
|
||||
|
||||
# Modify compose file if needed (port override, start command, working dir, volumes)
|
||||
if port_override or start_command or working_directory or extra_volumes:
|
||||
_modify_compose_file(
|
||||
instance.compose_path,
|
||||
port_override,
|
||||
start_command,
|
||||
working_directory,
|
||||
extra_volumes,
|
||||
)
|
||||
logger.debug("Modified compose file for instance %s", instance.id)
|
||||
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
||||
logger.info("Using manifest-based startup for instance %s", instance.id)
|
||||
|
||||
# Determine repo path
|
||||
repo = await session.get(GitRepository, instance.repository_id)
|
||||
repo_path = repo.path if repo else ""
|
||||
if instance.clone_mode == "clone":
|
||||
repo_path = os.path.join(instance_dir, "repo-clone")
|
||||
|
||||
try:
|
||||
(
|
||||
image_tag,
|
||||
compose_content,
|
||||
resolved_manifest,
|
||||
) = await _prepare_manifest_instance(
|
||||
session=session,
|
||||
instance=instance,
|
||||
instance_dir=instance_dir,
|
||||
repo_path=repo_path,
|
||||
configs=configs,
|
||||
env_vars=env_vars,
|
||||
extra_volumes=extra_volumes,
|
||||
working_directory=working_directory,
|
||||
)
|
||||
write_compose_file(instance_dir, compose_content)
|
||||
logger.debug(
|
||||
"Generated manifest-based compose for instance %s", instance.id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Manifest compilation failed for instance %s: %s", instance.id, exc
|
||||
)
|
||||
instance.status = "error"
|
||||
await session.commit()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Manifest compilation failed: {exc}",
|
||||
)
|
||||
else:
|
||||
# ── LEGACY FLOW ──────────────────────────────────────────
|
||||
# Mount SSH key for clone-mode instances
|
||||
if instance.clone_mode == "clone":
|
||||
repo = await session.get(GitRepository, instance.repository_id)
|
||||
if repo and repo.ssh_key_id:
|
||||
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
|
||||
if ssh_key:
|
||||
try:
|
||||
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
|
||||
extra_volumes.append(
|
||||
{
|
||||
"source": ssh_dir,
|
||||
"target": "/root/.ssh",
|
||||
"type": "ro",
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"Mounted SSH key for clone-mode instance %s", instance.id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to prepare SSH key for instance %s: %s",
|
||||
instance.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
# Modify compose file if needed (port override, start command, working dir, volumes)
|
||||
if port_override or start_command or working_directory or extra_volumes:
|
||||
_modify_compose_file(
|
||||
instance.compose_path,
|
||||
port_override,
|
||||
start_command,
|
||||
working_directory,
|
||||
extra_volumes,
|
||||
)
|
||||
logger.debug("Modified compose file for instance %s", instance.id)
|
||||
|
||||
# Sanitize compose file to remove invalid port mappings from old instances
|
||||
_sanitize_compose_file(instance.compose_path)
|
||||
@@ -1244,6 +1434,28 @@ async def start_instance(
|
||||
startup_result["waited_seconds"],
|
||||
)
|
||||
|
||||
# Apply mount permission fixes for manifest-based instances
|
||||
if resolved_manifest and instance.container_id:
|
||||
mounts = resolved_manifest.get("mounts", [])
|
||||
if mounts:
|
||||
logger.debug(
|
||||
"Applying permission fixes for instance %s (%d mounts)",
|
||||
instance.id,
|
||||
len(mounts),
|
||||
)
|
||||
permission_results = apply_mount_permissions(
|
||||
instance.container_id,
|
||||
mounts,
|
||||
)
|
||||
for result in permission_results:
|
||||
if not result["success"]:
|
||||
logger.warning(
|
||||
"Permission fix failed for mount %s on instance %s: %s",
|
||||
result["mount_name"],
|
||||
instance.id,
|
||||
result["error"],
|
||||
)
|
||||
|
||||
# Execute readiness probe if configured
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
if tool_type and instance.container_id:
|
||||
|
||||
@@ -18,6 +18,7 @@ from src.api.instance_proxy import router as instance_proxy_router
|
||||
from src.api.config_folders import router as config_folders_router
|
||||
from src.api.config_profiles import router as config_profiles_router
|
||||
from src.api.tool_configs import router as tool_configs_router
|
||||
from src.api.tool_definitions import router as tool_definitions_router
|
||||
from src.api.tool_instances import router as tool_instances_router
|
||||
from src.api.tool_instances import sessions_router
|
||||
from src.api.tool_types import router as tool_types_router
|
||||
@@ -66,7 +67,9 @@ def _sanitize_validation_errors(errors):
|
||||
"type": error.get("type"),
|
||||
"loc": error.get("loc"),
|
||||
"msg": error.get("msg"),
|
||||
"input": str(error.get("input")) if error.get("input") is not None else None,
|
||||
"input": str(error.get("input"))
|
||||
if error.get("input") is not None
|
||||
else None,
|
||||
}
|
||||
# Convert ctx to safe format
|
||||
ctx = error.get("ctx")
|
||||
@@ -110,10 +113,12 @@ async def on_startup():
|
||||
if not db_ready:
|
||||
logger.error("Database initialization failed. Shutting down.")
|
||||
import sys
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
logger.info("Startup complete.")
|
||||
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(dashboard_router)
|
||||
@@ -123,6 +128,7 @@ app.include_router(ssh_keys_router)
|
||||
app.include_router(git_repositories_router)
|
||||
app.include_router(user_config_router)
|
||||
app.include_router(tool_types_router)
|
||||
app.include_router(tool_definitions_router)
|
||||
app.include_router(config_folders_router)
|
||||
app.include_router(config_profiles_router)
|
||||
app.include_router(tool_instances_router)
|
||||
|
||||
@@ -4,9 +4,23 @@ from src.models.config_profile import ConfigProfile, ConfigProfileInclude
|
||||
from src.models.git_repository import GitRepository
|
||||
from src.models.project import Project
|
||||
from src.models.ssh_key import SSHKey
|
||||
from src.models.tool_definition_manifest import ToolDefinitionManifest
|
||||
from src.models.tool_instance import ToolInstance
|
||||
from src.models.tool_type import ToolType
|
||||
from src.models.user import User
|
||||
from src.models.user_config import UserConfig
|
||||
|
||||
__all__ = ["Base", "ConfigFolder", "ConfigProfile", "ConfigProfileInclude", "GitRepository", "Project", "SSHKey", "ToolInstance", "ToolType", "User", "UserConfig"]
|
||||
__all__ = [
|
||||
"Base",
|
||||
"ConfigFolder",
|
||||
"ConfigProfile",
|
||||
"ConfigProfileInclude",
|
||||
"GitRepository",
|
||||
"Project",
|
||||
"SSHKey",
|
||||
"ToolDefinitionManifest",
|
||||
"ToolInstance",
|
||||
"ToolType",
|
||||
"User",
|
||||
"UserConfig",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Tool Definition Manifest model."""
|
||||
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, JSON, String, Text
|
||||
from sqlalchemy import Uuid as UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.models.user import User
|
||||
|
||||
|
||||
class ToolDefinitionManifest(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""A declarative manifest that compiles to Dockerfile + Compose.
|
||||
|
||||
Can be either:
|
||||
- A base definition (is_base=True) with a FROM image and common packages
|
||||
- A tool definition (is_base=False) that references a base + adds specifics
|
||||
"""
|
||||
|
||||
__tablename__ = "tool_definition_manifests"
|
||||
|
||||
name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
category: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
interface_type: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
|
||||
# Base: either a direct image or a reference to another manifest
|
||||
base_image: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
base_definition_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(),
|
||||
ForeignKey("tool_definition_manifests.id"),
|
||||
nullable=True,
|
||||
)
|
||||
base_version: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="latest"
|
||||
)
|
||||
|
||||
# The full manifest JSON
|
||||
manifest: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
|
||||
# Caches for quick inspection
|
||||
dockerfile_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
compose_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# Versioning
|
||||
version: Mapped[str] = mapped_column(String(32), nullable=False, default="v1")
|
||||
is_base: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
|
||||
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(),
|
||||
ForeignKey("users.id"),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
# Relationships
|
||||
created_by: Mapped["User | None"] = relationship(
|
||||
foreign_keys=[created_by_id],
|
||||
)
|
||||
base_definition: Mapped["ToolDefinitionManifest | None"] = relationship(
|
||||
remote_side="ToolDefinitionManifest.id",
|
||||
foreign_keys=[base_definition_id],
|
||||
)
|
||||
@@ -33,42 +33,26 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(), ForeignKey("users.id"), nullable=False
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="pending"
|
||||
)
|
||||
container_id: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True
|
||||
)
|
||||
container_name: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True
|
||||
)
|
||||
compose_path: Mapped[str | None] = mapped_column(
|
||||
String(1024), nullable=True
|
||||
)
|
||||
url: Mapped[str | None] = mapped_column(
|
||||
String(1024), nullable=True
|
||||
)
|
||||
public_url: Mapped[str | None] = mapped_column(
|
||||
String(1024), nullable=True
|
||||
)
|
||||
tunnel_id: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True
|
||||
)
|
||||
port: Mapped[int | None] = mapped_column(
|
||||
Integer, nullable=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(50), nullable=False, default="pending")
|
||||
container_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
container_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
compose_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
public_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
tunnel_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
port: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
last_started_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
last_stopped_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
probe_result: Mapped[dict | None] = mapped_column(
|
||||
JSON, nullable=True
|
||||
)
|
||||
clone_mode: Mapped[str] = mapped_column(
|
||||
String(20), nullable=False, default="mount"
|
||||
manifest_compiled_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
image_tag: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
probe_result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
clone_mode: Mapped[str] = mapped_column(String(20), nullable=False, default="mount")
|
||||
branch: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, default="main"
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.models.tool_definition_manifest import ToolDefinitionManifest
|
||||
from src.models.user import User
|
||||
|
||||
|
||||
@@ -18,12 +19,19 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
category: Mapped[str] = mapped_column(String(50), nullable=False, default="other")
|
||||
interface_type: Mapped[str] = mapped_column(String(20), nullable=False, default="web")
|
||||
interface_type: Mapped[str] = mapped_column(
|
||||
String(20), nullable=False, default="web"
|
||||
)
|
||||
requires_port: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
||||
default_port: Mapped[int] = mapped_column(nullable=False)
|
||||
definition_type: Mapped[str] = mapped_column(
|
||||
String(20), nullable=False, default="compose"
|
||||
) # "compose" or "dockerfile"
|
||||
String(16), nullable=False, default="legacy"
|
||||
) # "legacy" | "manifest"
|
||||
manifest_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(),
|
||||
ForeignKey("tool_definition_manifests.id"),
|
||||
nullable=True,
|
||||
)
|
||||
compose_template: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
dockerfile_template: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
build_context: Mapped[dict | None] = mapped_column(
|
||||
@@ -31,11 +39,16 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
)
|
||||
readiness_probe: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
startup_command: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
required_variables: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||
required_variables: Mapped[list[str]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(),
|
||||
ForeignKey("users.id"),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
manifest: Mapped["ToolDefinitionManifest | None"] = relationship(
|
||||
foreign_keys=[manifest_id],
|
||||
)
|
||||
created_by: Mapped["User | None"] = relationship()
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""Manifest compiler: transforms ToolDefinitionManifest into Dockerfile + Compose."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shlex
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def resolve_base(manifest: dict) -> dict:
|
||||
"""Merge a base definition into a tool manifest.
|
||||
|
||||
If the manifest has base_definition_id, the base manifest is loaded
|
||||
and merged. Tool-specific values override base values.
|
||||
|
||||
Args:
|
||||
manifest: The tool manifest JSON (may reference a base)
|
||||
|
||||
Returns:
|
||||
A fully resolved manifest with base values merged in.
|
||||
"""
|
||||
result = deepcopy(manifest)
|
||||
|
||||
base_definition_id = result.pop("base_definition_id", None)
|
||||
base_version = result.pop("base_version", "latest")
|
||||
|
||||
if base_definition_id:
|
||||
# This will be provided by the caller (they have the DB session)
|
||||
# For now, we assume the manifest has been pre-resolved
|
||||
# or the caller provides the base manifest separately.
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def deep_merge(base: dict, override: dict) -> dict:
|
||||
"""Deep merge two manifests. Arrays are concatenated; dicts are merged.
|
||||
|
||||
Args:
|
||||
base: The base manifest.
|
||||
override: The tool-specific overrides.
|
||||
|
||||
Returns:
|
||||
Merged manifest.
|
||||
"""
|
||||
merged = deepcopy(base)
|
||||
|
||||
for key, value in override.items():
|
||||
if key == "mounts" and isinstance(value, list):
|
||||
# Concatenate mount arrays
|
||||
existing = merged.get("mounts", [])
|
||||
merged["mounts"] = existing + deepcopy(value)
|
||||
elif key == "scripts" and isinstance(value, dict):
|
||||
# Merge script categories
|
||||
if "scripts" not in merged:
|
||||
merged["scripts"] = {}
|
||||
for script_key, script_value in value.items():
|
||||
existing = merged["scripts"].get(script_key, [])
|
||||
merged["scripts"][script_key] = existing + deepcopy(script_value)
|
||||
elif key == "packages" and isinstance(value, dict):
|
||||
# Union package arrays
|
||||
if "packages" not in merged:
|
||||
merged["packages"] = {}
|
||||
for pkg_key, pkg_value in value.items():
|
||||
if (
|
||||
pkg_key in merged["packages"]
|
||||
and isinstance(merged["packages"][pkg_key], list)
|
||||
and isinstance(pkg_value, list)
|
||||
):
|
||||
merged["packages"][pkg_key] = merged["packages"][
|
||||
pkg_key
|
||||
] + deepcopy(pkg_value)
|
||||
else:
|
||||
merged["packages"][pkg_key] = deepcopy(pkg_value)
|
||||
elif key == "env" and isinstance(value, dict):
|
||||
# Dict merge: override wins on key conflict
|
||||
if "env" not in merged:
|
||||
merged["env"] = {}
|
||||
merged["env"].update(deepcopy(value))
|
||||
elif (
|
||||
isinstance(value, dict) and key in merged and isinstance(merged[key], dict)
|
||||
):
|
||||
# Generic dict merge
|
||||
merged[key] = {**merged[key], **deepcopy(value)}
|
||||
else:
|
||||
# Override entirely
|
||||
merged[key] = deepcopy(value)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def compile_dockerfile(manifest: dict) -> str:
|
||||
"""Compile a resolved manifest into a Dockerfile string.
|
||||
|
||||
Args:
|
||||
manifest: Fully resolved manifest JSON.
|
||||
|
||||
Returns:
|
||||
Dockerfile content.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
# FROM
|
||||
base_image = manifest.get("base_image", "ubuntu:24.04")
|
||||
lines.append(f"FROM {base_image}")
|
||||
lines.append("")
|
||||
|
||||
# Build-time environment
|
||||
env = manifest.get("env", {})
|
||||
for key, value in env.items():
|
||||
lines.append(f"ENV {key}={shlex.quote(value)}")
|
||||
if env:
|
||||
lines.append("")
|
||||
|
||||
# System packages (apt)
|
||||
apt_packages = manifest.get("packages", {}).get("apt", [])
|
||||
if apt_packages:
|
||||
lines.append("RUN apt-get update && apt-get install -y \\\\")
|
||||
for pkg in apt_packages[:-1]:
|
||||
lines.append(f" {pkg} \\\\")
|
||||
lines.append(f" {apt_packages[-1]} \\\\")
|
||||
lines.append(" && rm -rf /var/lib/apt/lists/*")
|
||||
lines.append("")
|
||||
|
||||
# Node.js
|
||||
node = manifest.get("packages", {}).get("node")
|
||||
if node:
|
||||
version = node.get("version", "20")
|
||||
lines.append(
|
||||
f"RUN curl -fsSL https://deb.nodesource.com/setup_{version}.x | bash - && \\\\"
|
||||
)
|
||||
lines.append(" apt-get install -y nodejs && \\\\")
|
||||
lines.append(" rm -rf /var/lib/apt/lists/*")
|
||||
lines.append("")
|
||||
|
||||
# NPM global packages
|
||||
npm_packages = manifest.get("packages", {}).get("npm_global", [])
|
||||
if npm_packages:
|
||||
pkg_list = " ".join(shlex.quote(p) for p in npm_packages)
|
||||
lines.append(f"RUN npm install -g {pkg_list}")
|
||||
lines.append("")
|
||||
|
||||
# Pip packages
|
||||
pip_packages = manifest.get("packages", {}).get("pip", [])
|
||||
if pip_packages:
|
||||
pkg_list = " ".join(shlex.quote(p) for p in pip_packages)
|
||||
lines.append(f"RUN pip install {pkg_list}")
|
||||
lines.append("")
|
||||
|
||||
# User creation
|
||||
user = manifest.get("user")
|
||||
if user:
|
||||
name = user["name"]
|
||||
uid = user["uid"]
|
||||
gid = user["gid"]
|
||||
create_home = "-m " if user.get("create_home", True) else ""
|
||||
shell = user.get("shell", "/bin/bash")
|
||||
lines.append(f"RUN groupadd -g {gid} {name} && \\\\")
|
||||
lines.append(f" useradd -u {uid} -g {gid} {create_home}-s {shell} {name}")
|
||||
lines.append("")
|
||||
|
||||
# Build scripts
|
||||
build_scripts = manifest.get("scripts", {}).get("build", [])
|
||||
for script in build_scripts:
|
||||
# Normalize multi-line scripts into single RUN command
|
||||
stripped_lines = [
|
||||
line.strip() for line in script.strip().split("\n") if line.strip()
|
||||
]
|
||||
if stripped_lines:
|
||||
normalized = " && ".join(stripped_lines)
|
||||
lines.append(f"RUN {normalized}")
|
||||
if build_scripts:
|
||||
lines.append("")
|
||||
|
||||
# Create mount target directories
|
||||
mounts = manifest.get("mounts", [])
|
||||
if mounts:
|
||||
dirs = [mount["target"] for mount in mounts]
|
||||
dir_str = " ".join(dirs)
|
||||
lines.append(f"RUN mkdir -p {dir_str}")
|
||||
if user:
|
||||
lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
|
||||
lines.append("")
|
||||
|
||||
# Entrypoint for startup scripts
|
||||
startup_scripts = manifest.get("scripts", {}).get("startup", [])
|
||||
if startup_scripts:
|
||||
lines.append(
|
||||
"COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint"
|
||||
)
|
||||
lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint")
|
||||
lines.append("")
|
||||
|
||||
# Switch to runtime user
|
||||
if user:
|
||||
lines.append(f"USER {user['name']}")
|
||||
lines.append(f"WORKDIR /home/{user['name']}")
|
||||
lines.append("")
|
||||
|
||||
# Entrypoint and CMD
|
||||
runtime = manifest.get("runtime", {})
|
||||
if startup_scripts:
|
||||
lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')
|
||||
|
||||
command = runtime.get("command", ["/bin/bash"])
|
||||
cmd_json = json.dumps(command)
|
||||
lines.append(f"CMD {cmd_json}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def compile_entrypoint(manifest: dict) -> str:
|
||||
"""Generate the startup entrypoint script from startup scripts.
|
||||
|
||||
Args:
|
||||
manifest: Fully resolved manifest JSON.
|
||||
|
||||
Returns:
|
||||
Shell script content.
|
||||
"""
|
||||
lines = ["#!/bin/bash", "set -e", ""]
|
||||
|
||||
startup_scripts = manifest.get("scripts", {}).get("startup", [])
|
||||
for script in startup_scripts:
|
||||
lines.append(script)
|
||||
lines.append("")
|
||||
|
||||
lines.append('exec "$@"')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:
|
||||
"""Compile a resolved manifest into a Docker Compose string.
|
||||
|
||||
Args:
|
||||
manifest: Fully resolved manifest JSON.
|
||||
variables: Resolved values: IMAGE_TAG, INSTANCE_NAME, REPO_PATH, etc.
|
||||
|
||||
Returns:
|
||||
Docker Compose YAML content.
|
||||
"""
|
||||
runtime = manifest.get("runtime", {})
|
||||
user = manifest.get("user")
|
||||
interface_type = manifest["interface_type"]
|
||||
|
||||
service: dict[str, Any] = {
|
||||
"image": variables["IMAGE_TAG"],
|
||||
"container_name": variables["INSTANCE_NAME"],
|
||||
"restart": "unless-stopped",
|
||||
}
|
||||
|
||||
# Terminal-specific fields
|
||||
if runtime.get("stdin_open", False):
|
||||
service["stdin_open"] = True
|
||||
if runtime.get("tty", False):
|
||||
service["tty"] = True
|
||||
if runtime.get("working_dir"):
|
||||
service["working_dir"] = runtime["working_dir"]
|
||||
|
||||
# User override
|
||||
if user:
|
||||
service["user"] = f"{user['uid']}:{user['gid']}"
|
||||
|
||||
# Ports for web tools
|
||||
default_port = manifest.get("default_port")
|
||||
if interface_type == "web" and default_port:
|
||||
service["ports"] = [f"{variables['TOOL_PORT']}:{default_port}"]
|
||||
|
||||
# Environment
|
||||
env = manifest.get("env", {})
|
||||
if env:
|
||||
service["environment"] = dict(env)
|
||||
|
||||
# Merge extra env from config
|
||||
extra_env = variables.get("EXTRA_ENV", {})
|
||||
if extra_env:
|
||||
if "environment" not in service:
|
||||
service["environment"] = {}
|
||||
service["environment"].update(extra_env)
|
||||
|
||||
# Volumes from mount schema
|
||||
volumes = []
|
||||
for mount in manifest.get("mounts", []):
|
||||
source = resolve_mount_source(mount, variables)
|
||||
if not source:
|
||||
continue
|
||||
target = mount["target"]
|
||||
readonly = ":ro" if mount.get("readonly", False) else ""
|
||||
volumes.append(f"{source}:{target}{readonly}")
|
||||
|
||||
# Append extra volumes from tool config / config profile
|
||||
for vol in variables.get("EXTRA_VOLUMES", []):
|
||||
vol_str = f"{vol['source']}:{vol['target']}"
|
||||
if vol.get("readonly"):
|
||||
vol_str += ":ro"
|
||||
volumes.append(vol_str)
|
||||
|
||||
if volumes:
|
||||
service["volumes"] = volumes
|
||||
|
||||
compose = {"services": {"app": service}}
|
||||
return yaml.dump(compose, default_flow_style=False)
|
||||
|
||||
|
||||
def resolve_mount_source(mount: dict, variables: dict[str, Any]) -> str:
|
||||
"""Resolve a mount's source_type to an actual host path.
|
||||
|
||||
Args:
|
||||
mount: Mount definition from manifest.
|
||||
variables: Resolved variables dict.
|
||||
|
||||
Returns:
|
||||
Host path string, or empty string if unresolved.
|
||||
"""
|
||||
source_type = mount.get("source_type", "host_path")
|
||||
|
||||
if source_type == "repo":
|
||||
return variables.get("REPO_PATH", "")
|
||||
elif source_type == "ssh_key":
|
||||
return variables.get("SSH_PATH", "")
|
||||
elif source_type == "instance":
|
||||
instance_dir = variables.get("INSTANCE_DIR", "")
|
||||
mount_name = mount.get("name", "unknown")
|
||||
return f"{instance_dir}/mounts/{mount_name}"
|
||||
elif source_type == "git_mount":
|
||||
ref = mount.get("git_mount_ref", "default")
|
||||
return variables.get(f"GIT_MOUNT_{ref}", "")
|
||||
elif source_type == "host_path":
|
||||
return mount.get("source", "")
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def compute_image_tag(tool_name: str, manifest: dict) -> str:
|
||||
"""Compute a deterministic image tag from manifest content.
|
||||
|
||||
Args:
|
||||
tool_name: Human-readable tool name.
|
||||
manifest: Fully resolved manifest JSON.
|
||||
|
||||
Returns:
|
||||
Docker image tag string.
|
||||
"""
|
||||
# Canonicalize: sort keys, stable JSON
|
||||
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
|
||||
hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8]
|
||||
safe_name = tool_name.lower().replace(" ", "-").replace("_", "-")
|
||||
return f"headquarter/{safe_name}-{hash_suffix}:latest"
|
||||
|
||||
|
||||
def merge_with_config(
|
||||
manifest: dict, tool_configs: list[dict], profile: dict | None = None
|
||||
) -> dict:
|
||||
"""Merge ToolConfig and ConfigProfile overrides into a manifest.
|
||||
|
||||
Args:
|
||||
manifest: Base manifest from tool definition.
|
||||
tool_configs: List of ToolConfig records.
|
||||
profile: Resolved ConfigProfile (optional).
|
||||
|
||||
Returns:
|
||||
Manifest with overrides applied.
|
||||
"""
|
||||
result = deepcopy(manifest)
|
||||
|
||||
# Apply ToolConfigs
|
||||
extra_env: dict[str, str] = {}
|
||||
extra_volumes: list[dict] = []
|
||||
|
||||
for config in tool_configs:
|
||||
if config.get("config_type") == "env":
|
||||
extra_env[config["key"]] = config["value"]
|
||||
elif config.get("config_type") == "file" and config.get("file_path"):
|
||||
# Files are handled outside the manifest (written to instance dir)
|
||||
pass
|
||||
if config.get("port_override"):
|
||||
result["default_port"] = config["port_override"]
|
||||
if config.get("start_command"):
|
||||
result["runtime"] = result.get("runtime", {})
|
||||
result["runtime"]["command"] = config["start_command"].split()
|
||||
if config.get("working_directory"):
|
||||
result["runtime"] = result.get("runtime", {})
|
||||
result["runtime"]["working_dir"] = config["working_directory"]
|
||||
if config.get("environment_variables"):
|
||||
extra_env.update(config["environment_variables"])
|
||||
if config.get("volumes"):
|
||||
extra_volumes.extend(config["volumes"])
|
||||
|
||||
# Apply ConfigProfile
|
||||
if profile:
|
||||
if profile.get("environment_variables"):
|
||||
extra_env.update(profile["environment_variables"])
|
||||
if profile.get("mounts"):
|
||||
extra_volumes.extend(profile["mounts"])
|
||||
# Profile hints override everything
|
||||
hints = profile.get("hints", {})
|
||||
if hints.get("start_command"):
|
||||
result["runtime"] = result.get("runtime", {})
|
||||
result["runtime"]["command"] = hints["start_command"].split()
|
||||
if hints.get("working_directory"):
|
||||
result["runtime"] = result.get("runtime", {})
|
||||
result["runtime"]["working_dir"] = hints["working_directory"]
|
||||
if hints.get("port_override"):
|
||||
result["default_port"] = hints["port_override"]
|
||||
|
||||
# Store merged extras for the compose compiler
|
||||
result["_extra_env"] = extra_env
|
||||
result["_extra_volumes"] = extra_volumes
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Permission fixer: applies mount permission policies post-start."""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def apply_mount_permissions(
|
||||
container_id: str,
|
||||
mounts: list[dict],
|
||||
timeout: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Apply permission policies to mounted directories in a running container.
|
||||
|
||||
Runs `chown`, `chmod`, and file-mode fixes for each mount that declares
|
||||
an owner, mode, or file_mode. Requires the container to have a root user.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
mounts: List of mount definitions from the manifest.
|
||||
timeout: Max seconds per docker exec command.
|
||||
|
||||
Returns:
|
||||
List of result dicts: [{mount_name, success, error}]
|
||||
"""
|
||||
results = []
|
||||
|
||||
for mount in mounts:
|
||||
name = mount.get("name", "unknown")
|
||||
target = mount["target"]
|
||||
owner = mount.get("owner")
|
||||
mode = mount.get("mode")
|
||||
file_mode = mount.get("file_mode")
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"mount_name": name,
|
||||
"success": True,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Skip if no permission policy defined
|
||||
if not owner and not mode and not file_mode:
|
||||
results.append(result)
|
||||
continue
|
||||
|
||||
try:
|
||||
if owner:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
["chown", "-R", f"{owner}:{owner}", target],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied owner %s to %s in container %s",
|
||||
owner,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
if mode and result["success"]:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
["chmod", mode, target],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied mode %s to %s in container %s",
|
||||
mode,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
if file_mode and result["success"]:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
[
|
||||
"sh",
|
||||
"-c",
|
||||
f"find {target} -type f -exec chmod {file_mode} {{}} +",
|
||||
],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied file_mode %s to files in %s in container %s",
|
||||
file_mode,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
except PermissionFixError as exc:
|
||||
result["success"] = False
|
||||
result["error"] = str(exc)
|
||||
logger.warning(
|
||||
"Permission fix failed for mount %s (target=%s): %s",
|
||||
name,
|
||||
target,
|
||||
exc,
|
||||
)
|
||||
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class PermissionFixError(Exception):
|
||||
"""Raised when a permission fix command fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def _run_in_container(
|
||||
container_id: str,
|
||||
command: list[str],
|
||||
timeout: int,
|
||||
) -> None:
|
||||
"""Run a command inside a container as root.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
command: Command + args to execute.
|
||||
timeout: Max seconds to wait.
|
||||
|
||||
Raises:
|
||||
PermissionFixError: If the command fails or times out.
|
||||
"""
|
||||
cmd = ["docker", "exec", "--user", "root", container_id] + command
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise PermissionFixError(
|
||||
f"Command timed out after {timeout}s: {' '.join(command)}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise PermissionFixError(f"Docker command not found: {' '.join(command)}")
|
||||
|
||||
if result.returncode != 0:
|
||||
raise PermissionFixError(
|
||||
f"Command failed (rc={result.returncode}): {result.stderr.strip()}"
|
||||
)
|
||||
|
||||
|
||||
def check_root_user_available(container_id: str, timeout: int = 5) -> bool:
|
||||
"""Check if the container has a root user we can exec as.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
timeout: Max seconds to wait.
|
||||
|
||||
Returns:
|
||||
True if root user exists and is usable.
|
||||
"""
|
||||
try:
|
||||
_run_in_container(container_id, ["id", "root"], timeout)
|
||||
return True
|
||||
except PermissionFixError:
|
||||
return False
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Unit tests for the manifest compiler."""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.manifest_compiler import (
|
||||
compile_compose,
|
||||
compile_dockerfile,
|
||||
compile_entrypoint,
|
||||
compute_image_tag,
|
||||
deep_merge,
|
||||
merge_with_config,
|
||||
resolve_base,
|
||||
)
|
||||
|
||||
|
||||
class TestResolveBase:
|
||||
"""Tests for resolve_base."""
|
||||
|
||||
def test_returns_manifest_unchanged_when_no_base(self) -> None:
|
||||
manifest = {"name": "test", "base_image": "ubuntu:24.04"}
|
||||
result = resolve_base(manifest)
|
||||
assert result["name"] == "test"
|
||||
assert "base_definition_id" not in result
|
||||
|
||||
|
||||
class TestDeepMerge:
|
||||
"""Tests for deep_merge."""
|
||||
|
||||
def test_packages_are_unioned(self) -> None:
|
||||
base = {"packages": {"apt": ["curl", "git"]}}
|
||||
override = {"packages": {"apt": ["neovim"]}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["packages"]["apt"] == ["curl", "git", "neovim"]
|
||||
|
||||
def test_node_version_overrides(self) -> None:
|
||||
base = {"packages": {"node": {"version": "18"}}}
|
||||
override = {"packages": {"node": {"version": "20"}}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["packages"]["node"]["version"] == "20"
|
||||
|
||||
def test_env_is_merged_with_override_winning(self) -> None:
|
||||
base = {"env": {"FOO": "base", "BAR": "base"}}
|
||||
override = {"env": {"FOO": "override"}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["env"]["FOO"] == "override"
|
||||
assert result["env"]["BAR"] == "base"
|
||||
|
||||
def test_build_scripts_are_concatenated(self) -> None:
|
||||
base = {"scripts": {"build": ["echo base"]}}
|
||||
override = {"scripts": {"build": ["echo override"]}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["scripts"]["build"] == ["echo base", "echo override"]
|
||||
|
||||
def test_mounts_are_concatenated(self) -> None:
|
||||
base = {"mounts": [{"name": "base-mount", "target": "/base"}]}
|
||||
override = {"mounts": [{"name": "tool-mount", "target": "/tool"}]}
|
||||
result = deep_merge(base, override)
|
||||
assert len(result["mounts"]) == 2
|
||||
|
||||
def test_user_is_overridden_entirely(self) -> None:
|
||||
base = {"user": {"name": "base", "uid": 1000}}
|
||||
override = {"user": {"name": "tool", "uid": 1001}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["user"]["name"] == "tool"
|
||||
assert result["user"]["uid"] == 1001
|
||||
|
||||
|
||||
class TestCompileDockerfile:
|
||||
"""Tests for compile_dockerfile."""
|
||||
|
||||
def test_includes_from(self) -> None:
|
||||
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "FROM ubuntu:24.04" in df
|
||||
|
||||
def test_installs_apt_packages(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"apt": ["curl", "git"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "apt-get install -y" in df
|
||||
assert "curl" in df
|
||||
assert "git" in df
|
||||
assert "rm -rf /var/lib/apt/lists/*" in df
|
||||
|
||||
def test_installs_node(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"node": {"version": "20"}},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "nodesource.com/setup_20.x" in df
|
||||
|
||||
def test_installs_npm_global(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"npm_global": ["@scope/pkg"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "npm install -g @scope/pkg" in df
|
||||
|
||||
def test_creates_user(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"user": {"name": "dev", "uid": 1001, "gid": 1001},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "groupadd -g 1001 dev" in df
|
||||
assert "useradd -u 1001 -g 1001" in df
|
||||
assert "USER dev" in df
|
||||
|
||||
def test_build_scripts_as_run_commands(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"scripts": {"build": ["echo hello", "echo world"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "RUN echo hello" in df
|
||||
assert "RUN echo world" in df
|
||||
|
||||
def test_creates_mount_directories(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"user": {"name": "dev", "uid": 1001, "gid": 1001},
|
||||
"mounts": [
|
||||
{"name": "ws", "target": "/workspace"},
|
||||
{"name": "cfg", "target": "/config"},
|
||||
],
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "mkdir -p /workspace /config" in df
|
||||
assert "chown -R dev:dev /workspace /config" in df
|
||||
|
||||
def test_entrypoint_for_startup_scripts(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"scripts": {"startup": ["echo start"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]' in df
|
||||
|
||||
def test_cmd_from_runtime(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"runtime": {"command": ["/bin/bash", "-il"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'CMD ["/bin/bash", "-il"]' in df
|
||||
|
||||
def test_default_cmd_when_no_runtime(self) -> None:
|
||||
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'CMD ["/bin/bash"]' in df
|
||||
|
||||
|
||||
class TestCompileEntrypoint:
|
||||
"""Tests for compile_entrypoint."""
|
||||
|
||||
def test_includes_shebang_and_set_e(self) -> None:
|
||||
manifest = {"scripts": {"startup": ["echo hello"]}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert "#!/bin/bash" in ep
|
||||
assert "set -e" in ep
|
||||
|
||||
def test_includes_startup_scripts(self) -> None:
|
||||
manifest = {"scripts": {"startup": ["echo hello", "echo world"]}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert "echo hello" in ep
|
||||
assert "echo world" in ep
|
||||
|
||||
def test_ends_with_exec(self) -> None:
|
||||
manifest: dict = {"scripts": {"startup": []}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert 'exec "$@"' in ep
|
||||
|
||||
|
||||
class TestCompileCompose:
|
||||
"""Tests for compile_compose."""
|
||||
|
||||
def test_includes_image_and_container_name(self) -> None:
|
||||
manifest = {"name": "test", "interface_type": "terminal"}
|
||||
vars_dict = {"IMAGE_TAG": "test:v1", "INSTANCE_NAME": "test-1"}
|
||||
compose = compile_compose(manifest, vars_dict)
|
||||
assert "image: test:v1" in compose
|
||||
assert "container_name: test-1" in compose
|
||||
|
||||
def test_terminal_fields(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"runtime": {"stdin_open": True, "tty": True, "working_dir": "/workspace"},
|
||||
}
|
||||
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
|
||||
assert "stdin_open: true" in compose
|
||||
assert "tty: true" in compose
|
||||
assert "working_dir: /workspace" in compose
|
||||
|
||||
def test_web_ports(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "web",
|
||||
"default_port": 8080,
|
||||
}
|
||||
compose = compile_compose(
|
||||
manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n", "TOOL_PORT": "3000"}
|
||||
)
|
||||
assert "3000:8080" in compose
|
||||
|
||||
def test_user_override(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"user": {"uid": 1001, "gid": 1001},
|
||||
}
|
||||
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
|
||||
assert "user: 1001:1001" in compose
|
||||
|
||||
def test_mounts_resolved(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"mounts": [
|
||||
{"name": "ws", "target": "/workspace", "source_type": "repo"},
|
||||
{
|
||||
"name": "ssh",
|
||||
"target": "/home/user/.ssh",
|
||||
"source_type": "ssh_key",
|
||||
"readonly": True,
|
||||
},
|
||||
],
|
||||
}
|
||||
compose = compile_compose(
|
||||
manifest,
|
||||
{
|
||||
"IMAGE_TAG": "t",
|
||||
"INSTANCE_NAME": "n",
|
||||
"REPO_PATH": "/repos/myrepo",
|
||||
"SSH_PATH": "/keys/ssh",
|
||||
},
|
||||
)
|
||||
assert "/repos/myrepo:/workspace" in compose
|
||||
assert "/keys/ssh:/home/user/.ssh:ro" in compose
|
||||
|
||||
def test_extra_volumes_appended(self) -> None:
|
||||
manifest = {"name": "test", "interface_type": "terminal"}
|
||||
compose = compile_compose(
|
||||
manifest,
|
||||
{
|
||||
"IMAGE_TAG": "t",
|
||||
"INSTANCE_NAME": "n",
|
||||
"EXTRA_VOLUMES": [{"source": "/host/x", "target": "/container/x"}],
|
||||
},
|
||||
)
|
||||
assert "/host/x:/container/x" in compose
|
||||
|
||||
|
||||
class TestComputeImageTag:
|
||||
"""Tests for compute_image_tag."""
|
||||
|
||||
def test_is_deterministic(self) -> None:
|
||||
manifest = {"name": "test", "packages": {"apt": ["curl"]}}
|
||||
tag1 = compute_image_tag("My Tool", manifest)
|
||||
tag2 = compute_image_tag("My Tool", manifest)
|
||||
assert tag1 == tag2
|
||||
|
||||
def test_changes_with_content(self) -> None:
|
||||
manifest1 = {"name": "test", "packages": {"apt": ["curl"]}}
|
||||
manifest2 = {"name": "test", "packages": {"apt": ["wget"]}}
|
||||
tag1 = compute_image_tag("test", manifest1)
|
||||
tag2 = compute_image_tag("test", manifest2)
|
||||
assert tag1 != tag2
|
||||
|
||||
def test_lowercases_name(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
tag = compute_image_tag("My Tool", manifest)
|
||||
assert "my-tool" in tag
|
||||
|
||||
def test_valid_docker_reference(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
tag = compute_image_tag("test", manifest)
|
||||
assert tag.startswith("headquarter/test-")
|
||||
assert tag.endswith(":latest")
|
||||
|
||||
|
||||
class TestMergeWithConfig:
|
||||
"""Tests for merge_with_config."""
|
||||
|
||||
def test_applies_tool_config_env(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
configs = [
|
||||
{"config_type": "env", "key": "FOO", "value": "bar"},
|
||||
]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["_extra_env"]["FOO"] == "bar"
|
||||
|
||||
def test_applies_port_override(self) -> None:
|
||||
manifest = {"name": "test", "default_port": 8080}
|
||||
configs = [{"port_override": 3000}]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["default_port"] == 3000
|
||||
|
||||
def test_applies_start_command(self) -> None:
|
||||
manifest = {"name": "test", "runtime": {"command": ["/bin/bash"]}}
|
||||
configs = [{"start_command": "/bin/sh"}]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["runtime"]["command"] == ["/bin/sh"]
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Unit tests for the permission fixer."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.permission_fixer import (
|
||||
PermissionFixError,
|
||||
apply_mount_permissions,
|
||||
check_root_user_available,
|
||||
_run_in_container,
|
||||
)
|
||||
|
||||
|
||||
class TestApplyMountPermissions:
|
||||
"""Tests for apply_mount_permissions."""
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_chown_when_owner_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "owner": "user"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["mount_name"] == "workspace"
|
||||
assert results[0]["success"] is True
|
||||
mock_run.assert_called_once()
|
||||
args = mock_run.call_args[0]
|
||||
assert args[0] == "abc123"
|
||||
assert args[1] == ["chown", "-R", "user:user", "/workspace"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_chmod_when_mode_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "ssh", "target": "/home/user/.ssh", "mode": "0700"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is True
|
||||
# Only chmod called (no owner, so no chown)
|
||||
assert mock_run.call_count == 1
|
||||
chmod_call = mock_run.call_args_list[0]
|
||||
assert chmod_call[0][1] == ["chmod", "0700", "/home/user/.ssh"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_file_mode_when_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{
|
||||
"name": "ssh",
|
||||
"target": "/home/user/.ssh",
|
||||
"file_mode": "0600",
|
||||
},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is True
|
||||
# Only file_mode called (no owner, no mode)
|
||||
assert mock_run.call_count == 1
|
||||
file_mode_call = mock_run.call_args_list[0]
|
||||
assert file_mode_call[0][1][0] == "sh"
|
||||
assert (
|
||||
"find /home/user/.ssh -type f -exec chmod 0600" in file_mode_call[0][1][2]
|
||||
)
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_skips_mount_with_no_policy(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "writable": True},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["success"] is True
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_reports_failure_on_command_error(self, mock_run) -> None:
|
||||
mock_run.side_effect = PermissionFixError("chown failed")
|
||||
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "owner": "user"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is False
|
||||
assert "chown failed" in results[0]["error"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_stops_on_first_failure(self, mock_run) -> None:
|
||||
"""If chown fails, chmod and file_mode should not run."""
|
||||
mock_run.side_effect = PermissionFixError("chown failed")
|
||||
|
||||
mounts = [
|
||||
{
|
||||
"name": "workspace",
|
||||
"target": "/workspace",
|
||||
"owner": "user",
|
||||
"mode": "0755",
|
||||
"file_mode": "0644",
|
||||
},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is False
|
||||
assert mock_run.call_count == 1 # Only chown attempted
|
||||
|
||||
|
||||
class TestRunInContainer:
|
||||
"""Tests for _run_in_container."""
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_success(self, mock_run) -> None:
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr="")
|
||||
_run_in_container("abc123", ["echo", "hello"], 10)
|
||||
mock_run.assert_called_once()
|
||||
cmd = mock_run.call_args[0][0]
|
||||
assert cmd == ["docker", "exec", "--user", "root", "abc123", "echo", "hello"]
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_failure_raises(self, mock_run) -> None:
|
||||
mock_run.return_value = MagicMock(returncode=1, stderr="permission denied")
|
||||
with pytest.raises(PermissionFixError, match="permission denied"):
|
||||
_run_in_container("abc123", ["chown", "x"], 10)
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_timeout_raises(self, mock_run) -> None:
|
||||
import subprocess
|
||||
|
||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd=["docker"], timeout=10)
|
||||
with pytest.raises(PermissionFixError, match="timed out"):
|
||||
_run_in_container("abc123", ["chown", "x"], 10)
|
||||
|
||||
|
||||
class TestCheckRootUserAvailable:
|
||||
"""Tests for check_root_user_available."""
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_returns_true_when_root_exists(self, mock_run) -> None:
|
||||
assert check_root_user_available("abc123") is True
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_returns_false_when_root_missing(self, mock_run) -> None:
|
||||
mock_run.side_effect = PermissionFixError("no such user")
|
||||
assert check_root_user_available("abc123") is False
|
||||
@@ -0,0 +1,37 @@
|
||||
project:
|
||||
name: Headquarter
|
||||
description: Docker-based development platform for managing coding agent tool instances
|
||||
repository: https://git.commumedia.org/alex/headquarter
|
||||
|
||||
stack:
|
||||
backend:
|
||||
framework: FastAPI
|
||||
language: Python 3.11
|
||||
database: PostgreSQL 15 (async SQLAlchemy)
|
||||
cache: Redis 7
|
||||
migrations: Alembic
|
||||
testing: pytest
|
||||
frontend:
|
||||
framework: React + Vite
|
||||
language: TypeScript
|
||||
infrastructure:
|
||||
local: Docker Compose
|
||||
production: Docker Compose + Traefik
|
||||
auth: Authentik SSO
|
||||
|
||||
sdd:
|
||||
execution_mode: interactive
|
||||
artifact_store: openspec
|
||||
chained_pr_strategy: auto-forecast
|
||||
review_budget_lines: 400
|
||||
|
||||
strict_tdd:
|
||||
enabled: true
|
||||
test_command: docker exec hq-api pytest
|
||||
evidence_required: red_green_triangulate_refactor
|
||||
|
||||
phase_rules:
|
||||
explore_before_proposal: true
|
||||
spec_before_design: true
|
||||
design_before_tasks: true
|
||||
verify_before_archive: true
|
||||
@@ -0,0 +1,751 @@
|
||||
# Design: Tool Definition Manifest System
|
||||
|
||||
## Status
|
||||
**Phase:** design
|
||||
**Date:** 2026-05-28
|
||||
**Owner:** el Gentleman
|
||||
**Based on:** Spec `tool-definition-manifest`
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Tool Workshop (Frontend) │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │
|
||||
│ │ Base Image │ │ Packages │ │ Mount Schema Designer │ │
|
||||
│ │ Selector │ │ Editors │ │ (target, owner, mode) │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────────────────┘ │
|
||||
│ ┌─────────────────────────────────────────────────────────────┐│
|
||||
│ │ Live Preview: Dockerfile + Compose ││
|
||||
│ └─────────────────────────────────────────────────────────────┘│
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼ POST /tool-definitions
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ API Backend │
|
||||
│ ┌─────────────────┐ ┌──────────────────┐ ┌────────────┐ │
|
||||
│ │ ManifestSchema │───▶│ ManifestCompiler │───▶│ LiveBuild │ │
|
||||
│ │ (validation) │ │ (Dockerfile + │ │ (optional) │ │
|
||||
│ │ │ │ Compose gen) │ │ │ │
|
||||
│ └─────────────────┘ └──────────────────┘ └────────────┘ │
|
||||
│ │
|
||||
│ ▼ save to DB
|
||||
│ ┌─────────────────────────────────────────────────────────────┐│
|
||||
│ │ ToolDefinitionManifest (JSONB in PostgreSQL) ││
|
||||
│ └─────────────────────────────────────────────────────────────┘│
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼ POST /instances/{id}/start
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Instance Startup Flow │
|
||||
│ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
|
||||
│ │ Resolve │──▶│ Compile │──▶│ Build Image │ │
|
||||
│ │ Manifest │ │ to Dockerfile│ │ (docker build) │ │
|
||||
│ │ (base merge)│ │ + Compose │ │ │ │
|
||||
│ └─────────────┘ └──────────────┘ └──────────────────────┘ │
|
||||
│ │
|
||||
│ ▼
|
||||
│ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
|
||||
│ │ Permission │◀──│ docker comp. │◀──│ Generate Compose │ │
|
||||
│ │ Fixer │ │ up │ │ (mount resolution) │ │
|
||||
│ │ (post-start)│ │ │ │ │ │
|
||||
│ └─────────────┘ └──────────────┘ └──────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Manifest JSON Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"required": ["name", "interface_type"],
|
||||
"oneOf": [
|
||||
{"required": ["base_image"]},
|
||||
{"required": ["base_definition_id"]}
|
||||
],
|
||||
"properties": {
|
||||
"name": {"type": "string", "pattern": "^[a-z0-9-]+$", "maxLength": 64},
|
||||
"display_name": {"type": "string", "maxLength": 128},
|
||||
"description": {"type": "string"},
|
||||
"category": {"type": "string", "maxLength": 64},
|
||||
"interface_type": {"type": "string", "enum": ["web", "terminal"]},
|
||||
"base_image": {"type": "string", "maxLength": 256},
|
||||
"base_definition_id": {"type": "string", "format": "uuid"},
|
||||
"base_version": {"type": "string", "maxLength": 32, "default": "latest"},
|
||||
"packages": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"apt": {"type": "array", "items": {"type": "string"}},
|
||||
"node": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"version": {"type": "string", "pattern": "^\\d+$"}
|
||||
},
|
||||
"required": ["version"]
|
||||
},
|
||||
"npm_global": {"type": "array", "items": {"type": "string"}},
|
||||
"pip": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
},
|
||||
"user": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "maxLength": 32},
|
||||
"uid": {"type": "integer", "minimum": 1, "maximum": 65535},
|
||||
"gid": {"type": "integer", "minimum": 1, "maximum": 65535},
|
||||
"create_home": {"type": "boolean", "default": true},
|
||||
"shell": {"type": "string", "maxLength": 64, "default": "/bin/bash"}
|
||||
},
|
||||
"required": ["name", "uid", "gid"]
|
||||
},
|
||||
"env": {"type": "object", "additionalProperties": {"type": "string"}},
|
||||
"scripts": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"build": {"type": "array", "items": {"type": "string"}},
|
||||
"startup": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
},
|
||||
"mounts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name", "target", "source_type"],
|
||||
"properties": {
|
||||
"name": {"type": "string", "maxLength": 64},
|
||||
"target": {"type": "string", "maxLength": 256},
|
||||
"source_type": {"type": "string", "enum": ["repo", "ssh_key", "instance", "git_mount", "host_path"]},
|
||||
"writable": {"type": "boolean", "default": true},
|
||||
"owner": {"type": "string", "maxLength": 32},
|
||||
"mode": {"type": "string", "pattern": "^[0-7]{3,4}$"},
|
||||
"file_mode": {"type": "string", "pattern": "^[0-7]{3,4}$"},
|
||||
"readonly": {"type": "boolean", "default": false},
|
||||
"git_mount_ref": {"type": "string", "maxLength": 64}
|
||||
}
|
||||
}
|
||||
},
|
||||
"runtime": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"command": {"type": "array", "items": {"type": "string"}},
|
||||
"stdin_open": {"type": "boolean", "default": false},
|
||||
"tty": {"type": "boolean", "default": false},
|
||||
"working_dir": {"type": "string", "maxLength": 256}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Manifest Compiler Algorithm
|
||||
|
||||
### Step 1: Resolve Base
|
||||
|
||||
```python
|
||||
def resolve_base(manifest: dict) -> dict:
|
||||
"""Merge base definition into the manifest."""
|
||||
if manifest.get("base_definition_id"):
|
||||
base = load_base_definition(manifest["base_definition_id"],
|
||||
manifest.get("base_version", "latest"))
|
||||
# Deep merge: base first, then tool-specific overrides
|
||||
merged = deep_merge(base, manifest)
|
||||
# Remove base fields from the merged result
|
||||
merged.pop("base_definition_id", None)
|
||||
merged.pop("base_version", None)
|
||||
return merged
|
||||
return manifest
|
||||
```
|
||||
|
||||
Merge rules:
|
||||
- `packages`: Union arrays (base apt + tool apt = combined apt)
|
||||
- `env`: Tool overrides base (dict merge, tool wins on key conflict)
|
||||
- `scripts.build`: Concatenate arrays (base scripts first, then tool)
|
||||
- `scripts.startup`: Concatenate arrays
|
||||
- `user`: Tool overrides base entirely
|
||||
- `mounts`: Concatenate arrays
|
||||
- `runtime`: Tool overrides base (dict merge)
|
||||
|
||||
### Step 2: Generate Dockerfile
|
||||
|
||||
```python
|
||||
def compile_dockerfile(manifest: dict) -> str:
|
||||
"""Compile a resolved manifest to a Dockerfile string."""
|
||||
lines = []
|
||||
|
||||
# FROM
|
||||
lines.append(f"FROM {manifest['base_image']}")
|
||||
lines.append("")
|
||||
|
||||
# ENV (build-time)
|
||||
for key, value in manifest.get("env", {}).items():
|
||||
lines.append(f"ENV {key}={shlex.quote(value)}")
|
||||
if manifest.get("env"):
|
||||
lines.append("")
|
||||
|
||||
# System packages (apt)
|
||||
apt_packages = manifest.get("packages", {}).get("apt", [])
|
||||
if apt_packages:
|
||||
lines.append("RUN apt-get update && apt-get install -y \\\\")
|
||||
for pkg in apt_packages:
|
||||
lines.append(f" {pkg} \\\\")
|
||||
lines.append(" && rm -rf /var/lib/apt/lists/*")
|
||||
lines.append("")
|
||||
|
||||
# Node.js
|
||||
node = manifest.get("packages", {}).get("node")
|
||||
if node:
|
||||
lines.append(
|
||||
f"RUN curl -fsSL https://deb.nodesource.com/setup_{node['version']}.x | bash - && \\\\")
|
||||
lines.append(" apt-get install -y nodejs && \\\\")
|
||||
lines.append(" rm -rf /var/lib/apt/lists/*")
|
||||
lines.append("")
|
||||
|
||||
# NPM global
|
||||
npm_packages = manifest.get("packages", {}).get("npm_global", [])
|
||||
if npm_packages:
|
||||
pkg_list = " ".join(shlex.quote(p) for p in npm_packages)
|
||||
lines.append(f"RUN npm install -g {pkg_list}")
|
||||
lines.append("")
|
||||
|
||||
# Pip
|
||||
pip_packages = manifest.get("packages", {}).get("pip", [])
|
||||
if pip_packages:
|
||||
pkg_list = " ".join(shlex.quote(p) for p in pip_packages)
|
||||
lines.append(f"RUN pip install {pkg_list}")
|
||||
lines.append("")
|
||||
|
||||
# User creation
|
||||
user = manifest.get("user")
|
||||
if user:
|
||||
lines.append(
|
||||
f"RUN groupadd -g {user['gid']} {user['name']} && \\\\")
|
||||
lines.append(
|
||||
f" useradd -u {user['uid']} -g {user['gid']} "
|
||||
f"{'-m ' if user.get('create_home', True) else ''}"
|
||||
f"-s {user['shell']} {user['name']}")
|
||||
lines.append("")
|
||||
|
||||
# Build scripts
|
||||
build_scripts = manifest.get("scripts", {}).get("build", [])
|
||||
for script in build_scripts:
|
||||
# Each script block becomes one RUN command
|
||||
# Normalize multi-line scripts
|
||||
normalized = " && ".join(line.strip() for line in script.strip().split("\n") if line.strip())
|
||||
lines.append(f"RUN {normalized}")
|
||||
if build_scripts:
|
||||
lines.append("")
|
||||
|
||||
# Create mount target directories and pre-set ownership
|
||||
mounts = manifest.get("mounts", [])
|
||||
if mounts:
|
||||
dirs = []
|
||||
for mount in mounts:
|
||||
dirs.append(mount["target"])
|
||||
if dirs:
|
||||
dir_str = " ".join(dirs)
|
||||
lines.append(f"RUN mkdir -p {dir_str}")
|
||||
if user:
|
||||
lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
|
||||
lines.append("")
|
||||
|
||||
# Entrypoint for startup scripts
|
||||
startup_scripts = manifest.get("scripts", {}).get("startup", [])
|
||||
if startup_scripts:
|
||||
lines.append("COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint")
|
||||
lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint")
|
||||
lines.append("")
|
||||
|
||||
# Switch to user
|
||||
if user:
|
||||
lines.append(f"USER {user['name']}")
|
||||
lines.append(f"WORKDIR /home/{user['name']}")
|
||||
lines.append("")
|
||||
|
||||
# Entrypoint and CMD
|
||||
runtime = manifest.get("runtime", {})
|
||||
if startup_scripts:
|
||||
lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')
|
||||
command = runtime.get("command", ["/bin/bash"])
|
||||
cmd_json = json.dumps(command)
|
||||
lines.append(f"CMD {cmd_json}")
|
||||
|
||||
return "\n".join(lines)
|
||||
```
|
||||
|
||||
### Step 3: Generate Entrypoint Script
|
||||
|
||||
```python
|
||||
def compile_entrypoint(manifest: dict) -> str:
|
||||
"""Generate the startup entrypoint script."""
|
||||
lines = ["#!/bin/bash", "set -e", ""]
|
||||
|
||||
startup_scripts = manifest.get("scripts", {}).get("startup", [])
|
||||
for script in startup_scripts:
|
||||
lines.append(script)
|
||||
lines.append("")
|
||||
|
||||
lines.append('exec "$@"')
|
||||
return "\n".join(lines)
|
||||
```
|
||||
|
||||
### Step 4: Generate Compose
|
||||
|
||||
```python
|
||||
def compile_compose(manifest: dict, variables: dict) -> str:
|
||||
"""Compile a resolved manifest to a Docker Compose string."""
|
||||
runtime = manifest.get("runtime", {})
|
||||
user = manifest.get("user")
|
||||
interface_type = manifest["interface_type"]
|
||||
|
||||
service = {
|
||||
"image": variables["IMAGE_TAG"],
|
||||
"container_name": variables["INSTANCE_NAME"],
|
||||
"restart": "unless-stopped",
|
||||
}
|
||||
|
||||
# Terminal-specific fields
|
||||
if runtime.get("stdin_open", False):
|
||||
service["stdin_open"] = True
|
||||
if runtime.get("tty", False):
|
||||
service["tty"] = True
|
||||
if runtime.get("working_dir"):
|
||||
service["working_dir"] = runtime["working_dir"]
|
||||
|
||||
# User override in compose (helps with permission consistency)
|
||||
if user:
|
||||
service["user"] = f"{user['uid']}:{user['gid']}"
|
||||
|
||||
# Ports for web tools
|
||||
if interface_type == "web" and manifest.get("default_port"):
|
||||
service["ports"] = [f"{variables['TOOL_PORT']}:{manifest['default_port']}"]
|
||||
|
||||
# Environment
|
||||
env = manifest.get("env", {})
|
||||
if env:
|
||||
service["environment"] = env
|
||||
|
||||
# Volumes from mounts
|
||||
volumes = []
|
||||
for mount in manifest.get("mounts", []):
|
||||
source = resolve_mount_source(mount, variables)
|
||||
target = mount["target"]
|
||||
readonly = ":ro" if mount.get("readonly", False) else ""
|
||||
volumes.append(f"{source}:{target}{readonly}")
|
||||
|
||||
# Append extra volumes from tool config / config profile
|
||||
for vol in variables.get("EXTRA_VOLUMES", []):
|
||||
vol_str = f"{vol['source']}:{vol['target']}"
|
||||
if vol.get("readonly"):
|
||||
vol_str += ":ro"
|
||||
volumes.append(vol_str)
|
||||
|
||||
if volumes:
|
||||
service["volumes"] = volumes
|
||||
|
||||
compose = {
|
||||
"services": {"app": service}
|
||||
}
|
||||
|
||||
return yaml.dump(compose, default_flow_style=False)
|
||||
```
|
||||
|
||||
### Step 5: Image Tag Hash
|
||||
|
||||
```python
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
def compute_image_tag(tool_name: str, manifest: dict) -> str:
|
||||
"""Deterministic image tag from manifest content."""
|
||||
# Normalize: sort keys, stable JSON
|
||||
canonical = json.dumps(manifest, sort_keys=True, separators=(',', ':'))
|
||||
hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8]
|
||||
safe_name = tool_name.lower().replace(" ", "-").replace("_", "-")
|
||||
return f"headquarter/{safe_name}-{hash_suffix}:latest"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Mount Resolution
|
||||
|
||||
Each `source_type` resolves differently at instance creation time:
|
||||
|
||||
| source_type | Resolution | Example |
|
||||
|-------------|------------|---------|
|
||||
| `repo` | `{repo_path}` (mount or clone) | `/data/repos/headquarter` |
|
||||
| `ssh_key` | `{instance_dir}/.ssh` | `/data/instances/.../.ssh` |
|
||||
| `instance` | `{instance_dir}/{name}` | `/data/instances/.../mounts/tmp_.pi_agents` |
|
||||
| `git_mount` | Resolved from ConfigProfile git_mounts | `/data/instances/.../git-mounts/...` |
|
||||
| `host_path` | Literal host path | `/var/run/docker.sock` |
|
||||
|
||||
```python
|
||||
def resolve_mount_source(mount: dict, variables: dict) -> str:
|
||||
source_type = mount["source_type"]
|
||||
if source_type == "repo":
|
||||
return variables["REPO_PATH"]
|
||||
elif source_type == "ssh_key":
|
||||
return variables["SSH_PATH"]
|
||||
elif source_type == "instance":
|
||||
instance_dir = variables["INSTANCE_DIR"]
|
||||
mount_name = mount["name"]
|
||||
return f"{instance_dir}/mounts/{mount_name}"
|
||||
elif source_type == "git_mount":
|
||||
ref = mount.get("git_mount_ref", "default")
|
||||
return variables.get(f"GIT_MOUNT_{ref}", "")
|
||||
elif source_type == "host_path":
|
||||
return mount.get("source", "")
|
||||
else:
|
||||
raise ValueError(f"Unknown source_type: {source_type}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Permission Fixer (Post-Start)
|
||||
|
||||
```python
|
||||
def apply_mount_permissions(
|
||||
container_id: str,
|
||||
mounts: list[dict],
|
||||
timeout: int = 10
|
||||
) -> list[dict]:
|
||||
"""Apply permission policies to mounted directories in a running container.
|
||||
|
||||
Returns a list of results: [{mount_name, success, error}]
|
||||
"""
|
||||
results = []
|
||||
for mount in mounts:
|
||||
name = mount["name"]
|
||||
target = mount["target"]
|
||||
owner = mount.get("owner")
|
||||
mode = mount.get("mode")
|
||||
file_mode = mount.get("file_mode")
|
||||
|
||||
result = {"mount_name": name, "success": True, "error": None}
|
||||
|
||||
try:
|
||||
if owner:
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", container_id, "chown", "-R",
|
||||
f"{owner}:{owner}", target],
|
||||
capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
result["success"] = False
|
||||
result["error"] = f"chown failed: {proc.stderr}"
|
||||
|
||||
if mode and result["success"]:
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", container_id, "chmod", mode, target],
|
||||
capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
result["success"] = False
|
||||
result["error"] = f"chmod failed: {proc.stderr}"
|
||||
|
||||
if file_mode and result["success"]:
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", container_id, "sh", "-c",
|
||||
f"find {target} -type f -exec chmod {file_mode} {{}} +"],
|
||||
capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
result["success"] = False
|
||||
result["error"] = f"file_mode chmod failed: {proc.stderr}"
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result["success"] = False
|
||||
result["error"] = "Permission fix timed out"
|
||||
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
```
|
||||
|
||||
**Important:** The permission fixer checks if `root` exists in the container before running. If the container has no `root` user (e.g., distroless images), it logs a warning and skips.
|
||||
|
||||
---
|
||||
|
||||
## Modified Startup Flow
|
||||
|
||||
The `start_instance` function in `tool_instances.py` is modified as follows:
|
||||
|
||||
```python
|
||||
async def start_instance(...):
|
||||
# ... existing validation ...
|
||||
|
||||
tool_type = await session.get(ToolType, instance.tool_type_id)
|
||||
|
||||
if tool_type.definition_type == "manifest" and tool_type.manifest_id:
|
||||
# NEW: Manifest-based startup flow
|
||||
manifest = await load_manifest(session, tool_type.manifest_id)
|
||||
|
||||
# Merge base definition
|
||||
resolved = resolve_base(manifest)
|
||||
|
||||
# Merge tool configs and config profile
|
||||
resolved = apply_tool_configs(resolved, configs)
|
||||
resolved = apply_config_profile(resolved, profile)
|
||||
|
||||
# Compile
|
||||
dockerfile = compile_dockerfile(resolved)
|
||||
entrypoint = compile_entrypoint(resolved)
|
||||
image_tag = compute_image_tag(tool_type.name, resolved)
|
||||
|
||||
# Build image
|
||||
build_context = {
|
||||
"Dockerfile": dockerfile,
|
||||
".headquarter/entrypoint.sh": entrypoint,
|
||||
}
|
||||
returncode, stdout, stderr = build_image(
|
||||
instance_dir=instance_dir,
|
||||
dockerfile=dockerfile, # The Dockerfile references entrypoint.sh
|
||||
tag=image_tag,
|
||||
build_context=build_context,
|
||||
)
|
||||
|
||||
# Generate compose with resolved variables
|
||||
variables = {
|
||||
"IMAGE_TAG": image_tag,
|
||||
"INSTANCE_NAME": instance.name.lower(),
|
||||
"REPO_PATH": repo_path,
|
||||
"SSH_PATH": ssh_dir,
|
||||
"INSTANCE_DIR": instance_dir,
|
||||
# ... git mount resolutions from config profile ...
|
||||
}
|
||||
compose_content = compile_compose(resolved, variables)
|
||||
write_compose_file(instance_dir, compose_content)
|
||||
|
||||
# Store image tag for reuse
|
||||
instance.image_tag = image_tag
|
||||
|
||||
else:
|
||||
# LEGACY: Existing dockerfile_template / compose_template flow
|
||||
...
|
||||
|
||||
# ... docker compose up ...
|
||||
# ... wait for running ...
|
||||
|
||||
# NEW: Apply mount permissions post-start
|
||||
if tool_type.definition_type == "manifest":
|
||||
manifest = await load_manifest(session, tool_type.manifest_id)
|
||||
resolved = resolve_base(manifest)
|
||||
permission_results = apply_mount_permissions(
|
||||
instance.container_id,
|
||||
resolved.get("mounts", [])
|
||||
)
|
||||
for result in permission_results:
|
||||
if not result["success"]:
|
||||
logger.warning(
|
||||
"Permission fix failed for mount %s: %s",
|
||||
result["mount_name"], result["error"]
|
||||
)
|
||||
|
||||
# ... readiness probe ...
|
||||
# ... tunnel creation ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
apps/api/src/
|
||||
├── models/
|
||||
│ ├── tool_definition_manifest.py # NEW: SQLAlchemy model
|
||||
│ └── tool_type.py # MOD: add manifest_id, definition_type
|
||||
├── services/
|
||||
│ ├── manifest_compiler.py # NEW: compile_dockerfile, compile_compose, resolve_base
|
||||
│ ├── permission_fixer.py # NEW: apply_mount_permissions
|
||||
│ └── docker_build.py # MOD: support build_context files
|
||||
├── api/
|
||||
│ ├── tool_definitions.py # NEW: CRUD + compile endpoints
|
||||
│ └── tool_instances.py # MOD: manifest-based startup flow
|
||||
├── schemas/
|
||||
│ └── manifest_schema.py # NEW: JSON Schema + Pydantic validators
|
||||
└── alembic/versions/
|
||||
└── 20260528_add_tool_definition_manifests.py # NEW: migration
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
| Module | Tests | Coverage |
|
||||
|--------|-------|----------|
|
||||
| `manifest_compiler.py` | Dockerfile generation for all package managers, base merging, entrypoint generation | All branches |
|
||||
| `permission_fixer.py` | chown/chmod success, failure, timeout, missing root user | All branches |
|
||||
| `manifest_schema.py` | Valid manifest acceptance, invalid manifest rejection (all error paths) | All validation rules |
|
||||
|
||||
### Integration Tests
|
||||
|
||||
| Scenario | Test |
|
||||
|----------|------|
|
||||
| Manifest → Dockerfile → Build | Create manifest, compile, build image, verify it runs |
|
||||
| ConfigProfile merge | Start instance with profile, verify mounts merged correctly |
|
||||
| Permission fix | Start non-root container, verify workspace is writable |
|
||||
| SSH key mount | Start clone-mode instance, verify SSH keys accessible and have correct permissions |
|
||||
| Legacy compatibility | Start instance from old dockerfile_template tool type, verify it still works |
|
||||
| Base versioning | Create tool with base v1, update base to v2, verify tool still uses v1 |
|
||||
|
||||
### E2E Tests
|
||||
|
||||
| Scenario | Test |
|
||||
|----------|------|
|
||||
| Tool Workshop CRUD | Create, edit, preview, delete a tool definition via UI |
|
||||
| Instance lifecycle | Create instance from manifest tool, start, terminal connect, stop, delete |
|
||||
|
||||
---
|
||||
|
||||
## Migration Plan
|
||||
|
||||
### Step 1: Schema Migration
|
||||
|
||||
```python
|
||||
# alembic migration
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"tool_definition_manifests",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("name", sa.String(64), nullable=False),
|
||||
sa.Column("display_name", sa.String(128), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("category", sa.String(64), nullable=True),
|
||||
sa.Column("interface_type", sa.String(16), nullable=False),
|
||||
sa.Column("base_image", sa.String(256), nullable=True),
|
||||
sa.Column("base_definition_id", sa.UUID(), nullable=True),
|
||||
sa.Column("base_version", sa.String(32), nullable=False, server_default="latest"),
|
||||
sa.Column("manifest", sa.JSON(), nullable=False),
|
||||
sa.Column("dockerfile_cache", sa.Text(), nullable=True),
|
||||
sa.Column("compose_cache", sa.Text(), nullable=True),
|
||||
sa.Column("version", sa.String(32), nullable=False, server_default="v1"),
|
||||
sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("created_by_id", sa.UUID(), nullable=True),
|
||||
sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("name"),
|
||||
sa.ForeignKeyConstraint(["base_definition_id"], ["tool_definition_manifests.id"]),
|
||||
sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]),
|
||||
sa.CheckConstraint(
|
||||
"(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)",
|
||||
name="ck_tool_definition_manifests_base_required"
|
||||
),
|
||||
)
|
||||
|
||||
op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True))
|
||||
op.add_column("tool_types", sa.Column("definition_type", sa.String(16), nullable=False, server_default="legacy"))
|
||||
op.create_foreign_key(
|
||||
"fk_tool_types_manifest_id",
|
||||
"tool_types", "tool_definition_manifests",
|
||||
["manifest_id"], ["id"]
|
||||
)
|
||||
|
||||
op.add_column("tool_instances", sa.Column("manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True))
|
||||
op.add_column("tool_instances", sa.Column("image_tag", sa.String(256), nullable=True))
|
||||
```
|
||||
|
||||
### Step 2: Data Migration
|
||||
|
||||
Convert the existing pi-agent tool type from `dockerfile_template` to manifest:
|
||||
|
||||
```python
|
||||
def upgrade_data():
|
||||
conn = op.get_bind()
|
||||
|
||||
# Create the base definition for ubuntu-24.04-dev
|
||||
base_id = uuid.uuid4()
|
||||
conn.execute(sa.text("""
|
||||
INSERT INTO tool_definition_manifests
|
||||
(id, name, display_name, description, interface_type, base_image, manifest, is_base, version)
|
||||
VALUES (:id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base', 'Base development environment',
|
||||
'terminal', 'ubuntu:24.04', :manifest, true, 'v1')
|
||||
"""), {
|
||||
"id": base_id,
|
||||
"manifest": json.dumps({
|
||||
"name": "ubuntu-24.04-dev",
|
||||
"base_image": "ubuntu:24.04",
|
||||
"packages": {"apt": ["curl", "wget", "git", "build-essential", "ca-certificates"]},
|
||||
"user": {"name": "user", "uid": 1000, "gid": 1000},
|
||||
})
|
||||
})
|
||||
|
||||
# Create pi-agent manifest referencing the base
|
||||
pi_agent_id = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3")
|
||||
conn.execute(sa.text("""
|
||||
INSERT INTO tool_definition_manifests
|
||||
(id, name, display_name, description, category, interface_type,
|
||||
base_definition_id, base_version, manifest, version)
|
||||
VALUES (:id, 'pi-agent', 'Pi Agent', 'Terminal-based coding harness',
|
||||
'development', 'terminal', :base_id, 'v1', :manifest, 'v1')
|
||||
"""), {
|
||||
"id": pi_agent_id,
|
||||
"base_id": base_id,
|
||||
"manifest": json.dumps({
|
||||
"name": "pi-agent",
|
||||
"display_name": "Pi Agent",
|
||||
"interface_type": "terminal",
|
||||
"packages": {
|
||||
"apt": ["neovim", "ranger", "tmux", "htop", "tree", "jq", "python3", "python3-pip"],
|
||||
"node": {"version": "20"},
|
||||
"npm_global": ["@earendil-works/pi-coding-agent"]
|
||||
},
|
||||
"user": {"name": "user", "uid": 1001, "gid": 1001, "create_home": True, "shell": "/bin/bash"},
|
||||
"env": {"DEBIAN_FRONTEND": "noninteractive"},
|
||||
"scripts": {
|
||||
"build": [
|
||||
"git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'",
|
||||
"mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf"
|
||||
],
|
||||
"startup": [
|
||||
"if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi"
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{"name": "workspace", "target": "/workspace", "source_type": "repo", "writable": True, "owner": "user"},
|
||||
{"name": "ssh_keys", "target": "/home/user/.ssh", "source_type": "ssh_key", "mode": "0700", "file_mode": "0600", "readonly": True},
|
||||
{"name": "pi_state", "target": "/tmp/.pi/agents", "source_type": "instance", "writable": True},
|
||||
{"name": "pi_config", "target": "/home/user/.pi", "source_type": "git_mount", "git_mount_ref": "dotfiles", "writable": True, "owner": "user"}
|
||||
],
|
||||
"runtime": {
|
||||
"command": ["/bin/bash"],
|
||||
"stdin_open": True,
|
||||
"tty": True,
|
||||
"working_dir": "/workspace"
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
# Update the existing tool_types row
|
||||
conn.execute(sa.text("""
|
||||
UPDATE tool_types
|
||||
SET manifest_id = :manifest_id,
|
||||
definition_type = 'manifest',
|
||||
dockerfile_template = NULL,
|
||||
compose_template = NULL
|
||||
WHERE name = 'pi-agent'
|
||||
"""), {"manifest_id": pi_agent_id})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Review Workload Forecast
|
||||
|
||||
| PR | Scope | Est. Lines | Review Risk |
|
||||
|----|-------|-----------|-------------|
|
||||
| PR 1: Backend | Compiler, fixer, API, tests, migration | ~1800 | Medium — core algorithm changes |
|
||||
| PR 2: Frontend | Tool Workshop UI | ~1200 | Medium — new feature, self-contained |
|
||||
| PR 3: Migration | Data migration, legacy fallback | ~300 | Low — additive only |
|
||||
|
||||
All PRs are under the 400-line budget individually. Chained PRs recommended for sequential review.
|
||||
@@ -0,0 +1,362 @@
|
||||
# SDD Exploration: Streamline Tool Container Definitions
|
||||
|
||||
## Status
|
||||
**Phase:** explore
|
||||
**Date:** 2026-05-28
|
||||
**Owner:** el Gentleman (parent session)
|
||||
**Scope:** Tool container definition, build, and mount system
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The current tool container system works for the happy path (pi-agent on Ubuntu) but has deep structural inflexibility:
|
||||
|
||||
1. **Monolithic Dockerfile strings** in the database — impossible to review, version, or compose
|
||||
2. **Ad-hoc compose generation** — string formatting with hardcoded fields (`stdin_open`, `tty`, `working_dir` missing for dockerfile types)
|
||||
3. **Hardcoded mount paths** — `/workspace` and `/root/.ssh` don't adapt to the container's runtime user
|
||||
4. **No package/base-image modularity** — every tool type carries a full Dockerfile copy
|
||||
5. **Permission mismatch** — bind mounts come in as root-owned; non-root container users can't write
|
||||
6. **Config overlap** — tool configs, config profiles, and compose templates fight for control of the same fields
|
||||
|
||||
This exploration proposes a **layered, declarative container definition system** where tool types compose from reusable base images, mount schemas, and permission policies.
|
||||
|
||||
---
|
||||
|
||||
## Current Architecture Map
|
||||
|
||||
### Data Model
|
||||
|
||||
```
|
||||
ToolType (DB table)
|
||||
├── name, display_name, description, category
|
||||
├── interface_type: "web" | "terminal"
|
||||
├── definition_type: "dockerfile" | "compose"
|
||||
├── dockerfile_template: TEXT (giant Dockerfile string)
|
||||
├── compose_template: TEXT (Jinja-like {{VAR}} string)
|
||||
├── build_context: JSON {path: content}
|
||||
├── required_variables: JSON ["REPO_PATH", ...]
|
||||
├── default_port: int
|
||||
└── readiness_probe: JSON
|
||||
|
||||
ToolInstance (DB table)
|
||||
├── name, display_name, status
|
||||
├── tool_type_id → ToolType
|
||||
├── repository_id → GitRepository
|
||||
├── compose_path: str
|
||||
├── container_id, container_name
|
||||
├── port, url, public_url, tunnel_id
|
||||
├── clone_mode: "mount" | "clone"
|
||||
├── branch, new_branch
|
||||
└── selected_config_profile_id → ConfigProfile
|
||||
|
||||
ToolConfig (DB table, per-user per-tool-type)
|
||||
├── config_type: "env" | "file"
|
||||
├── key, value, file_path
|
||||
├── port_override, start_command, working_directory
|
||||
├── environment_variables: JSON
|
||||
└── volumes: JSON [{source, target, type}]
|
||||
|
||||
ConfigProfile (DB table)
|
||||
├── name, description
|
||||
├── user_id, project_id, tool_type_id
|
||||
├── environment_variables: JSON
|
||||
├── files: JSON {path: content}
|
||||
├── mounts: JSON [{source, target, type}]
|
||||
├── git_mounts: JSON [{remote_url, source_path, target_path, branch}]
|
||||
└── parent_profile_id → ConfigProfile (hierarchy)
|
||||
```
|
||||
|
||||
### Creation Flow (`create_instance`)
|
||||
|
||||
```
|
||||
POST /projects/{id}/repositories/{id}/instances
|
||||
→ validate tool_type, repo, config_profile
|
||||
→ generate instance_name = "{tool_type}-{repo}-{uuid8}"
|
||||
→ ensure_instance_directory(instance_name)
|
||||
→ find_free_port()
|
||||
→ determine repo_path (mount = repo.path; clone = clone_repository())
|
||||
→ IF tool_type.definition_type == "dockerfile":
|
||||
build_image(instance_dir, dockerfile_template, tag, build_context)
|
||||
generate compose_content (HARDCODED STRING FORMATTING)
|
||||
ELSE:
|
||||
render_compose_template(tool_type.compose_template, variables)
|
||||
→ write_compose_file()
|
||||
→ create ToolInstance DB record (status="pending")
|
||||
```
|
||||
|
||||
### Startup Flow (`start_instance`)
|
||||
|
||||
```
|
||||
POST /instances/{id}/start
|
||||
→ fetch ToolConfigs (env, files, port_override, start_command, working_dir, volumes)
|
||||
→ IF selected_config_profile:
|
||||
resolve_profile() → env, files, mounts, git_mounts, hints
|
||||
→ write .env file, config files
|
||||
→ IF clone_mode: mount SSH keys at /root/.ssh (HARDCODED)
|
||||
→ _modify_compose_file(port, command, working_dir, extra_volumes)
|
||||
→ _sanitize_compose_file()
|
||||
→ execute_compose_command("up")
|
||||
→ get_container_id(instance.name) ← CASE-SENSITIVE BUG (fixed)
|
||||
→ get_container_name(instance.name)
|
||||
→ connect_container_to_network("backend")
|
||||
→ wait_for_container_running()
|
||||
→ IF web: start_cloudflared_tunnel()
|
||||
→ instance.status = "running"
|
||||
```
|
||||
|
||||
### Key Files
|
||||
|
||||
| File | Responsibility |
|
||||
|------|---------------|
|
||||
| `apps/api/src/api/tool_instances.py` | create_instance, start_instance, stop_instance, restart_instance, proxy, logs |
|
||||
| `apps/api/src/api/tool_types.py` | CRUD for ToolType (DB strings) |
|
||||
| `apps/api/src/services/docker.py` | compose execution, container queries, tunnel management |
|
||||
| `apps/api/src/services/docker_build.py` | `docker build` wrapper |
|
||||
| `apps/api/src/services/terminal_session.py` | PTY-based terminal over `docker exec` |
|
||||
| `apps/api/src/services/terminal_manager.py` | WebSocket ↔ terminal session lifecycle |
|
||||
| `apps/api/src/models/tool_type.py` | SQLAlchemy model |
|
||||
|
||||
---
|
||||
|
||||
## Pain Points (Detailed)
|
||||
|
||||
### 1. Monolithic Dockerfile Templates
|
||||
|
||||
The pi-agent Dockerfile template is a 40-line string stored in the DB migration:
|
||||
|
||||
```sql
|
||||
INSERT INTO tool_types (... dockerfile_template ...)
|
||||
VALUES ('...# Pi Coding Agent - Terminal-based coding harness\nFROM ubuntu:24.04\n...')
|
||||
```
|
||||
|
||||
**Problems:**
|
||||
- No syntax highlighting, linting, or `docker build` validation at edit time
|
||||
- Every tool type copies the entire Dockerfile; no reuse of common layers
|
||||
- Changes require a DB migration
|
||||
- No way for users to customize packages without forking the whole template
|
||||
|
||||
### 2. Ad-Hoc Compose Generation
|
||||
|
||||
For `dockerfile` type tools, the compose is generated by Python f-string:
|
||||
|
||||
```python
|
||||
compose_content = f"""version: "3.8"
|
||||
services:
|
||||
app:
|
||||
image: {image_tag}
|
||||
container_name: {instance_name.lower()}
|
||||
{ports_section} volumes:
|
||||
- {repo_path}:/workspace
|
||||
restart: unless-stopped
|
||||
"""
|
||||
```
|
||||
|
||||
**Problems:**
|
||||
- Missing `stdin_open: true` and `tty: true` (essential for terminal tools)
|
||||
- Missing `working_dir: /workspace`
|
||||
- No way to add labels, networks, healthchecks, or extra services
|
||||
- Port section is conditionally included with awkward string concatenation
|
||||
|
||||
### 3. Hardcoded Mount Paths
|
||||
|
||||
| Mount | Current Target | Problem |
|
||||
|-------|---------------|---------|
|
||||
| Repository | `/workspace` | Always root-owned; no permission fix for non-root users |
|
||||
| SSH keys (clone mode) | `/root/.ssh` | Invisible to containers running as `user` |
|
||||
| Git-mount configs | `/tmp/.pi` | May be root-owned; conflicts with user's `.pi` |
|
||||
| Config profile files | Instance-relative paths | No validation against container filesystem |
|
||||
|
||||
### 4. No Base Image / Layer Composition
|
||||
|
||||
Every tool type must specify a complete Dockerfile from `FROM` to `CMD`. There's no way to say:
|
||||
|
||||
```yaml
|
||||
base: ubuntu-24.04-dev # pre-built with curl, git, build-essential
|
||||
layers:
|
||||
- nodejs-20
|
||||
- pi-coding-agent
|
||||
- custom-packages: [neovim, ranger, tmux]
|
||||
```
|
||||
|
||||
### 5. Permission Mismatch (Non-Root Users)
|
||||
|
||||
The pi-agent Dockerfile creates a `user` account and uses `USER user`. Bind mounts from the host come in as root-owned. The API has **no automatic permission fix** — this caused the workspace-unwritable bug.
|
||||
|
||||
Workarounds considered:
|
||||
- Post-start `docker exec --user root chown` (current fix)
|
||||
- Dockerfile entrypoint script that chowns before dropping privileges
|
||||
- Matching container UID to host UID
|
||||
|
||||
None of these are systematic or configurable.
|
||||
|
||||
### 6. Config Overlap and Precedence Confusion
|
||||
|
||||
Three systems control the same container aspects:
|
||||
|
||||
| System | Controls | Stored |
|
||||
|--------|----------|--------|
|
||||
| ToolConfig | env vars, files, port, command, working_dir, volumes | DB (per-user per-tool) |
|
||||
| ConfigProfile | env vars, files, mounts, git_mounts, hints | DB (hierarchical) |
|
||||
| Compose template / generation | volumes, ports, command, working_dir | DB string / Python f-string |
|
||||
|
||||
**Precedence is unclear:**
|
||||
- ToolConfig `working_directory` vs ConfigProfile hint `working_directory` vs compose `working_dir`
|
||||
- ToolConfig `volumes` vs ConfigProfile `mounts` vs compose `volumes`
|
||||
- `start_command` from ToolConfig vs ConfigProfile vs Dockerfile `CMD`
|
||||
|
||||
### 7. Build Context Limitations
|
||||
|
||||
`build_context` is a JSON dictionary of `{relative_path: file_content}`. This is stored in the DB as text.
|
||||
|
||||
**Problems:**
|
||||
- Binary files (images, tarballs) can't be stored
|
||||
- Large files bloat the DB
|
||||
- No versioning or external reference (e.g., "use file from git repo")
|
||||
|
||||
---
|
||||
|
||||
## Extensibility Gaps
|
||||
|
||||
| Want | Current State | Gap |
|
||||
|------|--------------|-----|
|
||||
| Add a new language runtime (e.g., Go, Rust) | Copy entire Dockerfile, edit | No modular package/layer system |
|
||||
| Use a custom base image (e.g., `my-registry/dev-base:v2`) | Edit full Dockerfile | No base-image reference field |
|
||||
| Mount a second repo or a secrets file | Write ConfigProfile or ToolConfig JSON | No declarative mount schema |
|
||||
| Run as root instead of `user` | Edit full Dockerfile | No runtime-user field |
|
||||
| Add a sidecar (e.g., postgres for integration tests) | Edit compose_template string | No multi-service compose support |
|
||||
| Pre-install VS Code server | Edit full Dockerfile | No "feature" or "extension" mechanism |
|
||||
| Custom entrypoint script | Edit full Dockerfile | No entrypoint field |
|
||||
|
||||
---
|
||||
|
||||
## Design Directions (Pre-Proposal)
|
||||
|
||||
### Direction A: Declarative Tool Manifests
|
||||
|
||||
Replace the monolithic `dockerfile_template` with a structured manifest:
|
||||
|
||||
```yaml
|
||||
# Example: tool manifest for pi-agent
|
||||
name: pi-agent
|
||||
base_image: ubuntu:24.04
|
||||
user:
|
||||
name: user
|
||||
uid: 1000
|
||||
home: /home/user
|
||||
packages:
|
||||
apt: [curl, wget, git, neovim, ranger, tmux, htop, tree, jq, python3, python3-pip, build-essential]
|
||||
npm_global: [@earendil-works/pi-coding-agent]
|
||||
node_version: "20"
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
config_files:
|
||||
/home/user/.tmux.conf: "set -g mouse on\n..."
|
||||
/home/user/.config/ranger/rc.conf: "set preview_files true\n..."
|
||||
working_directory: /workspace
|
||||
command: ["/bin/bash"]
|
||||
ports: []
|
||||
mounts:
|
||||
repo: {target: /workspace, writable: true}
|
||||
ssh: {target: /home/user/.ssh, mode: "0600"}
|
||||
tmp_state: {target: /tmp/.pi, writable: true}
|
||||
```
|
||||
|
||||
**Pros:** Structured, reviewable, composable
|
||||
**Cons:** Requires a manifest-to-Dockerfile compiler; migration complexity
|
||||
|
||||
### Direction B: Base Image Registry + Layers
|
||||
|
||||
Maintain a registry of pre-built base images:
|
||||
|
||||
```
|
||||
headquarter/base/ubuntu-24.04-dev
|
||||
headquarter/base/nodejs-20
|
||||
headquarter/base/python-3.11
|
||||
```
|
||||
|
||||
Tool types reference a base image and a list of layers:
|
||||
|
||||
```yaml
|
||||
base_image: headquarter/base/ubuntu-24.04-dev
|
||||
layers:
|
||||
- type: npm_install
|
||||
package: @earendil-works/pi-coding-agent
|
||||
- type: config_file
|
||||
path: /home/user/.tmux.conf
|
||||
content: "..."
|
||||
```
|
||||
|
||||
**Pros:** Fast builds (base images cached), reusable, versioned
|
||||
**Cons:** Requires image registry management, layer ordering complexity
|
||||
|
||||
### Direction C: Compose-First with Dockerfile Overrides
|
||||
|
||||
Treat `compose_template` as the primary definition. For simple cases, use a pre-built image. For custom cases, allow an inline Dockerfile or a `build` section in the compose:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
app:
|
||||
build:
|
||||
context: .
|
||||
dockerfile_inline: |
|
||||
FROM ubuntu:24.04
|
||||
...
|
||||
stdin_open: true
|
||||
tty: true
|
||||
working_dir: /workspace
|
||||
volumes:
|
||||
- ${REPO_PATH}:/workspace
|
||||
- ${SSH_PATH}:/home/user/.ssh:ro
|
||||
user: "${CONTAINER_USER:-user}"
|
||||
```
|
||||
|
||||
**Pros:** Leverages Docker Compose native features, familiar to users
|
||||
**Cons:** Still string-based; inline Dockerfiles are hard to edit
|
||||
|
||||
### Direction D: Permission-Aware Mount Schema
|
||||
|
||||
Decouple mount declaration from mount implementation:
|
||||
|
||||
```python
|
||||
class MountPolicy:
|
||||
source: str # host path
|
||||
target: str # container path
|
||||
owner: str | None # container user to own the mount
|
||||
permissions: str # chmod string
|
||||
readonly: bool
|
||||
```
|
||||
|
||||
At startup, the API runs a post-start "permission fixer" that applies all policies:
|
||||
|
||||
```bash
|
||||
docker exec --user root <container> chown -R <owner> <target>
|
||||
docker exec --user root <container> chmod <permissions> <target>
|
||||
```
|
||||
|
||||
**Pros:** Systematic, works with any base image, configurable per mount
|
||||
**Cons:** Adds startup latency, requires root to exist in container
|
||||
|
||||
---
|
||||
|
||||
## Recommended Next Steps
|
||||
|
||||
1. **Proposal phase:** Evaluate Direction A (Declarative Manifests) vs Direction C (Compose-First) for the primary architecture
|
||||
2. **Design phase:** Detail the manifest schema or compose enhancement, migration path, and API changes
|
||||
3. **Consider Direction D** as a cross-cutting concern regardless of primary direction
|
||||
|
||||
---
|
||||
|
||||
## Risks
|
||||
|
||||
- **Migration risk:** Existing `dockerfile_template` and `compose_template` columns need backward-compatible migration
|
||||
- **Build cache invalidation:** Changing the build system may invalidate Docker layer caches
|
||||
- **User confusion:** Adding a manifest layer on top of Dockerfiles may feel like "yet another abstraction"
|
||||
- **Scope creep:** This touches tool types, tool configs, config profiles, compose generation, and the startup flow — high cross-cutting surface
|
||||
|
||||
---
|
||||
|
||||
## Artifacts
|
||||
|
||||
- `openspec/config.yaml` — SDD configuration
|
||||
- `openspec/explorations/streamline-tool-container-definitions.md` — This document
|
||||
@@ -0,0 +1,447 @@
|
||||
# SDD Proposal: Declarative Tool Container Compiler
|
||||
|
||||
## Status
|
||||
**Phase:** proposal
|
||||
**Date:** 2026-05-28
|
||||
**Owner:** el Gentleman
|
||||
**Based on:** Exploration `streamline-tool-container-definitions`
|
||||
|
||||
---
|
||||
|
||||
## User Story
|
||||
|
||||
As a platform operator, I want to define a tool container by specifying:
|
||||
- A **base image** (e.g. `ubuntu:24.04` or a pre-built `headquarter/base:dev-ubuntu`)
|
||||
- A **list of packages** to install (apt, npm, pip, etc.)
|
||||
- **Setup scripts** that run at build-time or container-startup
|
||||
- **Mount policies** that automatically fix permissions for the runtime user
|
||||
|
||||
I do **not** want to write Dockerfiles or Compose files by hand.
|
||||
|
||||
The system should compile these declarations into Dockerfiles and Compose files automatically, while remaining fully compatible with the existing ConfigProfile mount system.
|
||||
|
||||
---
|
||||
|
||||
## Core Concept: The Tool Definition Manifest
|
||||
|
||||
Replace the monolithic `dockerfile_template` and `compose_template` strings with a single structured **Tool Definition Manifest**.
|
||||
|
||||
```yaml
|
||||
# Tool Definition Manifest (stored as JSON in DB)
|
||||
name: pi-agent
|
||||
display_name: "Pi Agent"
|
||||
description: "Terminal-based coding harness"
|
||||
category: development
|
||||
interface_type: terminal # web | terminal
|
||||
|
||||
# ── Base Image ───────────────────────────────────────────────
|
||||
base_image: ubuntu:24.04
|
||||
# OR reference a pre-built base definition:
|
||||
# base_definition_id: "base-ubuntu-24.04-dev"
|
||||
|
||||
# ── Packages ─────────────────────────────────────────────────
|
||||
packages:
|
||||
apt:
|
||||
- curl
|
||||
- wget
|
||||
- git
|
||||
- neovim
|
||||
- ranger
|
||||
- tmux
|
||||
- htop
|
||||
- tree
|
||||
- jq
|
||||
- ca-certificates
|
||||
- python3
|
||||
- python3-pip
|
||||
- build-essential
|
||||
node:
|
||||
version: "20" # triggers nodesource setup
|
||||
npm_global:
|
||||
- "@earendil-works/pi-coding-agent"
|
||||
# pip:
|
||||
# - requests
|
||||
# - httpx
|
||||
|
||||
# ── Runtime User ─────────────────────────────────────────────
|
||||
user:
|
||||
name: user
|
||||
uid: 1001
|
||||
gid: 1001
|
||||
create_home: true
|
||||
shell: /bin/bash
|
||||
|
||||
# ── Environment ──────────────────────────────────────────────
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
|
||||
# ── Setup Scripts ────────────────────────────────────────────
|
||||
scripts:
|
||||
# build: runs during `docker build` → becomes RUN commands
|
||||
build:
|
||||
- |
|
||||
git config --global init.defaultBranch main
|
||||
git config --global user.email "dev@headquarter.local"
|
||||
git config --global user.name "Developer"
|
||||
- |
|
||||
mkdir -p /home/user/.config/ranger
|
||||
echo 'set preview_files true' > /home/user/.config/ranger/rc.conf
|
||||
|
||||
# startup: runs when container starts → becomes entrypoint script
|
||||
startup:
|
||||
- |
|
||||
# Ensure workspace is owned by runtime user
|
||||
if [ -d /workspace ]; then
|
||||
sudo chown -R user:user /workspace 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Mount Schema ─────────────────────────────────────────────
|
||||
mounts:
|
||||
- name: workspace
|
||||
target: /workspace
|
||||
source_type: repo # resolved from repository path at instance creation
|
||||
writable: true
|
||||
owner: user # post-start: chown -R user:user /workspace
|
||||
|
||||
- name: ssh_keys
|
||||
target: /home/user/.ssh
|
||||
source_type: ssh_key # resolved from repository's SSH key
|
||||
mode: "0700" # post-start: chmod 0700 /home/user/.ssh
|
||||
file_mode: "0600" # post-start: chmod 0600 files inside
|
||||
readonly: true
|
||||
|
||||
- name: pi_state
|
||||
target: /tmp/.pi/agents
|
||||
source_type: instance # resolved to {instance_dir}/mounts/tmp_.pi_agents
|
||||
writable: true
|
||||
|
||||
- name: pi_config
|
||||
target: /home/user/.pi
|
||||
source_type: git_mount # resolved from config profile git_mounts
|
||||
git_mount_ref: dotfiles # references a named git mount in the config profile
|
||||
writable: true
|
||||
owner: user
|
||||
|
||||
# ── Runtime ──────────────────────────────────────────────────
|
||||
runtime:
|
||||
command: ["/bin/bash"]
|
||||
stdin_open: true
|
||||
tty: true
|
||||
working_dir: /workspace
|
||||
# ports are auto-derived from interface_type:
|
||||
# web: expose default_port
|
||||
# terminal: no port mapping
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## How It Compiles
|
||||
|
||||
### 1. Dockerfile Generation
|
||||
|
||||
The manifest compiler transforms the spec into a Dockerfile:
|
||||
|
||||
```dockerfile
|
||||
# Generated Dockerfile — do not edit manually
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# ── System Packages ──
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl wget git neovim ranger tmux htop tree jq \
|
||||
ca-certificates python3 python3-pip build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ── Node.js ──
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
|
||||
&& apt-get install -y nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ── NPM Packages ──
|
||||
RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
|
||||
|
||||
# ── Runtime User ──
|
||||
RUN groupadd -g 1001 user && \
|
||||
useradd -u 1001 -g 1001 -m -s /bin/bash user
|
||||
|
||||
# ── Build Scripts ──
|
||||
RUN git config --global init.defaultBranch main && \
|
||||
git config --global user.email "dev@headquarter.local" && \
|
||||
git config --global user.name "Developer"
|
||||
|
||||
RUN mkdir -p /home/user/.config/ranger && \
|
||||
echo 'set preview_files true' > /home/user/.config/ranger/rc.conf
|
||||
|
||||
# ── Environment ──
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# ── Setup Directories ──
|
||||
RUN mkdir -p /workspace /tmp/.pi/agents /home/user/.pi && \
|
||||
chown -R user:user /workspace /tmp/.pi /home/user/.pi
|
||||
|
||||
# ── Entrypoint for Startup Scripts ──
|
||||
COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint
|
||||
RUN chmod +x /usr/local/bin/headquarter-entrypoint
|
||||
|
||||
USER user
|
||||
WORKDIR /home/user
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]
|
||||
CMD ["/bin/bash"]
|
||||
```
|
||||
|
||||
The generated `entrypoint.sh`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Run startup scripts
|
||||
echo "Ensure workspace is owned by runtime user"
|
||||
if [ -d /workspace ]; then
|
||||
sudo chown -R user:user /workspace 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Pass through to the main command
|
||||
exec "$@"
|
||||
```
|
||||
|
||||
### 2. Compose File Generation
|
||||
|
||||
The manifest compiler also generates the Compose file:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
app:
|
||||
image: ${IMAGE_TAG}
|
||||
container_name: ${INSTANCE_NAME}
|
||||
stdin_open: true
|
||||
tty: true
|
||||
working_dir: /workspace
|
||||
user: "1001:1001" # from manifest user.uid/gid
|
||||
volumes:
|
||||
- ${REPO_PATH}:/workspace
|
||||
- ${SSH_PATH}:/home/user/.ssh:ro
|
||||
- ${INSTANCE_DIR}/mounts/tmp_.pi_agents:/tmp/.pi/agents
|
||||
- ${GIT_MOUNT_dotfiles}:/home/user/.pi
|
||||
environment:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
Variables are resolved at instance creation time:
|
||||
- `${REPO_PATH}` → the repository working directory
|
||||
- `${SSH_PATH}` → prepared SSH key directory
|
||||
- `${INSTANCE_DIR}` → the instance working directory
|
||||
- `${GIT_MOUNT_dotfiles}` → resolved from config profile git_mounts
|
||||
|
||||
### 3. Permission Fixer (Post-Start)
|
||||
|
||||
After `docker compose up`, the API iterates over the mount schema and applies permission policies:
|
||||
|
||||
```python
|
||||
for mount in manifest.mounts:
|
||||
if mount.owner:
|
||||
docker_exec(f"chown -R {mount.owner}:{mount.owner} {mount.target}")
|
||||
if mount.mode:
|
||||
docker_exec(f"chmod {mount.mode} {mount.target}")
|
||||
if mount.file_mode:
|
||||
docker_exec(f"find {mount.target} -type f -exec chmod {mount.file_mode} {{}} +")
|
||||
```
|
||||
|
||||
This is **systematic and configurable** — not hardcoded to `/workspace`.
|
||||
|
||||
---
|
||||
|
||||
## Config Profile Compatibility
|
||||
|
||||
The existing ConfigProfile system provides:
|
||||
- `environment_variables` → merged into compose `environment`
|
||||
- `files` → written to instance dir, mounted via `volumes`
|
||||
- `mounts` → appended to compose `volumes`
|
||||
- `git_mounts` → resolved to host paths, appended to compose `volumes`
|
||||
- `hints.start_command` → overrides `runtime.command`
|
||||
- `hints.working_directory` → overrides `runtime.working_dir`
|
||||
|
||||
With the manifest system, ConfigProfiles **extend** the default mount schema:
|
||||
|
||||
1. Tool manifest defines the **default mount schema** (workspace, ssh, state)
|
||||
2. ConfigProfile can add **additional mounts** or **override runtime hints**
|
||||
3. Both are merged at instance-start time into the final compose file
|
||||
|
||||
The merge precedence:
|
||||
1. Tool manifest (defaults)
|
||||
2. ToolConfig overrides (per-user per-tool settings)
|
||||
3. ConfigProfile overrides (hierarchical, can inherit from parent)
|
||||
4. User-provided start options (e.g. branch selection)
|
||||
|
||||
---
|
||||
|
||||
## Base Image Definitions
|
||||
|
||||
A base definition is itself a manifest with no `runtime` section:
|
||||
|
||||
```yaml
|
||||
# Base Definition: "ubuntu-24.04-dev"
|
||||
name: ubuntu-24.04-dev
|
||||
description: "Ubuntu 24.04 with build tools"
|
||||
base_image: ubuntu:24.04
|
||||
packages:
|
||||
apt:
|
||||
- curl
|
||||
- wget
|
||||
- git
|
||||
- build-essential
|
||||
- ca-certificates
|
||||
user:
|
||||
name: user
|
||||
uid: 1000
|
||||
gid: 1000
|
||||
create_home: true
|
||||
```
|
||||
|
||||
A tool definition can reference it:
|
||||
|
||||
```yaml
|
||||
base_definition_id: "ubuntu-24.04-dev"
|
||||
packages:
|
||||
apt:
|
||||
- neovim
|
||||
- ranger
|
||||
- tmux
|
||||
node:
|
||||
version: "20"
|
||||
```
|
||||
|
||||
The compiler **merges** the base definition with the tool-specific overrides:
|
||||
- Packages are **unioned** (base apt + tool apt)
|
||||
- Scripts are **appended** (base build scripts, then tool build scripts)
|
||||
- User/env are **overridden** (tool wins)
|
||||
|
||||
This enables a family of tool types to share a common base.
|
||||
|
||||
---
|
||||
|
||||
## Database Schema (Proposed)
|
||||
|
||||
### New Table: `tool_definition_manifests`
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `id` | UUID | PK |
|
||||
| `name` | str | Unique identifier |
|
||||
| `display_name` | str | Human-readable |
|
||||
| `description` | str | |
|
||||
| `category` | str | development, data-science, etc. |
|
||||
| `interface_type` | enum | web, terminal |
|
||||
| `base_image` | str | e.g. `ubuntu:24.04` |
|
||||
| `base_definition_id` | UUID? | FK to another manifest |
|
||||
| `manifest` | JSONB | The full manifest JSON |
|
||||
| `dockerfile_cache` | TEXT | Last generated Dockerfile (for inspection) |
|
||||
| `created_at` | datetime | |
|
||||
| `updated_at` | datetime | |
|
||||
|
||||
### Migration: `tool_types` table
|
||||
|
||||
Add a nullable `manifest_id` column to `tool_types`.
|
||||
|
||||
For backward compatibility:
|
||||
- If `manifest_id` is set → use the new manifest system
|
||||
- If `manifest_id` is null → fall back to `dockerfile_template` / `compose_template`
|
||||
|
||||
A data migration converts existing pi-agent to the new manifest format.
|
||||
|
||||
---
|
||||
|
||||
## API Changes
|
||||
|
||||
### New Endpoints
|
||||
|
||||
```
|
||||
GET /tool-definitions → list all base/tool definitions
|
||||
GET /tool-definitions/{id} → get a definition
|
||||
POST /tool-definitions → create a new definition
|
||||
PUT /tool-definitions/{id} → update a definition
|
||||
DELETE /tool-definitions/{id} → delete (if not in use)
|
||||
POST /tool-definitions/{id}/compile → preview generated Dockerfile + compose
|
||||
```
|
||||
|
||||
### Modified Endpoints
|
||||
|
||||
```
|
||||
POST /tool-types → can now accept manifest_id instead of templates
|
||||
GET /tool-types/{id} → includes manifest if available
|
||||
```
|
||||
|
||||
### Frontend Changes
|
||||
|
||||
New UI page: **Tool Workshop**
|
||||
|
||||
- Base image selector (dropdown of existing bases or custom FROM)
|
||||
- Package manager tabs (apt, npm, pip, etc.)
|
||||
- Script editor (build vs startup)
|
||||
- Mount schema designer (drag-drop or form)
|
||||
- Live preview of generated Dockerfile
|
||||
- Test build button (builds image and reports success/failure)
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Should we support multi-stage builds?**
|
||||
- Pros: smaller images, separation of build deps from runtime
|
||||
- Cons: more complexity in the manifest schema
|
||||
|
||||
2. **Should base definitions be versioned?**
|
||||
- Pros: reproducible builds, safe updates
|
||||
- Cons: more DB complexity
|
||||
|
||||
3. **How do we handle binary build context files?**
|
||||
- Current: JSON text in DB
|
||||
- Option A: Store in filesystem, reference by path
|
||||
- Option B: Upload to object storage (S3/minio)
|
||||
|
||||
4. **Should generated images be cached/pushed to a registry?**
|
||||
- Currently: built locally per instance
|
||||
- Option: push to `headquarter/tools/{tool-name}:{hash}` for reuse
|
||||
|
||||
---
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Severity | Mitigation |
|
||||
|------|----------|------------|
|
||||
| Migration complexity | Medium | Keep old fields nullable; gradual adoption |
|
||||
| Build cache invalidation | Medium | Use deterministic Dockerfile generation; hash manifest for image tag |
|
||||
| User confusion ("yet another abstraction") | Low | Provide live preview + "view generated Dockerfile" button |
|
||||
| Scope creep into full CI/CD | High | Keep scope to container definition only; no pipeline/orchestration |
|
||||
| Binary files in manifests | Medium | Limit build context to text; document workaround for binaries |
|
||||
|
||||
---
|
||||
|
||||
## Effort Estimate
|
||||
|
||||
| Phase | Files | Lines (est) | Complexity |
|
||||
|-------|-------|-------------|------------|
|
||||
| DB migration + models | 3 | 200 | Low |
|
||||
| Manifest compiler (Dockerfile) | 2 | 400 | Medium |
|
||||
| Manifest compiler (Compose) | 2 | 300 | Medium |
|
||||
| Permission fixer refactor | 2 | 200 | Low |
|
||||
| API endpoints | 3 | 400 | Medium |
|
||||
| Frontend Tool Workshop | 8 | 1200 | High |
|
||||
| Tests | 4 | 600 | Medium |
|
||||
| **Total** | **24** | **~3300** | **High** |
|
||||
|
||||
**Review workload forecast:** ~3300 lines is well above the 400-line budget. This should be split into **chained PRs**:
|
||||
1. Backend: manifest schema, compiler, API (PR 1)
|
||||
2. Frontend: Tool Workshop UI (PR 2)
|
||||
3. Migration + data conversion (PR 3)
|
||||
|
||||
---
|
||||
|
||||
## Next Recommended Phase
|
||||
|
||||
**Design** — Detail the manifest JSON schema, compiler internals, and migration plan.
|
||||
|
||||
Should I proceed to design?
|
||||
@@ -0,0 +1,323 @@
|
||||
# Spec: Tool Definition Manifest System
|
||||
|
||||
## Status
|
||||
**Phase:** spec
|
||||
**Date:** 2026-05-28
|
||||
**Owner:** el Gentleman
|
||||
**Based on:** Proposal `streamline-tool-container-definitions`
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
### R1. Declarative Tool Definitions
|
||||
Users must be able to define a tool container without writing Dockerfiles or Compose files. The definition is a structured manifest specifying base image, packages, scripts, mounts, and runtime configuration.
|
||||
|
||||
### R2. Base Image Versioning
|
||||
Base definitions must be versioned. Tool definitions reference a specific base version. Updating a base creates a new version; existing tools remain pinned to their version until explicitly updated.
|
||||
|
||||
### R3. Package Managers
|
||||
The manifest must support multiple package managers: `apt`, `npm` (global), `pip`, and `node` (version installation).
|
||||
|
||||
### R4. Build vs Startup Scripts
|
||||
Scripts are categorized by execution phase:
|
||||
- **Build scripts**: Run during `docker build` (e.g., `git config`, config file setup)
|
||||
- **Startup scripts**: Run when the container starts (e.g., permission fixes, dynamic setup)
|
||||
|
||||
### R5. Mount Schema with Permission Policies
|
||||
Mounts declare:
|
||||
- `target`: Container path
|
||||
- `source_type`: How the source is resolved (`repo`, `ssh_key`, `instance`, `git_mount`, `host_path`)
|
||||
- `writable`: Whether the mount is read-write
|
||||
- `owner`: Container user to own the target path (post-start chown)
|
||||
- `mode`: Directory permissions (post-start chmod)
|
||||
- `file_mode`: File permissions inside the directory
|
||||
- `readonly`: Whether mounted read-only in compose
|
||||
|
||||
### R6. Config Profile Compatibility
|
||||
ConfigProfiles continue to add `env`, `files`, `mounts`, and `git_mounts` on top of the manifest defaults. The merge precedence is: manifest defaults → ToolConfig → ConfigProfile → user options.
|
||||
|
||||
### R7. Backward Compatibility
|
||||
Existing `dockerfile_template` and `compose_template` columns remain functional. New tool types use the manifest system; old types continue to work. A data migration converts the existing pi-agent to the new format.
|
||||
|
||||
### R8. Local Builds
|
||||
Images are built locally per instance using the standard `docker build` command. No registry integration in this phase.
|
||||
|
||||
### R9. Live Preview
|
||||
The API provides a `compile` endpoint that returns the generated Dockerfile and Compose file without building.
|
||||
|
||||
### R10. Deterministic Image Tags
|
||||
The image tag is derived from a hash of the manifest content, enabling build cache reuse when the manifest hasn't changed.
|
||||
|
||||
---
|
||||
|
||||
## Scenarios
|
||||
|
||||
### S1. Creating a New Tool Definition
|
||||
|
||||
**Given** a user on the Tool Workshop page
|
||||
**When** they select base "ubuntu-24.04-dev:v1", add packages `[neovim, tmux]`, add a build script for git config, and define mounts for workspace + ssh
|
||||
**Then** the system generates a manifest, compiles a Dockerfile + Compose preview, and upon save stores the manifest in the database.
|
||||
|
||||
### S2. Building an Instance from a Manifest
|
||||
|
||||
**Given** a tool instance created from a manifest-based tool type
|
||||
**When** `start_instance` is called
|
||||
**Then** the API compiles the manifest to a Dockerfile, builds the image, generates the Compose file with resolved mount paths, starts the container, and applies permission policies post-start.
|
||||
|
||||
### S3. Permission Fix on Non-Root Containers
|
||||
|
||||
**Given** a manifest with `user: {name: user, uid: 1001}` and a mount `target: /workspace, owner: user`
|
||||
**When** the container starts with the workspace bind-mounted from host (root-owned)
|
||||
**Then** the post-start permission fixer runs `docker exec --user root chown -R user:user /workspace`, making the directory writable for the container user.
|
||||
|
||||
### S4. SSH Key Mount for Non-Root User
|
||||
|
||||
**Given** a manifest with a mount `target: /home/user/.ssh, source_type: ssh_key, mode: "0700"`
|
||||
**When** the container starts
|
||||
**Then** SSH keys are mounted from the instance `.ssh` directory to `/home/user/.ssh`, and post-start fixes permissions to `0700` for the directory and `0600` for key files.
|
||||
|
||||
### S5. Config Profile Extends Manifest
|
||||
|
||||
**Given** a manifest with default mount `workspace: /workspace` and a ConfigProfile that adds `git_mounts: [{remote_url: "...", target_path: "/home/user/.config"}]`
|
||||
**When** the instance starts with that profile selected
|
||||
**Then** the final Compose includes both the workspace mount and the config git mount, merged in the correct precedence.
|
||||
|
||||
### S6. Base Version Pinning
|
||||
|
||||
**Given** a tool definition referencing `base_definition_id: "ubuntu-24.04-dev", base_version: "v1"`
|
||||
**When** the base definition is updated to "v2"
|
||||
**Then** the tool definition continues to use "v1" until explicitly updated. New tool definitions default to the latest version.
|
||||
|
||||
### S7. Deterministic Image Tag
|
||||
|
||||
**Given** a manifest with specific packages and scripts
|
||||
**When** compiled
|
||||
**Then** the generated image tag is `headquarter/{tool-name}-{manifest-hash}:latest`, and rebuilding the same manifest reuses the cached image layer.
|
||||
|
||||
### S8. Live Preview Without Build
|
||||
|
||||
**Given** a manifest being edited
|
||||
**When** the user clicks "Preview"
|
||||
**Then** the API returns the generated Dockerfile and Compose file within 500ms, without invoking Docker.
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
### A1. Manifest Schema Validation
|
||||
- [ ] The manifest JSON must validate against a defined JSON Schema
|
||||
- [ ] Invalid manifests return 400 with detailed field-level errors
|
||||
- [ ] Missing required fields (name, base_image or base_definition_id) are rejected
|
||||
|
||||
### A2. Dockerfile Compilation
|
||||
- [ ] Generated Dockerfile builds successfully with `docker build`
|
||||
- [ ] Build scripts appear as `RUN` commands in order
|
||||
- [ ] Startup scripts appear in the generated entrypoint script
|
||||
- [ ] Packages are installed in a single layer per package manager
|
||||
- [ ] User creation uses the declared uid/gid
|
||||
|
||||
### A3. Compose Compilation
|
||||
- [ ] Generated Compose file starts successfully with `docker compose up`
|
||||
- [ ] Mounts are resolved from `source_type` to actual host paths
|
||||
- [ ] `stdin_open` and `tty` are set for terminal interface types
|
||||
- [ ] Ports are only included for web interface types
|
||||
|
||||
### A4. Permission Fixer
|
||||
- [ ] Post-start chown runs for all mounts with an `owner` declared
|
||||
- [ ] Post-start chmod runs for all mounts with `mode` or `file_mode` declared
|
||||
- [ ] Permission fixes complete within 5 seconds of container start
|
||||
- [ ] If the container has no `root` user, permission fixes are skipped with a warning
|
||||
|
||||
### A5. Config Profile Merge
|
||||
- [ ] ConfigProfile env vars override manifest defaults
|
||||
- [ ] ConfigProfile mounts are appended to manifest mounts
|
||||
- [ ] ConfigProfile git_mounts are resolved and appended
|
||||
- [ ] ToolConfig values override both manifest and ConfigProfile
|
||||
|
||||
### A6. Backward Compatibility
|
||||
- [ ] Existing tool types with `dockerfile_template` continue to work
|
||||
- [ ] Existing tool types with `compose_template` continue to work
|
||||
- [ ] The pi-agent tool type is migrated to the new manifest format
|
||||
- [ ] Old and new tool types can coexist in the same project
|
||||
|
||||
### A7. Base Versioning
|
||||
- [ ] Base definitions store a version string
|
||||
- [ ] Tool definitions store the base version they reference
|
||||
- [ ] Updating a base creates a new version; old versions remain accessible
|
||||
- [ ] The "latest" version can be referenced explicitly or by omission
|
||||
|
||||
### A8. Image Tag Determinism
|
||||
- [ ] Same manifest produces the same image tag
|
||||
- [ ] Changing any field (package, script, env) produces a different tag
|
||||
- [ ] The tag is lowercased and valid as a Docker image reference
|
||||
|
||||
### A9. API Endpoints
|
||||
- [ ] `POST /tool-definitions` creates a definition (201)
|
||||
- [ ] `GET /tool-definitions/{id}` returns the definition with compiled preview
|
||||
- [ ] `POST /tool-definitions/{id}/compile` returns Dockerfile + Compose (no build)
|
||||
- [ ] `PUT /tool-definitions/{id}` updates and re-validates
|
||||
|
||||
### A10. Frontend Tool Workshop
|
||||
- [ ] Users can create a tool definition via form (no raw JSON editing required)
|
||||
- [ ] Live preview shows generated Dockerfile and Compose
|
||||
- [ ] Package lists support add/remove/reorder
|
||||
- [ ] Mount schema supports add/remove with visual feedback
|
||||
- [ ] Base image selector shows available versions
|
||||
|
||||
---
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- **Multi-stage builds** — Out of scope for this phase. The compiler generates single-stage Dockerfiles.
|
||||
- **Docker registry integration** — Images are built locally per instance.
|
||||
- **Binary build context files** — Build context is limited to text files stored in the DB.
|
||||
- **Custom Dockerfile editing** — Users work exclusively through the manifest; no raw Dockerfile editing.
|
||||
- **Container orchestration beyond Compose** — No Kubernetes, Swarm, or other orchestrators.
|
||||
- **Real-time collaborative editing** — Tool Workshop is single-user editing.
|
||||
|
||||
---
|
||||
|
||||
## API Contract
|
||||
|
||||
### POST /tool-definitions
|
||||
|
||||
**Request:**
|
||||
```json
|
||||
{
|
||||
"name": "pi-agent",
|
||||
"display_name": "Pi Agent",
|
||||
"description": "Terminal coding harness",
|
||||
"category": "development",
|
||||
"interface_type": "terminal",
|
||||
"base_image": "ubuntu:24.04",
|
||||
"packages": {
|
||||
"apt": ["curl", "git", "neovim", "tmux"],
|
||||
"node": {"version": "20"},
|
||||
"npm_global": ["@earendil-works/pi-coding-agent"]
|
||||
},
|
||||
"user": {
|
||||
"name": "user",
|
||||
"uid": 1001,
|
||||
"gid": 1001,
|
||||
"create_home": true,
|
||||
"shell": "/bin/bash"
|
||||
},
|
||||
"env": {"DEBIAN_FRONTEND": "noninteractive"},
|
||||
"scripts": {
|
||||
"build": [
|
||||
"git config --global init.defaultBranch main",
|
||||
"mkdir -p /home/user/.config/ranger"
|
||||
],
|
||||
"startup": [
|
||||
"if [ -d /workspace ]; then sudo chown -R user:user /workspace; fi"
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "workspace",
|
||||
"target": "/workspace",
|
||||
"source_type": "repo",
|
||||
"writable": true,
|
||||
"owner": "user"
|
||||
},
|
||||
{
|
||||
"name": "ssh",
|
||||
"target": "/home/user/.ssh",
|
||||
"source_type": "ssh_key",
|
||||
"mode": "0700",
|
||||
"file_mode": "0600",
|
||||
"readonly": true
|
||||
}
|
||||
],
|
||||
"runtime": {
|
||||
"command": ["/bin/bash"],
|
||||
"stdin_open": true,
|
||||
"tty": true,
|
||||
"working_dir": "/workspace"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response (201):**
|
||||
```json
|
||||
{
|
||||
"id": "d07b8376-2151-4119-8c1d-27f792aae9a3",
|
||||
"name": "pi-agent",
|
||||
"display_name": "Pi Agent",
|
||||
"manifest": { ... },
|
||||
"dockerfile_preview": "FROM ubuntu:24.04\n...",
|
||||
"compose_preview": "services:\n app:\n image: ...",
|
||||
"created_at": "2026-05-28T10:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
### POST /tool-definitions/{id}/compile
|
||||
|
||||
**Response (200):**
|
||||
```json
|
||||
{
|
||||
"dockerfile": "FROM ubuntu:24.04\n...",
|
||||
"compose": "services:\n app:\n...",
|
||||
"image_tag": "headquarter/pi-agent-a3f7c2d9:latest",
|
||||
"mounts_resolved": [
|
||||
{"name": "workspace", "source": "/data/repos/headquarter", "target": "/workspace"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Schema
|
||||
|
||||
### `tool_definition_manifests`
|
||||
|
||||
| Column | Type | Constraints |
|
||||
|--------|------|-------------|
|
||||
| `id` | UUID | PK |
|
||||
| `name` | VARCHAR(64) | UNIQUE, NOT NULL |
|
||||
| `display_name` | VARCHAR(128) | NOT NULL |
|
||||
| `description` | TEXT | |
|
||||
| `category` | VARCHAR(64) | |
|
||||
| `interface_type` | VARCHAR(16) | CHECK IN ('web', 'terminal') |
|
||||
| `base_image` | VARCHAR(256) | |
|
||||
| `base_definition_id` | UUID | FK → `tool_definition_manifests.id` |
|
||||
| `base_version` | VARCHAR(32) | DEFAULT 'latest' |
|
||||
| `manifest` | JSONB | NOT NULL |
|
||||
| `dockerfile_cache` | TEXT | |
|
||||
| `compose_cache` | TEXT | |
|
||||
| `version` | VARCHAR(32) | DEFAULT 'v1' |
|
||||
| `is_base` | BOOLEAN | DEFAULT FALSE |
|
||||
| `created_by_id` | UUID | FK → `users.id` |
|
||||
| `created_at` | TIMESTAMPTZ | DEFAULT now() |
|
||||
| `updated_at` | TIMESTAMPTZ | DEFAULT now() |
|
||||
|
||||
**Check constraint:** Exactly one of `base_image` or `base_definition_id` must be set.
|
||||
|
||||
### Alter `tool_types`
|
||||
|
||||
```sql
|
||||
ALTER TABLE tool_types
|
||||
ADD COLUMN manifest_id UUID REFERENCES tool_definition_manifests(id),
|
||||
ADD COLUMN definition_type VARCHAR(16) DEFAULT 'legacy'; -- 'legacy' | 'manifest'
|
||||
```
|
||||
|
||||
### Alter `tool_instances`
|
||||
|
||||
```sql
|
||||
ALTER TABLE tool_instances
|
||||
ADD COLUMN manifest_compiled_at TIMESTAMPTZ,
|
||||
ADD COLUMN image_tag VARCHAR(256);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Related Files
|
||||
|
||||
- `apps/api/src/models/tool_definition_manifest.py` — New model
|
||||
- `apps/api/src/models/tool_type.py` — Add manifest_id, definition_type
|
||||
- `apps/api/src/services/manifest_compiler.py` — New compiler
|
||||
- `apps/api/src/services/permission_fixer.py` — Refactored mount policy applier
|
||||
- `apps/api/src/api/tool_definitions.py` — New endpoints
|
||||
- `apps/api/src/api/tool_instances.py` — Modified start_instance flow
|
||||
- `apps/api/alembic/versions/20260528_add_tool_definition_manifests.py` — Migration
|
||||
@@ -0,0 +1,89 @@
|
||||
# Tasks: Tool Definition Manifest System
|
||||
|
||||
## PR 1: Backend Manifest System
|
||||
|
||||
### T1.1 Database Migration
|
||||
- [ ] Create `tool_definition_manifests` table
|
||||
- [ ] Add `manifest_id`, `definition_type` to `tool_types`
|
||||
- [ ] Add `manifest_compiled_at`, `image_tag` to `tool_instances`
|
||||
- [ ] Data migration: convert pi-agent to manifest
|
||||
|
||||
### T1.2 Models
|
||||
- [ ] `ToolDefinitionManifest` SQLAlchemy model
|
||||
- [ ] Update `ToolType` model with manifest relationship
|
||||
- [ ] Update `ToolInstance` model with image_tag
|
||||
|
||||
### T1.3 Manifest Compiler
|
||||
- [ ] `resolve_base()` — deep merge base + tool manifest
|
||||
- [ ] `compile_dockerfile()` — generate Dockerfile from manifest
|
||||
- [ ] `compile_entrypoint()` — generate startup entrypoint script
|
||||
- [ ] `compile_compose()` — generate Compose from manifest
|
||||
- [ ] `compute_image_tag()` — deterministic hash-based tag
|
||||
- [ ] `resolve_mount_source()` — mount source resolution
|
||||
|
||||
### T1.4 Permission Fixer
|
||||
- [ ] `apply_mount_permissions()` — post-start chown/chmod
|
||||
- [ ] Handle missing root user gracefully
|
||||
- [ ] Timeout and error reporting
|
||||
|
||||
### T1.5 API Endpoints
|
||||
- [ ] `POST /tool-definitions` — create
|
||||
- [ ] `GET /tool-definitions` — list
|
||||
- [ ] `GET /tool-definitions/{id}` — get
|
||||
- [ ] `PUT /tool-definitions/{id}` — update
|
||||
- [ ] `DELETE /tool-definitions/{id}` — delete
|
||||
- [ ] `POST /tool-definitions/{id}/compile` — preview
|
||||
|
||||
### T1.6 Modified Startup Flow
|
||||
- [ ] Update `start_instance` to use manifest when `definition_type == "manifest"`
|
||||
- [ ] Integrate permission fixer post-start
|
||||
- [ ] Store image_tag on instance for reuse
|
||||
|
||||
### T1.7 Tests
|
||||
- [ ] Unit: manifest compiler (all package managers, base merge)
|
||||
- [ ] Unit: permission fixer (success, failure, timeout)
|
||||
- [ ] Unit: mount resolution (all source types)
|
||||
- [ ] Integration: manifest → build → start → terminal works
|
||||
- [ ] Integration: legacy tool types still work
|
||||
|
||||
---
|
||||
|
||||
## PR 2: Frontend Tool Workshop
|
||||
|
||||
### T2.1 Tool Definitions API Client
|
||||
- [ ] Add tool definition endpoints to `client.ts`
|
||||
- [ ] Type definitions for manifest schema
|
||||
|
||||
### T2.2 Tool Workshop Page
|
||||
- [ ] Base image selector (with version dropdown)
|
||||
- [ ] Package manager editors (apt list, npm list, node version)
|
||||
- [ ] Script editors (build vs startup, tabbed)
|
||||
- [ ] Mount schema designer (form table with add/remove)
|
||||
- [ ] Runtime config (command, working_dir, stdin_open, tty)
|
||||
|
||||
### T2.3 Live Preview
|
||||
- [ ] Preview panel showing generated Dockerfile
|
||||
- [ ] Preview panel showing generated Compose
|
||||
- [ ] "Compile" button calling API preview endpoint
|
||||
|
||||
### T2.4 Tool Definitions List
|
||||
- [ ] Table view of all definitions
|
||||
- [ ] Create / Edit / Delete actions
|
||||
- [ ] Base indicator (shows if it's a base definition)
|
||||
|
||||
---
|
||||
|
||||
## PR 3: Migration & Legacy Fallback
|
||||
|
||||
### T3.1 Data Migration
|
||||
- [ ] Alembic migration creating base definition + pi-agent manifest
|
||||
- [ ] Update existing pi-agent tool_type row
|
||||
|
||||
### T3.2 Legacy Fallback
|
||||
- [ ] Ensure `definition_type == "legacy"` still uses old flow
|
||||
- [ ] Ensure `dockerfile_template` / `compose_template` still work
|
||||
- [ ] Tests for legacy path
|
||||
|
||||
### T3.3 Documentation
|
||||
- [ ] Update API docs
|
||||
- [ ] Add Tool Workshop user guide
|
||||
Reference in New Issue
Block a user