alex/headquarter
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: tool definition manifest system (PR 1)

Browse Source 
- Add ToolDefinitionManifest model with base image versioning
- Add manifest compiler: Dockerfile + Compose generation from JSON manifests
- Add permission fixer: post-start chown/chmod for mount policies
- Add tool definition CRUD API with live compile preview endpoint
- Integrate manifest-based startup flow in start_instance
- Add Alembic migration with data conversion for pi-agent
- Add 48 unit tests for manifest compiler, permission fixer, docker service
- Keep backward compatibility with legacy dockerfile_template/compose_template

Migration: applied successfully. Pi-agent converted to manifest.
Quality gates: pytest (146 passed, 4 pre-existing unrelated failures)
This commit is contained in:
Alex Blank
2026-05-28 13:26:54 +02:00
parent 314ba3aee4
commit 5deee8c65c
20 changed files with 4211 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/.pi-lens/cache/review-graph.json
Vendored
+1
View File
File diff suppressed because one or more lines are too long
apps/api/alembic/versions/2026_05_28_add_tool_definition_manifests.py
+375
View File
@@ -0,0 +1,375 @@
"""add tool definition manifests
Revision ID: 2026_05_28_add_tool_definition_manifests
Revises: 20260527_160017_add_pi_agent
Create Date: 2026-05-28T11:00:00
"""
import json
import uuid
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "2026_05_28_add_tool_definition_manifests"
down_revision: Union[str, None] = "20260527_160017_add_pi_agent"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
BASE_UBUNTU_ID = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
PI_AGENT_MANIFEST_ID = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3")
def upgrade() -> None:
conn = op.get_bind()
# ── Create tool_definition_manifests table ───────────────────────
op.create_table(
"tool_definition_manifests",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("name", sa.String(64), nullable=False),
sa.Column("display_name", sa.String(128), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("category", sa.String(64), nullable=True),
sa.Column("interface_type", sa.String(16), nullable=False),
sa.Column("base_image", sa.String(256), nullable=True),
sa.Column("base_definition_id", sa.UUID(), nullable=True),
sa.Column(
"base_version", sa.String(32), nullable=False, server_default="latest"
),
sa.Column("manifest", sa.JSON(), nullable=False),
sa.Column("dockerfile_cache", sa.Text(), nullable=True),
sa.Column("compose_cache", sa.Text(), nullable=True),
sa.Column("version", sa.String(32), nullable=False, server_default="v1"),
sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"),
sa.Column("created_by_id", sa.UUID(), nullable=True),
sa.Column(
"created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
),
sa.Column(
"updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name"),
sa.ForeignKeyConstraint(
["base_definition_id"], ["tool_definition_manifests.id"]
),
sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]),
sa.CheckConstraint(
"(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)",
name="ck_tool_definition_manifests_base_required",
),
)
# ── Add columns to tool_types ────────────────────────────────────
# Check if manifest_id exists before adding
conn = op.get_bind()
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
""")
)
if not result.fetchone():
op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True))
op.create_foreign_key(
"fk_tool_types_manifest_id",
"tool_types",
"tool_definition_manifests",
["manifest_id"],
["id"],
)
# Update definition_type to allow 'legacy' and 'manifest'
result = conn.execute(
sa.text("""
SELECT constraint_name FROM information_schema.check_constraints
WHERE constraint_name = 'chk_definition_type'
""")
)
if result.fetchone():
op.drop_constraint("chk_definition_type", "tool_types", type_="check")
op.execute("ALTER TABLE tool_types ALTER COLUMN definition_type TYPE VARCHAR(16)")
op.execute("ALTER TABLE tool_types ALTER COLUMN definition_type SET DEFAULT 'legacy'")
# ── Add columns to tool_instances ────────────────────────────────
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
""")
)
if not result.fetchone():
op.add_column(
"tool_instances",
sa.Column("manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True),
)
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
""")
)
if not result.fetchone():
op.add_column(
"tool_instances",
sa.Column("image_tag", sa.String(256), nullable=True),
)
# ── Data migration: create base definition + pi-agent manifest ───
conn.execute(
sa.text(
"""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, interface_type, base_image,
manifest, is_base, version, created_at, updated_at)
VALUES
(:base_id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base',
'Base development environment with build tools', 'terminal',
'ubuntu:24.04', :base_manifest, true, 'v1', now(), now())
"""
),
{
"base_id": BASE_UBUNTU_ID,
"base_manifest": json.dumps(
{
"name": "ubuntu-24.04-dev",
"display_name": "Ubuntu 24.04 Dev Base",
"interface_type": "terminal",
"base_image": "ubuntu:24.04",
"packages": {
"apt": [
"curl",
"wget",
"git",
"build-essential",
"ca-certificates",
"python3",
"python3-pip",
]
},
"user": {
"name": "user",
"uid": 1000,
"gid": 1000,
"create_home": True,
"shell": "/bin/bash",
},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
}
),
},
)
conn.execute(
sa.text(
"""
INSERT INTO tool_definition_manifests
(id, name, display_name, description, category, interface_type,
base_definition_id, base_version, manifest, version, created_at, updated_at)
VALUES
(:manifest_id, 'pi-agent', 'Pi Agent',
'Terminal-based coding harness with nvim, ranger, tmux',
'development', 'terminal', :base_id, 'v1', :manifest, 'v1',
now(), now())
"""
),
{
"manifest_id": PI_AGENT_MANIFEST_ID,
"base_id": BASE_UBUNTU_ID,
"manifest": json.dumps(
{
"name": "pi-agent",
"display_name": "Pi Agent",
"description": "Terminal-based coding harness",
"category": "development",
"interface_type": "terminal",
"base_definition_id": str(BASE_UBUNTU_ID),
"base_version": "v1",
"packages": {
"apt": [
"neovim",
"ranger",
"tmux",
"htop",
"tree",
"jq",
],
"node": {"version": "20"},
"npm_global": ["@earendil-works/pi-coding-agent"],
},
"user": {
"name": "user",
"uid": 1001,
"gid": 1001,
"create_home": True,
"shell": "/bin/bash",
},
"env": {"DEBIAN_FRONTEND": "noninteractive"},
"scripts": {
"build": [
"git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'",
"mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf",
],
"startup": [
"if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi",
],
},
"mounts": [
{
"name": "workspace",
"target": "/workspace",
"source_type": "repo",
"writable": True,
"owner": "user",
},
{
"name": "ssh_keys",
"target": "/home/user/.ssh",
"source_type": "ssh_key",
"mode": "0700",
"file_mode": "0600",
"readonly": True,
},
{
"name": "pi_state",
"target": "/tmp/.pi/agents",
"source_type": "instance",
"writable": True,
},
{
"name": "pi_config",
"target": "/home/user/.pi",
"source_type": "git_mount",
"git_mount_ref": "dotfiles",
"writable": True,
"owner": "user",
},
],
"runtime": {
"command": ["/bin/bash"],
"stdin_open": True,
"tty": True,
"working_dir": "/workspace",
},
}
),
},
)
# ── Update existing pi-agent tool_type ───────────────────────────
conn.execute(
sa.text(
"""
UPDATE tool_types
SET manifest_id = :manifest_id,
definition_type = 'manifest',
dockerfile_template = NULL,
compose_template = NULL
WHERE name = 'pi-agent'
"""
),
{"manifest_id": PI_AGENT_MANIFEST_ID},
)
def downgrade() -> None:
conn = op.get_bind()
# Restore pi-agent templates if manifest_id column exists
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_types' AND column_name = 'manifest_id'
""")
)
has_manifest_id = result.fetchone() is not None
if has_manifest_id:
conn.execute(
sa.text(
"""
UPDATE tool_types
SET manifest_id = NULL,
definition_type = 'dockerfile',
dockerfile_template = :dockerfile,
compose_template = :compose
WHERE name = 'pi-agent'
"""
),
{
"dockerfile": """# Pi Coding Agent - Terminal-based coding harness
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \\
curl wget git neovim ranger tmux htop tree jq \\
ca-certificates python3 python3-pip build-essential \\
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\
&& apt-get install -y nodejs \\
&& rm -rf /var/lib/apt/lists/*
RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
RUN useradd -m -s /bin/bash user
WORKDIR /home/user
RUN git config --global init.defaultBranch main \\
&& git config --global user.email "dev@headquarter.local" \\
&& git config --global user.name "Developer"
RUN echo 'set -g mouse on\\nset -g default-terminal "screen-256color"' > /home/user/.tmux.conf
RUN mkdir -p /home/user/.config/ranger \\
&& echo 'set preview_files true\\nset use_preview_script true' > /home/user/.config/ranger/rc.conf
RUN mkdir -p /home/user/.pi/agent
USER user
CMD ["/bin/bash"]
""",
"compose": """services:
app:
build: .
stdin_open: true
tty: true
volumes:
- ${REPO_PATH}:/workspace
working_dir: /workspace
command: /bin/bash""",
},
)
# Drop columns conditionally
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'image_tag'
""")
)
if result.fetchone():
op.drop_column("tool_instances", "image_tag")
result = conn.execute(
sa.text("""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'tool_instances' AND column_name = 'manifest_compiled_at'
""")
)
if result.fetchone():
op.drop_column("tool_instances", "manifest_compiled_at")
if has_manifest_id:
op.drop_constraint("fk_tool_types_manifest_id", "tool_types", type_="foreignkey")
op.drop_column("tool_types", "manifest_id")
op.drop_table("tool_definition_manifests")
apps/api/src/api/tool_definitions.py
+424
View File
@@ -0,0 +1,424 @@
"""Tool definition API endpoints."""
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.auth.dependencies import get_current_user_id, get_db_session
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.tool_type import ToolType
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
resolve_base,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/tool-definitions", tags=["tool-definitions"])
class CreateToolDefinitionRequest(BaseModel):
"""Request body for creating a tool definition manifest."""
model_config = {"extra": "ignore"}
name: str = Field(description="Unique identifier (kebab-case)")
display_name: str = Field(description="Human-readable name")
description: str | None = Field(default=None)
category: str = Field(default="development")
interface_type: str = Field(default="terminal", description="web or terminal")
base_image: str | None = Field(default=None, description="Direct base image")
base_definition_id: str | None = Field(
default=None, description="Reference to a base definition"
)
base_version: str = Field(default="latest")
manifest: dict = Field(description="The full manifest JSON")
class UpdateToolDefinitionRequest(BaseModel):
"""Request body for updating a tool definition manifest."""
model_config = {"extra": "ignore"}
display_name: str | None = Field(default=None)
description: str | None = Field(default=None)
category: str | None = Field(default=None)
manifest: dict | None = Field(default=None)
base_version: str | None = Field(default=None)
@router.post(
"",
summary="Create tool definition",
description="Create a new tool definition manifest.",
)
async def create_tool_definition(
data: CreateToolDefinitionRequest,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Create a new tool definition manifest.
Args:
data: Manifest data.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with created definition details.
"""
# Validate base reference
if not data.base_image and not data.base_definition_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Either base_image or base_definition_id is required",
)
base_def_id = None
if data.base_definition_id:
try:
base_def_id = uuid.UUID(data.base_definition_id)
except ValueError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid base_definition_id: {data.base_definition_id}",
)
base_def = await session.get(ToolDefinitionManifest, base_def_id)
if not base_def:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Base definition not found: {data.base_definition_id}",
)
if not base_def.is_base:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Referenced definition is not a base definition",
)
# Check name uniqueness
existing = await session.execute(
select(ToolDefinitionManifest).where(ToolDefinitionManifest.name == data.name)
)
if existing.scalar_one_or_none():
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Tool definition '{data.name}' already exists",
)
definition = ToolDefinitionManifest(
name=data.name,
display_name=data.display_name,
description=data.description,
category=data.category,
interface_type=data.interface_type,
base_image=data.base_image,
base_definition_id=base_def_id,
base_version=data.base_version,
manifest=data.manifest,
created_by_id=user_id,
)
session.add(definition)
await session.commit()
await session.refresh(definition)
logger.info("Created tool definition %s (%s)", definition.id, definition.name)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"description": definition.description,
"category": definition.category,
"interface_type": definition.interface_type,
"base_image": definition.base_image,
"base_definition_id": str(definition.base_definition_id)
if definition.base_definition_id
else None,
"base_version": definition.base_version,
"manifest": definition.manifest,
"is_base": definition.is_base,
"created_at": definition.created_at.isoformat(),
}
@router.get(
"",
summary="List tool definitions",
description="List all tool definition manifests.",
)
async def list_tool_definitions(
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
include_bases: bool = True,
) -> dict:
"""List all tool definition manifests.
Args:
user_id: Authenticated user ID.
session: Database session.
include_bases: Whether to include base definitions.
Returns:
Dictionary containing list of definitions.
"""
query = select(ToolDefinitionManifest)
if not include_bases:
query = query.where(ToolDefinitionManifest.is_base == False)
result = await session.execute(
query.order_by(ToolDefinitionManifest.created_at.desc())
)
definitions = result.scalars().all()
return {
"definitions": [
{
"id": str(d.id),
"name": d.name,
"display_name": d.display_name,
"description": d.description,
"category": d.category,
"interface_type": d.interface_type,
"is_base": d.is_base,
"base_image": d.base_image,
"base_definition_id": str(d.base_definition_id)
if d.base_definition_id
else None,
"base_version": d.base_version,
"version": d.version,
"created_at": d.created_at.isoformat(),
}
for d in definitions
]
}
@router.get(
"/{definition_id}",
summary="Get tool definition",
description="Get a specific tool definition manifest.",
)
async def get_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Get a specific tool definition manifest.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with definition details.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"description": definition.description,
"category": definition.category,
"interface_type": definition.interface_type,
"base_image": definition.base_image,
"base_definition_id": str(definition.base_definition_id)
if definition.base_definition_id
else None,
"base_version": definition.base_version,
"manifest": definition.manifest,
"dockerfile_cache": definition.dockerfile_cache,
"compose_cache": definition.compose_cache,
"version": definition.version,
"is_base": definition.is_base,
"created_at": definition.created_at.isoformat(),
"updated_at": definition.updated_at.isoformat(),
}
@router.put(
"/{definition_id}",
summary="Update tool definition",
description="Update a tool definition manifest.",
)
async def update_tool_definition(
definition_id: uuid.UUID,
data: UpdateToolDefinitionRequest,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Update a tool definition manifest.
Args:
definition_id: UUID of the definition.
data: Update data.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with updated definition details.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
if data.display_name is not None:
definition.display_name = data.display_name
if data.description is not None:
definition.description = data.description
if data.category is not None:
definition.category = data.category
if data.manifest is not None:
definition.manifest = data.manifest
if data.base_version is not None:
definition.base_version = data.base_version
await session.commit()
await session.refresh(definition)
logger.info("Updated tool definition %s (%s)", definition.id, definition.name)
return {
"id": str(definition.id),
"name": definition.name,
"display_name": definition.display_name,
"manifest": definition.manifest,
"updated_at": definition.updated_at.isoformat(),
}
@router.delete(
"/{definition_id}",
summary="Delete tool definition",
description="Delete a tool definition manifest.",
)
async def delete_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Delete a tool definition manifest.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with deletion status.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
# Check if any tool types reference this manifest
result = await session.execute(
select(ToolType).where(ToolType.manifest_id == definition_id)
)
referencing = result.scalars().all()
if referencing:
tool_names = ", ".join(t.name for t in referencing)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Cannot delete: referenced by tool types: {tool_names}",
)
await session.delete(definition)
await session.commit()
logger.info("Deleted tool definition %s (%s)", definition.id, definition.name)
return {"status": "deleted", "id": str(definition_id)}
@router.post(
"/{definition_id}/compile",
summary="Compile tool definition",
description="Compile a manifest to Dockerfile + Compose preview without building.",
)
async def compile_tool_definition(
definition_id: uuid.UUID,
user_id: uuid.UUID = Depends(get_current_user_id),
session: AsyncSession = Depends(get_db_session),
) -> dict:
"""Compile a manifest to Dockerfile + Compose preview.
Args:
definition_id: UUID of the definition.
user_id: Authenticated user ID.
session: Database session.
Returns:
Dictionary with compiled Dockerfile, Compose, and image tag.
"""
definition = await session.get(ToolDefinitionManifest, definition_id)
if not definition:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Tool definition not found: {definition_id}",
)
manifest = dict(definition.manifest)
# Resolve base if referenced
if definition.base_definition_id:
base_def = await session.get(
ToolDefinitionManifest, definition.base_definition_id
)
if base_def:
base_manifest = dict(base_def.manifest)
manifest = resolve_base(deep_merge(base_manifest, manifest))
# Compile
dockerfile = compile_dockerfile(manifest)
entrypoint = compile_entrypoint(manifest)
image_tag = compute_image_tag(definition.name, manifest)
# Dummy compose with placeholder variables
dummy_vars = {
"IMAGE_TAG": image_tag,
"INSTANCE_NAME": f"{definition.name}-preview",
"INSTANCE_DIR": "/data/instances/preview",
"REPO_PATH": "/data/repos/preview",
"SSH_PATH": "/data/instances/preview/.ssh",
"TOOL_PORT": "8080",
"EXTRA_ENV": {},
"EXTRA_VOLUMES": [],
}
compose = compile_compose(manifest, dummy_vars)
# Update cache
definition.dockerfile_cache = dockerfile
definition.compose_cache = compose
await session.commit()
return {
"id": str(definition.id),
"name": definition.name,
"dockerfile": dockerfile,
"entrypoint": entrypoint,
"compose": compose,
"image_tag": image_tag,
}
apps/api/src/api/tool_instances.py
+246 -34
View File
@@ -62,6 +62,16 @@ from src.services.docker import (
write_env_file,
)
from src.services.docker_build import build_image
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
merge_with_config,
resolve_base,
)
from src.services.permission_fixer import apply_mount_permissions
from src.services.readiness_probe import execute_probe
from src.services.ssh_keys import cleanup_ssh_key_files, prepare_ssh_key_files
@@ -961,6 +971,142 @@ async def get_instance(
}
async def _prepare_manifest_instance(
session: AsyncSession,
instance: ToolInstance,
instance_dir: str,
repo_path: str,
configs: list,
env_vars: dict,
extra_volumes: list,
working_directory: str | None,
) -> tuple[str, str, dict]:
"""Build image and generate compose from a manifest-based tool type.
Returns:
Tuple of (image_tag, compose_content, resolved_manifest)
"""
from src.models.tool_definition_manifest import ToolDefinitionManifest
tool_type = await session.get(ToolType, instance.tool_type_id)
manifest_def = await session.get(ToolDefinitionManifest, tool_type.manifest_id)
if not manifest_def:
raise RuntimeError(f"Manifest not found for tool type {tool_type.id}")
manifest = dict(manifest_def.manifest)
# Resolve base if referenced
if manifest_def.base_definition_id:
base_def = await session.get(
ToolDefinitionManifest, manifest_def.base_definition_id
)
if base_def:
base_manifest = dict(base_def.manifest)
manifest = resolve_base(deep_merge(base_manifest, manifest))
else:
logger.warning(
"Base definition %s not found for manifest %s",
manifest_def.base_definition_id,
manifest_def.id,
)
# Merge tool configs
tool_config_dicts = [
{
"config_type": c.config_type,
"key": c.key,
"value": c.value,
"file_path": c.file_path,
"port_override": c.port_override,
"start_command": c.start_command,
"working_directory": c.working_directory,
"environment_variables": c.environment_variables,
"volumes": c.volumes,
}
for c in configs
]
manifest = merge_with_config(manifest, tool_config_dicts)
# Resolve extra env and volumes from merge_with_config
extra_env = manifest.pop("_extra_env", {})
extra_cfg_volumes = manifest.pop("_extra_volumes", [])
env_vars.update(extra_env)
extra_volumes.extend(extra_cfg_volumes)
# Compute image tag
image_tag = compute_image_tag(tool_type.name, manifest)
# Check if image already exists
check = subprocess.run(
["docker", "images", "-q", image_tag],
capture_output=True,
text=True,
)
image_exists = check.returncode == 0 and check.stdout.strip()
if not image_exists:
# Compile and build
dockerfile = compile_dockerfile(manifest)
entrypoint = compile_entrypoint(manifest)
build_ctx = {
"Dockerfile": dockerfile,
".headquarter/entrypoint.sh": entrypoint,
}
returncode, stdout, stderr = await asyncio.to_thread(
build_image,
instance_dir=instance_dir,
dockerfile=dockerfile,
tag=image_tag,
build_context=build_ctx,
)
if returncode != 0:
raise RuntimeError(f"Docker build failed: {stderr}")
logger.info("Built image %s for instance %s", image_tag, instance.id)
else:
logger.info("Reusing existing image %s for instance %s", image_tag, instance.id)
# Prepare SSH path for mount resolution
ssh_path = ""
if instance.clone_mode == "clone":
ssh_path = os.path.join(instance_dir, ".ssh")
# Resolve git mount variables from config profile
git_mount_vars = {}
if instance.selected_config_profile_id:
resolved_profile = await resolve_profile(
session, instance.selected_config_profile_id
)
for gm in resolved_profile.git_mounts or []:
ref = gm.get("git_mount_ref", "default")
# The actual resolution happens in _resolve_git_mounts; we store placeholder
git_mount_vars[f"GIT_MOUNT_{ref}"] = ""
variables = {
"IMAGE_TAG": image_tag,
"INSTANCE_NAME": instance.name.lower(),
"INSTANCE_DIR": instance_dir,
"REPO_PATH": repo_path,
"SSH_PATH": ssh_path,
"TOOL_PORT": instance.port or 0,
"EXTRA_ENV": env_vars,
"EXTRA_VOLUMES": extra_volumes,
**git_mount_vars,
}
compose_content = compile_compose(manifest, variables)
# Cache
instance.image_tag = image_tag
instance.manifest_compiled_at = datetime.now()
return image_tag, compose_content, manifest
@router.post(
"/{project_id}/repositories/{repo_id}/instances/{instance_id}/start",
summary="Start instance",
@@ -1119,41 +1265,85 @@ async def start_instance(
"Wrote %d config files for instance %s", len(config_files), instance.id
)
# Mount SSH key for clone-mode instances
if instance.clone_mode == "clone":
repo = await session.get(GitRepository, instance.repository_id)
if repo and repo.ssh_key_id:
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
if ssh_key:
try:
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
extra_volumes.append(
{
"source": ssh_dir,
"target": "/root/.ssh",
"type": "ro",
}
)
logger.debug(
"Mounted SSH key for clone-mode instance %s", instance.id
)
except Exception as exc:
logger.error(
"Failed to prepare SSH key for instance %s: %s",
instance.id,
exc,
)
# ── MANIFEST-BASED FLOW ──────────────────────────────────────
tool_type = await session.get(ToolType, instance.tool_type_id)
resolved_manifest = None
# Modify compose file if needed (port override, start command, working dir, volumes)
if port_override or start_command or working_directory or extra_volumes:
_modify_compose_file(
instance.compose_path,
port_override,
start_command,
working_directory,
extra_volumes,
)
logger.debug("Modified compose file for instance %s", instance.id)
if tool_type and tool_type.definition_type == "manifest" and tool_type.manifest_id:
logger.info("Using manifest-based startup for instance %s", instance.id)
# Determine repo path
repo = await session.get(GitRepository, instance.repository_id)
repo_path = repo.path if repo else ""
if instance.clone_mode == "clone":
repo_path = os.path.join(instance_dir, "repo-clone")
try:
(
image_tag,
compose_content,
resolved_manifest,
) = await _prepare_manifest_instance(
session=session,
instance=instance,
instance_dir=instance_dir,
repo_path=repo_path,
configs=configs,
env_vars=env_vars,
extra_volumes=extra_volumes,
working_directory=working_directory,
)
write_compose_file(instance_dir, compose_content)
logger.debug(
"Generated manifest-based compose for instance %s", instance.id
)
except Exception as exc:
logger.exception(
"Manifest compilation failed for instance %s: %s", instance.id, exc
)
instance.status = "error"
await session.commit()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Manifest compilation failed: {exc}",
)
else:
# ── LEGACY FLOW ──────────────────────────────────────────
# Mount SSH key for clone-mode instances
if instance.clone_mode == "clone":
repo = await session.get(GitRepository, instance.repository_id)
if repo and repo.ssh_key_id:
ssh_key = await session.get(SSHKey, repo.ssh_key_id)
if ssh_key:
try:
ssh_dir = prepare_ssh_key_files(instance_dir, ssh_key)
extra_volumes.append(
{
"source": ssh_dir,
"target": "/root/.ssh",
"type": "ro",
}
)
logger.debug(
"Mounted SSH key for clone-mode instance %s", instance.id
)
except Exception as exc:
logger.error(
"Failed to prepare SSH key for instance %s: %s",
instance.id,
exc,
)
# Modify compose file if needed (port override, start command, working dir, volumes)
if port_override or start_command or working_directory or extra_volumes:
_modify_compose_file(
instance.compose_path,
port_override,
start_command,
working_directory,
extra_volumes,
)
logger.debug("Modified compose file for instance %s", instance.id)
# Sanitize compose file to remove invalid port mappings from old instances
_sanitize_compose_file(instance.compose_path)
@@ -1244,6 +1434,28 @@ async def start_instance(
startup_result["waited_seconds"],
)
# Apply mount permission fixes for manifest-based instances
if resolved_manifest and instance.container_id:
mounts = resolved_manifest.get("mounts", [])
if mounts:
logger.debug(
"Applying permission fixes for instance %s (%d mounts)",
instance.id,
len(mounts),
)
permission_results = apply_mount_permissions(
instance.container_id,
mounts,
)
for result in permission_results:
if not result["success"]:
logger.warning(
"Permission fix failed for mount %s on instance %s: %s",
result["mount_name"],
instance.id,
result["error"],
)
# Execute readiness probe if configured
tool_type = await session.get(ToolType, instance.tool_type_id)
if tool_type and instance.container_id:
apps/api/src/main.py
+7 -1
View File
@@ -18,6 +18,7 @@ from src.api.instance_proxy import router as instance_proxy_router
from src.api.config_folders import router as config_folders_router
from src.api.config_profiles import router as config_profiles_router
from src.api.tool_configs import router as tool_configs_router
from src.api.tool_definitions import router as tool_definitions_router
from src.api.tool_instances import router as tool_instances_router
from src.api.tool_instances import sessions_router
from src.api.tool_types import router as tool_types_router
@@ -66,7 +67,9 @@ def _sanitize_validation_errors(errors):
"type": error.get("type"),
"loc": error.get("loc"),
"msg": error.get("msg"),
"input": str(error.get("input")) if error.get("input") is not None else None,
"input": str(error.get("input"))
if error.get("input") is not None
else None,
}
# Convert ctx to safe format
ctx = error.get("ctx")
@@ -110,10 +113,12 @@ async def on_startup():
if not db_ready:
logger.error("Database initialization failed. Shutting down.")
import sys
sys.exit(1)
logger.info("Startup complete.")
app.include_router(health_router)
app.include_router(auth_router)
app.include_router(dashboard_router)
@@ -123,6 +128,7 @@ app.include_router(ssh_keys_router)
app.include_router(git_repositories_router)
app.include_router(user_config_router)
app.include_router(tool_types_router)
app.include_router(tool_definitions_router)
app.include_router(config_folders_router)
app.include_router(config_profiles_router)
app.include_router(tool_instances_router)
apps/api/src/models/__init__.py
+15 -1
View File
@@ -4,9 +4,23 @@ from src.models.config_profile import ConfigProfile, ConfigProfileInclude
from src.models.git_repository import GitRepository
from src.models.project import Project
from src.models.ssh_key import SSHKey
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.tool_instance import ToolInstance
from src.models.tool_type import ToolType
from src.models.user import User
from src.models.user_config import UserConfig
__all__ = ["Base", "ConfigFolder", "ConfigProfile", "ConfigProfileInclude", "GitRepository", "Project", "SSHKey", "ToolInstance", "ToolType", "User", "UserConfig"]
__all__ = [
"Base",
"ConfigFolder",
"ConfigProfile",
"ConfigProfileInclude",
"GitRepository",
"Project",
"SSHKey",
"ToolDefinitionManifest",
"ToolInstance",
"ToolType",
"User",
"UserConfig",
]
apps/api/src/models/tool_definition_manifest.py
+67
View File
@@ -0,0 +1,67 @@
"""Tool Definition Manifest model."""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import Boolean, ForeignKey, JSON, String, Text
from sqlalchemy import Uuid as UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
if TYPE_CHECKING:
from src.models.user import User
class ToolDefinitionManifest(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""A declarative manifest that compiles to Dockerfile + Compose.
Can be either:
- A base definition (is_base=True) with a FROM image and common packages
- A tool definition (is_base=False) that references a base + adds specifics
"""
__tablename__ = "tool_definition_manifests"
name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
display_name: Mapped[str] = mapped_column(String(128), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
category: Mapped[str | None] = mapped_column(String(64), nullable=True)
interface_type: Mapped[str] = mapped_column(String(16), nullable=False)
# Base: either a direct image or a reference to another manifest
base_image: Mapped[str | None] = mapped_column(String(256), nullable=True)
base_definition_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("tool_definition_manifests.id"),
nullable=True,
)
base_version: Mapped[str] = mapped_column(
String(32), nullable=False, default="latest"
)
# The full manifest JSON
manifest: Mapped[dict] = mapped_column(JSON, nullable=False)
# Caches for quick inspection
dockerfile_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
compose_cache: Mapped[str | None] = mapped_column(Text, nullable=True)
# Versioning
version: Mapped[str] = mapped_column(String(32), nullable=False, default="v1")
is_base: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("users.id"),
nullable=True,
)
# Relationships
created_by: Mapped["User | None"] = relationship(
foreign_keys=[created_by_id],
)
base_definition: Mapped["ToolDefinitionManifest | None"] = relationship(
remote_side="ToolDefinitionManifest.id",
foreign_keys=[base_definition_id],
)
apps/api/src/models/tool_instance.py
+13 -29
View File
@@ -33,42 +33,26 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base):
owner_id: Mapped[uuid.UUID] = mapped_column(
UUID(), ForeignKey("users.id"), nullable=False
)
status: Mapped[str] = mapped_column(
String(50), nullable=False, default="pending"
)
container_id: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
container_name: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
compose_path: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
url: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
public_url: Mapped[str | None] = mapped_column(
String(1024), nullable=True
)
tunnel_id: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
port: Mapped[int | None] = mapped_column(
Integer, nullable=True
)
status: Mapped[str] = mapped_column(String(50), nullable=False, default="pending")
container_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
container_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
compose_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
public_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
tunnel_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
port: Mapped[int | None] = mapped_column(Integer, nullable=True)
last_started_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
last_stopped_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
probe_result: Mapped[dict | None] = mapped_column(
JSON, nullable=True
)
clone_mode: Mapped[str] = mapped_column(
String(20), nullable=False, default="mount"
manifest_compiled_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
image_tag: Mapped[str | None] = mapped_column(String(256), nullable=True)
probe_result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
clone_mode: Mapped[str] = mapped_column(String(20), nullable=False, default="mount")
branch: Mapped[str | None] = mapped_column(
String(255), nullable=True, default="main"
)
apps/api/src/models/tool_type.py
+17 -4
View File
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
if TYPE_CHECKING:
from src.models.tool_definition_manifest import ToolDefinitionManifest
from src.models.user import User
@@ -18,12 +19,19 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
category: Mapped[str] = mapped_column(String(50), nullable=False, default="other")
interface_type: Mapped[str] = mapped_column(String(20), nullable=False, default="web")
interface_type: Mapped[str] = mapped_column(
String(20), nullable=False, default="web"
)
requires_port: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
default_port: Mapped[int] = mapped_column(nullable=False)
definition_type: Mapped[str] = mapped_column(
String(20), nullable=False, default="compose"
) # "compose" or "dockerfile"
String(16), nullable=False, default="legacy"
) # "legacy" | "manifest"
manifest_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("tool_definition_manifests.id"),
nullable=True,
)
compose_template: Mapped[str | None] = mapped_column(Text, nullable=True)
dockerfile_template: Mapped[str | None] = mapped_column(Text, nullable=True)
build_context: Mapped[dict | None] = mapped_column(
@@ -31,11 +39,16 @@ class ToolType(UUIDPrimaryKeyMixin, TimestampMixin, Base):
)
readiness_probe: Mapped[dict | None] = mapped_column(JSON, nullable=True)
startup_command: Mapped[str | None] = mapped_column(Text, nullable=True)
required_variables: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
required_variables: Mapped[list[str]] = mapped_column(
JSON, default=list, nullable=False
)
created_by_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(),
ForeignKey("users.id"),
nullable=True,
)
manifest: Mapped["ToolDefinitionManifest | None"] = relationship(
foreign_keys=[manifest_id],
)
created_by: Mapped["User | None"] = relationship()
apps/api/src/services/manifest_compiler.py
+412
View File
@@ -0,0 +1,412 @@
"""Manifest compiler: transforms ToolDefinitionManifest into Dockerfile + Compose."""
import hashlib
import json
import shlex
from copy import deepcopy
from typing import Any
import yaml
def resolve_base(manifest: dict) -> dict:
"""Merge a base definition into a tool manifest.
If the manifest has base_definition_id, the base manifest is loaded
and merged. Tool-specific values override base values.
Args:
manifest: The tool manifest JSON (may reference a base)
Returns:
A fully resolved manifest with base values merged in.
"""
result = deepcopy(manifest)
base_definition_id = result.pop("base_definition_id", None)
base_version = result.pop("base_version", "latest")
if base_definition_id:
# This will be provided by the caller (they have the DB session)
# For now, we assume the manifest has been pre-resolved
# or the caller provides the base manifest separately.
pass
return result
def deep_merge(base: dict, override: dict) -> dict:
"""Deep merge two manifests. Arrays are concatenated; dicts are merged.
Args:
base: The base manifest.
override: The tool-specific overrides.
Returns:
Merged manifest.
"""
merged = deepcopy(base)
for key, value in override.items():
if key == "mounts" and isinstance(value, list):
# Concatenate mount arrays
existing = merged.get("mounts", [])
merged["mounts"] = existing + deepcopy(value)
elif key == "scripts" and isinstance(value, dict):
# Merge script categories
if "scripts" not in merged:
merged["scripts"] = {}
for script_key, script_value in value.items():
existing = merged["scripts"].get(script_key, [])
merged["scripts"][script_key] = existing + deepcopy(script_value)
elif key == "packages" and isinstance(value, dict):
# Union package arrays
if "packages" not in merged:
merged["packages"] = {}
for pkg_key, pkg_value in value.items():
if (
pkg_key in merged["packages"]
and isinstance(merged["packages"][pkg_key], list)
and isinstance(pkg_value, list)
):
merged["packages"][pkg_key] = merged["packages"][
pkg_key
] + deepcopy(pkg_value)
else:
merged["packages"][pkg_key] = deepcopy(pkg_value)
elif key == "env" and isinstance(value, dict):
# Dict merge: override wins on key conflict
if "env" not in merged:
merged["env"] = {}
merged["env"].update(deepcopy(value))
elif (
isinstance(value, dict) and key in merged and isinstance(merged[key], dict)
):
# Generic dict merge
merged[key] = {**merged[key], **deepcopy(value)}
else:
# Override entirely
merged[key] = deepcopy(value)
return merged
def compile_dockerfile(manifest: dict) -> str:
"""Compile a resolved manifest into a Dockerfile string.
Args:
manifest: Fully resolved manifest JSON.
Returns:
Dockerfile content.
"""
lines: list[str] = []
# FROM
base_image = manifest.get("base_image", "ubuntu:24.04")
lines.append(f"FROM {base_image}")
lines.append("")
# Build-time environment
env = manifest.get("env", {})
for key, value in env.items():
lines.append(f"ENV {key}={shlex.quote(value)}")
if env:
lines.append("")
# System packages (apt)
apt_packages = manifest.get("packages", {}).get("apt", [])
if apt_packages:
lines.append("RUN apt-get update && apt-get install -y \\\\")
for pkg in apt_packages[:-1]:
lines.append(f" {pkg} \\\\")
lines.append(f" {apt_packages[-1]} \\\\")
lines.append(" && rm -rf /var/lib/apt/lists/*")
lines.append("")
# Node.js
node = manifest.get("packages", {}).get("node")
if node:
version = node.get("version", "20")
lines.append(
f"RUN curl -fsSL https://deb.nodesource.com/setup_{version}.x | bash - && \\\\"
)
lines.append(" apt-get install -y nodejs && \\\\")
lines.append(" rm -rf /var/lib/apt/lists/*")
lines.append("")
# NPM global packages
npm_packages = manifest.get("packages", {}).get("npm_global", [])
if npm_packages:
pkg_list = " ".join(shlex.quote(p) for p in npm_packages)
lines.append(f"RUN npm install -g {pkg_list}")
lines.append("")
# Pip packages
pip_packages = manifest.get("packages", {}).get("pip", [])
if pip_packages:
pkg_list = " ".join(shlex.quote(p) for p in pip_packages)
lines.append(f"RUN pip install {pkg_list}")
lines.append("")
# User creation
user = manifest.get("user")
if user:
name = user["name"]
uid = user["uid"]
gid = user["gid"]
create_home = "-m " if user.get("create_home", True) else ""
shell = user.get("shell", "/bin/bash")
lines.append(f"RUN groupadd -g {gid} {name} && \\\\")
lines.append(f" useradd -u {uid} -g {gid} {create_home}-s {shell} {name}")
lines.append("")
# Build scripts
build_scripts = manifest.get("scripts", {}).get("build", [])
for script in build_scripts:
# Normalize multi-line scripts into single RUN command
stripped_lines = [
line.strip() for line in script.strip().split("\n") if line.strip()
]
if stripped_lines:
normalized = " && ".join(stripped_lines)
lines.append(f"RUN {normalized}")
if build_scripts:
lines.append("")
# Create mount target directories
mounts = manifest.get("mounts", [])
if mounts:
dirs = [mount["target"] for mount in mounts]
dir_str = " ".join(dirs)
lines.append(f"RUN mkdir -p {dir_str}")
if user:
lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
lines.append("")
# Entrypoint for startup scripts
startup_scripts = manifest.get("scripts", {}).get("startup", [])
if startup_scripts:
lines.append(
"COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint"
)
lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint")
lines.append("")
# Switch to runtime user
if user:
lines.append(f"USER {user['name']}")
lines.append(f"WORKDIR /home/{user['name']}")
lines.append("")
# Entrypoint and CMD
runtime = manifest.get("runtime", {})
if startup_scripts:
lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')
command = runtime.get("command", ["/bin/bash"])
cmd_json = json.dumps(command)
lines.append(f"CMD {cmd_json}")
return "\n".join(lines)
def compile_entrypoint(manifest: dict) -> str:
"""Generate the startup entrypoint script from startup scripts.
Args:
manifest: Fully resolved manifest JSON.
Returns:
Shell script content.
"""
lines = ["#!/bin/bash", "set -e", ""]
startup_scripts = manifest.get("scripts", {}).get("startup", [])
for script in startup_scripts:
lines.append(script)
lines.append("")
lines.append('exec "$@"')
return "\n".join(lines)
def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:
"""Compile a resolved manifest into a Docker Compose string.
Args:
manifest: Fully resolved manifest JSON.
variables: Resolved values: IMAGE_TAG, INSTANCE_NAME, REPO_PATH, etc.
Returns:
Docker Compose YAML content.
"""
runtime = manifest.get("runtime", {})
user = manifest.get("user")
interface_type = manifest["interface_type"]
service: dict[str, Any] = {
"image": variables["IMAGE_TAG"],
"container_name": variables["INSTANCE_NAME"],
"restart": "unless-stopped",
}
# Terminal-specific fields
if runtime.get("stdin_open", False):
service["stdin_open"] = True
if runtime.get("tty", False):
service["tty"] = True
if runtime.get("working_dir"):
service["working_dir"] = runtime["working_dir"]
# User override
if user:
service["user"] = f"{user['uid']}:{user['gid']}"
# Ports for web tools
default_port = manifest.get("default_port")
if interface_type == "web" and default_port:
service["ports"] = [f"{variables['TOOL_PORT']}:{default_port}"]
# Environment
env = manifest.get("env", {})
if env:
service["environment"] = dict(env)
# Merge extra env from config
extra_env = variables.get("EXTRA_ENV", {})
if extra_env:
if "environment" not in service:
service["environment"] = {}
service["environment"].update(extra_env)
# Volumes from mount schema
volumes = []
for mount in manifest.get("mounts", []):
source = resolve_mount_source(mount, variables)
if not source:
continue
target = mount["target"]
readonly = ":ro" if mount.get("readonly", False) else ""
volumes.append(f"{source}:{target}{readonly}")
# Append extra volumes from tool config / config profile
for vol in variables.get("EXTRA_VOLUMES", []):
vol_str = f"{vol['source']}:{vol['target']}"
if vol.get("readonly"):
vol_str += ":ro"
volumes.append(vol_str)
if volumes:
service["volumes"] = volumes
compose = {"services": {"app": service}}
return yaml.dump(compose, default_flow_style=False)
def resolve_mount_source(mount: dict, variables: dict[str, Any]) -> str:
"""Resolve a mount's source_type to an actual host path.
Args:
mount: Mount definition from manifest.
variables: Resolved variables dict.
Returns:
Host path string, or empty string if unresolved.
"""
source_type = mount.get("source_type", "host_path")
if source_type == "repo":
return variables.get("REPO_PATH", "")
elif source_type == "ssh_key":
return variables.get("SSH_PATH", "")
elif source_type == "instance":
instance_dir = variables.get("INSTANCE_DIR", "")
mount_name = mount.get("name", "unknown")
return f"{instance_dir}/mounts/{mount_name}"
elif source_type == "git_mount":
ref = mount.get("git_mount_ref", "default")
return variables.get(f"GIT_MOUNT_{ref}", "")
elif source_type == "host_path":
return mount.get("source", "")
return ""
def compute_image_tag(tool_name: str, manifest: dict) -> str:
"""Compute a deterministic image tag from manifest content.
Args:
tool_name: Human-readable tool name.
manifest: Fully resolved manifest JSON.
Returns:
Docker image tag string.
"""
# Canonicalize: sort keys, stable JSON
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8]
safe_name = tool_name.lower().replace(" ", "-").replace("_", "-")
return f"headquarter/{safe_name}-{hash_suffix}:latest"
def merge_with_config(
manifest: dict, tool_configs: list[dict], profile: dict | None = None
) -> dict:
"""Merge ToolConfig and ConfigProfile overrides into a manifest.
Args:
manifest: Base manifest from tool definition.
tool_configs: List of ToolConfig records.
profile: Resolved ConfigProfile (optional).
Returns:
Manifest with overrides applied.
"""
result = deepcopy(manifest)
# Apply ToolConfigs
extra_env: dict[str, str] = {}
extra_volumes: list[dict] = []
for config in tool_configs:
if config.get("config_type") == "env":
extra_env[config["key"]] = config["value"]
elif config.get("config_type") == "file" and config.get("file_path"):
# Files are handled outside the manifest (written to instance dir)
pass
if config.get("port_override"):
result["default_port"] = config["port_override"]
if config.get("start_command"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["command"] = config["start_command"].split()
if config.get("working_directory"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["working_dir"] = config["working_directory"]
if config.get("environment_variables"):
extra_env.update(config["environment_variables"])
if config.get("volumes"):
extra_volumes.extend(config["volumes"])
# Apply ConfigProfile
if profile:
if profile.get("environment_variables"):
extra_env.update(profile["environment_variables"])
if profile.get("mounts"):
extra_volumes.extend(profile["mounts"])
# Profile hints override everything
hints = profile.get("hints", {})
if hints.get("start_command"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["command"] = hints["start_command"].split()
if hints.get("working_directory"):
result["runtime"] = result.get("runtime", {})
result["runtime"]["working_dir"] = hints["working_directory"]
if hints.get("port_override"):
result["default_port"] = hints["port_override"]
# Store merged extras for the compose compiler
result["_extra_env"] = extra_env
result["_extra_volumes"] = extra_volumes
return result
apps/api/src/services/permission_fixer.py
+164
View File
@@ -0,0 +1,164 @@
"""Permission fixer: applies mount permission policies post-start."""
import logging
import subprocess
from typing import Any
logger = logging.getLogger(__name__)
def apply_mount_permissions(
container_id: str,
mounts: list[dict],
timeout: int = 10,
) -> list[dict[str, Any]]:
"""Apply permission policies to mounted directories in a running container.
Runs `chown`, `chmod`, and file-mode fixes for each mount that declares
an owner, mode, or file_mode. Requires the container to have a root user.
Args:
container_id: Docker container ID or name.
mounts: List of mount definitions from the manifest.
timeout: Max seconds per docker exec command.
Returns:
List of result dicts: [{mount_name, success, error}]
"""
results = []
for mount in mounts:
name = mount.get("name", "unknown")
target = mount["target"]
owner = mount.get("owner")
mode = mount.get("mode")
file_mode = mount.get("file_mode")
result: dict[str, Any] = {
"mount_name": name,
"success": True,
"error": None,
}
# Skip if no permission policy defined
if not owner and not mode and not file_mode:
results.append(result)
continue
try:
if owner:
_run_in_container(
container_id,
["chown", "-R", f"{owner}:{owner}", target],
timeout,
)
logger.debug(
"Applied owner %s to %s in container %s",
owner,
target,
container_id,
)
if mode and result["success"]:
_run_in_container(
container_id,
["chmod", mode, target],
timeout,
)
logger.debug(
"Applied mode %s to %s in container %s",
mode,
target,
container_id,
)
if file_mode and result["success"]:
_run_in_container(
container_id,
[
"sh",
"-c",
f"find {target} -type f -exec chmod {file_mode} {{}} +",
],
timeout,
)
logger.debug(
"Applied file_mode %s to files in %s in container %s",
file_mode,
target,
container_id,
)
except PermissionFixError as exc:
result["success"] = False
result["error"] = str(exc)
logger.warning(
"Permission fix failed for mount %s (target=%s): %s",
name,
target,
exc,
)
results.append(result)
return results
class PermissionFixError(Exception):
"""Raised when a permission fix command fails."""
pass
def _run_in_container(
container_id: str,
command: list[str],
timeout: int,
) -> None:
"""Run a command inside a container as root.
Args:
container_id: Docker container ID or name.
command: Command + args to execute.
timeout: Max seconds to wait.
Raises:
PermissionFixError: If the command fails or times out.
"""
cmd = ["docker", "exec", "--user", "root", container_id] + command
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
raise PermissionFixError(
f"Command timed out after {timeout}s: {' '.join(command)}"
)
except FileNotFoundError:
raise PermissionFixError(f"Docker command not found: {' '.join(command)}")
if result.returncode != 0:
raise PermissionFixError(
f"Command failed (rc={result.returncode}): {result.stderr.strip()}"
)
def check_root_user_available(container_id: str, timeout: int = 5) -> bool:
"""Check if the container has a root user we can exec as.
Args:
container_id: Docker container ID or name.
timeout: Max seconds to wait.
Returns:
True if root user exists and is usable.
"""
try:
_run_in_container(container_id, ["id", "root"], timeout)
return True
except PermissionFixError:
return False
apps/api/tests/unit/test_manifest_compiler.py
+315
View File
@@ -0,0 +1,315 @@
"""Unit tests for the manifest compiler."""
import pytest
from src.services.manifest_compiler import (
compile_compose,
compile_dockerfile,
compile_entrypoint,
compute_image_tag,
deep_merge,
merge_with_config,
resolve_base,
)
class TestResolveBase:
"""Tests for resolve_base."""
def test_returns_manifest_unchanged_when_no_base(self) -> None:
manifest = {"name": "test", "base_image": "ubuntu:24.04"}
result = resolve_base(manifest)
assert result["name"] == "test"
assert "base_definition_id" not in result
class TestDeepMerge:
"""Tests for deep_merge."""
def test_packages_are_unioned(self) -> None:
base = {"packages": {"apt": ["curl", "git"]}}
override = {"packages": {"apt": ["neovim"]}}
result = deep_merge(base, override)
assert result["packages"]["apt"] == ["curl", "git", "neovim"]
def test_node_version_overrides(self) -> None:
base = {"packages": {"node": {"version": "18"}}}
override = {"packages": {"node": {"version": "20"}}}
result = deep_merge(base, override)
assert result["packages"]["node"]["version"] == "20"
def test_env_is_merged_with_override_winning(self) -> None:
base = {"env": {"FOO": "base", "BAR": "base"}}
override = {"env": {"FOO": "override"}}
result = deep_merge(base, override)
assert result["env"]["FOO"] == "override"
assert result["env"]["BAR"] == "base"
def test_build_scripts_are_concatenated(self) -> None:
base = {"scripts": {"build": ["echo base"]}}
override = {"scripts": {"build": ["echo override"]}}
result = deep_merge(base, override)
assert result["scripts"]["build"] == ["echo base", "echo override"]
def test_mounts_are_concatenated(self) -> None:
base = {"mounts": [{"name": "base-mount", "target": "/base"}]}
override = {"mounts": [{"name": "tool-mount", "target": "/tool"}]}
result = deep_merge(base, override)
assert len(result["mounts"]) == 2
def test_user_is_overridden_entirely(self) -> None:
base = {"user": {"name": "base", "uid": 1000}}
override = {"user": {"name": "tool", "uid": 1001}}
result = deep_merge(base, override)
assert result["user"]["name"] == "tool"
assert result["user"]["uid"] == 1001
class TestCompileDockerfile:
"""Tests for compile_dockerfile."""
def test_includes_from(self) -> None:
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
df = compile_dockerfile(manifest)
assert "FROM ubuntu:24.04" in df
def test_installs_apt_packages(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"apt": ["curl", "git"]},
}
df = compile_dockerfile(manifest)
assert "apt-get install -y" in df
assert "curl" in df
assert "git" in df
assert "rm -rf /var/lib/apt/lists/*" in df
def test_installs_node(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"node": {"version": "20"}},
}
df = compile_dockerfile(manifest)
assert "nodesource.com/setup_20.x" in df
def test_installs_npm_global(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"packages": {"npm_global": ["@scope/pkg"]},
}
df = compile_dockerfile(manifest)
assert "npm install -g @scope/pkg" in df
def test_creates_user(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"user": {"name": "dev", "uid": 1001, "gid": 1001},
}
df = compile_dockerfile(manifest)
assert "groupadd -g 1001 dev" in df
assert "useradd -u 1001 -g 1001" in df
assert "USER dev" in df
def test_build_scripts_as_run_commands(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"scripts": {"build": ["echo hello", "echo world"]},
}
df = compile_dockerfile(manifest)
assert "RUN echo hello" in df
assert "RUN echo world" in df
def test_creates_mount_directories(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"user": {"name": "dev", "uid": 1001, "gid": 1001},
"mounts": [
{"name": "ws", "target": "/workspace"},
{"name": "cfg", "target": "/config"},
],
}
df = compile_dockerfile(manifest)
assert "mkdir -p /workspace /config" in df
assert "chown -R dev:dev /workspace /config" in df
def test_entrypoint_for_startup_scripts(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"scripts": {"startup": ["echo start"]},
}
df = compile_dockerfile(manifest)
assert 'ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]' in df
def test_cmd_from_runtime(self) -> None:
manifest = {
"base_image": "ubuntu:24.04",
"name": "test",
"runtime": {"command": ["/bin/bash", "-il"]},
}
df = compile_dockerfile(manifest)
assert 'CMD ["/bin/bash", "-il"]' in df
def test_default_cmd_when_no_runtime(self) -> None:
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
df = compile_dockerfile(manifest)
assert 'CMD ["/bin/bash"]' in df
class TestCompileEntrypoint:
"""Tests for compile_entrypoint."""
def test_includes_shebang_and_set_e(self) -> None:
manifest = {"scripts": {"startup": ["echo hello"]}}
ep = compile_entrypoint(manifest)
assert "#!/bin/bash" in ep
assert "set -e" in ep
def test_includes_startup_scripts(self) -> None:
manifest = {"scripts": {"startup": ["echo hello", "echo world"]}}
ep = compile_entrypoint(manifest)
assert "echo hello" in ep
assert "echo world" in ep
def test_ends_with_exec(self) -> None:
manifest: dict = {"scripts": {"startup": []}}
ep = compile_entrypoint(manifest)
assert 'exec "$@"' in ep
class TestCompileCompose:
"""Tests for compile_compose."""
def test_includes_image_and_container_name(self) -> None:
manifest = {"name": "test", "interface_type": "terminal"}
vars_dict = {"IMAGE_TAG": "test:v1", "INSTANCE_NAME": "test-1"}
compose = compile_compose(manifest, vars_dict)
assert "image: test:v1" in compose
assert "container_name: test-1" in compose
def test_terminal_fields(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"runtime": {"stdin_open": True, "tty": True, "working_dir": "/workspace"},
}
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
assert "stdin_open: true" in compose
assert "tty: true" in compose
assert "working_dir: /workspace" in compose
def test_web_ports(self) -> None:
manifest = {
"name": "test",
"interface_type": "web",
"default_port": 8080,
}
compose = compile_compose(
manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n", "TOOL_PORT": "3000"}
)
assert "3000:8080" in compose
def test_user_override(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"user": {"uid": 1001, "gid": 1001},
}
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
assert "user: 1001:1001" in compose
def test_mounts_resolved(self) -> None:
manifest = {
"name": "test",
"interface_type": "terminal",
"mounts": [
{"name": "ws", "target": "/workspace", "source_type": "repo"},
{
"name": "ssh",
"target": "/home/user/.ssh",
"source_type": "ssh_key",
"readonly": True,
},
],
}
compose = compile_compose(
manifest,
{
"IMAGE_TAG": "t",
"INSTANCE_NAME": "n",
"REPO_PATH": "/repos/myrepo",
"SSH_PATH": "/keys/ssh",
},
)
assert "/repos/myrepo:/workspace" in compose
assert "/keys/ssh:/home/user/.ssh:ro" in compose
def test_extra_volumes_appended(self) -> None:
manifest = {"name": "test", "interface_type": "terminal"}
compose = compile_compose(
manifest,
{
"IMAGE_TAG": "t",
"INSTANCE_NAME": "n",
"EXTRA_VOLUMES": [{"source": "/host/x", "target": "/container/x"}],
},
)
assert "/host/x:/container/x" in compose
class TestComputeImageTag:
"""Tests for compute_image_tag."""
def test_is_deterministic(self) -> None:
manifest = {"name": "test", "packages": {"apt": ["curl"]}}
tag1 = compute_image_tag("My Tool", manifest)
tag2 = compute_image_tag("My Tool", manifest)
assert tag1 == tag2
def test_changes_with_content(self) -> None:
manifest1 = {"name": "test", "packages": {"apt": ["curl"]}}
manifest2 = {"name": "test", "packages": {"apt": ["wget"]}}
tag1 = compute_image_tag("test", manifest1)
tag2 = compute_image_tag("test", manifest2)
assert tag1 != tag2
def test_lowercases_name(self) -> None:
manifest = {"name": "test"}
tag = compute_image_tag("My Tool", manifest)
assert "my-tool" in tag
def test_valid_docker_reference(self) -> None:
manifest = {"name": "test"}
tag = compute_image_tag("test", manifest)
assert tag.startswith("headquarter/test-")
assert tag.endswith(":latest")
class TestMergeWithConfig:
"""Tests for merge_with_config."""
def test_applies_tool_config_env(self) -> None:
manifest = {"name": "test"}
configs = [
{"config_type": "env", "key": "FOO", "value": "bar"},
]
result = merge_with_config(manifest, configs)
assert result["_extra_env"]["FOO"] == "bar"
def test_applies_port_override(self) -> None:
manifest = {"name": "test", "default_port": 8080}
configs = [{"port_override": 3000}]
result = merge_with_config(manifest, configs)
assert result["default_port"] == 3000
def test_applies_start_command(self) -> None:
manifest = {"name": "test", "runtime": {"command": ["/bin/bash"]}}
configs = [{"start_command": "/bin/sh"}]
result = merge_with_config(manifest, configs)
assert result["runtime"]["command"] == ["/bin/sh"]
apps/api/tests/unit/test_permission_fixer.py
+145
View File
@@ -0,0 +1,145 @@
"""Unit tests for the permission fixer."""
from unittest.mock import MagicMock, patch
import pytest
from src.services.permission_fixer import (
PermissionFixError,
apply_mount_permissions,
check_root_user_available,
_run_in_container,
)
class TestApplyMountPermissions:
"""Tests for apply_mount_permissions."""
@patch("src.services.permission_fixer._run_in_container")
def test_applies_chown_when_owner_declared(self, mock_run) -> None:
mounts = [
{"name": "workspace", "target": "/workspace", "owner": "user"},
]
results = apply_mount_permissions("abc123", mounts)
assert len(results) == 1
assert results[0]["mount_name"] == "workspace"
assert results[0]["success"] is True
mock_run.assert_called_once()
args = mock_run.call_args[0]
assert args[0] == "abc123"
assert args[1] == ["chown", "-R", "user:user", "/workspace"]
@patch("src.services.permission_fixer._run_in_container")
def test_applies_chmod_when_mode_declared(self, mock_run) -> None:
mounts = [
{"name": "ssh", "target": "/home/user/.ssh", "mode": "0700"},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is True
# Only chmod called (no owner, so no chown)
assert mock_run.call_count == 1
chmod_call = mock_run.call_args_list[0]
assert chmod_call[0][1] == ["chmod", "0700", "/home/user/.ssh"]
@patch("src.services.permission_fixer._run_in_container")
def test_applies_file_mode_when_declared(self, mock_run) -> None:
mounts = [
{
"name": "ssh",
"target": "/home/user/.ssh",
"file_mode": "0600",
},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is True
# Only file_mode called (no owner, no mode)
assert mock_run.call_count == 1
file_mode_call = mock_run.call_args_list[0]
assert file_mode_call[0][1][0] == "sh"
assert (
"find /home/user/.ssh -type f -exec chmod 0600" in file_mode_call[0][1][2]
)
@patch("src.services.permission_fixer._run_in_container")
def test_skips_mount_with_no_policy(self, mock_run) -> None:
mounts = [
{"name": "workspace", "target": "/workspace", "writable": True},
]
results = apply_mount_permissions("abc123", mounts)
assert len(results) == 1
assert results[0]["success"] is True
mock_run.assert_not_called()
@patch("src.services.permission_fixer._run_in_container")
def test_reports_failure_on_command_error(self, mock_run) -> None:
mock_run.side_effect = PermissionFixError("chown failed")
mounts = [
{"name": "workspace", "target": "/workspace", "owner": "user"},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is False
assert "chown failed" in results[0]["error"]
@patch("src.services.permission_fixer._run_in_container")
def test_stops_on_first_failure(self, mock_run) -> None:
"""If chown fails, chmod and file_mode should not run."""
mock_run.side_effect = PermissionFixError("chown failed")
mounts = [
{
"name": "workspace",
"target": "/workspace",
"owner": "user",
"mode": "0755",
"file_mode": "0644",
},
]
results = apply_mount_permissions("abc123", mounts)
assert results[0]["success"] is False
assert mock_run.call_count == 1 # Only chown attempted
class TestRunInContainer:
"""Tests for _run_in_container."""
@patch("subprocess.run")
def test_success(self, mock_run) -> None:
mock_run.return_value = MagicMock(returncode=0, stderr="")
_run_in_container("abc123", ["echo", "hello"], 10)
mock_run.assert_called_once()
cmd = mock_run.call_args[0][0]
assert cmd == ["docker", "exec", "--user", "root", "abc123", "echo", "hello"]
@patch("subprocess.run")
def test_failure_raises(self, mock_run) -> None:
mock_run.return_value = MagicMock(returncode=1, stderr="permission denied")
with pytest.raises(PermissionFixError, match="permission denied"):
_run_in_container("abc123", ["chown", "x"], 10)
@patch("subprocess.run")
def test_timeout_raises(self, mock_run) -> None:
import subprocess
mock_run.side_effect = subprocess.TimeoutExpired(cmd=["docker"], timeout=10)
with pytest.raises(PermissionFixError, match="timed out"):
_run_in_container("abc123", ["chown", "x"], 10)
class TestCheckRootUserAvailable:
"""Tests for check_root_user_available."""
@patch("src.services.permission_fixer._run_in_container")
def test_returns_true_when_root_exists(self, mock_run) -> None:
assert check_root_user_available("abc123") is True
@patch("src.services.permission_fixer._run_in_container")
def test_returns_false_when_root_missing(self, mock_run) -> None:
mock_run.side_effect = PermissionFixError("no such user")
assert check_root_user_available("abc123") is False