# Design: Tool Definition Manifest System ## Status **Phase:** design **Date:** 2026-05-28 **Owner:** el Gentleman **Based on:** Spec `tool-definition-manifest` --- ## Architecture Overview ``` ┌─────────────────────────────────────────────────────────────────┐ │ Tool Workshop (Frontend) │ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │ │ │ Base Image │ │ Packages │ │ Mount Schema Designer │ │ │ │ Selector │ │ Editors │ │ (target, owner, mode) │ │ │ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ │ ┌─────────────────────────────────────────────────────────────┐│ │ │ Live Preview: Dockerfile + Compose ││ │ └─────────────────────────────────────────────────────────────┘│ └─────────────────────────────────────────────────────────────────┘ │ ▼ POST /tool-definitions ┌─────────────────────────────────────────────────────────────────┐ │ API Backend │ │ ┌─────────────────┐ ┌──────────────────┐ ┌────────────┐ │ │ │ ManifestSchema │───▶│ ManifestCompiler │───▶│ LiveBuild │ │ │ │ (validation) │ │ (Dockerfile + │ │ (optional) │ │ │ │ │ │ Compose gen) │ │ │ │ │ └─────────────────┘ └──────────────────┘ └────────────┘ │ │ │ │ ▼ save to DB │ ┌─────────────────────────────────────────────────────────────┐│ │ │ ToolDefinitionManifest (JSONB in PostgreSQL) ││ │ └─────────────────────────────────────────────────────────────┘│ └─────────────────────────────────────────────────────────────────┘ │ ▼ POST /instances/{id}/start ┌─────────────────────────────────────────────────────────────────┐ │ Instance Startup Flow │ │ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ │ │ Resolve │──▶│ Compile │──▶│ Build Image │ │ │ │ Manifest │ │ to Dockerfile│ │ (docker build) │ │ │ │ (base merge)│ │ + Compose │ │ │ │ │ └─────────────┘ └──────────────┘ └──────────────────────┘ │ │ │ │ ▼ │ ┌─────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ │ │ Permission │◀──│ docker comp. │◀──│ Generate Compose │ │ │ │ Fixer │ │ up │ │ (mount resolution) │ │ │ │ (post-start)│ │ │ │ │ │ │ └─────────────┘ └──────────────┘ └──────────────────────┘ │ └─────────────────────────────────────────────────────────────────┘ ``` --- ## Manifest JSON Schema ```json { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "required": ["name", "interface_type"], "oneOf": [ {"required": ["base_image"]}, {"required": ["base_definition_id"]} ], "properties": { "name": {"type": "string", "pattern": "^[a-z0-9-]+$", "maxLength": 64}, "display_name": {"type": "string", "maxLength": 128}, "description": {"type": "string"}, "category": {"type": "string", "maxLength": 64}, "interface_type": {"type": "string", "enum": ["web", "terminal"]}, "base_image": {"type": "string", "maxLength": 256}, "base_definition_id": {"type": "string", "format": "uuid"}, "base_version": {"type": "string", "maxLength": 32, "default": "latest"}, "packages": { "type": "object", "properties": { "apt": {"type": "array", "items": {"type": "string"}}, "node": { "type": "object", "properties": { "version": {"type": "string", "pattern": "^\\d+$"} }, "required": ["version"] }, "npm_global": {"type": "array", "items": {"type": "string"}}, "pip": {"type": "array", "items": {"type": "string"}} } }, "user": { "type": "object", "properties": { "name": {"type": "string", "maxLength": 32}, "uid": {"type": "integer", "minimum": 1, "maximum": 65535}, "gid": {"type": "integer", "minimum": 1, "maximum": 65535}, "create_home": {"type": "boolean", "default": true}, "shell": {"type": "string", "maxLength": 64, "default": "/bin/bash"} }, "required": ["name", "uid", "gid"] }, "env": {"type": "object", "additionalProperties": {"type": "string"}}, "scripts": { "type": "object", "properties": { "build": {"type": "array", "items": {"type": "string"}}, "startup": {"type": "array", "items": {"type": "string"}} } }, "mounts": { "type": "array", "items": { "type": "object", "required": ["name", "target", "source_type"], "properties": { "name": {"type": "string", "maxLength": 64}, "target": {"type": "string", "maxLength": 256}, "source_type": {"type": "string", "enum": ["repo", "ssh_key", "instance", "git_mount", "host_path"]}, "writable": {"type": "boolean", "default": true}, "owner": {"type": "string", "maxLength": 32}, "mode": {"type": "string", "pattern": "^[0-7]{3,4}$"}, "file_mode": {"type": "string", "pattern": "^[0-7]{3,4}$"}, "readonly": {"type": "boolean", "default": false}, "git_mount_ref": {"type": "string", "maxLength": 64} } } }, "runtime": { "type": "object", "properties": { "command": {"type": "array", "items": {"type": "string"}}, "stdin_open": {"type": "boolean", "default": false}, "tty": {"type": "boolean", "default": false}, "working_dir": {"type": "string", "maxLength": 256} } } } } ``` --- ## Manifest Compiler Algorithm ### Step 1: Resolve Base ```python def resolve_base(manifest: dict) -> dict: """Merge base definition into the manifest.""" if manifest.get("base_definition_id"): base = load_base_definition(manifest["base_definition_id"], manifest.get("base_version", "latest")) # Deep merge: base first, then tool-specific overrides merged = deep_merge(base, manifest) # Remove base fields from the merged result merged.pop("base_definition_id", None) merged.pop("base_version", None) return merged return manifest ``` Merge rules: - `packages`: Union arrays (base apt + tool apt = combined apt) - `env`: Tool overrides base (dict merge, tool wins on key conflict) - `scripts.build`: Concatenate arrays (base scripts first, then tool) - `scripts.startup`: Concatenate arrays - `user`: Tool overrides base entirely - `mounts`: Concatenate arrays - `runtime`: Tool overrides base (dict merge) ### Step 2: Generate Dockerfile ```python def compile_dockerfile(manifest: dict) -> str: """Compile a resolved manifest to a Dockerfile string.""" lines = [] # FROM lines.append(f"FROM {manifest['base_image']}") lines.append("") # ENV (build-time) for key, value in manifest.get("env", {}).items(): lines.append(f"ENV {key}={shlex.quote(value)}") if manifest.get("env"): lines.append("") # System packages (apt) apt_packages = manifest.get("packages", {}).get("apt", []) if apt_packages: lines.append("RUN apt-get update && apt-get install -y \\\\") for pkg in apt_packages: lines.append(f" {pkg} \\\\") lines.append(" && rm -rf /var/lib/apt/lists/*") lines.append("") # Node.js node = manifest.get("packages", {}).get("node") if node: lines.append( f"RUN curl -fsSL https://deb.nodesource.com/setup_{node['version']}.x | bash - && \\\\") lines.append(" apt-get install -y nodejs && \\\\") lines.append(" rm -rf /var/lib/apt/lists/*") lines.append("") # NPM global npm_packages = manifest.get("packages", {}).get("npm_global", []) if npm_packages: pkg_list = " ".join(shlex.quote(p) for p in npm_packages) lines.append(f"RUN npm install -g {pkg_list}") lines.append("") # Pip pip_packages = manifest.get("packages", {}).get("pip", []) if pip_packages: pkg_list = " ".join(shlex.quote(p) for p in pip_packages) lines.append(f"RUN pip install {pkg_list}") lines.append("") # User creation user = manifest.get("user") if user: lines.append( f"RUN groupadd -g {user['gid']} {user['name']} && \\\\") lines.append( f" useradd -u {user['uid']} -g {user['gid']} " f"{'-m ' if user.get('create_home', True) else ''}" f"-s {user['shell']} {user['name']}") lines.append("") # Build scripts build_scripts = manifest.get("scripts", {}).get("build", []) for script in build_scripts: # Each script block becomes one RUN command # Normalize multi-line scripts normalized = " && ".join(line.strip() for line in script.strip().split("\n") if line.strip()) lines.append(f"RUN {normalized}") if build_scripts: lines.append("") # Create mount target directories and pre-set ownership mounts = manifest.get("mounts", []) if mounts: dirs = [] for mount in mounts: dirs.append(mount["target"]) if dirs: dir_str = " ".join(dirs) lines.append(f"RUN mkdir -p {dir_str}") if user: lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}") lines.append("") # Entrypoint for startup scripts startup_scripts = manifest.get("scripts", {}).get("startup", []) if startup_scripts: lines.append("COPY .headquarter/entrypoint.sh /usr/local/bin/headquarter-entrypoint") lines.append("RUN chmod +x /usr/local/bin/headquarter-entrypoint") lines.append("") # Switch to user if user: lines.append(f"USER {user['name']}") lines.append(f"WORKDIR /home/{user['name']}") lines.append("") # Entrypoint and CMD runtime = manifest.get("runtime", {}) if startup_scripts: lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]') command = runtime.get("command", ["/bin/bash"]) cmd_json = json.dumps(command) lines.append(f"CMD {cmd_json}") return "\n".join(lines) ``` ### Step 3: Generate Entrypoint Script ```python def compile_entrypoint(manifest: dict) -> str: """Generate the startup entrypoint script.""" lines = ["#!/bin/bash", "set -e", ""] startup_scripts = manifest.get("scripts", {}).get("startup", []) for script in startup_scripts: lines.append(script) lines.append("") lines.append('exec "$@"') return "\n".join(lines) ``` ### Step 4: Generate Compose ```python def compile_compose(manifest: dict, variables: dict) -> str: """Compile a resolved manifest to a Docker Compose string.""" runtime = manifest.get("runtime", {}) user = manifest.get("user") interface_type = manifest["interface_type"] service = { "image": variables["IMAGE_TAG"], "container_name": variables["INSTANCE_NAME"], "restart": "unless-stopped", } # Terminal-specific fields if runtime.get("stdin_open", False): service["stdin_open"] = True if runtime.get("tty", False): service["tty"] = True if runtime.get("working_dir"): service["working_dir"] = runtime["working_dir"] # User override in compose (helps with permission consistency) if user: service["user"] = f"{user['uid']}:{user['gid']}" # Ports for web tools if interface_type == "web" and manifest.get("default_port"): service["ports"] = [f"{variables['TOOL_PORT']}:{manifest['default_port']}"] # Environment env = manifest.get("env", {}) if env: service["environment"] = env # Volumes from mounts volumes = [] for mount in manifest.get("mounts", []): source = resolve_mount_source(mount, variables) target = mount["target"] readonly = ":ro" if mount.get("readonly", False) else "" volumes.append(f"{source}:{target}{readonly}") # Append extra volumes from tool config / config profile for vol in variables.get("EXTRA_VOLUMES", []): vol_str = f"{vol['source']}:{vol['target']}" if vol.get("readonly"): vol_str += ":ro" volumes.append(vol_str) if volumes: service["volumes"] = volumes compose = { "services": {"app": service} } return yaml.dump(compose, default_flow_style=False) ``` ### Step 5: Image Tag Hash ```python import hashlib import json def compute_image_tag(tool_name: str, manifest: dict) -> str: """Deterministic image tag from manifest content.""" # Normalize: sort keys, stable JSON canonical = json.dumps(manifest, sort_keys=True, separators=(',', ':')) hash_suffix = hashlib.sha256(canonical.encode()).hexdigest()[:8] safe_name = tool_name.lower().replace(" ", "-").replace("_", "-") return f"headquarter/{safe_name}-{hash_suffix}:latest" ``` --- ## Mount Resolution Each `source_type` resolves differently at instance creation time: | source_type | Resolution | Example | |-------------|------------|---------| | `repo` | `{repo_path}` (mount or clone) | `/data/repos/headquarter` | | `ssh_key` | `{instance_dir}/.ssh` | `/data/instances/.../.ssh` | | `instance` | `{instance_dir}/{name}` | `/data/instances/.../mounts/tmp_.pi_agents` | | `git_mount` | Resolved from ConfigProfile git_mounts | `/data/instances/.../git-mounts/...` | | `host_path` | Literal host path | `/var/run/docker.sock` | ```python def resolve_mount_source(mount: dict, variables: dict) -> str: source_type = mount["source_type"] if source_type == "repo": return variables["REPO_PATH"] elif source_type == "ssh_key": return variables["SSH_PATH"] elif source_type == "instance": instance_dir = variables["INSTANCE_DIR"] mount_name = mount["name"] return f"{instance_dir}/mounts/{mount_name}" elif source_type == "git_mount": ref = mount.get("git_mount_ref", "default") return variables.get(f"GIT_MOUNT_{ref}", "") elif source_type == "host_path": return mount.get("source", "") else: raise ValueError(f"Unknown source_type: {source_type}") ``` --- ## Permission Fixer (Post-Start) ```python def apply_mount_permissions( container_id: str, mounts: list[dict], timeout: int = 10 ) -> list[dict]: """Apply permission policies to mounted directories in a running container. Returns a list of results: [{mount_name, success, error}] """ results = [] for mount in mounts: name = mount["name"] target = mount["target"] owner = mount.get("owner") mode = mount.get("mode") file_mode = mount.get("file_mode") result = {"mount_name": name, "success": True, "error": None} try: if owner: proc = subprocess.run( ["docker", "exec", container_id, "chown", "-R", f"{owner}:{owner}", target], capture_output=True, text=True, timeout=timeout ) if proc.returncode != 0: result["success"] = False result["error"] = f"chown failed: {proc.stderr}" if mode and result["success"]: proc = subprocess.run( ["docker", "exec", container_id, "chmod", mode, target], capture_output=True, text=True, timeout=timeout ) if proc.returncode != 0: result["success"] = False result["error"] = f"chmod failed: {proc.stderr}" if file_mode and result["success"]: proc = subprocess.run( ["docker", "exec", container_id, "sh", "-c", f"find {target} -type f -exec chmod {file_mode} {{}} +"], capture_output=True, text=True, timeout=timeout ) if proc.returncode != 0: result["success"] = False result["error"] = f"file_mode chmod failed: {proc.stderr}" except subprocess.TimeoutExpired: result["success"] = False result["error"] = "Permission fix timed out" results.append(result) return results ``` **Important:** The permission fixer checks if `root` exists in the container before running. If the container has no `root` user (e.g., distroless images), it logs a warning and skips. --- ## Modified Startup Flow The `start_instance` function in `tool_instances.py` is modified as follows: ```python async def start_instance(...): # ... existing validation ... tool_type = await session.get(ToolType, instance.tool_type_id) if tool_type.definition_type == "manifest" and tool_type.manifest_id: # NEW: Manifest-based startup flow manifest = await load_manifest(session, tool_type.manifest_id) # Merge base definition resolved = resolve_base(manifest) # Merge tool configs and config profile resolved = apply_tool_configs(resolved, configs) resolved = apply_config_profile(resolved, profile) # Compile dockerfile = compile_dockerfile(resolved) entrypoint = compile_entrypoint(resolved) image_tag = compute_image_tag(tool_type.name, resolved) # Build image build_context = { "Dockerfile": dockerfile, ".headquarter/entrypoint.sh": entrypoint, } returncode, stdout, stderr = build_image( instance_dir=instance_dir, dockerfile=dockerfile, # The Dockerfile references entrypoint.sh tag=image_tag, build_context=build_context, ) # Generate compose with resolved variables variables = { "IMAGE_TAG": image_tag, "INSTANCE_NAME": instance.name.lower(), "REPO_PATH": repo_path, "SSH_PATH": ssh_dir, "INSTANCE_DIR": instance_dir, # ... git mount resolutions from config profile ... } compose_content = compile_compose(resolved, variables) write_compose_file(instance_dir, compose_content) # Store image tag for reuse instance.image_tag = image_tag else: # LEGACY: Existing dockerfile_template / compose_template flow ... # ... docker compose up ... # ... wait for running ... # NEW: Apply mount permissions post-start if tool_type.definition_type == "manifest": manifest = await load_manifest(session, tool_type.manifest_id) resolved = resolve_base(manifest) permission_results = apply_mount_permissions( instance.container_id, resolved.get("mounts", []) ) for result in permission_results: if not result["success"]: logger.warning( "Permission fix failed for mount %s: %s", result["mount_name"], result["error"] ) # ... readiness probe ... # ... tunnel creation ... ``` --- ## File Layout ``` apps/api/src/ ├── models/ │ ├── tool_definition_manifest.py # NEW: SQLAlchemy model │ └── tool_type.py # MOD: add manifest_id, definition_type ├── services/ │ ├── manifest_compiler.py # NEW: compile_dockerfile, compile_compose, resolve_base │ ├── permission_fixer.py # NEW: apply_mount_permissions │ └── docker_build.py # MOD: support build_context files ├── api/ │ ├── tool_definitions.py # NEW: CRUD + compile endpoints │ └── tool_instances.py # MOD: manifest-based startup flow ├── schemas/ │ └── manifest_schema.py # NEW: JSON Schema + Pydantic validators └── alembic/versions/ └── 20260528_add_tool_definition_manifests.py # NEW: migration ``` --- ## Testing Strategy ### Unit Tests | Module | Tests | Coverage | |--------|-------|----------| | `manifest_compiler.py` | Dockerfile generation for all package managers, base merging, entrypoint generation | All branches | | `permission_fixer.py` | chown/chmod success, failure, timeout, missing root user | All branches | | `manifest_schema.py` | Valid manifest acceptance, invalid manifest rejection (all error paths) | All validation rules | ### Integration Tests | Scenario | Test | |----------|------| | Manifest → Dockerfile → Build | Create manifest, compile, build image, verify it runs | | ConfigProfile merge | Start instance with profile, verify mounts merged correctly | | Permission fix | Start non-root container, verify workspace is writable | | SSH key mount | Start clone-mode instance, verify SSH keys accessible and have correct permissions | | Legacy compatibility | Start instance from old dockerfile_template tool type, verify it still works | | Base versioning | Create tool with base v1, update base to v2, verify tool still uses v1 | ### E2E Tests | Scenario | Test | |----------|------| | Tool Workshop CRUD | Create, edit, preview, delete a tool definition via UI | | Instance lifecycle | Create instance from manifest tool, start, terminal connect, stop, delete | --- ## Migration Plan ### Step 1: Schema Migration ```python # alembic migration from alembic import op import sqlalchemy as sa def upgrade(): op.create_table( "tool_definition_manifests", sa.Column("id", sa.UUID(), nullable=False), sa.Column("name", sa.String(64), nullable=False), sa.Column("display_name", sa.String(128), nullable=False), sa.Column("description", sa.Text(), nullable=True), sa.Column("category", sa.String(64), nullable=True), sa.Column("interface_type", sa.String(16), nullable=False), sa.Column("base_image", sa.String(256), nullable=True), sa.Column("base_definition_id", sa.UUID(), nullable=True), sa.Column("base_version", sa.String(32), nullable=False, server_default="latest"), sa.Column("manifest", sa.JSON(), nullable=False), sa.Column("dockerfile_cache", sa.Text(), nullable=True), sa.Column("compose_cache", sa.Text(), nullable=True), sa.Column("version", sa.String(32), nullable=False, server_default="v1"), sa.Column("is_base", sa.Boolean(), nullable=False, server_default="false"), sa.Column("created_by_id", sa.UUID(), nullable=True), sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()), sa.Column("updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now()), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("name"), sa.ForeignKeyConstraint(["base_definition_id"], ["tool_definition_manifests.id"]), sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]), sa.CheckConstraint( "(base_image IS NOT NULL) OR (base_definition_id IS NOT NULL)", name="ck_tool_definition_manifests_base_required" ), ) op.add_column("tool_types", sa.Column("manifest_id", sa.UUID(), nullable=True)) op.add_column("tool_types", sa.Column("definition_type", sa.String(16), nullable=False, server_default="legacy")) op.create_foreign_key( "fk_tool_types_manifest_id", "tool_types", "tool_definition_manifests", ["manifest_id"], ["id"] ) op.add_column("tool_instances", sa.Column("manifest_compiled_at", sa.TIMESTAMP(timezone=True), nullable=True)) op.add_column("tool_instances", sa.Column("image_tag", sa.String(256), nullable=True)) ``` ### Step 2: Data Migration Convert the existing pi-agent tool type from `dockerfile_template` to manifest: ```python def upgrade_data(): conn = op.get_bind() # Create the base definition for ubuntu-24.04-dev base_id = uuid.uuid4() conn.execute(sa.text(""" INSERT INTO tool_definition_manifests (id, name, display_name, description, interface_type, base_image, manifest, is_base, version) VALUES (:id, 'ubuntu-24.04-dev', 'Ubuntu 24.04 Dev Base', 'Base development environment', 'terminal', 'ubuntu:24.04', :manifest, true, 'v1') """), { "id": base_id, "manifest": json.dumps({ "name": "ubuntu-24.04-dev", "base_image": "ubuntu:24.04", "packages": {"apt": ["curl", "wget", "git", "build-essential", "ca-certificates"]}, "user": {"name": "user", "uid": 1000, "gid": 1000}, }) }) # Create pi-agent manifest referencing the base pi_agent_id = uuid.UUID("d07b8376-2151-4119-8c1d-27f792aae9a3") conn.execute(sa.text(""" INSERT INTO tool_definition_manifests (id, name, display_name, description, category, interface_type, base_definition_id, base_version, manifest, version) VALUES (:id, 'pi-agent', 'Pi Agent', 'Terminal-based coding harness', 'development', 'terminal', :base_id, 'v1', :manifest, 'v1') """), { "id": pi_agent_id, "base_id": base_id, "manifest": json.dumps({ "name": "pi-agent", "display_name": "Pi Agent", "interface_type": "terminal", "packages": { "apt": ["neovim", "ranger", "tmux", "htop", "tree", "jq", "python3", "python3-pip"], "node": {"version": "20"}, "npm_global": ["@earendil-works/pi-coding-agent"] }, "user": {"name": "user", "uid": 1001, "gid": 1001, "create_home": True, "shell": "/bin/bash"}, "env": {"DEBIAN_FRONTEND": "noninteractive"}, "scripts": { "build": [ "git config --global init.defaultBranch main && git config --global user.email 'dev@headquarter.local' && git config --global user.name 'Developer'", "mkdir -p /home/user/.config/ranger && echo 'set preview_files true' > /home/user/.config/ranger/rc.conf" ], "startup": [ "if [ -d /workspace ]; then sudo chown -R user:user /workspace 2>/dev/null || true; fi" ] }, "mounts": [ {"name": "workspace", "target": "/workspace", "source_type": "repo", "writable": True, "owner": "user"}, {"name": "ssh_keys", "target": "/home/user/.ssh", "source_type": "ssh_key", "mode": "0700", "file_mode": "0600", "readonly": True}, {"name": "pi_state", "target": "/tmp/.pi/agents", "source_type": "instance", "writable": True}, {"name": "pi_config", "target": "/home/user/.pi", "source_type": "git_mount", "git_mount_ref": "dotfiles", "writable": True, "owner": "user"} ], "runtime": { "command": ["/bin/bash"], "stdin_open": True, "tty": True, "working_dir": "/workspace" } }) }) # Update the existing tool_types row conn.execute(sa.text(""" UPDATE tool_types SET manifest_id = :manifest_id, definition_type = 'manifest', dockerfile_template = NULL, compose_template = NULL WHERE name = 'pi-agent' """), {"manifest_id": pi_agent_id}) ``` --- ## Review Workload Forecast | PR | Scope | Est. Lines | Review Risk | |----|-------|-----------|-------------| | PR 1: Backend | Compiler, fixer, API, tests, migration | ~1800 | Medium — core algorithm changes | | PR 2: Frontend | Tool Workshop UI | ~1200 | Medium — new feature, self-contained | | PR 3: Migration | Data migration, legacy fallback | ~300 | Low — additive only | All PRs are under the 400-line budget individually. Chained PRs recommended for sequential review.