feat: implement configurable tool container home directory

- Add ToolType.home_directory column with default /home/user - Add Alembic migration to add column, set existing rows, and rewrite /workspace to /home/user/{{WORKSPACE_NAME}} in legacy templates - Add merge migration fc8f1a20cbf6 to resolve Alembic multiple heads - Update manifest compiler to honor manifest.home_directory for HOME, WORKDIR, /workspace symlink, and default repo mount target - Update legacy dockerfile/compose instance generation to use tool_type.home_directory - Thread resolved home_dir through config profile and git mount expansion - Generate entrypoint permission fixer to chown home/mounts at startup - Update base.dockerfile with sudo/passwordless sudo for permission fixer - Add unit tests for manifest compiler, instance service, and migrations - Add placeholder integration test for container lifecycle - Update openspec/tasks/home-path-expansion.md task checkboxes - Update project maps for modified files Quality gates: py_compile, ruff, mypy, pytest tests/unit (205 passed), pytest tests/integration (110 passed, 35 skipped). Alembic round-trip and container lifecycle integration tests require Docker/PostgreSQL.
2026-06-14 13:09:41 +00:00
parent b4203a4a09
commit ddd92e3dd4
51 changed files with 846 additions and 121 deletions
@@ -2,7 +2,7 @@
 dir: apps/api/src/services

 ## role
-Marks the services directory as a Python package for business logic layer components.
+Marks the services directory as a Python package for organizing business logic and service-layer abstractions.
 ## parent
 index: apps/api/src/.pi-map.index.md
 map: apps/api/src/.pi-map.md
@@ -4,11 +4,11 @@ dir: apps/api/src/services
 index: apps/api/src/services/.pi-map.index.md

 ## role
-Provides a Python package namespace for organizing service-layer modules in the API application.
+This directory serves as a Python package namespace for organizing service-layer modules in the API application.
 ## files
 - __init__.py | Empty file with no functionality
 ## arch
-Standard Python package structure using __init__.py for directory-based module organization, following conventional layered architecture patterns.
+Standard Python package structure using __init__.py to define an importable module directory, following conventional layered architecture patterns.
 ## tags
 init, empty, functionality
 ## symbols
@@ -8,6 +8,7 @@ from typing import Any

 import yaml

+from src.services.config.config_profile_resolver import expand_container_path
 from src.services.docker import sort_volumes_by_specificity


@@ -158,6 +159,8 @@ def compile_dockerfile(manifest: dict) -> str:

    # User creation
    user = manifest.get("user")
+    home_dir = get_manifest_home_dir(manifest)
+    workspace_name = manifest.get("workspace_name", "{{WORKSPACE_NAME}}")
    if user:
        name = user["name"]
        uid = user["uid"]
@@ -168,22 +171,21 @@ def compile_dockerfile(manifest: dict) -> str:
        lines.append(f"    useradd -u {uid} -g {gid} {create_home}-s {shell} {name}")
        lines.append("")
        # Set HOME and USER for runtime compatibility
-        home = f"/home/{name}"
-        lines.append(f"ENV HOME={home}")
+        lines.append(f"ENV HOME={home_dir}")
        lines.append(f"ENV USER={name}")
        lines.append("")
        # Ensure home directory exists and is writable by the user.
        # Recursively chown so any files copied from /etc/skel by useradd -m
        # (e.g. .bashrc, .config) are owned by the container user.
        lines.append(
-            f"RUN mkdir -p {home} && chown -R {name}:{name} {home} && chmod 755 {home}"
+            f"RUN mkdir -p {home_dir} && chown -R {name}:{name} {home_dir} && chmod 755 {home_dir}"
        )
        # Pre-create common config directories so apps like ranger can write
        # their configs on first run without permission errors.
        common_dirs = [".config", ".local/share", ".cache"]
        for d in common_dirs:
            lines.append(
-                f"RUN mkdir -p {home}/{d} && chown -R {name}:{name} {home}/{d}"
+                f"RUN mkdir -p {home_dir}/{d} && chown -R {name}:{name} {home_dir}/{d}"
            )
        lines.append("")

@@ -208,10 +210,12 @@ def compile_dockerfile(manifest: dict) -> str:

    # After build scripts, ensure everything in home is owned by the user
    if user and build_scripts:
-        lines.append(f"RUN chown -R {name}:{name} {home}")
+        lines.append(f"RUN chown -R {name}:{name} {home_dir}")
        lines.append("")

-    # Create mount target directories
+    # Create mount target directories and /workspace compatibility symlink.
+    # The symlink target includes the workspace/repo name so legacy scripts
+    # that cd into /workspace still land on the right project.
    mounts = manifest.get("mounts", [])
    if mounts:
        dirs = [mount["target"] for mount in mounts]
@@ -221,6 +225,16 @@ def compile_dockerfile(manifest: dict) -> str:
            lines.append(f"RUN chown -R {user['name']}:{user['name']} {dir_str}")
        lines.append("")

+    workspace_target = f"{home_dir}/{workspace_name}"
+    lines.append(f"RUN mkdir -p {workspace_target}")
+    if user:
+        lines.append(
+            f"RUN ln -sfn {workspace_target} /workspace && chown -R {user['name']}:{user['name']} {home_dir}"
+        )
+    else:
+        lines.append(f"RUN ln -sfn {workspace_target} /workspace")
+    lines.append("")
+
    # Entrypoint for startup scripts
    startup_scripts = manifest.get("scripts", {}).get("startup", [])
    if startup_scripts:
@@ -233,11 +247,18 @@ def compile_dockerfile(manifest: dict) -> str:
    # Switch to runtime user
    if user:
        lines.append(f"USER {user['name']}")
-        lines.append(f"WORKDIR /home/{user['name']}")
-        lines.append("")
+
+    # Set WORKDIR to the configured home directory unless runtime.working_dir
+    # explicitly overrides it.
+    runtime = manifest.get("runtime", {})
+    working_dir = runtime.get("working_dir")
+    if working_dir:
+        lines.append(f"WORKDIR {expand_container_path(working_dir, home_dir)}")
+    else:
+        lines.append(f"WORKDIR {home_dir}")
+    lines.append("")

    # Entrypoint and CMD
-    runtime = manifest.get("runtime", {})
    if startup_scripts:
        lines.append('ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]')

@@ -251,6 +272,11 @@ def compile_dockerfile(manifest: dict) -> str:
 def compile_entrypoint(manifest: dict) -> str:
    """Generate the startup entrypoint script from startup scripts.

+    Injects a permission-fixer preamble that runs as root (or via sudo) before
+    any user-defined startup script. It chowns the home directory and a safe
+    subset of mount parents to the container user, creates the /workspace
+    compatibility symlink, and avoids recursive chown of large repo subtrees.
+
    Args:
        manifest: Fully resolved manifest JSON.

@@ -259,6 +285,70 @@ def compile_entrypoint(manifest: dict) -> str:
    """
    lines = ["#!/bin/bash", "set -e", ""]

+    user = manifest.get("user")
+    home_dir = get_manifest_home_dir(manifest)
+    workspace_name = manifest.get("workspace_name", "{{WORKSPACE_NAME}}")
+    workspace_target = f"{home_dir}/{workspace_name}"
+
+    # Permission fixer preamble: run as root when possible, else fall back to
+    # passwordless sudo configured in the Dockerfile.
+    lines.append("# Permission fixer preamble")
+    lines.append("CONTAINER_USER=''")
+    lines.append('if [ "$(id -u)" = '"'"'0'"'"' ]; then')
+    if user:
+        lines.append(f"    CONTAINER_USER='{user['name']}'")
+    lines.append("else")
+    lines.append("    # Try passwordless sudo; ignore failure so the container still starts")
+    lines.append("    if sudo -n true 2>/dev/null; then")
+    lines.append("        SUDO='sudo'")
+    lines.append("    else")
+    lines.append("        SUDO=''")
+    lines.append("    fi")
+    lines.append("fi")
+    lines.append("")
+
+    if user:
+        name = user["name"]
+        uid = user["uid"]
+        gid = user["gid"]
+        lines.append(f"USER_NAME='{name}'")
+        lines.append(f"USER_UID='{uid}'")
+        lines.append(f"USER_GID='{gid}'")
+        lines.append(f"HOME_DIR='{home_dir}'")
+        lines.append(f"WORKSPACE_TARGET='{workspace_target}'")
+        lines.append("")
+        lines.append("fix_owner() {")
+        lines.append("    local path=\"$1\"")
+        lines.append('    [ -e "$path" ] || return 0')
+        lines.append('    if [ -n "$SUDO" ]; then')
+        lines.append('        sudo chown "$USER_UID:$USER_GID" "$path" 2>/dev/null || true')
+        lines.append('    elif [ "$(id -u)" = "0" ]; then')
+        lines.append('        chown "$USER_UID:$USER_GID" "$path" 2>/dev/null || true')
+        lines.append('    fi')
+        lines.append("}")
+        lines.append("")
+        lines.append("# Ensure home directory exists and is owned by the container user")
+        lines.append('mkdir -p "$HOME_DIR"')
+        lines.append('fix_owner "$HOME_DIR"')
+        lines.append("")
+        lines.append("# Ensure workspace target exists and is owned by the container user")
+        lines.append('mkdir -p "$WORKSPACE_TARGET"')
+        lines.append('fix_owner "$WORKSPACE_TARGET"')
+        lines.append("")
+        lines.append("# Create /workspace compatibility symlink")
+        lines.append('ln -sfn "$WORKSPACE_TARGET" /workspace')
+        lines.append("")
+        lines.append("# Fix ownership of declared mount targets (top-level only)")
+        for mount in manifest.get("mounts", []):
+            target = mount.get("target")
+            if not target:
+                continue
+            # Expand any ~/$HOME placeholders in the mount target.
+            expanded = target.replace("~", home_dir).replace("$HOME", home_dir)
+            if expanded.startswith(home_dir) and not mount.get("readonly", False):
+                lines.append(f'fix_owner "{expanded}"')
+        lines.append("")
+
    startup_scripts = manifest.get("scripts", {}).get("startup", [])
    for script in startup_scripts:
        lines.append(script)
@@ -281,6 +371,13 @@ def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:
    runtime = manifest.get("runtime", {})
    user = manifest.get("user")
    interface_type = manifest["interface_type"]
+    home_dir = get_manifest_home_dir(manifest)
+
+    # Determine the workspace/repo name from variables when available.
+    workspace_name = variables.get(
+        "WORKSPACE_NAME",
+        variables.get("REPO_NAME", "workspace"),
+    )

    service: dict[str, Any] = {
        "image": variables["IMAGE_TAG"],
@@ -294,7 +391,9 @@ def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:
    if runtime.get("tty", False):
        service["tty"] = True
    if runtime.get("working_dir"):
-        service["working_dir"] = runtime["working_dir"]
+        service["working_dir"] = expand_container_path(
+            runtime["working_dir"], home_dir
+        )

    # User override
    if user:
@@ -319,14 +418,24 @@ def compile_compose(manifest: dict, variables: dict[str, Any]) -> str:

    # Volumes from mount schema
    volumes = []
+    has_explicit_repo_mount = False
    for mount in manifest.get("mounts", []):
        source = resolve_mount_source(mount, variables)
        if not source:
            continue
-        target = mount["target"]
+        if mount.get("source_type") == "repo":
+            has_explicit_repo_mount = True
+        target = expand_container_path(mount["target"], home_dir)
        readonly = ":ro" if mount.get("readonly", False) else ""
        volumes.append(f"{source}:{target}{readonly}")

+    # Synthesize a default repo/workspace mount when the manifest does not
+    # declare an explicit repo mount. This preserves the repo root directory
+    # name under the configured home directory.
+    if not has_explicit_repo_mount and variables.get("REPO_PATH"):
+        target = f"{home_dir}/{workspace_name}"
+        volumes.append(f"{variables['REPO_PATH']}:{target}")
+
    # Append extra volumes from tool config / config profile
    for vol in variables.get("EXTRA_VOLUMES", []):
        vol_str = f"{vol['source']}:{vol['target']}"
@@ -385,7 +494,12 @@ def resolve_mount_source(mount: dict, variables: dict[str, Any]) -> str:


 def get_manifest_home_dir(manifest: dict) -> str:
-    """Get the home directory for a container based on manifest user config.
+    """Get the home directory for a container based on manifest config.
+
+    Precedence:
+      1. manifest["home_directory"] if present and non-empty.
+      2. /home/{user.name} if manifest.user.name is present.
+      3. /root otherwise.

    Args:
        manifest: Fully resolved manifest JSON.
@@ -393,6 +507,10 @@ def get_manifest_home_dir(manifest: dict) -> str:
    Returns:
        Home directory path (e.g., /home/user or /root).
    """
+    home_directory = manifest.get("home_directory")
+    if home_directory and isinstance(home_directory, str) and home_directory.strip():
+        return home_directory.strip()
+
    user = manifest.get("user")
    if user and user.get("name"):
        return f"/home/{user['name']}"
@@ -2,7 +2,7 @@
 dir: apps/api/src/services/docker

 ## role
-Provides Docker infrastructure services for container lifecycle management, compose orchestration, configuration deployment, and secure tunneling to expose internal services.
+Provides Docker infrastructure services for container lifecycle management, compose orchestration, secure configuration deployment, and external tunnel exposure.
 ## parent
 index: apps/api/src/services/.pi-map.index.md
 map: apps/api/src/services/.pi-map.md
@@ -4,15 +4,15 @@ dir: apps/api/src/services/docker
 index: apps/api/src/services/docker/.pi-map.index.md

 ## role
-Provides Docker infrastructure services for container lifecycle management, compose orchestration, configuration deployment, and secure tunneling to expose internal services.
+Provides Docker infrastructure services for container lifecycle management, compose orchestration, secure configuration deployment, and external tunnel exposure.
 ## files
 - __init__.py | Package initialization file that exposes Docker-related service functions for container operations, compose management, configuration staging, and tunnel management. | dep: src.services.docker.compose, src.services.docker.config_staging, src.services.docker.container, src.services.docker.tunnel
- compose.py | Generates, renders, and executes Docker Compose commands for container orchestration with volume sorting and template substitution. | exp: func:sort_volumes_by_specificity(volumes: list[str]) → list[str], call:vol.split, call:len, call:parts[1].rstrip, call:target.count, call:targets.append, call:Counter(targets).items, call:logger.warning, call:sorted, func:_target_depth(vol: str) → int, call:vol.split, call:len, call:parts[1].rstrip, call:target.count, func:render_compose_template(template: str, variables: dict[str, Any]) → str, call:variables.items, call:result.replace, call:str, func:write_compose_file(instance_dir: str, content: str) → str, call:Path, call:compose_path.write_text, call:str, func:execute_compose_command(compose_path: str, action: str, timeout, env_file) → tuple[int, str, str], call:Path, call:cmd.extend, call:cmd.append, call:subprocess.run, call:str, raise:ValueError | dep: logging, subprocess, collections, pathlib, typing, collections.Counter, pathlib.Path, typing.Any
+- compose.py | Generates, renders, and executes Docker Compose files with volume sorting and template variable substitution. | exp: func:sort_volumes_by_specificity(volumes: list[str]) → list[str], call:vol.split, call:len, call:parts[1].rstrip, call:target.count, call:targets.append, call:Counter(targets).items, call:logger.warning, call:sorted, func:_target_depth(vol: str) → int, call:vol.split, call:len, call:parts[1].rstrip, call:target.count, func:render_compose_template(template: str, variables: dict[str, Any]) → str, call:variables.items, call:result.replace, call:str, call:aliases.items, call:variables.get, func:write_compose_file(instance_dir: str, content: str) → str, call:Path, call:compose_path.write_text, call:str, func:execute_compose_command(compose_path: str, action: str, timeout, env_file) → tuple[int, str, str], call:Path, call:cmd.extend, call:cmd.append, call:subprocess.run, call:str, raise:ValueError | dep: logging, subprocess, collections, pathlib, typing, collections.Counter, pathlib.Path, typing.Any
 - config_staging.py | Stages configuration files into instance directories with security checks for path traversal. | exp: func:ensure_instance_directory(instance_id: str, base_path) → str, call:Settings, call:Path, call:instance_dir.mkdir, call:str, call:instance_dir.absolute, func:write_env_file(instance_dir: str, env_vars: dict[str, str]) → str, call:Path, call:env_vars.items, call:env_path.write_text, call:"\n".join, call:str, func:write_config_files(instance_dir: str, files: dict[str, str]) → None, call:Path, call:files.items, call:full_path.resolve().relative_to, call:instance_path.resolve, call:full_path.parent.mkdir, call:full_path.write_text, raise:ValueError | dep: logging, pathlib, src.config, src.config.Settings
 - container.py | Provides Docker container runtime queries and network management utilities via subprocess calls to the Docker CLI. | exp: func:get_container_id(instance_name: str) → str | None, call:instance_name.lower, call:subprocess.run, call:result.stdout.strip, call:ps_result.stdout.strip().splitlines, call:line.split, call:len, call:name.lower, func:get_container_name(instance_name: str) → str | None, call:subprocess.run, call:instance_name.lower, call:result.stdout.strip().lstrip, func:get_backend_network_name() → str, call:subprocess.run, call:result.stdout.strip().split, call:net.lower, func:connect_container_to_network(container_name: str, network_name) → bool, call:get_backend_network_name, call:subprocess.run, func:get_container_ip_on_network(container_id: str, network_name) → str | None, call:get_backend_network_name, call:subprocess.run, call:result.stdout.strip, func:is_container_on_network(container_id: str, network_name) → bool, call:get_backend_network_name, call:subprocess.run, func:get_container_status(container_id: str) → dict[str, Any], call:subprocess.run, call:result.stdout.strip().split, call:int, call:len, call:parts[1].isdigit, func:wait_for_container_running(container_id: str, timeout, interval) → dict[str, Any], call:time.time, call:get_container_status, call:time.sleep, func:get_container_logs(container_id: str, tail) → str, call:subprocess.run, call:str, func:find_free_port(start, end) → int, call:range, call:socket.socket, call:s.connect_ex, raise:RuntimeError | dep: logging, subprocess, time, typing, socket
 - tunnel.py | Manages Cloudflare tunnels by orchestrating cloudflared Docker containers to expose internal services via temporary public URLs. | exp: func:_tunnel_container_name(instance_name: str) → str, call:instance_name.lower, func:_ensure_image() → None, call:subprocess.run, call:result.stdout.strip, call:logger.info, call:logger.warning, func:_cleanup_stale_tunnel(tunnel_name: str) → None, call:subprocess.run, func:_get_tunnel_logs(tunnel_name: str) → tuple[str, str], call:subprocess.run, func:_get_tunnel_exit_code(tunnel_name: str) → int | None, call:subprocess.run, call:int, call:result.stdout.strip, func:start_tunnel(instance_name: str, container_port: int, timeout, target_url) → dict[str, str], call:_ensure_image, call:_tunnel_container_name, call:_cleanup_stale_tunnel, call:instance_name.lower, call:get_backend_network_name, call:logger.debug, call:" ".join, call:subprocess.run, call:proc.stdout.strip, call:re.compile, call:__import__("time").time, call:_get_tunnel_logs, call:url_pattern.search, call:match.group, call:_get_tunnel_exit_code, call:__import__("time").sleep, call:logger.info, raise:RuntimeError, func:stop_tunnel(instance_name: str) → None, call:_tunnel_container_name, call:_cleanup_stale_tunnel, call:logger.debug, func:recreate_tunnel(instance_name: str, container_port: int, target_url) → dict[str, str], call:stop_tunnel, call:start_tunnel, func:check_tunnel_health(url: str, timeout) → dict[str, Any], call:subprocess.run, call:int, call:result.stdout.strip, call:str(exc).lower, call:any | dep: logging, re, subprocess, typing, src.services.docker.container
 ## arch
-Subprocess-based CLI wrapper architecture around Docker/cloudflared tools with template rendering, path-traversal-safe file staging, and functional decomposition into single-responsibility modules.
+Service-oriented utility modules with subprocess-based Docker CLI integration, Jinja2 templating for compose generation, and security-hardened file operations with path traversal validation.
 ## tags
 tunnel, container, call:subprocess.run, get, name, call:, network, call:result.stdout.strip
 ## symbols
@@ -61,6 +61,19 @@ def render_compose_template(template: str, variables: dict[str, Any]) -> str:
    for key, value in variables.items():
        placeholder = f"{{{{{key}}}}}"
        result = result.replace(placeholder, str(value))
+
+    # Convenience aliases so legacy and migrated templates can use lowercase
+    # placeholders without changing every stored template.
+    aliases = {
+        "{{workspace_name}}": "WORKSPACE_NAME",
+        "{{home_directory}}": "HOME_DIRECTORY",
+    }
+    for alias_placeholder, key in aliases.items():
+        if alias_placeholder in result:
+            result = result.replace(
+                alias_placeholder, str(variables.get(key, "workspace"))
+            )
+
    return result


@@ -2,7 +2,7 @@
 dir: apps/api/src/services/tool

 ## role
-Provides backend infrastructure for provisioning and managing isolated development tool instances with their dependencies and network access.
+Orchestrates end-to-end deployment and runtime management of development tool instances via containerized environments with remote access capabilities.
 ## parent
 index: apps/api/src/services/.pi-map.index.md
 map: apps/api/src/services/.pi-map.md
@@ -9,7 +9,6 @@ import subprocess
 import uuid
 from datetime import datetime

-import httpx
 from fastapi import HTTPException, status
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -33,7 +32,6 @@ from src.services.docker import (
    get_container_id,
    get_container_ip_on_network,
    get_container_logs,
-    get_container_status,
    is_container_on_network,
    render_compose_template,
    sort_volumes_by_specificity,
@@ -64,10 +62,9 @@ from src.services.shared.permission_fixer import (
    apply_ssh_permissions,
 )
 from src.services.shared.readiness_probe import execute_probe
-from src.services.shared.ssh_keys import cleanup_ssh_key_files, prepare_ssh_key_files
+from src.services.shared.ssh_keys import prepare_ssh_key_files
 from src.services.instance.event_bus import InstanceEventBus
 from src.services.instance.lifecycle_hooks import publish_lifecycle_event
-from src.auth.dependencies import _get_owned_project, _get_user

 logger = logging.getLogger(__name__)
 _event_bus = InstanceEventBus()
@@ -139,7 +136,7 @@ async def resolve_git_mounts(
        if isinstance(result, Exception):
            logger.warning("Git mount failed: %s", result)
            continue
-        if result:
+        if isinstance(result, list):
            volume_mounts.extend(result)

    return volume_mounts
@@ -1043,7 +1040,11 @@ async def create_tool_instance(
            else ""
        )

-        compose_content = f"""version: "3.8"\nservices:\n  app:\n    image: {image_tag}\n    container_name: {instance_name.lower()}\n    stdin_open: true\n    tty: true\n{ports_section}    volumes:\n      - {repo_path}:/workspace\n    restart: unless-stopped\n"""
+        home_dir = tool_type.home_directory or "/home/user"
+        repo_name = os.path.basename(os.path.normpath(repo_path))
+        workspace_target = f"{home_dir}/{repo_name}"
+
+        compose_content = f"""version: "3.8"\nservices:\n  app:\n    image: {image_tag}\n    container_name: {instance_name.lower()}\n    stdin_open: true\n    tty: true\n{ports_section}    environment:\n      - HOME={home_dir}\n    volumes:\n      - {repo_path}:{workspace_target}\n    working_dir: {workspace_target}\n    restart: unless-stopped\n"""
        write_compose_file(instance_dir, compose_content)

    elif tool_type.definition_type == "manifest":
@@ -1095,6 +1096,8 @@ async def create_tool_instance(
            "TOOL_PORT": tool_port,
            "USER_ID": str(user_id),
            "PROJECT_ID": str(project_id),
+            "WORKSPACE_NAME": os.path.basename(os.path.normpath(repo_path)),
+            "HOME_DIRECTORY": tool_type.home_directory or "/home/user",
        }
        compose_content = render_compose_template(
            tool_type.compose_template, variables