feat: tool definition manifest system (PR 1)
- Add ToolDefinitionManifest model with base image versioning - Add manifest compiler: Dockerfile + Compose generation from JSON manifests - Add permission fixer: post-start chown/chmod for mount policies - Add tool definition CRUD API with live compile preview endpoint - Integrate manifest-based startup flow in start_instance - Add Alembic migration with data conversion for pi-agent - Add 48 unit tests for manifest compiler, permission fixer, docker service - Keep backward compatibility with legacy dockerfile_template/compose_template Migration: applied successfully. Pi-agent converted to manifest. Quality gates: pytest (146 passed, 4 pre-existing unrelated failures)
This commit is contained in:
Alex Blank
@@ -0,0 +1,315 @@
|
||||
"""Unit tests for the manifest compiler."""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.manifest_compiler import (
|
||||
compile_compose,
|
||||
compile_dockerfile,
|
||||
compile_entrypoint,
|
||||
compute_image_tag,
|
||||
deep_merge,
|
||||
merge_with_config,
|
||||
resolve_base,
|
||||
)
|
||||
|
||||
|
||||
class TestResolveBase:
|
||||
"""Tests for resolve_base."""
|
||||
|
||||
def test_returns_manifest_unchanged_when_no_base(self) -> None:
|
||||
manifest = {"name": "test", "base_image": "ubuntu:24.04"}
|
||||
result = resolve_base(manifest)
|
||||
assert result["name"] == "test"
|
||||
assert "base_definition_id" not in result
|
||||
|
||||
|
||||
class TestDeepMerge:
|
||||
"""Tests for deep_merge."""
|
||||
|
||||
def test_packages_are_unioned(self) -> None:
|
||||
base = {"packages": {"apt": ["curl", "git"]}}
|
||||
override = {"packages": {"apt": ["neovim"]}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["packages"]["apt"] == ["curl", "git", "neovim"]
|
||||
|
||||
def test_node_version_overrides(self) -> None:
|
||||
base = {"packages": {"node": {"version": "18"}}}
|
||||
override = {"packages": {"node": {"version": "20"}}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["packages"]["node"]["version"] == "20"
|
||||
|
||||
def test_env_is_merged_with_override_winning(self) -> None:
|
||||
base = {"env": {"FOO": "base", "BAR": "base"}}
|
||||
override = {"env": {"FOO": "override"}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["env"]["FOO"] == "override"
|
||||
assert result["env"]["BAR"] == "base"
|
||||
|
||||
def test_build_scripts_are_concatenated(self) -> None:
|
||||
base = {"scripts": {"build": ["echo base"]}}
|
||||
override = {"scripts": {"build": ["echo override"]}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["scripts"]["build"] == ["echo base", "echo override"]
|
||||
|
||||
def test_mounts_are_concatenated(self) -> None:
|
||||
base = {"mounts": [{"name": "base-mount", "target": "/base"}]}
|
||||
override = {"mounts": [{"name": "tool-mount", "target": "/tool"}]}
|
||||
result = deep_merge(base, override)
|
||||
assert len(result["mounts"]) == 2
|
||||
|
||||
def test_user_is_overridden_entirely(self) -> None:
|
||||
base = {"user": {"name": "base", "uid": 1000}}
|
||||
override = {"user": {"name": "tool", "uid": 1001}}
|
||||
result = deep_merge(base, override)
|
||||
assert result["user"]["name"] == "tool"
|
||||
assert result["user"]["uid"] == 1001
|
||||
|
||||
|
||||
class TestCompileDockerfile:
|
||||
"""Tests for compile_dockerfile."""
|
||||
|
||||
def test_includes_from(self) -> None:
|
||||
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "FROM ubuntu:24.04" in df
|
||||
|
||||
def test_installs_apt_packages(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"apt": ["curl", "git"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "apt-get install -y" in df
|
||||
assert "curl" in df
|
||||
assert "git" in df
|
||||
assert "rm -rf /var/lib/apt/lists/*" in df
|
||||
|
||||
def test_installs_node(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"node": {"version": "20"}},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "nodesource.com/setup_20.x" in df
|
||||
|
||||
def test_installs_npm_global(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"packages": {"npm_global": ["@scope/pkg"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "npm install -g @scope/pkg" in df
|
||||
|
||||
def test_creates_user(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"user": {"name": "dev", "uid": 1001, "gid": 1001},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "groupadd -g 1001 dev" in df
|
||||
assert "useradd -u 1001 -g 1001" in df
|
||||
assert "USER dev" in df
|
||||
|
||||
def test_build_scripts_as_run_commands(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"scripts": {"build": ["echo hello", "echo world"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "RUN echo hello" in df
|
||||
assert "RUN echo world" in df
|
||||
|
||||
def test_creates_mount_directories(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"user": {"name": "dev", "uid": 1001, "gid": 1001},
|
||||
"mounts": [
|
||||
{"name": "ws", "target": "/workspace"},
|
||||
{"name": "cfg", "target": "/config"},
|
||||
],
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert "mkdir -p /workspace /config" in df
|
||||
assert "chown -R dev:dev /workspace /config" in df
|
||||
|
||||
def test_entrypoint_for_startup_scripts(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"scripts": {"startup": ["echo start"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'ENTRYPOINT ["/usr/local/bin/headquarter-entrypoint"]' in df
|
||||
|
||||
def test_cmd_from_runtime(self) -> None:
|
||||
manifest = {
|
||||
"base_image": "ubuntu:24.04",
|
||||
"name": "test",
|
||||
"runtime": {"command": ["/bin/bash", "-il"]},
|
||||
}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'CMD ["/bin/bash", "-il"]' in df
|
||||
|
||||
def test_default_cmd_when_no_runtime(self) -> None:
|
||||
manifest = {"base_image": "ubuntu:24.04", "name": "test"}
|
||||
df = compile_dockerfile(manifest)
|
||||
assert 'CMD ["/bin/bash"]' in df
|
||||
|
||||
|
||||
class TestCompileEntrypoint:
|
||||
"""Tests for compile_entrypoint."""
|
||||
|
||||
def test_includes_shebang_and_set_e(self) -> None:
|
||||
manifest = {"scripts": {"startup": ["echo hello"]}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert "#!/bin/bash" in ep
|
||||
assert "set -e" in ep
|
||||
|
||||
def test_includes_startup_scripts(self) -> None:
|
||||
manifest = {"scripts": {"startup": ["echo hello", "echo world"]}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert "echo hello" in ep
|
||||
assert "echo world" in ep
|
||||
|
||||
def test_ends_with_exec(self) -> None:
|
||||
manifest: dict = {"scripts": {"startup": []}}
|
||||
ep = compile_entrypoint(manifest)
|
||||
assert 'exec "$@"' in ep
|
||||
|
||||
|
||||
class TestCompileCompose:
|
||||
"""Tests for compile_compose."""
|
||||
|
||||
def test_includes_image_and_container_name(self) -> None:
|
||||
manifest = {"name": "test", "interface_type": "terminal"}
|
||||
vars_dict = {"IMAGE_TAG": "test:v1", "INSTANCE_NAME": "test-1"}
|
||||
compose = compile_compose(manifest, vars_dict)
|
||||
assert "image: test:v1" in compose
|
||||
assert "container_name: test-1" in compose
|
||||
|
||||
def test_terminal_fields(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"runtime": {"stdin_open": True, "tty": True, "working_dir": "/workspace"},
|
||||
}
|
||||
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
|
||||
assert "stdin_open: true" in compose
|
||||
assert "tty: true" in compose
|
||||
assert "working_dir: /workspace" in compose
|
||||
|
||||
def test_web_ports(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "web",
|
||||
"default_port": 8080,
|
||||
}
|
||||
compose = compile_compose(
|
||||
manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n", "TOOL_PORT": "3000"}
|
||||
)
|
||||
assert "3000:8080" in compose
|
||||
|
||||
def test_user_override(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"user": {"uid": 1001, "gid": 1001},
|
||||
}
|
||||
compose = compile_compose(manifest, {"IMAGE_TAG": "t", "INSTANCE_NAME": "n"})
|
||||
assert "user: 1001:1001" in compose
|
||||
|
||||
def test_mounts_resolved(self) -> None:
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"interface_type": "terminal",
|
||||
"mounts": [
|
||||
{"name": "ws", "target": "/workspace", "source_type": "repo"},
|
||||
{
|
||||
"name": "ssh",
|
||||
"target": "/home/user/.ssh",
|
||||
"source_type": "ssh_key",
|
||||
"readonly": True,
|
||||
},
|
||||
],
|
||||
}
|
||||
compose = compile_compose(
|
||||
manifest,
|
||||
{
|
||||
"IMAGE_TAG": "t",
|
||||
"INSTANCE_NAME": "n",
|
||||
"REPO_PATH": "/repos/myrepo",
|
||||
"SSH_PATH": "/keys/ssh",
|
||||
},
|
||||
)
|
||||
assert "/repos/myrepo:/workspace" in compose
|
||||
assert "/keys/ssh:/home/user/.ssh:ro" in compose
|
||||
|
||||
def test_extra_volumes_appended(self) -> None:
|
||||
manifest = {"name": "test", "interface_type": "terminal"}
|
||||
compose = compile_compose(
|
||||
manifest,
|
||||
{
|
||||
"IMAGE_TAG": "t",
|
||||
"INSTANCE_NAME": "n",
|
||||
"EXTRA_VOLUMES": [{"source": "/host/x", "target": "/container/x"}],
|
||||
},
|
||||
)
|
||||
assert "/host/x:/container/x" in compose
|
||||
|
||||
|
||||
class TestComputeImageTag:
|
||||
"""Tests for compute_image_tag."""
|
||||
|
||||
def test_is_deterministic(self) -> None:
|
||||
manifest = {"name": "test", "packages": {"apt": ["curl"]}}
|
||||
tag1 = compute_image_tag("My Tool", manifest)
|
||||
tag2 = compute_image_tag("My Tool", manifest)
|
||||
assert tag1 == tag2
|
||||
|
||||
def test_changes_with_content(self) -> None:
|
||||
manifest1 = {"name": "test", "packages": {"apt": ["curl"]}}
|
||||
manifest2 = {"name": "test", "packages": {"apt": ["wget"]}}
|
||||
tag1 = compute_image_tag("test", manifest1)
|
||||
tag2 = compute_image_tag("test", manifest2)
|
||||
assert tag1 != tag2
|
||||
|
||||
def test_lowercases_name(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
tag = compute_image_tag("My Tool", manifest)
|
||||
assert "my-tool" in tag
|
||||
|
||||
def test_valid_docker_reference(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
tag = compute_image_tag("test", manifest)
|
||||
assert tag.startswith("headquarter/test-")
|
||||
assert tag.endswith(":latest")
|
||||
|
||||
|
||||
class TestMergeWithConfig:
|
||||
"""Tests for merge_with_config."""
|
||||
|
||||
def test_applies_tool_config_env(self) -> None:
|
||||
manifest = {"name": "test"}
|
||||
configs = [
|
||||
{"config_type": "env", "key": "FOO", "value": "bar"},
|
||||
]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["_extra_env"]["FOO"] == "bar"
|
||||
|
||||
def test_applies_port_override(self) -> None:
|
||||
manifest = {"name": "test", "default_port": 8080}
|
||||
configs = [{"port_override": 3000}]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["default_port"] == 3000
|
||||
|
||||
def test_applies_start_command(self) -> None:
|
||||
manifest = {"name": "test", "runtime": {"command": ["/bin/bash"]}}
|
||||
configs = [{"start_command": "/bin/sh"}]
|
||||
result = merge_with_config(manifest, configs)
|
||||
assert result["runtime"]["command"] == ["/bin/sh"]
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Unit tests for the permission fixer."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.permission_fixer import (
|
||||
PermissionFixError,
|
||||
apply_mount_permissions,
|
||||
check_root_user_available,
|
||||
_run_in_container,
|
||||
)
|
||||
|
||||
|
||||
class TestApplyMountPermissions:
|
||||
"""Tests for apply_mount_permissions."""
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_chown_when_owner_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "owner": "user"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["mount_name"] == "workspace"
|
||||
assert results[0]["success"] is True
|
||||
mock_run.assert_called_once()
|
||||
args = mock_run.call_args[0]
|
||||
assert args[0] == "abc123"
|
||||
assert args[1] == ["chown", "-R", "user:user", "/workspace"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_chmod_when_mode_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "ssh", "target": "/home/user/.ssh", "mode": "0700"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is True
|
||||
# Only chmod called (no owner, so no chown)
|
||||
assert mock_run.call_count == 1
|
||||
chmod_call = mock_run.call_args_list[0]
|
||||
assert chmod_call[0][1] == ["chmod", "0700", "/home/user/.ssh"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_applies_file_mode_when_declared(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{
|
||||
"name": "ssh",
|
||||
"target": "/home/user/.ssh",
|
||||
"file_mode": "0600",
|
||||
},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is True
|
||||
# Only file_mode called (no owner, no mode)
|
||||
assert mock_run.call_count == 1
|
||||
file_mode_call = mock_run.call_args_list[0]
|
||||
assert file_mode_call[0][1][0] == "sh"
|
||||
assert (
|
||||
"find /home/user/.ssh -type f -exec chmod 0600" in file_mode_call[0][1][2]
|
||||
)
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_skips_mount_with_no_policy(self, mock_run) -> None:
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "writable": True},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["success"] is True
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_reports_failure_on_command_error(self, mock_run) -> None:
|
||||
mock_run.side_effect = PermissionFixError("chown failed")
|
||||
|
||||
mounts = [
|
||||
{"name": "workspace", "target": "/workspace", "owner": "user"},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is False
|
||||
assert "chown failed" in results[0]["error"]
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_stops_on_first_failure(self, mock_run) -> None:
|
||||
"""If chown fails, chmod and file_mode should not run."""
|
||||
mock_run.side_effect = PermissionFixError("chown failed")
|
||||
|
||||
mounts = [
|
||||
{
|
||||
"name": "workspace",
|
||||
"target": "/workspace",
|
||||
"owner": "user",
|
||||
"mode": "0755",
|
||||
"file_mode": "0644",
|
||||
},
|
||||
]
|
||||
results = apply_mount_permissions("abc123", mounts)
|
||||
|
||||
assert results[0]["success"] is False
|
||||
assert mock_run.call_count == 1 # Only chown attempted
|
||||
|
||||
|
||||
class TestRunInContainer:
|
||||
"""Tests for _run_in_container."""
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_success(self, mock_run) -> None:
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr="")
|
||||
_run_in_container("abc123", ["echo", "hello"], 10)
|
||||
mock_run.assert_called_once()
|
||||
cmd = mock_run.call_args[0][0]
|
||||
assert cmd == ["docker", "exec", "--user", "root", "abc123", "echo", "hello"]
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_failure_raises(self, mock_run) -> None:
|
||||
mock_run.return_value = MagicMock(returncode=1, stderr="permission denied")
|
||||
with pytest.raises(PermissionFixError, match="permission denied"):
|
||||
_run_in_container("abc123", ["chown", "x"], 10)
|
||||
|
||||
@patch("subprocess.run")
|
||||
def test_timeout_raises(self, mock_run) -> None:
|
||||
import subprocess
|
||||
|
||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd=["docker"], timeout=10)
|
||||
with pytest.raises(PermissionFixError, match="timed out"):
|
||||
_run_in_container("abc123", ["chown", "x"], 10)
|
||||
|
||||
|
||||
class TestCheckRootUserAvailable:
|
||||
"""Tests for check_root_user_available."""
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_returns_true_when_root_exists(self, mock_run) -> None:
|
||||
assert check_root_user_available("abc123") is True
|
||||
|
||||
@patch("src.services.permission_fixer._run_in_container")
|
||||
def test_returns_false_when_root_missing(self, mock_run) -> None:
|
||||
mock_run.side_effect = PermissionFixError("no such user")
|
||||
assert check_root_user_available("abc123") is False
|
||||
Reference in New Issue
Block a user