refactor: organize API routers and services into subpackages
Service organization (19 files moved into 6 subpackages): - services/instance/ — event_bus, health_monitor, lifecycle_hooks - services/config/ — config_profile_resolver - services/git/ — clone, git_operations, git_service - services/build/ — docker_build, manifest_compiler - services/terminal/ — terminal_manager, terminal_session - services/shared/ — correlation, file_service, notification_service, permission_fixer, readiness_probe, ssh_keys, tunnel, workspace_manager API router organization (16 files moved into 6 subpackages): - api/tool/ — tool_instances, tool_types, tool_definitions, tool_types_validation, sessions (extracted from tool_instances) - api/config/ — config_profiles, user_config - api/workspace/ — workspaces, workspace_files, workspace_git, workspace_instances - api/user/ — users, auth, ssh_keys - api/project/ — projects, git_repositories - api/system/ — health, events, notifications, dashboard, terminal, instance_proxy Updated main.py imports and all __init__.py re-exports. Sessions router extracted from tool_instances.py into api/tool/sessions.py. Quality gates: py_compile passed, ruff passed.
This commit is contained in:
@@ -1 +1,51 @@
|
||||
"""Shared module."""
|
||||
"""Shared services module."""
|
||||
|
||||
from src.services.shared.correlation import CorrelationIdMiddleware, get_correlation_id
|
||||
from src.services.shared.file_service import FileEntry, FileService
|
||||
from src.services.shared.notification_service import NotificationService
|
||||
from src.services.shared.permission_fixer import (
|
||||
PermissionFixError,
|
||||
apply_mount_permissions,
|
||||
apply_ssh_permissions,
|
||||
check_root_user_available,
|
||||
)
|
||||
from src.services.shared.readiness_probe import execute_probe
|
||||
from src.services.shared.ssh_keys import (
|
||||
cleanup_ssh_key_files,
|
||||
prepare_ssh_key_files,
|
||||
write_ssh_config,
|
||||
)
|
||||
from src.services.shared.tunnel import (
|
||||
check_tunnel_health,
|
||||
recreate_tunnel,
|
||||
start_tunnel,
|
||||
stop_tunnel,
|
||||
)
|
||||
from src.services.shared.workspace_manager import (
|
||||
SyncResult,
|
||||
WorkspaceHasInstancesError,
|
||||
WorkspaceManager,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CorrelationIdMiddleware",
|
||||
"FileEntry",
|
||||
"FileService",
|
||||
"NotificationService",
|
||||
"PermissionFixError",
|
||||
"SyncResult",
|
||||
"WorkspaceHasInstancesError",
|
||||
"WorkspaceManager",
|
||||
"apply_mount_permissions",
|
||||
"apply_ssh_permissions",
|
||||
"check_root_user_available",
|
||||
"check_tunnel_health",
|
||||
"cleanup_ssh_key_files",
|
||||
"execute_probe",
|
||||
"get_correlation_id",
|
||||
"prepare_ssh_key_files",
|
||||
"recreate_tunnel",
|
||||
"start_tunnel",
|
||||
"stop_tunnel",
|
||||
"write_ssh_config",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Async correlation ID context variable and helpers."""
|
||||
|
||||
import contextvars
|
||||
import uuid
|
||||
|
||||
from fastapi import Request
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
CORRELATION_ID: contextvars.ContextVar[str] = contextvars.ContextVar("correlation_id")
|
||||
|
||||
|
||||
def get_correlation_id() -> str:
|
||||
"""Return the current correlation ID or generate a new UUID."""
|
||||
try:
|
||||
return CORRELATION_ID.get()
|
||||
except LookupError:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class CorrelationIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Set correlation ID from X-Request-ID header or generate a new UUID."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
request_id = request.headers.get("X-Request-ID")
|
||||
correlation_id = request_id or str(uuid.uuid4())
|
||||
token = CORRELATION_ID.set(correlation_id)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-ID"] = correlation_id
|
||||
return response
|
||||
finally:
|
||||
CORRELATION_ID.reset(token)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""File operations scoped to a workspace directory."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from src.models import Workspace
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileEntry:
|
||||
"""A single file or directory entry."""
|
||||
|
||||
name: str
|
||||
path: str
|
||||
type: str # "file" or "directory"
|
||||
size: int | None = None
|
||||
|
||||
|
||||
class FileService:
|
||||
"""Read and write files within a workspace directory."""
|
||||
|
||||
def list_directory(
|
||||
self,
|
||||
workspace: Workspace,
|
||||
relative_path: str = "",
|
||||
) -> list[FileEntry]:
|
||||
"""List entries in a workspace directory.
|
||||
|
||||
Args:
|
||||
workspace: The workspace to list files in.
|
||||
relative_path: Path relative to workspace root.
|
||||
|
||||
Returns:
|
||||
List of file entries sorted by name (directories first).
|
||||
"""
|
||||
abs_path = os.path.join(workspace.path, relative_path)
|
||||
abs_path = os.path.normpath(abs_path)
|
||||
|
||||
# Security: ensure we stay within workspace
|
||||
if not abs_path.startswith(os.path.normpath(workspace.path)):
|
||||
raise ValueError("Path escapes workspace directory")
|
||||
|
||||
if not os.path.exists(abs_path):
|
||||
return []
|
||||
|
||||
entries = []
|
||||
for item in sorted(os.listdir(abs_path)):
|
||||
full = os.path.join(abs_path, item)
|
||||
rel = os.path.join(relative_path, item) if relative_path else item
|
||||
is_dir = os.path.isdir(full)
|
||||
size = os.path.getsize(full) if os.path.isfile(full) else None
|
||||
entries.append(
|
||||
FileEntry(
|
||||
name=item,
|
||||
path=rel.replace("\\", "/"),
|
||||
type="directory" if is_dir else "file",
|
||||
size=size,
|
||||
)
|
||||
)
|
||||
|
||||
# Directories first, then files, both alphabetical
|
||||
entries.sort(key=lambda e: (0 if e.type == "directory" else 1, e.name.lower()))
|
||||
return entries
|
||||
|
||||
def read_file(self, workspace: Workspace, relative_path: str) -> str:
|
||||
"""Read a text file from the workspace.
|
||||
|
||||
Args:
|
||||
workspace: The workspace to read from.
|
||||
relative_path: Path relative to workspace root.
|
||||
|
||||
Returns:
|
||||
File contents as string.
|
||||
|
||||
Raises:
|
||||
ValueError: If path escapes workspace or file is binary.
|
||||
FileNotFoundError: If file does not exist.
|
||||
"""
|
||||
abs_path = self._resolve_path(workspace, relative_path)
|
||||
|
||||
if not os.path.isfile(abs_path):
|
||||
raise FileNotFoundError(f"Not a file: {relative_path}")
|
||||
|
||||
# Basic binary check — read first 8KB and look for null bytes
|
||||
with open(abs_path, "rb") as f:
|
||||
chunk = f.read(8192)
|
||||
if b"\x00" in chunk:
|
||||
raise ValueError("Binary files cannot be viewed")
|
||||
|
||||
with open(abs_path, encoding="utf-8", errors="replace") as f:
|
||||
return f.read()
|
||||
|
||||
def write_file(
|
||||
self,
|
||||
workspace: Workspace,
|
||||
relative_path: str,
|
||||
content: str,
|
||||
) -> None:
|
||||
"""Write a text file to the workspace.
|
||||
|
||||
Args:
|
||||
workspace: The workspace to write to.
|
||||
relative_path: Path relative to workspace root.
|
||||
content: File contents.
|
||||
|
||||
Raises:
|
||||
ValueError: If path escapes workspace.
|
||||
"""
|
||||
abs_path = self._resolve_path(workspace, relative_path)
|
||||
os.makedirs(os.path.dirname(abs_path), exist_ok=True)
|
||||
|
||||
with open(abs_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
logger.info("Wrote file %s in workspace %s", relative_path, workspace.id)
|
||||
|
||||
def _resolve_path(self, workspace: Workspace, relative_path: str) -> str:
|
||||
"""Resolve a relative path to absolute, with security check."""
|
||||
abs_path = os.path.normpath(os.path.join(workspace.path, relative_path))
|
||||
workspace_root = os.path.normpath(workspace.path)
|
||||
|
||||
if not abs_path.startswith(workspace_root):
|
||||
raise ValueError("Path escapes workspace directory")
|
||||
|
||||
return abs_path
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Notification persistence service."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.models import Notification
|
||||
|
||||
|
||||
class NotificationService:
|
||||
"""Singleton notification persistence service.
|
||||
|
||||
All methods filter by user_id to enforce strict ownership isolation.
|
||||
"""
|
||||
|
||||
async def create_notification(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
*,
|
||||
category: str,
|
||||
severity: str,
|
||||
title: str,
|
||||
message: str | None = None,
|
||||
source_type: str | None = None,
|
||||
source_id: uuid.UUID | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> Notification:
|
||||
"""Insert a new notification row.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
user_id: Owner of the notification.
|
||||
category: Notification category (e.g., instance, system, health).
|
||||
severity: Severity level (e.g., info, warning, error, success).
|
||||
title: Short notification title.
|
||||
message: Optional longer message body.
|
||||
source_type: Optional source entity type.
|
||||
source_id: Optional source entity UUID.
|
||||
metadata: Optional JSON metadata dictionary.
|
||||
|
||||
Returns:
|
||||
The newly created Notification instance.
|
||||
"""
|
||||
notification = Notification(
|
||||
user_id=user_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
title=title,
|
||||
message=message,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
notification_metadata=metadata or {},
|
||||
)
|
||||
session.add(notification)
|
||||
await session.commit()
|
||||
await session.refresh(notification)
|
||||
return notification
|
||||
|
||||
async def list_notifications(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
*,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
unread_only: bool = False,
|
||||
mute_categories: list[str] | None = None,
|
||||
) -> tuple[list[Notification], int]:
|
||||
"""Return paginated notifications for a user.
|
||||
|
||||
Excludes dismissed notifications and applies optional filtering.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
user_id: Owner of the notifications.
|
||||
limit: Maximum number of items to return.
|
||||
offset: Number of items to skip.
|
||||
unread_only: If True, only return unread notifications.
|
||||
mute_categories: Categories to exclude from results.
|
||||
|
||||
Returns:
|
||||
A tuple of (items, total_count).
|
||||
"""
|
||||
where_clauses = [
|
||||
Notification.user_id == user_id,
|
||||
Notification.dismissed_at.is_(None),
|
||||
]
|
||||
|
||||
if unread_only:
|
||||
where_clauses.append(Notification.read_at.is_(None))
|
||||
|
||||
if mute_categories:
|
||||
where_clauses.append(Notification.category.not_in(mute_categories))
|
||||
|
||||
total_stmt = (
|
||||
select(func.count()).select_from(Notification).where(*where_clauses)
|
||||
)
|
||||
total_result = await session.execute(total_stmt)
|
||||
total = total_result.scalar_one()
|
||||
|
||||
items_stmt = (
|
||||
select(Notification)
|
||||
.where(*where_clauses)
|
||||
.order_by(Notification.created_at.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
items_result = await session.execute(items_stmt)
|
||||
items = list(items_result.scalars().all())
|
||||
|
||||
return items, total
|
||||
|
||||
async def get_unread_count(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
) -> int:
|
||||
"""Count unread, non-dismissed notifications for a user.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
user_id: Owner of the notifications.
|
||||
|
||||
Returns:
|
||||
Number of unread notifications.
|
||||
"""
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Notification)
|
||||
.where(
|
||||
Notification.user_id == user_id,
|
||||
Notification.read_at.is_(None),
|
||||
Notification.dismissed_at.is_(None),
|
||||
)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one()
|
||||
|
||||
async def mark_read(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
notification_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
) -> Notification:
|
||||
"""Mark a single notification as read.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
notification_id: UUID of the notification to mark.
|
||||
user_id: Owner of the notification.
|
||||
|
||||
Returns:
|
||||
The updated Notification instance.
|
||||
|
||||
Raises:
|
||||
ValueError: If the notification does not exist or is not owned by the user.
|
||||
"""
|
||||
notification = await self._get_owned_notification(
|
||||
session, notification_id, user_id
|
||||
)
|
||||
notification.read_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(notification)
|
||||
return notification
|
||||
|
||||
async def mark_all_read(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
) -> int:
|
||||
"""Mark all unread notifications as read for a user.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
user_id: Owner of the notifications.
|
||||
|
||||
Returns:
|
||||
Number of rows updated.
|
||||
"""
|
||||
stmt = (
|
||||
update(Notification)
|
||||
.where(
|
||||
Notification.user_id == user_id,
|
||||
Notification.read_at.is_(None),
|
||||
Notification.dismissed_at.is_(None),
|
||||
)
|
||||
.values(read_at=datetime.now(timezone.utc))
|
||||
)
|
||||
result: CursorResult[Any] = await session.execute(stmt) # type: ignore[assignment]
|
||||
await session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
async def dismiss_all(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
) -> int:
|
||||
"""Soft-delete all non-dismissed notifications for a user.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
user_id: Owner of the notifications.
|
||||
|
||||
Returns:
|
||||
Number of rows updated.
|
||||
"""
|
||||
stmt = (
|
||||
update(Notification)
|
||||
.where(
|
||||
Notification.user_id == user_id,
|
||||
Notification.dismissed_at.is_(None),
|
||||
)
|
||||
.values(dismissed_at=datetime.now(timezone.utc))
|
||||
)
|
||||
result: CursorResult[Any] = await session.execute(stmt) # type: ignore[assignment]
|
||||
await session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
async def dismiss(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
notification_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
) -> None:
|
||||
"""Soft-delete a notification by setting dismissed_at.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
notification_id: UUID of the notification to dismiss.
|
||||
user_id: Owner of the notification.
|
||||
|
||||
Raises:
|
||||
ValueError: If the notification does not exist or is not owned by the user.
|
||||
"""
|
||||
notification = await self._get_owned_notification(
|
||||
session, notification_id, user_id
|
||||
)
|
||||
notification.dismissed_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
|
||||
async def _get_owned_notification(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
notification_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
) -> Notification:
|
||||
"""Fetch a notification and verify ownership.
|
||||
|
||||
Args:
|
||||
session: Database session.
|
||||
notification_id: UUID of the notification.
|
||||
user_id: Expected owner.
|
||||
|
||||
Returns:
|
||||
The Notification instance.
|
||||
|
||||
Raises:
|
||||
ValueError: If the notification does not exist or is not owned.
|
||||
"""
|
||||
notification = await session.get(Notification, notification_id)
|
||||
if notification is None or notification.user_id != user_id:
|
||||
raise ValueError("Notification not found")
|
||||
return notification
|
||||
|
||||
|
||||
# Module-level singleton instance
|
||||
notification_service = NotificationService()
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Permission fixer: applies mount permission policies post-start."""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def apply_mount_permissions(
|
||||
container_id: str,
|
||||
mounts: list[dict],
|
||||
timeout: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Apply permission policies to mounted directories in a running container.
|
||||
|
||||
Runs `chown`, `chmod`, and file-mode fixes for each mount that declares
|
||||
an owner, mode, or file_mode. Requires the container to have a root user.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
mounts: List of mount definitions from the manifest.
|
||||
timeout: Max seconds per docker exec command.
|
||||
|
||||
Returns:
|
||||
List of result dicts: [{mount_name, success, error}]
|
||||
"""
|
||||
results = []
|
||||
|
||||
for mount in mounts:
|
||||
name = mount.get("name", "unknown")
|
||||
target = mount["target"]
|
||||
owner = mount.get("owner")
|
||||
mode = mount.get("mode")
|
||||
file_mode = mount.get("file_mode")
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"mount_name": name,
|
||||
"success": True,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Skip read-only mounts — their permissions cannot be changed
|
||||
# post-start because the bind mount is locked.
|
||||
if mount.get("readonly", False):
|
||||
logger.debug(
|
||||
"Skipping permission fix for read-only mount %s (target=%s)",
|
||||
name,
|
||||
target,
|
||||
)
|
||||
results.append(result)
|
||||
continue
|
||||
|
||||
# Skip if no permission policy defined
|
||||
if not owner and not mode and not file_mode:
|
||||
results.append(result)
|
||||
continue
|
||||
|
||||
try:
|
||||
if owner:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
["chown", "-R", f"{owner}:{owner}", target],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied owner %s to %s in container %s",
|
||||
owner,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
if mode and result["success"]:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
["chmod", mode, target],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied mode %s to %s in container %s",
|
||||
mode,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
if file_mode and result["success"]:
|
||||
_run_in_container(
|
||||
container_id,
|
||||
[
|
||||
"sh",
|
||||
"-c",
|
||||
f"find {target} -type f -exec chmod {file_mode} {{}} +",
|
||||
],
|
||||
timeout,
|
||||
)
|
||||
logger.debug(
|
||||
"Applied file_mode %s to files in %s in container %s",
|
||||
file_mode,
|
||||
target,
|
||||
container_id,
|
||||
)
|
||||
|
||||
except PermissionFixError as exc:
|
||||
result["success"] = False
|
||||
result["error"] = str(exc)
|
||||
logger.warning(
|
||||
"Permission fix failed for mount %s (target=%s): %s",
|
||||
name,
|
||||
target,
|
||||
exc,
|
||||
)
|
||||
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _exec_and_log(
|
||||
container_id: str,
|
||||
command: list[str],
|
||||
timeout: int,
|
||||
description: str,
|
||||
) -> str:
|
||||
"""Run a docker exec command and log stdout/stderr for debugging."""
|
||||
cmd = ["docker", "exec", "--user", "root", container_id] + command
|
||||
logger.debug("[SSH-fix] %s: %s", description, " ".join(cmd))
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise PermissionFixError(
|
||||
f"Command timed out after {timeout}s: {' '.join(command)}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise PermissionFixError(f"Docker command not found: {' '.join(command)}")
|
||||
|
||||
stdout = result.stdout.strip()
|
||||
stderr = result.stderr.strip()
|
||||
if stdout:
|
||||
logger.debug("[SSH-fix] %s stdout: %s", description, stdout)
|
||||
if stderr:
|
||||
logger.debug("[SSH-fix] %s stderr: %s", description, stderr)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise PermissionFixError(
|
||||
f"Command failed (rc={result.returncode}): {stderr or '(no stderr)'}"
|
||||
)
|
||||
return stdout
|
||||
|
||||
|
||||
def apply_ssh_permissions(
|
||||
container_id: str,
|
||||
ssh_target: str,
|
||||
container_user: str,
|
||||
timeout: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
"""Fix SSH directory ownership and permissions in a running container.
|
||||
|
||||
Runs chown and chmod on the ~/.ssh directory so the container user
|
||||
can use the keys (SSH requires the private key to be owned by the
|
||||
user with mode 600).
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
ssh_target: Absolute path to the .ssh directory inside the container.
|
||||
container_user: The container user that should own the keys.
|
||||
timeout: Max seconds per docker exec command.
|
||||
|
||||
Returns:
|
||||
Result dict with keys: success, error.
|
||||
"""
|
||||
result: dict[str, Any] = {"success": True, "error": None}
|
||||
try:
|
||||
# 1. Ensure directory is owned by the container user
|
||||
_exec_and_log(
|
||||
container_id,
|
||||
["chown", "-R", f"{container_user}:{container_user}", ssh_target],
|
||||
timeout,
|
||||
"chown",
|
||||
)
|
||||
|
||||
# 2. Set directory permissions
|
||||
_exec_and_log(
|
||||
container_id,
|
||||
["chmod", "700", ssh_target],
|
||||
timeout,
|
||||
"chmod-dir",
|
||||
)
|
||||
|
||||
# 3. Set private key permissions (id_ed25519, id_rsa, etc.)
|
||||
_exec_and_log(
|
||||
container_id,
|
||||
[
|
||||
"sh",
|
||||
"-c",
|
||||
f"find {ssh_target} -name 'id_*' -type f -exec chmod 600 {{}} +",
|
||||
],
|
||||
timeout,
|
||||
"chmod-keys",
|
||||
)
|
||||
|
||||
# 4. Verify final state
|
||||
ls_output = _exec_and_log(
|
||||
container_id,
|
||||
["ls", "-la", ssh_target],
|
||||
timeout,
|
||||
"verify-ls",
|
||||
)
|
||||
stat_output = _exec_and_log(
|
||||
container_id,
|
||||
["stat", "-c", "%U:%G %a %n", ssh_target],
|
||||
timeout,
|
||||
"verify-stat-dir",
|
||||
)
|
||||
key_stat = _exec_and_log(
|
||||
container_id,
|
||||
[
|
||||
"sh",
|
||||
"-c",
|
||||
f"stat -c '%U:%G %a %n' {ssh_target}/id_* 2>/dev/null || echo 'no id_* files found'",
|
||||
],
|
||||
timeout,
|
||||
"verify-stat-keys",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"SSH permissions fixed for container %s (user=%s, target=%s). "
|
||||
"ls:\n%s\nstat-dir: %s\nstat-keys: %s",
|
||||
container_id,
|
||||
container_user,
|
||||
ssh_target,
|
||||
ls_output,
|
||||
stat_output,
|
||||
key_stat,
|
||||
)
|
||||
except PermissionFixError as exc:
|
||||
result["success"] = False
|
||||
result["error"] = str(exc)
|
||||
logger.warning(
|
||||
"SSH permission fix failed for container %s (target=%s): %s",
|
||||
container_id,
|
||||
ssh_target,
|
||||
exc,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class PermissionFixError(Exception):
|
||||
"""Raised when a permission fix command fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def _run_in_container(
|
||||
container_id: str,
|
||||
command: list[str],
|
||||
timeout: int,
|
||||
) -> None:
|
||||
"""Run a command inside a container as root.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
command: Command + args to execute.
|
||||
timeout: Max seconds to wait.
|
||||
|
||||
Raises:
|
||||
PermissionFixError: If the command fails or times out.
|
||||
"""
|
||||
cmd = ["docker", "exec", "--user", "root", container_id] + command
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise PermissionFixError(
|
||||
f"Command timed out after {timeout}s: {' '.join(command)}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise PermissionFixError(f"Docker command not found: {' '.join(command)}")
|
||||
|
||||
if result.returncode != 0:
|
||||
raise PermissionFixError(
|
||||
f"Command failed (rc={result.returncode}): {result.stderr.strip()}"
|
||||
)
|
||||
|
||||
|
||||
def check_root_user_available(container_id: str, timeout: int = 5) -> bool:
|
||||
"""Check if the container has a root user we can exec as.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name.
|
||||
timeout: Max seconds to wait.
|
||||
|
||||
Returns:
|
||||
True if root user exists and is usable.
|
||||
"""
|
||||
try:
|
||||
_run_in_container(container_id, ["id", "root"], timeout)
|
||||
return True
|
||||
except PermissionFixError:
|
||||
return False
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Readiness probe service for checking if containers are ready."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def execute_probe(
|
||||
container_id: str,
|
||||
command: str,
|
||||
timeout: int = 30,
|
||||
interval: int = 2,
|
||||
) -> tuple[bool, list[str]]:
|
||||
"""Execute a readiness probe command inside a container.
|
||||
|
||||
Args:
|
||||
container_id: Docker container ID or name
|
||||
command: Command to execute inside the container
|
||||
timeout: Maximum total time to wait (seconds)
|
||||
interval: Time between retries (seconds)
|
||||
|
||||
Returns:
|
||||
Tuple of (success, logs)
|
||||
"""
|
||||
logs = []
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
attempt = 0
|
||||
|
||||
while True:
|
||||
attempt += 1
|
||||
elapsed = asyncio.get_event_loop().time() - start_time
|
||||
|
||||
if elapsed >= timeout:
|
||||
logs.append(f"Probe timed out after {timeout}s ({attempt} attempts)")
|
||||
return False, logs
|
||||
|
||||
try:
|
||||
logger.debug("Probe attempt %d: %s", attempt, command)
|
||||
|
||||
# Execute command inside container
|
||||
result = subprocess.run(
|
||||
["docker", "exec", container_id, "sh", "-c", command],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=interval, # Each attempt has its own timeout
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logs.append(f"Attempt {attempt}: Success")
|
||||
if result.stdout:
|
||||
logs.append(f"Output: {result.stdout.strip()}")
|
||||
return True, logs
|
||||
else:
|
||||
logs.append(f"Attempt {attempt}: Failed (exit code {result.returncode})")
|
||||
if result.stderr:
|
||||
logs.append(f"Stderr: {result.stderr.strip()[:200]}")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logs.append(f"Attempt {attempt}: Command timed out")
|
||||
except Exception as exc:
|
||||
logs.append(f"Attempt {attempt}: Error - {exc}")
|
||||
|
||||
# Wait before next attempt
|
||||
await asyncio.sleep(interval)
|
||||
@@ -0,0 +1,182 @@
|
||||
"""SSH key service utilities for preparing keys for container use."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from src.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_fernet() -> Fernet:
|
||||
"""Generate a valid Fernet key from the session secret."""
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
settings = Settings()
|
||||
key_bytes = hashlib.sha256(settings.session_secret.encode()).digest()
|
||||
key = base64.urlsafe_b64encode(key_bytes)
|
||||
return Fernet(key)
|
||||
|
||||
|
||||
def _sanitize_filename(name: str) -> str:
|
||||
"""Sanitize a string for use as a filename.
|
||||
|
||||
Replaces non-alphanumeric characters with underscores and strips
|
||||
leading/trailing underscores.
|
||||
"""
|
||||
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", name)
|
||||
sanitized = sanitized.strip("_")
|
||||
# Ensure it's not empty
|
||||
if not sanitized:
|
||||
sanitized = "key"
|
||||
return sanitized
|
||||
|
||||
|
||||
def prepare_ssh_key_files(
|
||||
instance_dir: str,
|
||||
ssh_key,
|
||||
subdir: str = ".ssh",
|
||||
uid: int | None = None,
|
||||
gid: int | None = None,
|
||||
key_filename: str = "id_ed25519",
|
||||
write_config: bool = True,
|
||||
) -> str:
|
||||
"""Decrypt and write SSH key files to instance directory for container mounting.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
ssh_key: SSHKey model instance with encrypted private key
|
||||
subdir: Subdirectory within instance_dir to write to (default: ".ssh")
|
||||
uid: Optional UID to own the files (for bind-mount into non-root container)
|
||||
gid: Optional GID to own the files
|
||||
key_filename: Base filename for the key pair (default: "id_ed25519").
|
||||
The private key will be named "{key_filename}" and the public key
|
||||
"{key_filename}.pub".
|
||||
write_config: Whether to write an SSH config file (default: True).
|
||||
Set to False when combining multiple keys into one directory,
|
||||
then call write_ssh_config() separately.
|
||||
|
||||
Returns:
|
||||
Path to the .ssh directory
|
||||
"""
|
||||
ssh_dir = Path(instance_dir) / subdir
|
||||
ssh_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Decrypt private key
|
||||
fernet = _get_fernet()
|
||||
private_key = fernet.decrypt(ssh_key.private_key_encrypted.encode()).decode()
|
||||
|
||||
# Write private key with restricted permissions
|
||||
private_key_path = ssh_dir / key_filename
|
||||
private_key_path.write_text(private_key)
|
||||
os.chmod(private_key_path, 0o600)
|
||||
|
||||
# Write public key
|
||||
public_key_path = ssh_dir / f"{key_filename}.pub"
|
||||
public_key_path.write_text(ssh_key.public_key)
|
||||
os.chmod(public_key_path, 0o644)
|
||||
|
||||
# Write SSH config (only if requested)
|
||||
if write_config:
|
||||
config_path = ssh_dir / "config"
|
||||
config_content = f"""Host *
|
||||
StrictHostKeyChecking no
|
||||
UserKnownHostsFile /dev/null
|
||||
IdentityFile ~/.ssh/{key_filename}
|
||||
IdentitiesOnly yes
|
||||
"""
|
||||
config_path.write_text(config_content)
|
||||
os.chmod(config_path, 0o644)
|
||||
|
||||
# Set ownership to target container user if requested
|
||||
if uid is not None or gid is not None:
|
||||
effective_uid = uid if uid is not None else -1
|
||||
effective_gid = gid if gid is not None else -1
|
||||
try:
|
||||
os.chown(ssh_dir, effective_uid, effective_gid)
|
||||
os.chown(private_key_path, effective_uid, effective_gid)
|
||||
os.chown(public_key_path, effective_uid, effective_gid)
|
||||
os.chown(config_path, effective_uid, effective_gid)
|
||||
logger.debug(
|
||||
"Set SSH key ownership to uid=%s gid=%s for %s",
|
||||
effective_uid,
|
||||
effective_gid,
|
||||
ssh_dir,
|
||||
)
|
||||
except PermissionError as exc:
|
||||
logger.warning(
|
||||
"Cannot chown SSH keys to uid=%s gid=%s (running as uid=%s): %s",
|
||||
effective_uid,
|
||||
effective_gid,
|
||||
os.getuid(),
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
# Still chown the key files even if we didn't write config
|
||||
if uid is not None or gid is not None:
|
||||
effective_uid = uid if uid is not None else -1
|
||||
effective_gid = gid if gid is not None else -1
|
||||
try:
|
||||
os.chown(private_key_path, effective_uid, effective_gid)
|
||||
os.chown(public_key_path, effective_uid, effective_gid)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
return str(ssh_dir)
|
||||
|
||||
|
||||
def write_ssh_config(
|
||||
ssh_dir: str,
|
||||
key_filenames: list[str],
|
||||
uid: int | None = None,
|
||||
gid: int | None = None,
|
||||
) -> None:
|
||||
"""Write an SSH config file that includes multiple IdentityFile entries.
|
||||
|
||||
Args:
|
||||
ssh_dir: Path to the .ssh directory
|
||||
key_filenames: List of key filenames (without .pub extension)
|
||||
uid: Optional UID to own the config file
|
||||
gid: Optional GID to own the config file
|
||||
"""
|
||||
ssh_dir_path = Path(ssh_dir)
|
||||
ssh_dir_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
config_path = ssh_dir_path / "config"
|
||||
lines = ["Host *"]
|
||||
lines.append(" StrictHostKeyChecking no")
|
||||
lines.append(" UserKnownHostsFile /dev/null")
|
||||
lines.append(" IdentitiesOnly yes")
|
||||
for filename in key_filenames:
|
||||
lines.append(f" IdentityFile ~/.ssh/{filename}")
|
||||
lines.append("")
|
||||
|
||||
config_content = "\n".join(lines)
|
||||
config_path.write_text(config_content)
|
||||
os.chmod(config_path, 0o644)
|
||||
|
||||
if uid is not None or gid is not None:
|
||||
effective_uid = uid if uid is not None else -1
|
||||
effective_gid = gid if gid is not None else -1
|
||||
try:
|
||||
os.chown(config_path, effective_uid, effective_gid)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
|
||||
def cleanup_ssh_key_files(instance_dir: str) -> None:
|
||||
"""Remove temporary SSH key files from instance directory.
|
||||
|
||||
Args:
|
||||
instance_dir: Path to instance directory
|
||||
"""
|
||||
ssh_dir = Path(instance_dir) / ".ssh"
|
||||
if ssh_dir.exists():
|
||||
for file_path in ssh_dir.iterdir():
|
||||
file_path.unlink()
|
||||
ssh_dir.rmdir()
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Tunnel service — re-exported from docker.tunnel for backward compatibility."""
|
||||
|
||||
from src.services.docker.tunnel import ( # noqa: F401
|
||||
check_tunnel_health,
|
||||
recreate_tunnel,
|
||||
start_tunnel,
|
||||
stop_tunnel,
|
||||
)
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Workspace lifecycle management service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.models import Workspace
|
||||
from src.services.git.git_service import GitService
|
||||
from src.services.shared.ssh_keys import _get_fernet
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.models import GitRepository
|
||||
from src.models import ToolInstance
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
"""Result of a workspace sync operation."""
|
||||
|
||||
branch_deleted: bool = False
|
||||
|
||||
|
||||
class WorkspaceHasInstancesError(Exception):
|
||||
"""Raised when attempting to delete a workspace with running instances."""
|
||||
|
||||
def __init__(self, instances: list[dict]) -> None:
|
||||
self.instances = instances
|
||||
super().__init__(f"Workspace has {len(instances)} running tool instance(s)")
|
||||
|
||||
|
||||
class WorkspaceManager:
|
||||
"""Manages workspace lifecycle: create, delete, sync, validate."""
|
||||
|
||||
BASE_PATH = "/data/working-copies"
|
||||
|
||||
def _workspace_path(self, repo_id: uuid.UUID, name: str) -> str:
|
||||
"""Return the filesystem path for a workspace."""
|
||||
return os.path.join(self.BASE_PATH, str(repo_id), name)
|
||||
|
||||
async def create(
|
||||
self,
|
||||
repo: GitRepository,
|
||||
user_id: uuid.UUID,
|
||||
name: str,
|
||||
branch: str = "main",
|
||||
session: AsyncSession | None = None,
|
||||
) -> Workspace:
|
||||
"""Clone repo to workspace path and create DB record.
|
||||
|
||||
Args:
|
||||
repo: The git repository to clone.
|
||||
user_id: The owner user ID.
|
||||
name: The workspace name (unique per repo).
|
||||
branch: The branch to clone (default: "main").
|
||||
session: Database session for loading SSH keys.
|
||||
|
||||
Returns:
|
||||
The created Workspace record.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If git clone fails.
|
||||
"""
|
||||
path = self._workspace_path(repo.id, name)
|
||||
parent = os.path.dirname(path)
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
# Ensure container users (various UIDs) can write to workspace dirs
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(parent, 0o777)
|
||||
|
||||
logger.info(
|
||||
"Creating workspace: name=%s, repo=%s, branch=%s", name, repo.id, branch
|
||||
)
|
||||
|
||||
if not repo.remote_url:
|
||||
raise ValueError("Repository has no remote URL")
|
||||
|
||||
# Remove stale directory from previous failed/aborted clone
|
||||
if os.path.exists(path):
|
||||
logger.warning("Removing stale workspace directory: %s", path)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
# Load SSH key if repo has one
|
||||
ssh_key = None
|
||||
if getattr(repo, "ssh_key_id", None) and session is not None:
|
||||
from src.models import SSHKey
|
||||
|
||||
result = await session.execute(
|
||||
select(SSHKey).where(SSHKey.id == repo.ssh_key_id)
|
||||
)
|
||||
ssh_key_obj = result.scalar_one_or_none()
|
||||
if ssh_key_obj:
|
||||
fernet = _get_fernet()
|
||||
ssh_key = fernet.decrypt(
|
||||
ssh_key_obj.private_key_encrypted.encode()
|
||||
).decode()
|
||||
|
||||
await GitService.clone(repo.remote_url, branch, path, ssh_key=ssh_key)
|
||||
self._make_world_writable(path)
|
||||
|
||||
workspace = Workspace(
|
||||
name=name,
|
||||
repo_id=repo.id,
|
||||
user_id=user_id,
|
||||
branch=branch,
|
||||
path=path,
|
||||
status="ready",
|
||||
last_sync_at=datetime.now(),
|
||||
)
|
||||
logger.info("Workspace created: %s", workspace.id)
|
||||
return workspace
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
workspace: Workspace,
|
||||
force: bool = False,
|
||||
session: AsyncSession | None = None,
|
||||
) -> None:
|
||||
"""Delete a workspace and all associated tool instances.
|
||||
|
||||
Args:
|
||||
workspace: The workspace to delete.
|
||||
force: If True, delete even if instances exist.
|
||||
session: The database session (required for checking instances).
|
||||
|
||||
Raises:
|
||||
WorkspaceHasInstancesError: If instances exist and force=False.
|
||||
"""
|
||||
if session is None:
|
||||
raise ValueError("session is required for delete")
|
||||
|
||||
instances = await self._get_instances(workspace, session)
|
||||
if instances and not force:
|
||||
raise WorkspaceHasInstancesError(
|
||||
[{"id": str(i.id), "name": i.name} for i in instances]
|
||||
)
|
||||
|
||||
# Stop and delete all instances
|
||||
for instance in instances:
|
||||
await self._stop_and_delete_instance(instance)
|
||||
|
||||
# Delete directory
|
||||
if os.path.exists(workspace.path):
|
||||
shutil.rmtree(workspace.path, ignore_errors=True)
|
||||
logger.info("Deleted workspace directory: %s", workspace.path)
|
||||
|
||||
# Delete record
|
||||
await session.delete(workspace)
|
||||
logger.info("Deleted workspace record: %s", workspace.id)
|
||||
|
||||
async def sync(
|
||||
self, workspace: Workspace, session: AsyncSession | None = None
|
||||
) -> SyncResult:
|
||||
"""Sync a workspace with its remote.
|
||||
|
||||
Args:
|
||||
workspace: The workspace to sync.
|
||||
session: Database session for loading SSH keys.
|
||||
|
||||
Returns:
|
||||
SyncResult indicating whether the branch was deleted.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If git operations fail.
|
||||
"""
|
||||
logger.info("Syncing workspace: %s", workspace.id)
|
||||
|
||||
# Load SSH key if repo has one
|
||||
ssh_key = None
|
||||
if session is not None:
|
||||
from src.models import GitRepository
|
||||
from src.models import SSHKey
|
||||
|
||||
repo = await session.get(GitRepository, workspace.repo_id)
|
||||
if repo and getattr(repo, "ssh_key_id", None):
|
||||
result = await session.execute(
|
||||
select(SSHKey).where(SSHKey.id == repo.ssh_key_id)
|
||||
)
|
||||
ssh_key_obj = result.scalar_one_or_none()
|
||||
if ssh_key_obj:
|
||||
fernet = _get_fernet()
|
||||
ssh_key = fernet.decrypt(
|
||||
ssh_key_obj.private_key_encrypted.encode()
|
||||
).decode()
|
||||
|
||||
await GitService.fetch(workspace.path, ssh_key=ssh_key)
|
||||
|
||||
if not GitService.branch_exists_remotely(
|
||||
workspace.path, workspace.branch, ssh_key=ssh_key
|
||||
):
|
||||
return SyncResult(branch_deleted=True)
|
||||
|
||||
await GitService.pull(workspace.path, workspace.branch, ssh_key=ssh_key)
|
||||
self._make_world_writable(workspace.path)
|
||||
|
||||
workspace.last_sync_at = datetime.now()
|
||||
logger.info("Workspace synced: %s", workspace.id)
|
||||
return SyncResult(branch_deleted=False)
|
||||
|
||||
def _make_world_writable(self, path: str) -> None:
|
||||
"""Recursively make path readable/writable/traversable by any UID.
|
||||
|
||||
Directories get 777 (traversable). Files get rw for all while
|
||||
preserving any existing execute bits.
|
||||
"""
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(path, 0o777)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for d in dirs:
|
||||
dpath = os.path.join(root, d)
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(dpath, 0o777)
|
||||
for f in files:
|
||||
fpath = os.path.join(root, f)
|
||||
with contextlib.suppress(OSError):
|
||||
mode = os.stat(fpath).st_mode
|
||||
# Preserve execute bits, ensure read+write for all
|
||||
new_mode = (mode & stat.S_IXUSR) | 0o666
|
||||
if mode & stat.S_IXGRP:
|
||||
new_mode |= stat.S_IXGRP
|
||||
if mode & stat.S_IXOTH:
|
||||
new_mode |= stat.S_IXOTH
|
||||
os.chmod(fpath, new_mode)
|
||||
|
||||
async def _get_instances(
|
||||
self,
|
||||
workspace: Workspace,
|
||||
session: AsyncSession,
|
||||
) -> list[ToolInstance]:
|
||||
"""Get all tool instances associated with this workspace."""
|
||||
from src.models import ToolInstance
|
||||
|
||||
result = await session.execute(
|
||||
select(ToolInstance).where(ToolInstance.workspace_id == workspace.id)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def _stop_and_delete_instance(self, instance: ToolInstance) -> None:
|
||||
"""Stop and delete a tool instance.
|
||||
|
||||
TODO(PR-2): Wire up to actual instance stop/delete logic.
|
||||
For now, this is a placeholder.
|
||||
"""
|
||||
logger.warning("Placeholder: stopping and deleting instance %s", instance.id)
|
||||
Reference in New Issue
Block a user