feat: responsive web terminal with auto-reconnect, heartbeat, and local echo

Implements a resilient, responsive web terminal that survives network blips,
provides instant typing feedback, and restores scrollback on reconnect.

Backend changes:
- Add heartbeat tracking (15s ping interval, 60s idle timeout)
- Add message batching (16ms flush window) for efficient I/O
- Add termios echo detection and set_echo_state control messages
- Add graceful session_ended notification before close
- Add ping/pong protocol support

Frontend changes:
- Rewrite TerminalComponent with status bar, connection indicator,
  session-ended overlay, reconnect banner, and ResizeObserver
- Add useTerminalConnection hook with:
  - Exponential backoff auto-reconnect (1s → 30s max, 10 attempts)
  - Heartbeat/ping-pong with latency tracking
  - Local echo for printable ASCII with server deduplication
  - Resize debounce (200ms) + throttle (500ms)
  - Scrollback serialization via xterm-addon-serialize
  - Ctrl+Shift+R manual reconnect shortcut
- Add WebSocket protocol types and encoding utilities
- Add xterm-addon-serialize dependency

Tests:
- 16 backend unit tests (TerminalSession + TerminalManager)
- 13 frontend hook tests (connection lifecycle, reconnect, resize,
  scrollback, callbacks)

Quality gates:
- Frontend typecheck: clean
- Frontend lint: clean
- Frontend tests: 48 passed
- Backend unit tests: 101 passed
- Backend ruff: clean

SDD artifacts: openspec/changes/responsive-terminal/
This commit is contained in:
2026-05-27 21:27:49 +02:00
parent 48fa858090
commit 6c8cfe9157
18 changed files with 2776 additions and 350 deletions
+52 -18
View File
@@ -4,7 +4,7 @@ import asyncio
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, status
from fastapi import APIRouter, Depends, WebSocket
from sqlalchemy.ext.asyncio import AsyncSession
from src.auth.dependencies import get_db_session
@@ -26,6 +26,11 @@ async def terminal_websocket(
"""WebSocket endpoint for terminal access to a tool instance.
Provides an interactive terminal session inside a running tool instance container.
Supports:
- Auto-reconnection (client reconnects, server spawns new session)
- Heartbeat ping/pong
- Binary and text input frames
- Graceful session end notifications
Args:
websocket: The WebSocket connection.
@@ -34,26 +39,27 @@ async def terminal_websocket(
Returns:
None. Communicates via WebSocket messages.
"""
logger.info("Terminal WebSocket connection attempt for instance %s", instance_id)
await websocket.accept()
try:
# Parse instance_id
instance_uuid = uuid.UUID(instance_id)
except ValueError:
logger.error("Invalid instance ID: %s", instance_id)
await websocket.close(code=4001, reason="Invalid instance ID")
return
# Authenticate user from session cookie
user_id = await _get_user_from_websocket(websocket, db_session)
if user_id is None:
logger.warning("Unauthorized terminal access attempt for instance %s", instance_id)
logger.warning(
"Unauthorized terminal access attempt for instance %s",
instance_id,
)
await websocket.close(code=4003, reason="Unauthorized")
return
# Get instance and verify ownership
instance = await db_session.get(ToolInstance, instance_uuid)
if instance is None:
logger.warning("Instance %s not found", instance_id)
@@ -61,38 +67,65 @@ async def terminal_websocket(
return
if instance.owner_id != user_id:
logger.warning("Forbidden terminal access for instance %s by user %s", instance_id, user_id)
logger.warning(
"Forbidden terminal access for instance %s by user %s",
instance_id,
user_id,
)
await websocket.close(code=4003, reason="Forbidden")
return
if instance.status != "running" or not instance.container_id:
logger.warning("Instance %s not running (status=%s, container_id=%s)", instance_id, instance.status, instance.container_id)
logger.warning(
"Instance %s not running (status=%s, container_id=%s)",
instance_id,
instance.status,
instance.container_id,
)
await websocket.close(code=4004, reason="Instance not running")
return
logger.info("Creating terminal session for instance %s (container_id=%s)", instance_id, instance.container_id)
# Create terminal session
logger.info(
"Creating terminal session for instance %s (container_id=%s)",
instance_id,
instance.container_id,
)
try:
session = await terminal_manager.create_session(
instance_uuid,
instance.container_id,
websocket,
)
logger.info("Terminal session created successfully for instance %s", instance_id)
logger.info(
"Terminal session created successfully for instance %s",
instance_id,
)
# Send connected status
await websocket.send_json({"type": "status", "status": "connected"})
# Keep connection alive until session ends
# The terminal_manager handles I/O loops, we just wait here
while session.is_alive() and not session._closed:
await asyncio.sleep(0.5)
# Monitor session health and echo state
while session.is_alive() and not session.closed:
# Check echo state periodically
new_echo_state = await session.check_echo_state()
if new_echo_state is not None:
await websocket.send_json(
{"type": "set_echo_state", "enabled": new_echo_state},
)
await asyncio.sleep(1.0)
except Exception as exc:
logger.error("Terminal session error for instance %s: %s", instance_id, str(exc), exc_info=True)
await websocket.close(code=4000, reason=f"Error: {exc}")
# Session ended — determine reason and notify client
exit_reason = session.get_exit_reason() or "process_exit"
await websocket.send_json({"type": "session_ended", "reason": exit_reason})
await websocket.close(code=1000, reason=f"Session ended: {exit_reason}")
except Exception:
logger.exception(
"Terminal session error for instance %s",
instance_id,
)
await websocket.close(code=4000, reason="Terminal session error")
finally:
# Cleanup will be handled by the session manager
pass
@@ -108,6 +141,7 @@ async def _get_user_from_websocket(
Returns:
The user's UUID if authenticated, None otherwise.
"""
from src.auth.session import decode_session_cookie
from src.config import Settings
+118 -21
View File
@@ -1,19 +1,35 @@
"""Terminal session manager for WebSocket connections."""
import asyncio
import contextlib
import json
import logging
import time
import uuid
from collections.abc import Coroutine
from typing import Any
from fastapi import WebSocket
from src.services.terminal_session import TerminalSession
logger = logging.getLogger(__name__)
_READ_BATCH_INTERVAL_S = 0.016 # 16ms max batching delay
_READ_POLL_TIMEOUT_S = 0.005
_READ_POLL_SLEEP_S = 0.001
_HEARTBEAT_INTERVAL_S = 15.0
_IDLE_TIMEOUT_S = 60.0
class TerminalManager:
"""Manages active terminal sessions."""
def __init__(self) -> None:
"""Initialise the terminal manager."""
self._sessions: dict[str, TerminalSession] = {}
self._last_client_message: dict[str, float] = {}
self._background_tasks: set[asyncio.Task[Any]] = set()
async def create_session(
self,
@@ -26,55 +42,134 @@ class TerminalManager:
session = TerminalSession(session_id, instance_id, container_id)
await session.start()
self._sessions[session_id] = session
self._last_client_message[session_id] = time.monotonic()
# Start background tasks for I/O streaming
asyncio.create_task(self._read_loop(session, websocket))
asyncio.create_task(self._write_loop(session, websocket))
self._start_task(self._read_loop(session, websocket))
self._start_task(self._write_loop(session, websocket))
self._start_task(self._heartbeat_loop(session, websocket))
return session
async def _read_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
"""Read output from the container and send to WebSocket."""
def _start_task(self, coro: Coroutine[Any, Any, None]) -> None:
"""Start a background task and store a reference to prevent GC."""
task = asyncio.create_task(coro)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
async def _read_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Read output from the container and send to WebSocket with batching."""
try:
while session.is_alive() and not session._closed:
data = await session.read_output()
buffer = bytearray()
last_flush = time.monotonic()
while session.is_alive() and not session.closed:
data = await session.read_output(select_timeout=_READ_POLL_TIMEOUT_S)
if data:
await websocket.send_bytes(data)
else:
await asyncio.sleep(0.01)
buffer.extend(data)
now = time.monotonic()
flush_due = buffer and (
now - last_flush >= _READ_BATCH_INTERVAL_S or not data
)
if flush_due:
await websocket.send_bytes(bytes(buffer))
buffer.clear()
last_flush = now
elif not data:
await asyncio.sleep(_READ_POLL_SLEEP_S)
# Flush any remaining data
if buffer:
with contextlib.suppress(Exception):
await websocket.send_bytes(bytes(buffer))
except Exception:
pass
logger.exception("Read loop error for session %s", session.session_id)
finally:
await self._cleanup_session(session)
async def _write_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
async def _write_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Read input from WebSocket and send to container."""
try:
while session.is_alive() and not session._closed:
while session.is_alive() and not session.closed:
message = await websocket.receive()
self._last_client_message[session.session_id] = time.monotonic()
if message["type"] == "websocket.receive":
if "bytes" in message:
await session.write_input(message["bytes"])
elif "text" in message:
text = message["text"]
if text.startswith("{"):
# Control message (JSON)
import json
try:
ctrl = json.loads(text)
if ctrl.get("type") == "resize":
await session.resize(
ctrl.get("cols", 80),
ctrl.get("rows", 24),
)
await self._handle_control_message(
session,
websocket,
ctrl,
)
except json.JSONDecodeError:
pass
logger.debug("Invalid JSON control message: %s", text)
else:
await session.write_input(text.encode("utf-8"))
elif message["type"] == "websocket.disconnect":
break
except Exception:
pass
logger.exception("Write loop error for session %s", session.session_id)
finally:
await self._cleanup_session(session)
async def _handle_control_message(
self,
session: TerminalSession,
websocket: WebSocket,
ctrl: dict[str, Any],
) -> None:
"""Handle a JSON control message from the client."""
msg_type = ctrl.get("type")
if msg_type == "resize":
await session.resize(
ctrl.get("cols", 80),
ctrl.get("rows", 24),
)
elif msg_type == "ping":
await websocket.send_json(
{"type": "pong", "id": ctrl.get("id")},
)
async def _heartbeat_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Monitor client activity and close idle connections."""
try:
while session.is_alive() and not session.closed:
await asyncio.sleep(_HEARTBEAT_INTERVAL_S)
last_msg = self._last_client_message.get(session.session_id, 0)
if time.monotonic() - last_msg > _IDLE_TIMEOUT_S:
# Client has been silent for 60s — close connection
with contextlib.suppress(Exception):
await websocket.close(
code=1000,
reason="Idle timeout",
)
break
except Exception:
logger.exception(
"Heartbeat loop error for session %s",
session.session_id,
)
finally:
await self._cleanup_session(session)
@@ -82,12 +177,14 @@ class TerminalManager:
"""Clean up a session."""
if session.session_id in self._sessions:
del self._sessions[session.session_id]
self._last_client_message.pop(session.session_id, None)
await session.close()
async def close_all(self) -> None:
"""Close all active sessions."""
sessions = list(self._sessions.values())
self._sessions.clear()
self._last_client_message.clear()
for session in sessions:
await session.close()
+76 -31
View File
@@ -1,19 +1,29 @@
"""Terminal session management for tool instances."""
import asyncio
import contextlib
import fcntl
import logging
import os
import pty
import select
import struct
import fcntl
import termios
import uuid
from typing import Any
logger = logging.getLogger(__name__)
class TerminalSession:
"""Manages a single terminal session connected to a docker container."""
def __init__(self, session_id: str, instance_id: uuid.UUID, container_id: str) -> None:
def __init__(
self,
session_id: str,
instance_id: uuid.UUID,
container_id: str,
) -> None:
"""Initialize a terminal session."""
self.session_id = session_id
self.instance_id = instance_id
self.container_id = container_id
@@ -21,23 +31,20 @@ class TerminalSession:
self._closed = False
self._master_fd: int | None = None
self._slave_fd: int | None = None
self._echo_enabled = True
self._exit_reason: str | None = None
async def start(self) -> None:
"""Start the docker exec process with a shell using a PTY."""
# Create a pseudo-terminal on the host
self._master_fd, self._slave_fd = pty.openpty()
# Set the terminal size initially
self._set_terminal_size(80, 24)
# Start docker exec with the slave fd as stdin/stdout/stderr
# Using -it because the slave fd IS a TTY
self.process = await asyncio.create_subprocess_exec(
"docker",
"exec",
"-it",
"-e",
"TERM=xterm",
"TERM=xterm-256color",
self.container_id,
"bash",
"-il",
@@ -45,44 +52,71 @@ class TerminalSession:
stdout=self._slave_fd,
stderr=self._slave_fd,
)
# Close slave fd in parent process
os.close(self._slave_fd)
self._slave_fd = None
self._echo_enabled = self._detect_echo_state()
def _set_terminal_size(self, cols: int, rows: int) -> None:
"""Set the terminal size using TIOCSWINSZ."""
if self._master_fd is None:
return
# TIOCSWINSZ = 0x5414 on Linux
TIOCSWINSZ = 0x5414
size = struct.pack('HHHH', rows, cols, 0, 0)
try:
fcntl.ioctl(self._master_fd, TIOCSWINSZ, size)
except (OSError, IOError):
pass
tiocswinsz = 0x5414
size = struct.pack("HHHH", rows, cols, 0, 0)
with contextlib.suppress(OSError):
fcntl.ioctl(self._master_fd, tiocswinsz, size)
async def read_output(self) -> bytes:
def _detect_echo_state(self) -> bool:
"""Detect whether the PTY has echo enabled via termios."""
if self._master_fd is None:
return True
try:
attrs = termios.tcgetattr(self._master_fd)
return bool(attrs[3] & termios.ECHO)
except OSError:
return True
async def check_echo_state(self) -> bool | None:
"""Check if echo state changed. Returns new state if changed, None otherwise."""
current = self._detect_echo_state()
if current != self._echo_enabled:
self._echo_enabled = current
return current
return None
@property
def echo_enabled(self) -> bool:
"""Return whether the PTY currently has echo enabled."""
return self._echo_enabled
@property
def closed(self) -> bool:
"""Return whether the session has been closed."""
return self._closed
async def read_output(self, select_timeout: float = 0.1) -> bytes:
"""Read output from the PTY master."""
if self._master_fd is None or self._closed:
return b""
try:
# Use select to check if data is available
readable, _, _ = select.select([self._master_fd], [], [], 0.1)
readable, _, _ = select.select(
[self._master_fd],
[],
[],
select_timeout,
)
if readable:
return os.read(self._master_fd, 4096)
return os.read(self._master_fd, 8192)
return b""
except (OSError, IOError, ValueError):
except (OSError, ValueError):
return b""
async def write_input(self, data: bytes) -> None:
"""Write input to the PTY master."""
if self._master_fd is None or self._closed:
return
try:
with contextlib.suppress(OSError):
os.write(self._master_fd, data)
except (OSError, IOError):
pass
async def resize(self, cols: int, rows: int) -> None:
"""Resize the terminal."""
@@ -90,24 +124,35 @@ class TerminalSession:
return
self._set_terminal_size(cols, rows)
def get_exit_reason(self) -> str | None:
"""Return the reason the session ended, if known."""
return self._exit_reason
async def close(self) -> None:
"""Close the session and cleanup."""
if self._closed:
return
self._closed = True
# Determine exit reason
if self.process is not None and self.process.returncode is not None:
if self.process.returncode == 0:
self._exit_reason = "process_exit"
else:
self._exit_reason = "process_exit"
else:
self._exit_reason = "timeout"
if self._master_fd is not None:
try:
with contextlib.suppress(OSError):
os.close(self._master_fd)
except OSError:
pass
self._master_fd = None
if self.process is not None:
try:
self.process.kill()
await asyncio.wait_for(self.process.wait(), timeout=2.0)
except (asyncio.TimeoutError, ProcessLookupError):
except (TimeoutError, ProcessLookupError):
pass
def is_alive(self) -> bool:
@@ -0,0 +1,112 @@
"""Unit tests for TerminalManager."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.services.terminal_manager import TerminalManager
from src.services.terminal_session import TerminalSession
@pytest.fixture
def manager():
return TerminalManager()
@pytest.fixture
def mock_websocket():
ws = AsyncMock()
ws.send_bytes = AsyncMock()
ws.send_json = AsyncMock()
ws.close = AsyncMock()
ws.receive = AsyncMock()
return ws
@pytest.fixture
def mock_session():
session = MagicMock(spec=TerminalSession)
session.session_id = "sess-123"
session.is_alive.return_value = True
session._closed = False
session.read_output = AsyncMock(return_value=b"")
session.write_input = AsyncMock()
session.resize = AsyncMock()
session.close = AsyncMock()
session.get_exit_reason.return_value = None
return session
class TestCreateSession:
@patch("src.services.terminal_manager.asyncio.create_task")
@patch("src.services.terminal_manager.uuid.uuid4", return_value="sess-123")
async def test_create_session_registers_and_starts_loops(
self, mock_uuid, mock_create_task, manager, mock_websocket
):
instance_id = __import__("uuid").uuid4()
mock_sess = MagicMock()
mock_sess.session_id = "sess-123"
mock_sess.is_alive.return_value = True
mock_sess._closed = False
mock_sess.start = AsyncMock()
mock_sess.read_output = AsyncMock(return_value=b"")
mock_sess.write_input = AsyncMock()
mock_sess.resize = AsyncMock()
mock_sess.close = AsyncMock()
mock_sess.get_exit_reason.return_value = None
with (
patch.object(manager, "_read_loop", new=AsyncMock()),
patch.object(manager, "_write_loop", new=AsyncMock()),
patch.object(manager, "_heartbeat_loop", new=AsyncMock()),
patch(
"src.services.terminal_manager.TerminalSession",
return_value=mock_sess,
),
):
session = await manager.create_session(
instance_id, "container-abc", mock_websocket
)
assert session.session_id == "sess-123"
assert "sess-123" in manager._sessions
assert "sess-123" in manager._last_client_message
class TestHandleControlMessage:
async def test_handle_resize(self, manager, mock_session, mock_websocket):
ctrl = {"type": "resize", "cols": 120, "rows": 40}
await manager._handle_control_message(mock_session, mock_websocket, ctrl)
mock_session.resize.assert_awaited_once_with(120, 40)
async def test_handle_ping(self, manager, mock_session, mock_websocket):
ctrl = {"type": "ping", "id": 42}
await manager._handle_control_message(mock_session, mock_websocket, ctrl)
mock_websocket.send_json.assert_awaited_once_with({"type": "pong", "id": 42})
async def test_handle_unknown_type(self, manager, mock_session, mock_websocket):
ctrl = {"type": "unknown", "data": "test"}
await manager._handle_control_message(mock_session, mock_websocket, ctrl)
mock_websocket.send_json.assert_not_awaited()
mock_session.resize.assert_not_awaited()
class TestCleanupSession:
async def test_cleanup_removes_session(self, manager, mock_session):
manager._sessions["sess-123"] = mock_session
manager._last_client_message["sess-123"] = 123.0
await manager._cleanup_session(mock_session)
assert "sess-123" not in manager._sessions
assert "sess-123" not in manager._last_client_message
mock_session.close.assert_awaited_once()
class TestCloseAll:
async def test_close_all_clears_sessions(self, manager, mock_session):
manager._sessions["sess-123"] = mock_session
manager._last_client_message["sess-123"] = 123.0
await manager.close_all()
assert len(manager._sessions) == 0
assert len(manager._last_client_message) == 0
mock_session.close.assert_awaited_once()
@@ -0,0 +1,168 @@
"""Unit tests for TerminalSession."""
import asyncio
from unittest.mock import MagicMock, patch
import pytest
from src.services.terminal_session import TerminalSession
@pytest.fixture
def mock_pty():
"""Mock pty.openpty to return predictable fds."""
master_fd = 10
slave_fd = 11
with (
patch(
"src.services.terminal_session.pty.openpty",
return_value=(master_fd, slave_fd),
),
patch("src.services.terminal_session.os.close") as mock_close,
):
yield master_fd, slave_fd, mock_close
class TestTerminalSessionStart:
def test_init_state(self, mock_pty):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
assert session.session_id == "sess-1"
assert session.container_id == "container-abc"
assert session._echo_enabled is True
assert session._exit_reason is None
class TestTerminalSessionEchoDetection:
@patch("src.services.terminal_session.termios.tcgetattr")
def test_detect_echo_state_enabled(self, mock_tcgetattr):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
# termios.ECHO flag set
attrs = [[], [], [], __import__("termios").ECHO, [], [], []]
mock_tcgetattr.return_value = attrs
result = session._detect_echo_state()
assert result is True
@patch("src.services.terminal_session.termios.tcgetattr")
def test_detect_echo_state_disabled(self, mock_tcgetattr):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
# termios.ECHO flag NOT set
attrs = [[], [], [], 0, [], [], []]
mock_tcgetattr.return_value = attrs
result = session._detect_echo_state()
assert result is False
def test_detect_echo_state_no_master_fd(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = None
result = session._detect_echo_state()
assert result is True # default
class TestTerminalSessionResize:
@patch("src.services.terminal_session.fcntl.ioctl")
def test_resize_sets_size(self, mock_ioctl):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
# Should not raise
asyncio.run(session.resize(120, 40))
mock_ioctl.assert_called_once()
def test_resize_when_closed(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._closed = True
# Should not raise
asyncio.run(session.resize(120, 40))
class TestTerminalSessionWriteInput:
@patch("src.services.terminal_session.os.write")
def test_write_input(self, mock_write):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
asyncio.run(session.write_input(b"hello"))
mock_write.assert_called_once_with(10, b"hello")
def test_write_input_when_closed(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._closed = True
# Should not raise
asyncio.run(session.write_input(b"hello"))
class TestTerminalSessionReadOutput:
@patch("src.services.terminal_session.select.select")
@patch("src.services.terminal_session.os.read")
def test_read_output_with_data(self, mock_read, mock_select):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
mock_select.return_value = ([10], [], [])
mock_read.return_value = b"output"
result = asyncio.run(session.read_output())
assert result == b"output"
@patch("src.services.terminal_session.select.select")
def test_read_output_no_data(self, mock_select):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
mock_select.return_value = ([], [], [])
result = asyncio.run(session.read_output())
assert result == b""
class TestTerminalSessionClose:
@patch("src.services.terminal_session.os.close")
@patch("src.services.terminal_session.asyncio.wait_for")
async def test_close_sets_exit_reason(self, mock_wait_for, mock_close):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._master_fd = 10
session.process = MagicMock()
session.process.returncode = 0
await session.close()
assert session._exit_reason == "process_exit"
assert session._closed is True
async def test_close_idempotent(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session._closed = True
# Should not raise
await session.close()
class TestTerminalSessionIsAlive:
def test_is_alive_with_running_process(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session.process = MagicMock()
session.process.returncode = None
assert session.is_alive() is True
def test_is_alive_with_exited_process(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session.process = MagicMock()
session.process.returncode = 0
assert session.is_alive() is False
def test_is_alive_no_process(self):
session = TerminalSession("sess-1", __import__("uuid").uuid4(), "container-abc")
session.process = None
assert session.is_alive() is False