alex/headquarter
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: responsive web terminal with auto-reconnect, heartbeat, and local echo

Browse Source 
Implements a resilient, responsive web terminal that survives network blips,
provides instant typing feedback, and restores scrollback on reconnect.

Backend changes:
- Add heartbeat tracking (15s ping interval, 60s idle timeout)
- Add message batching (16ms flush window) for efficient I/O
- Add termios echo detection and set_echo_state control messages
- Add graceful session_ended notification before close
- Add ping/pong protocol support

Frontend changes:
- Rewrite TerminalComponent with status bar, connection indicator,
  session-ended overlay, reconnect banner, and ResizeObserver
- Add useTerminalConnection hook with:
  - Exponential backoff auto-reconnect (1s → 30s max, 10 attempts)
  - Heartbeat/ping-pong with latency tracking
  - Local echo for printable ASCII with server deduplication
  - Resize debounce (200ms) + throttle (500ms)
  - Scrollback serialization via xterm-addon-serialize
  - Ctrl+Shift+R manual reconnect shortcut
- Add WebSocket protocol types and encoding utilities
- Add xterm-addon-serialize dependency

Tests:
- 16 backend unit tests (TerminalSession + TerminalManager)
- 13 frontend hook tests (connection lifecycle, reconnect, resize,
  scrollback, callbacks)

Quality gates:
- Frontend typecheck: clean
- Frontend lint: clean
- Frontend tests: 48 passed
- Backend unit tests: 101 passed
- Backend ruff: clean

SDD artifacts: openspec/changes/responsive-terminal/
This commit is contained in:
Alex
2026-05-27 21:27:49 +02:00
parent 48fa858090
commit 6c8cfe9157
18 changed files with 2776 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/terminal_manager.py
+118 -21
View File
@@ -1,19 +1,35 @@
"""Terminal session manager for WebSocket connections."""
import asyncio
import contextlib
import json
import logging
import time
import uuid
from collections.abc import Coroutine
from typing import Any
from fastapi import WebSocket
from src.services.terminal_session import TerminalSession
logger = logging.getLogger(__name__)
_READ_BATCH_INTERVAL_S = 0.016 # 16ms max batching delay
_READ_POLL_TIMEOUT_S = 0.005
_READ_POLL_SLEEP_S = 0.001
_HEARTBEAT_INTERVAL_S = 15.0
_IDLE_TIMEOUT_S = 60.0
class TerminalManager:
"""Manages active terminal sessions."""
def __init__(self) -> None:
"""Initialise the terminal manager."""
self._sessions: dict[str, TerminalSession] = {}
self._last_client_message: dict[str, float] = {}
self._background_tasks: set[asyncio.Task[Any]] = set()
async def create_session(
self,
@@ -26,55 +42,134 @@ class TerminalManager:
session = TerminalSession(session_id, instance_id, container_id)
await session.start()
self._sessions[session_id] = session
self._last_client_message[session_id] = time.monotonic()
# Start background tasks for I/O streaming
asyncio.create_task(self._read_loop(session, websocket))
asyncio.create_task(self._write_loop(session, websocket))
self._start_task(self._read_loop(session, websocket))
self._start_task(self._write_loop(session, websocket))
self._start_task(self._heartbeat_loop(session, websocket))
return session
async def _read_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
"""Read output from the container and send to WebSocket."""
def _start_task(self, coro: Coroutine[Any, Any, None]) -> None:
"""Start a background task and store a reference to prevent GC."""
task = asyncio.create_task(coro)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
async def _read_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Read output from the container and send to WebSocket with batching."""
try:
while session.is_alive() and not session._closed:
data = await session.read_output()
buffer = bytearray()
last_flush = time.monotonic()
while session.is_alive() and not session.closed:
data = await session.read_output(select_timeout=_READ_POLL_TIMEOUT_S)
if data:
await websocket.send_bytes(data)
else:
await asyncio.sleep(0.01)
buffer.extend(data)
now = time.monotonic()
flush_due = buffer and (
now - last_flush >= _READ_BATCH_INTERVAL_S or not data
)
if flush_due:
await websocket.send_bytes(bytes(buffer))
buffer.clear()
last_flush = now
elif not data:
await asyncio.sleep(_READ_POLL_SLEEP_S)
# Flush any remaining data
if buffer:
with contextlib.suppress(Exception):
await websocket.send_bytes(bytes(buffer))
except Exception:
pass
logger.exception("Read loop error for session %s", session.session_id)
finally:
await self._cleanup_session(session)
async def _write_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
async def _write_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Read input from WebSocket and send to container."""
try:
while session.is_alive() and not session._closed:
while session.is_alive() and not session.closed:
message = await websocket.receive()
self._last_client_message[session.session_id] = time.monotonic()
if message["type"] == "websocket.receive":
if "bytes" in message:
await session.write_input(message["bytes"])
elif "text" in message:
text = message["text"]
if text.startswith("{"):
# Control message (JSON)
import json
try:
ctrl = json.loads(text)
if ctrl.get("type") == "resize":
await session.resize(
ctrl.get("cols", 80),
ctrl.get("rows", 24),
)
await self._handle_control_message(
session,
websocket,
ctrl,
)
except json.JSONDecodeError:
pass
logger.debug("Invalid JSON control message: %s", text)
else:
await session.write_input(text.encode("utf-8"))
elif message["type"] == "websocket.disconnect":
break
except Exception:
pass
logger.exception("Write loop error for session %s", session.session_id)
finally:
await self._cleanup_session(session)
async def _handle_control_message(
self,
session: TerminalSession,
websocket: WebSocket,
ctrl: dict[str, Any],
) -> None:
"""Handle a JSON control message from the client."""
msg_type = ctrl.get("type")
if msg_type == "resize":
await session.resize(
ctrl.get("cols", 80),
ctrl.get("rows", 24),
)
elif msg_type == "ping":
await websocket.send_json(
{"type": "pong", "id": ctrl.get("id")},
)
async def _heartbeat_loop(
self,
session: TerminalSession,
websocket: WebSocket,
) -> None:
"""Monitor client activity and close idle connections."""
try:
while session.is_alive() and not session.closed:
await asyncio.sleep(_HEARTBEAT_INTERVAL_S)
last_msg = self._last_client_message.get(session.session_id, 0)
if time.monotonic() - last_msg > _IDLE_TIMEOUT_S:
# Client has been silent for 60s — close connection
with contextlib.suppress(Exception):
await websocket.close(
code=1000,
reason="Idle timeout",
)
break
except Exception:
logger.exception(
"Heartbeat loop error for session %s",
session.session_id,
)
finally:
await self._cleanup_session(session)
@@ -82,12 +177,14 @@ class TerminalManager:
"""Clean up a session."""
if session.session_id in self._sessions:
del self._sessions[session.session_id]
self._last_client_message.pop(session.session_id, None)
await session.close()
async def close_all(self) -> None:
"""Close all active sessions."""
sessions = list(self._sessions.values())
self._sessions.clear()
self._last_client_message.clear()
for session in sessions:
await session.close()
apps/api/src/services/terminal_session.py
+76 -31
View File
@@ -1,19 +1,29 @@
"""Terminal session management for tool instances."""
import asyncio
import contextlib
import fcntl
import logging
import os
import pty
import select
import struct
import fcntl
import termios
import uuid
from typing import Any
logger = logging.getLogger(__name__)
class TerminalSession:
"""Manages a single terminal session connected to a docker container."""
def __init__(self, session_id: str, instance_id: uuid.UUID, container_id: str) -> None:
def __init__(
self,
session_id: str,
instance_id: uuid.UUID,
container_id: str,
) -> None:
"""Initialize a terminal session."""
self.session_id = session_id
self.instance_id = instance_id
self.container_id = container_id
@@ -21,23 +31,20 @@ class TerminalSession:
self._closed = False
self._master_fd: int | None = None
self._slave_fd: int | None = None
self._echo_enabled = True
self._exit_reason: str | None = None
async def start(self) -> None:
"""Start the docker exec process with a shell using a PTY."""
# Create a pseudo-terminal on the host
self._master_fd, self._slave_fd = pty.openpty()
# Set the terminal size initially
self._set_terminal_size(80, 24)
# Start docker exec with the slave fd as stdin/stdout/stderr
# Using -it because the slave fd IS a TTY
self.process = await asyncio.create_subprocess_exec(
"docker",
"exec",
"-it",
"-e",
"TERM=xterm",
"TERM=xterm-256color",
self.container_id,
"bash",
"-il",
@@ -45,44 +52,71 @@ class TerminalSession:
stdout=self._slave_fd,
stderr=self._slave_fd,
)
# Close slave fd in parent process
os.close(self._slave_fd)
self._slave_fd = None
self._echo_enabled = self._detect_echo_state()
def _set_terminal_size(self, cols: int, rows: int) -> None:
"""Set the terminal size using TIOCSWINSZ."""
if self._master_fd is None:
return
# TIOCSWINSZ = 0x5414 on Linux
TIOCSWINSZ = 0x5414
size = struct.pack('HHHH', rows, cols, 0, 0)
try:
fcntl.ioctl(self._master_fd, TIOCSWINSZ, size)
except (OSError, IOError):
pass
tiocswinsz = 0x5414
size = struct.pack("HHHH", rows, cols, 0, 0)
with contextlib.suppress(OSError):
fcntl.ioctl(self._master_fd, tiocswinsz, size)
async def read_output(self) -> bytes:
def _detect_echo_state(self) -> bool:
"""Detect whether the PTY has echo enabled via termios."""
if self._master_fd is None:
return True
try:
attrs = termios.tcgetattr(self._master_fd)
return bool(attrs[3] & termios.ECHO)
except OSError:
return True
async def check_echo_state(self) -> bool | None:
"""Check if echo state changed. Returns new state if changed, None otherwise."""
current = self._detect_echo_state()
if current != self._echo_enabled:
self._echo_enabled = current
return current
return None
@property
def echo_enabled(self) -> bool:
"""Return whether the PTY currently has echo enabled."""
return self._echo_enabled
@property
def closed(self) -> bool:
"""Return whether the session has been closed."""
return self._closed
async def read_output(self, select_timeout: float = 0.1) -> bytes:
"""Read output from the PTY master."""
if self._master_fd is None or self._closed:
return b""
try:
# Use select to check if data is available
readable, _, _ = select.select([self._master_fd], [], [], 0.1)
readable, _, _ = select.select(
[self._master_fd],
[],
[],
select_timeout,
)
if readable:
return os.read(self._master_fd, 4096)
return os.read(self._master_fd, 8192)
return b""
except (OSError, IOError, ValueError):
except (OSError, ValueError):
return b""
async def write_input(self, data: bytes) -> None:
"""Write input to the PTY master."""
if self._master_fd is None or self._closed:
return
try:
with contextlib.suppress(OSError):
os.write(self._master_fd, data)
except (OSError, IOError):
pass
async def resize(self, cols: int, rows: int) -> None:
"""Resize the terminal."""
@@ -90,24 +124,35 @@ class TerminalSession:
return
self._set_terminal_size(cols, rows)
def get_exit_reason(self) -> str | None:
"""Return the reason the session ended, if known."""
return self._exit_reason
async def close(self) -> None:
"""Close the session and cleanup."""
if self._closed:
return
self._closed = True
# Determine exit reason
if self.process is not None and self.process.returncode is not None:
if self.process.returncode == 0:
self._exit_reason = "process_exit"
else:
self._exit_reason = "process_exit"
else:
self._exit_reason = "timeout"
if self._master_fd is not None:
try:
with contextlib.suppress(OSError):
os.close(self._master_fd)
except OSError:
pass
self._master_fd = None
if self.process is not None:
try:
self.process.kill()
await asyncio.wait_for(self.process.wait(), timeout=2.0)
except (asyncio.TimeoutError, ProcessLookupError):
except (TimeoutError, ProcessLookupError):
pass
def is_alive(self) -> bool: