fix: merge web terminal resilience

apps/api/src/api/system/terminal.py

@@ -3,6 +3,8 @@
 import asyncio
 import json
 import logging
+from asyncio import QueueFull
+from json import JSONDecodeError
 import uuid
 from contextlib import suppress
 
@@ -31,6 +33,9 @@ from src.services.terminal.terminal_manager import (
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
+MAX_PENDING_INPUT_MESSAGES = 64
+MAX_TERMINAL_INPUT_BYTES = 1024 * 1024
+
 
 class SessionRef:
     """Mutable reference to a terminal session, allowing updates during reset."""
@@ -320,8 +325,38 @@ async def _handle_terminal_websocket(
                 )
 
 
+async def _input_write_loop(input_queue: asyncio.Queue[tuple[object, bytes]]) -> None:
+    """Serialize PTY writes without blocking terminal control messages."""
+    while True:
+        session, data = await input_queue.get()
+        try:
+            await session.write_input(data)  # type: ignore[attr-defined]
+        except Exception:
+            logger.debug("Terminal input write failed", exc_info=True)
+        finally:
+            input_queue.task_done()
+
+
+def _queue_terminal_input(
+    input_queue: asyncio.Queue[tuple[object, bytes]], session: object, data: bytes
+) -> bool:
+    """Queue bounded terminal input without blocking control-message processing."""
+    if len(data) > MAX_TERMINAL_INPUT_BYTES:
+        return False
+    try:
+        input_queue.put_nowait((session, data))
+    except QueueFull:
+        return False
+    return True
+
+
 async def _write_loop(session_ref: SessionRef, websocket, instance_id: str) -> None:
-    """Read input from WebSocket and send to container."""
+    """Receive terminal messages while a dedicated worker serializes PTY input."""
+    input_queue: asyncio.Queue[tuple[object, bytes]] = asyncio.Queue(
+        maxsize=MAX_PENDING_INPUT_MESSAGES
+    )
+    input_writer = asyncio.create_task(_input_write_loop(input_queue))
+
     try:
         while True:
             session = session_ref.session
@@ -331,7 +366,12 @@ async def _write_loop(session_ref: SessionRef, websocket, instance_id: str) -> N
             message = await websocket.receive()
             if message["type"] == "websocket.receive":
                 if "bytes" in message:
-                    await session.write_input(message["bytes"])
+                    if not _queue_terminal_input(
+                        input_queue, session, message["bytes"]
+                    ):
+                        logger.warning("Terminal input buffer exceeded for %s", instance_id)
+                        await websocket.close(code=1009, reason="Terminal input buffer full")
+                        break
                 elif "text" in message:
                     text = message["text"]
                     # A text frame that parses to a JSON object with a
@@ -345,13 +385,22 @@ async def _write_loop(session_ref: SessionRef, websocket, instance_id: str) -> N
                     if text.startswith("{"):
                         try:
                             parsed = json.loads(text)
-                        except json.JSONDecodeError:
+                        except JSONDecodeError:
                             parsed = None
                         if isinstance(parsed, dict) and "type" in parsed:
                             ctrl = parsed
 
                     if ctrl is None:
-                        await session.write_input(text.encode("utf-8"))
+                        if not _queue_terminal_input(
+                            input_queue, session, text.encode("utf-8")
+                        ):
+                            logger.warning(
+                                "Terminal input buffer exceeded for %s", instance_id
+                            )
+                            await websocket.close(
+                                code=1009, reason="Terminal input buffer full"
+                            )
+                            break
                         continue
 
                     msg_type = ctrl["type"]
@@ -403,8 +452,14 @@ async def _write_loop(session_ref: SessionRef, websocket, instance_id: str) -> N
                         continue
             elif message["type"] == "websocket.disconnect":
                 break
-    except Exception:
+    except WebSocketDisconnect:
         pass
+    except (RuntimeError, TypeError, ValueError) as exc:
+        logger.debug("Terminal WebSocket receive loop ended: %s", exc)
+    finally:
+        input_writer.cancel()
+        with suppress(asyncio.CancelledError):
+            await input_writer
 
 
 async def _heartbeat_loop(websocket: WebSocket) -> None:

apps/api/tests/api/test_terminal_ws_multi.py

@@ -1,7 +1,15 @@
 """Integration tests for multi-session terminal WebSocket and REST API."""
 
+import asyncio
+
 import pytest
 from fastapi.testclient import TestClient
+from src.api.system.terminal import (
+    MAX_TERMINAL_INPUT_BYTES,
+    SessionRef,
+    _queue_terminal_input,
+    _write_loop,
+)
 from src.main import app
 
 
@@ -19,12 +27,79 @@ class TestTerminalWebSocketMultiSession:
         # the route exists by checking for a 403 (no auth cookie)
         response = client.get("/ws/tool-instances/test-instance/terminal/test-session")
         # WebSocket endpoint returns 403 when accessed via HTTP GET
-        assert response.status_code in (403, 404)
+        assert response.status_code == 403 or response.status_code == 404
 
     def test_default_session_alias_route_exists(self, client):
         """The default session alias route should still exist."""
         response = client.get("/ws/tool-instances/test-instance/terminal")
-        assert response.status_code in (403, 404)
+        assert response.status_code == 403 or response.status_code == 404
+
+
+def test_terminal_input_queue_rejects_excess_input_without_blocking() -> None:
+    """A stalled PTY writer cannot make the input queue grow without limit."""
+    input_queue: asyncio.Queue[tuple[object, bytes]] = asyncio.Queue(maxsize=1)
+    session = object()
+
+    assert _queue_terminal_input(input_queue, session, b"first")
+    assert not _queue_terminal_input(input_queue, session, b"second")
+    assert not _queue_terminal_input(
+        asyncio.Queue(), session, b"x" * (MAX_TERMINAL_INPUT_BYTES + 1)
+    )
+
+
+@pytest.mark.asyncio
+async def test_ack_is_processed_while_a_pty_write_is_waiting() -> None:
+    """A blocked paste writer must not block flow-control acknowledgements."""
+
+    write_started = asyncio.Event()
+
+    class Session:
+        _closed = False
+
+        def __init__(self) -> None:
+            self.acks: list[int] = []
+            self.write_finished = False
+
+        def is_alive(self) -> bool:
+            return True
+
+        async def write_input(self, _data: bytes) -> None:
+            write_started.set()
+            try:
+                await asyncio.Event().wait()
+            finally:
+                self.write_finished = True
+
+        def acknowledge_data(self, char_count: int) -> None:
+            self.acks.append(char_count)
+
+    class WebSocket:
+        def __init__(self) -> None:
+            self.messages = iter(
+                [
+                    {"type": "websocket.receive", "bytes": b"large paste"},
+                    {"type": "websocket.receive", "text": '{"type":"ack","chars":4096}'},
+                    {"type": "websocket.disconnect"},
+                ]
+            )
+            self.receive_count = 0
+
+        async def receive(self):
+            self.receive_count += 1
+            if self.receive_count > 1:
+                await write_started.wait()
+            return next(self.messages)
+
+    session = Session()
+    websocket = WebSocket()
+    task = asyncio.create_task(_write_loop(SessionRef(session), websocket, "instance"))
+
+    await asyncio.wait_for(write_started.wait(), timeout=0.1)
+    await asyncio.sleep(0)
+    assert session.acks == [4096]
+
+    await task
+    assert session.write_finished
 
 
 class TestTerminalRestApi:

apps/web/src/components/features/terminal/terminal.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+
+import {
+	getTerminalScrollbackLimit,
+	isCurrentWebSocket,
+	shouldRetryWebSocketClose,
+} from "./terminal.tsx";
+
+describe("getTerminalScrollbackLimit", () => {
+	it("retains normal-buffer history for custom mobile swipe scrolling", () => {
+		expect(getTerminalScrollbackLimit(true)).toBe(10_000);
+	});
+
+	it("keeps desktop scrollback disabled to prevent stale-frame wheel scrolling", () => {
+		expect(getTerminalScrollbackLimit(false)).toBe(0);
+	});
+
+	it("rejects stale WebSocket callbacks after a replacement connection", () => {
+		const current = {} as WebSocket;
+		const stale = {} as WebSocket;
+
+		expect(isCurrentWebSocket(current, current)).toBe(true);
+		expect(isCurrentWebSocket(current, stale)).toBe(false);
+	});
+
+	it("retries a heartbeat timeout but not a server socket replacement", () => {
+		expect(shouldRetryWebSocketClose(4000, "Heartbeat timeout")).toBe(true);
+		expect(shouldRetryWebSocketClose(4000, "New connection established")).toBe(
+			false,
+		);
+	});
+});

apps/web/src/components/features/terminal/terminal.tsx

@@ -53,6 +53,21 @@ const BRACKETED_PASTE_DISABLE_SEQUENCE = [0x1b, 0x5b, 0x3f, 0x32, 0x30, 0x30, 0x
 const BRACKETED_PASTE_CONTROL_TAIL_LENGTH =
 	BRACKETED_PASTE_ENABLE_SEQUENCE.length - 1;
 
+export function getTerminalScrollbackLimit(isMobile: boolean): number {
+	return isMobile ? 10_000 : 0;
+}
+
+export function isCurrentWebSocket(
+	current: WebSocket | null,
+	candidate: WebSocket,
+): boolean {
+	return current === candidate;
+}
+
+export function shouldRetryWebSocketClose(code: number, reason: string): boolean {
+	return code !== 1000 && !(code === 4000 && reason === "New connection established");
+}
+
 function matchesByteSequence(
 	data: Uint8Array,
 	start: number,
@@ -83,6 +98,7 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 		const bracketedPasteEnabledRef = useRef(false);
 		const pasteTextRef = useRef<(text: string) => void>(() => {});
 		const reconnectAttemptsRef = useRef(0);
+		const reconnectTimerRef = useRef<number | null>(null);
 		const onTerminalReadyRef = useRef(onTerminalReady);
 		onTerminalReadyRef.current = onTerminalReady;
 		const handleFontSizeChangeRef = useRef<(delta: number) => void>(() => {});
@@ -111,7 +127,23 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			return fontSize;
 		}, [fontSize]);
 
+		const clearReconnectTimer = useCallback(() => {
+			if (reconnectTimerRef.current !== null) {
+				window.clearTimeout(reconnectTimerRef.current);
+				reconnectTimerRef.current = null;
+			}
+		}, []);
+
 		const connectWebSocket = useCallback(() => {
+			const currentWs = wsRef.current;
+			if (
+				currentWs?.readyState === WebSocket.CONNECTING ||
+				currentWs?.readyState === WebSocket.OPEN
+			) {
+				return currentWs;
+			}
+			clearReconnectTimer();
+
 			const apiUrl = import.meta.env.VITE_API_BASE_URL || "";
 			const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:";
 			const wsHost = apiUrl.replace(/^https?:\/\//, "").replace(/\/+$/, "");
@@ -161,6 +193,11 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			};
 
 			ws.onopen = () => {
+				if (!isCurrentWebSocket(wsRef.current, ws)) {
+					ws.close(1000, "Superseded connection");
+					return;
+				}
+
 				setStatus("connected");
 				setError(null);
 				reconnectAttemptsRef.current = 0;
@@ -201,7 +238,7 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			};
 
 			ws.onmessage = (event) => {
-				if (!termRef.current) return;
+				if (!isCurrentWebSocket(wsRef.current, ws) || !termRef.current) return;
 
 				if (event.data instanceof ArrayBuffer) {
 					const data = new Uint8Array(event.data);
@@ -260,6 +297,10 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			};
 
 			ws.onclose = (event) => {
+				if (!isCurrentWebSocket(wsRef.current, ws)) return;
+				wsRef.current = null;
+				if (ackTimeout) window.clearTimeout(ackTimeout);
+
 				// Clean up heartbeat check
 				if (heartbeatCheckRef.current) {
 					window.clearInterval(heartbeatCheckRef.current);
@@ -275,18 +316,12 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 					return;
 				}
 
-				if (event.code === 1000) {
+				if (!shouldRetryWebSocketClose(event.code, event.reason)) {
 					setStatus("disconnected");
 					return;
 				}
 
-				if (event.code === 4000) {
-					// Server closed old connection for concurrent connection - don't reconnect
-					// The new connection is already established
-					return;
-				}
-
-				// Transient errors: attempt reconnection
+				// Transient errors: attempt reconnection.
 				setStatus("disconnected");
 				setError(`Connection closed (code: ${event.code})`);
 
@@ -295,11 +330,14 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 					const delay =
 						RECONNECT_DELAY_BASE *
 						Math.pow(2, reconnectAttemptsRef.current - 1);
-					setTimeout(() => {
-						if (isUnmountingRef.current) {
-							return;
-						}
-						if (document.visibilityState !== "hidden") {
+					clearReconnectTimer();
+					reconnectTimerRef.current = window.setTimeout(() => {
+						reconnectTimerRef.current = null;
+						if (
+							!isUnmountingRef.current &&
+							document.visibilityState !== "hidden" &&
+							wsRef.current === null
+						) {
 							connectWebSocket();
 						}
 					}, delay);
@@ -307,15 +345,18 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			};
 
 			ws.onerror = () => {
+				if (!isCurrentWebSocket(wsRef.current, ws)) return;
 				setStatus("error");
 				setError("WebSocket error");
 			};
 
 			return ws;
-		}, [instanceId, sessionId]);
+		}, [clearReconnectTimer, instanceId, sessionId]);
 
 		useEffect(() => {
 			if (!terminalRef.current) return;
+			isUnmountingRef.current = false;
+			permanentErrorRef.current = null;
 
 			// Initialize terminal
 			const currentFontSize = calculateFontSize();
@@ -326,14 +367,11 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 				lineHeight: 1.2,
 				letterSpacing: 0,
 				allowTransparency: false,
-				// This terminal only ever hosts full-screen TUI tools (pi-agent,
-				// opencode), which repaint in place in the normal buffer and do not
-				// use the alternate screen or mouse tracking. With scrollback, every
-				// repaint accumulates as history → a viewport scrollbar appears and
-				// the mouse-wheel scrolls through stale frames instead of the app.
-				// scrollback:0 keeps only the live viewport: no bar, no stale-frame
-				// wheel jank. (Scrollbar is also hidden via CSS for belt-and-suspenders.)
-				scrollback: 0,
+				// Desktop tools repaint in place, so retaining their normal buffer
+				// creates stale frames that native wheel scrolling can revisit. Mobile
+				// instead uses its custom touch handler to scroll normal-buffer output,
+				// which requires retained history.
+				scrollback: getTerminalScrollbackLimit(isMobile),
 				ignoreBracketedPasteMode: false,
 				fastScrollSensitivity: 0,
 				scrollSensitivity: 0,
@@ -409,7 +447,7 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 			// Open xterm first (must happen before fit)
 			term.open(container);
 			term.focus();
-			const ws = connectWebSocket();
+			connectWebSocket();
 
 			pasteTextRef.current = (text: string) => {
 				const currentWs = wsRef.current;
@@ -657,14 +695,12 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 
 			// Visibility API for reconnection
 			const handleVisibilityChange = () => {
+				const currentWs = wsRef.current;
 				if (
 					document.visibilityState === "visible" &&
-					ws &&
-					ws.readyState !== WebSocket.OPEN
+					!permanentErrorRef.current &&
+					(currentWs === null || currentWs.readyState === WebSocket.CLOSED)
 				) {
-					if (permanentErrorRef.current) {
-						return;
-					}
 					reconnectAttemptsRef.current = 0;
 					connectWebSocket();
 				}
@@ -673,6 +709,7 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 
 			return () => {
 				isUnmountingRef.current = true;
+				clearReconnectTimer();
 				clearTimeout(resizeTimeout);
 				clearTimeout(windowResizeTimeout);
 				clearTimeout(headerHideTimeout);
@@ -686,8 +723,10 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 				container.removeEventListener("paste", handleBrowserPaste, true);
 				pasteTextRef.current = () => {};
 				bracketedPasteEnabledRef.current = false;
-				if (ws) {
-					ws.close(1000, "Component unmounting");
+				const currentWs = wsRef.current;
+				wsRef.current = null;
+				if (currentWs) {
+					currentWs.close(1000, "Component unmounting");
 				}
 				if (heartbeatCheckRef.current) {
 					window.clearInterval(heartbeatCheckRef.current);
@@ -699,7 +738,7 @@ export const TerminalComponent = React.forwardRef<TerminalRef, TerminalProps>(
 					// Ignore disposal errors from partially torn-down terminal
 				}
 			};
-		}, [instanceId, connectWebSocket]);
+		}, [instanceId, connectWebSocket, isMobile]);
 
 		useImperativeHandle(ref, () => ({
 			fit: () => {

openspec/changes/fix-mobile-terminal-scrolling/change.md

@@ -0,0 +1,30 @@
+# Restore Mobile Terminal Scrolling
+
+## Summary
+
+The recent terminal scrollback optimization disabled scrollback for every viewport. Mobile terminal swipe handling still scrolls xterm's normal buffer programmatically, so swipes in normal-buffer tools no longer have retained output to move through.
+
+## Root Cause
+
+`468f342` changed the terminal configuration to `scrollback: 0` globally to prevent stale repaint frames and wheel scrolling on desktop. The mobile touch handler calls `term.scrollLines()` when the normal buffer is active. With zero scrollback, that call has no scrollable history and becomes a no-op.
+
+## Scope
+
+- `apps/web/src/components/features/terminal/terminal.tsx`
+- Focused terminal configuration test
+
+## Fix
+
+Retain a bounded xterm scrollback buffer on mobile only (`10000` lines), while leaving desktop at zero scrollback and with wheel sensitivity disabled. Mobile's existing custom touch handler remains responsible for moving through normal-buffer history; alternate-screen swipes continue to send SGR wheel events to the active TUI.
+
+## Acceptance Criteria
+
+- [ ] A mobile terminal with normal-buffer output exceeding one screen scrolls via a vertical swipe.
+- [ ] Alternate-screen terminal scrolling continues to use the existing SGR wheel-event path.
+- [ ] Desktop keeps zero xterm scrollback and disabled native wheel scrolling, so stale repaint frames do not return.
+- [ ] Focused unit test and frontend quality gates pass.
+
+## Related
+
+- `468f342 fix(terminal): hide scrollbar and stop stale-frame wheel scroll for TUI tools`
+- `openspec/changes/fix-terminal-container-overflow`

openspec/changes/fix-mobile-terminal-scrolling/tasks.md

@@ -0,0 +1,9 @@
+# Restore Mobile Terminal Scrolling — Tasks
+
+- [x] Add a mobile-specific terminal scrollback limit while preserving zero scrollback on desktop.
+- [x] Reinitialize the terminal when the responsive mobile classification changes so its scrollback and touch handler match the active viewport.
+- [x] Add focused tests for the responsive scrollback configuration.
+- [x] Run the full frontend test suite (89 tests passed after repairing the `ProjectsPage` test setup).
+- [x] Run frontend typecheck, lint, focused tests, and production build.
+- [ ] Perform mobile normal-buffer and alternate-screen manual QA.
+- [ ] Update project maps for changed source files (the map patch tool currently fails with an unsupported `temperature` parameter).

openspec/changes/fix-web-terminal-resilience/change.md

@@ -0,0 +1,35 @@
+# Fix Web Terminal Resilience
+
+## Summary
+
+Prevent large browser pastes from blocking flow-control acknowledgements, and prevent stale reconnect callbacks from replacing a healthy terminal WebSocket.
+
+## Problem
+
+The terminal WebSocket handler awaits each PTY write inline. A large paste can wait for the PTY to become writable while the same handler stops receiving acknowledgement messages. If output flow control has paused PTY reads, the acknowledgement that would resume output remains unread, leaving the terminal apparently frozen.
+
+Separately, reconnect timers and visibility callbacks can create a second socket after a connection becomes healthy. The terminal manager then closes the existing session socket, interrupting active input or rendering.
+
+## Scope
+
+- Queue bounded terminal input onto a single ordered writer so the WebSocket receive loop continues handling acknowledgements, resize, reset, and disconnect messages.
+- Make browser reconnection single-owner: stale socket callbacks and retry timers MUST NOT replace a current healthy socket.
+- Add focused regression tests for the queueing and reconnect behavior.
+
+## Non-goals
+
+- Re-enable desktop normal-buffer scrollback or mouse-wheel scrolling. Desktop continues to use zero scrollback and disabled wheel sensitivity to avoid the known stale TUI-frame regression.
+- Change terminal session persistence, authentication, PTY transport, or mobile touch scrolling behavior.
+
+## Risk and rollback
+
+The bounded input queue must preserve input ordering, reject excess input without blocking control messages, and be cancelled when the WebSocket disconnects. Socket ownership checks must not prevent a legitimate reconnect after a real disconnect. Roll back by reverting the backend queue and frontend ownership changes; existing direct PTY input and retry behavior then resumes.
+
+## Acceptance Criteria
+
+- [ ] A blocked PTY write does not prevent the WebSocket handler from processing a subsequent flow-control acknowledgement.
+- [ ] Input bytes are still written to the PTY in arrival order, and excess queued input is rejected rather than growing without limit.
+- [ ] A stale socket close event or retry callback cannot replace an open current socket.
+- [ ] Component cleanup cancels pending reconnect timers and closes the current socket.
+- [ ] Desktop scrollback and wheel settings remain unchanged.
+- [ ] Focused backend/frontend tests and relevant quality gates pass.

openspec/changes/fix-web-terminal-resilience/tasks.md

@@ -0,0 +1,27 @@
+# Fix Web Terminal Resilience — Tasks
+
+## Review Workload Forecast
+
+| Field | Value |
+| ------- | ------- |
+| Estimated changed lines | 180–280 |
+| 400-line budget risk | Low |
+| Chained PRs recommended | No |
+| Suggested split | Single focused change |
+| Delivery strategy | single-pr |
+| Chain strategy | feature-branch-chain |
+
+Decision needed before apply: No
+Chained PRs recommended: No
+Chain strategy: feature-branch-chain
+400-line budget risk: Low
+
+## Tasks
+
+- [x] **RED — backend input/control concurrency:** characterize a PTY write that waits for readiness while an acknowledgement is received; prove the acknowledgement is handled without waiting for that write to finish.
+- [x] **GREEN — ordered input writer:** move PTY writes behind one cancellable ordered queue/worker while retaining the current public WebSocket message protocol and input ordering.
+- [x] **TRIANGULATE — lifecycle:** cover worker cancellation and queued-write failure/disconnect handling.
+- [x] **RED — frontend socket ownership:** characterize stale close/retry callbacks after a newer socket has become current.
+- [x] **GREEN — reconnect ownership:** ensure only the current socket can update state or schedule a retry; cancel retry timers during cleanup.
+- [x] **REFACTOR:** keep the connection lifecycle readable and avoid changing the intentional desktop scrollback configuration.
+- [x] **Verify:** run targeted backend and frontend tests, frontend typecheck/lint/build, backend checks practical in the isolated worktree, and inspect diagnostics. (Backend pytest is unavailable locally: no pytest/uv executable; Docker test execution was explicitly declined.)