Merge branch 'dev' of ssh://git.commumedia.org:2222/alex/headquarter into dev
This commit is contained in:
Fusion
+154
-10
@@ -26,6 +26,7 @@ async def terminal_websocket(
|
||||
"""WebSocket endpoint for terminal access to a tool instance.
|
||||
|
||||
Provides an interactive terminal session inside a running tool instance container.
|
||||
Sessions persist across WebSocket disconnections.
|
||||
|
||||
Args:
|
||||
websocket: The WebSocket connection.
|
||||
@@ -70,32 +71,175 @@ async def terminal_websocket(
|
||||
await websocket.close(code=4004, reason="Instance not running")
|
||||
return
|
||||
|
||||
logger.info("Creating terminal session for instance %s (container_id=%s)", instance_id, instance.container_id)
|
||||
# Create terminal session
|
||||
# Get or create terminal session
|
||||
try:
|
||||
session = await terminal_manager.create_session(
|
||||
session = await terminal_manager.get_or_create_session(
|
||||
instance_uuid,
|
||||
instance.container_id,
|
||||
websocket,
|
||||
)
|
||||
logger.info("Terminal session created successfully for instance %s", instance_id)
|
||||
logger.info("Terminal session ready for instance %s (session_id=%s)", instance_id, session.session_id)
|
||||
|
||||
# Attach WebSocket to session
|
||||
await terminal_manager.attach_websocket(session, websocket)
|
||||
logger.info("WebSocket attached to session for instance %s", instance_id)
|
||||
|
||||
# Send connected status
|
||||
await websocket.send_json({"type": "status", "status": "connected"})
|
||||
|
||||
# Keep connection alive until session ends
|
||||
# The terminal_manager handles I/O loops, we just wait here
|
||||
while session.is_alive() and not session._closed:
|
||||
await asyncio.sleep(0.5)
|
||||
# Start I/O loops
|
||||
read_task = asyncio.create_task(_read_loop(session, websocket))
|
||||
write_task = asyncio.create_task(_write_loop(session, websocket))
|
||||
|
||||
# Wait for either task to complete (indicating disconnect or error)
|
||||
done, pending = await asyncio.wait(
|
||||
[read_task, write_task],
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
# Cancel remaining tasks
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Terminal session error for instance %s: %s", instance_id, str(exc), exc_info=True)
|
||||
await websocket.close(code=4000, reason=f"Error: {exc}")
|
||||
finally:
|
||||
# Cleanup will be handled by the session manager
|
||||
# Detach WebSocket, don't kill session
|
||||
try:
|
||||
if 'session' in locals():
|
||||
await terminal_manager.detach_websocket(session, websocket)
|
||||
logger.info("WebSocket detached from session for instance %s", instance_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _read_loop(session, websocket) -> None:
|
||||
"""Read output from the container and send to WebSocket."""
|
||||
try:
|
||||
while session.is_alive() and not session._closed:
|
||||
data = await session.read_output()
|
||||
if data:
|
||||
try:
|
||||
await websocket.send_bytes(data)
|
||||
except Exception:
|
||||
break
|
||||
else:
|
||||
await asyncio.sleep(0.01)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _write_loop(session, websocket) -> None:
|
||||
"""Read input from WebSocket and send to container."""
|
||||
try:
|
||||
while session.is_alive() and not session._closed:
|
||||
message = await websocket.receive()
|
||||
if message["type"] == "websocket.receive":
|
||||
if "bytes" in message:
|
||||
await session.write_input(message["bytes"])
|
||||
elif "text" in message:
|
||||
text = message["text"]
|
||||
if text.startswith("{"):
|
||||
# Control message (JSON)
|
||||
import json
|
||||
try:
|
||||
ctrl = json.loads(text)
|
||||
msg_type = ctrl.get("type")
|
||||
|
||||
if msg_type == "resize":
|
||||
await session.resize(
|
||||
ctrl.get("cols", 80),
|
||||
ctrl.get("rows", 24),
|
||||
)
|
||||
elif msg_type == "reset":
|
||||
# Reset terminal session
|
||||
logger.info("Resetting terminal session for instance %s", session.instance_id)
|
||||
await websocket.send_json({"type": "status", "status": "resetting"})
|
||||
|
||||
# Reset the session
|
||||
new_session = await terminal_manager.reset_session(
|
||||
session.instance_id,
|
||||
session.container_id,
|
||||
)
|
||||
|
||||
# Attach to new session
|
||||
await terminal_manager.attach_websocket(new_session, websocket)
|
||||
await websocket.send_json({"type": "status", "status": "connected"})
|
||||
|
||||
# Update session reference and restart loops
|
||||
# Note: This will cause the current loops to exit
|
||||
# The WebSocket handler will create new ones
|
||||
return
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
await session.write_input(text.encode("utf-8"))
|
||||
elif message["type"] == "websocket.disconnect":
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.post(
|
||||
"/projects/{project_id}/repositories/{repo_id}/instances/{instance_id}/terminal/reset",
|
||||
summary="Reset terminal session",
|
||||
description="Reset the terminal session for a tool instance, killing the current shell and starting fresh.",
|
||||
)
|
||||
async def reset_terminal_session(
|
||||
project_id: uuid.UUID,
|
||||
repo_id: uuid.UUID,
|
||||
instance_id: uuid.UUID,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
) -> dict:
|
||||
"""Reset the terminal session for an instance.
|
||||
|
||||
Args:
|
||||
project_id: UUID of the project.
|
||||
repo_id: UUID of the repository.
|
||||
instance_id: UUID of the tool instance.
|
||||
db_session: Database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with status message.
|
||||
"""
|
||||
# Get instance and verify it exists and is running
|
||||
instance = await db_session.get(ToolInstance, instance_id)
|
||||
if instance is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Instance not found"
|
||||
)
|
||||
|
||||
if instance.status != "running" or not instance.container_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Instance is not running"
|
||||
)
|
||||
|
||||
try:
|
||||
# Reset the session
|
||||
new_session = await terminal_manager.reset_session(
|
||||
instance_id,
|
||||
instance.container_id,
|
||||
)
|
||||
|
||||
logger.info("Terminal session reset for instance %s (new session_id=%s)", instance_id, new_session.session_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Terminal session reset successfully",
|
||||
"instance_id": str(instance_id),
|
||||
"session_id": new_session.session_id,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error("Failed to reset terminal session for instance %s: %s", instance_id, str(exc), exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to reset terminal session: {exc}"
|
||||
)
|
||||
|
||||
|
||||
async def _get_user_from_websocket(
|
||||
websocket: WebSocket,
|
||||
db_session: AsyncSession,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Terminal session manager for WebSocket connections."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
@@ -8,81 +9,132 @@ from fastapi import WebSocket
|
||||
|
||||
from src.services.terminal_session import TerminalSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TerminalManager:
|
||||
"""Manages active terminal sessions."""
|
||||
"""Manages active terminal sessions with persistence support."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Track sessions by instance_id for persistence
|
||||
self._sessions: dict[str, TerminalSession] = {}
|
||||
self._idle_check_task: asyncio.Task | None = None
|
||||
self._start_idle_check()
|
||||
|
||||
async def create_session(
|
||||
def _start_idle_check(self) -> None:
|
||||
"""Start the idle timeout background task."""
|
||||
if self._idle_check_task is None or self._idle_check_task.done():
|
||||
self._idle_check_task = asyncio.create_task(self._idle_check_loop())
|
||||
|
||||
async def _idle_check_loop(self) -> None:
|
||||
"""Periodically check for idle sessions and clean them up."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(60) # Check every minute
|
||||
await self._cleanup_idle_sessions()
|
||||
except Exception as exc:
|
||||
logger.error("Error in idle check loop: %s", exc)
|
||||
|
||||
async def _cleanup_idle_sessions(self) -> None:
|
||||
"""Clean up sessions that have been idle for too long."""
|
||||
idle_sessions = []
|
||||
for instance_id, session in list(self._sessions.items()):
|
||||
if session.is_idle():
|
||||
idle_sessions.append(instance_id)
|
||||
|
||||
for instance_id in idle_sessions:
|
||||
logger.info("Cleaning up idle terminal session for instance %s", instance_id)
|
||||
session = self._sessions.pop(instance_id, None)
|
||||
if session:
|
||||
await session.close()
|
||||
|
||||
async def get_or_create_session(
|
||||
self,
|
||||
instance_id: uuid.UUID,
|
||||
container_id: str,
|
||||
websocket: WebSocket,
|
||||
) -> TerminalSession:
|
||||
"""Create a new terminal session."""
|
||||
"""Get existing session or create a new one."""
|
||||
instance_id_str = str(instance_id)
|
||||
|
||||
# Check for existing session
|
||||
if instance_id_str in self._sessions:
|
||||
session = self._sessions[instance_id_str]
|
||||
|
||||
# Check if session is still alive
|
||||
if session.is_alive():
|
||||
logger.info("Reattaching to existing terminal session for instance %s", instance_id)
|
||||
return session
|
||||
else:
|
||||
# Session died, clean it up
|
||||
logger.info("Existing session for instance %s is dead, cleaning up", instance_id)
|
||||
await session.close()
|
||||
del self._sessions[instance_id_str]
|
||||
|
||||
# Create new session
|
||||
logger.info("Creating new terminal session for instance %s", instance_id)
|
||||
session_id = str(uuid.uuid4())
|
||||
session = TerminalSession(session_id, instance_id, container_id)
|
||||
await session.start()
|
||||
self._sessions[session_id] = session
|
||||
|
||||
# Start background tasks for I/O streaming
|
||||
asyncio.create_task(self._read_loop(session, websocket))
|
||||
asyncio.create_task(self._write_loop(session, websocket))
|
||||
|
||||
self._sessions[instance_id_str] = session
|
||||
|
||||
return session
|
||||
|
||||
async def _read_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
|
||||
"""Read output from the container and send to WebSocket."""
|
||||
try:
|
||||
while session.is_alive() and not session._closed:
|
||||
data = await session.read_output()
|
||||
if data:
|
||||
await websocket.send_bytes(data)
|
||||
else:
|
||||
await asyncio.sleep(0.01)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
await self._cleanup_session(session)
|
||||
async def attach_websocket(
|
||||
self,
|
||||
session: TerminalSession,
|
||||
websocket: WebSocket,
|
||||
) -> None:
|
||||
"""Attach a WebSocket to an existing session."""
|
||||
# Handle concurrent connections - close existing ones
|
||||
if session.has_websockets():
|
||||
logger.info("Closing existing WebSocket connections for instance %s", session.instance_id)
|
||||
for ws in list(session._websockets):
|
||||
try:
|
||||
await ws.close(code=4000, reason="New connection established")
|
||||
except Exception:
|
||||
pass
|
||||
session._websockets.clear()
|
||||
|
||||
# Attach new WebSocket
|
||||
session.attach_websocket(websocket)
|
||||
|
||||
# Replay buffer
|
||||
buffer = session.get_buffer()
|
||||
if buffer:
|
||||
try:
|
||||
await websocket.send_bytes(buffer)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _write_loop(self, session: TerminalSession, websocket: WebSocket) -> None:
|
||||
"""Read input from WebSocket and send to container."""
|
||||
try:
|
||||
while session.is_alive() and not session._closed:
|
||||
message = await websocket.receive()
|
||||
if message["type"] == "websocket.receive":
|
||||
if "bytes" in message:
|
||||
await session.write_input(message["bytes"])
|
||||
elif "text" in message:
|
||||
text = message["text"]
|
||||
if text.startswith("{"):
|
||||
# Control message (JSON)
|
||||
import json
|
||||
try:
|
||||
ctrl = json.loads(text)
|
||||
if ctrl.get("type") == "resize":
|
||||
await session.resize(
|
||||
ctrl.get("cols", 80),
|
||||
ctrl.get("rows", 24),
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
await session.write_input(text.encode("utf-8"))
|
||||
elif message["type"] == "websocket.disconnect":
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
await self._cleanup_session(session)
|
||||
async def detach_websocket(
|
||||
self,
|
||||
session: TerminalSession,
|
||||
websocket: WebSocket,
|
||||
) -> None:
|
||||
"""Detach a WebSocket from a session."""
|
||||
session.detach_websocket(websocket)
|
||||
|
||||
async def _cleanup_session(self, session: TerminalSession) -> None:
|
||||
"""Clean up a session."""
|
||||
if session.session_id in self._sessions:
|
||||
del self._sessions[session.session_id]
|
||||
await session.close()
|
||||
async def reset_session(
|
||||
self,
|
||||
instance_id: uuid.UUID,
|
||||
container_id: str,
|
||||
) -> TerminalSession:
|
||||
"""Reset a session by killing it and creating a new one."""
|
||||
instance_id_str = str(instance_id)
|
||||
|
||||
# Close existing session if any
|
||||
if instance_id_str in self._sessions:
|
||||
logger.info("Resetting terminal session for instance %s", instance_id)
|
||||
old_session = self._sessions.pop(instance_id_str)
|
||||
await old_session.close()
|
||||
|
||||
# Create new session
|
||||
session_id = str(uuid.uuid4())
|
||||
session = TerminalSession(session_id, instance_id, container_id)
|
||||
await session.start()
|
||||
self._sessions[instance_id_str] = session
|
||||
|
||||
return session
|
||||
|
||||
async def close_all(self) -> None:
|
||||
"""Close all active sessions."""
|
||||
@@ -90,6 +142,9 @@ class TerminalManager:
|
||||
self._sessions.clear()
|
||||
for session in sessions:
|
||||
await session.close()
|
||||
|
||||
if self._idle_check_task and not self._idle_check_task.done():
|
||||
self._idle_check_task.cancel()
|
||||
|
||||
|
||||
# Global terminal manager instance
|
||||
|
||||
@@ -6,12 +6,24 @@ import pty
|
||||
import select
|
||||
import struct
|
||||
import fcntl
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TerminalSession:
|
||||
"""Manages a single terminal session connected to a docker container."""
|
||||
"""Manages a single terminal session connected to a docker container.
|
||||
|
||||
Supports persistent sessions that survive WebSocket disconnections.
|
||||
Multiple WebSocket connections can attach/detach from the same session.
|
||||
"""
|
||||
|
||||
# Circular buffer size (10KB)
|
||||
BUFFER_SIZE = 10 * 1024
|
||||
|
||||
# Idle timeout in seconds (30 minutes)
|
||||
IDLE_TIMEOUT = 30 * 60
|
||||
|
||||
def __init__(self, session_id: str, instance_id: uuid.UUID, container_id: str) -> None:
|
||||
self.session_id = session_id
|
||||
@@ -21,6 +33,20 @@ class TerminalSession:
|
||||
self._closed = False
|
||||
self._master_fd: int | None = None
|
||||
self._slave_fd: int | None = None
|
||||
|
||||
# Circular buffer for output replay
|
||||
self._output_buffer: deque[bytes] = deque(maxlen=self.BUFFER_SIZE)
|
||||
self._buffer_size = 0
|
||||
|
||||
# WebSocket connections
|
||||
self._websockets: set[Any] = set()
|
||||
|
||||
# Activity tracking
|
||||
self.last_activity = time.time()
|
||||
|
||||
# Terminal size
|
||||
self._cols = 80
|
||||
self._rows = 24
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the docker exec process with a shell using a PTY."""
|
||||
@@ -28,7 +54,7 @@ class TerminalSession:
|
||||
self._master_fd, self._slave_fd = pty.openpty()
|
||||
|
||||
# Set the terminal size initially
|
||||
self._set_terminal_size(80, 24)
|
||||
self._set_terminal_size(self._cols, self._rows)
|
||||
|
||||
# Start docker exec with the slave fd as stdin/stdout/stderr
|
||||
# Using -it because the slave fd IS a TTY
|
||||
@@ -49,6 +75,8 @@ class TerminalSession:
|
||||
# Close slave fd in parent process
|
||||
os.close(self._slave_fd)
|
||||
self._slave_fd = None
|
||||
|
||||
self.last_activity = time.time()
|
||||
|
||||
def _set_terminal_size(self, cols: int, rows: int) -> None:
|
||||
"""Set the terminal size using TIOCSWINSZ."""
|
||||
@@ -63,24 +91,43 @@ class TerminalSession:
|
||||
pass
|
||||
|
||||
async def read_output(self) -> bytes:
|
||||
"""Read output from the PTY master."""
|
||||
"""Read output from the PTY master and store in buffer."""
|
||||
if self._master_fd is None or self._closed:
|
||||
return b""
|
||||
try:
|
||||
# Use select to check if data is available
|
||||
readable, _, _ = select.select([self._master_fd], [], [], 0.1)
|
||||
if readable:
|
||||
return os.read(self._master_fd, 4096)
|
||||
data = os.read(self._master_fd, 4096)
|
||||
if data:
|
||||
self._add_to_buffer(data)
|
||||
self.last_activity = time.time()
|
||||
return data
|
||||
return b""
|
||||
except (OSError, IOError, ValueError):
|
||||
return b""
|
||||
|
||||
def _add_to_buffer(self, data: bytes) -> None:
|
||||
"""Add data to circular buffer, maintaining size limit."""
|
||||
self._output_buffer.append(data)
|
||||
self._buffer_size += len(data)
|
||||
|
||||
# Trim if exceeds max size
|
||||
while self._buffer_size > self.BUFFER_SIZE and self._output_buffer:
|
||||
removed = self._output_buffer.popleft()
|
||||
self._buffer_size -= len(removed)
|
||||
|
||||
def get_buffer(self) -> bytes:
|
||||
"""Get buffered output for replay."""
|
||||
return b"".join(self._output_buffer)
|
||||
|
||||
async def write_input(self, data: bytes) -> None:
|
||||
"""Write input to the PTY master."""
|
||||
if self._master_fd is None or self._closed:
|
||||
return
|
||||
try:
|
||||
os.write(self._master_fd, data)
|
||||
self.last_activity = time.time()
|
||||
except (OSError, IOError):
|
||||
pass
|
||||
|
||||
@@ -88,8 +135,21 @@ class TerminalSession:
|
||||
"""Resize the terminal."""
|
||||
if self._closed:
|
||||
return
|
||||
self._cols = cols
|
||||
self._rows = rows
|
||||
self._set_terminal_size(cols, rows)
|
||||
|
||||
async def reset(self) -> None:
|
||||
"""Reset the session by killing the process and clearing state."""
|
||||
await self.close()
|
||||
self._closed = False
|
||||
self._output_buffer.clear()
|
||||
self._buffer_size = 0
|
||||
self._websockets.clear()
|
||||
self.process = None
|
||||
self._master_fd = None
|
||||
self._slave_fd = None
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the session and cleanup."""
|
||||
if self._closed:
|
||||
@@ -115,3 +175,35 @@ class TerminalSession:
|
||||
if self.process is None:
|
||||
return False
|
||||
return self.process.returncode is None
|
||||
|
||||
def is_idle(self) -> bool:
|
||||
"""Check if the session has been idle for too long."""
|
||||
if self._websockets:
|
||||
return False
|
||||
return time.time() - self.last_activity > self.IDLE_TIMEOUT
|
||||
|
||||
def attach_websocket(self, websocket: Any) -> None:
|
||||
"""Attach a WebSocket to this session."""
|
||||
self._websockets.add(websocket)
|
||||
self.last_activity = time.time()
|
||||
|
||||
def detach_websocket(self, websocket: Any) -> None:
|
||||
"""Detach a WebSocket from this session."""
|
||||
self._websockets.discard(websocket)
|
||||
|
||||
def has_websockets(self) -> bool:
|
||||
"""Check if any WebSockets are attached."""
|
||||
return len(self._websockets) > 0
|
||||
|
||||
async def send_to_all(self, data: bytes) -> None:
|
||||
"""Send data to all attached WebSockets."""
|
||||
dead_sockets = set()
|
||||
for ws in self._websockets:
|
||||
try:
|
||||
await ws.send_bytes(data)
|
||||
except Exception:
|
||||
dead_sockets.add(ws)
|
||||
|
||||
# Clean up dead sockets
|
||||
for ws in dead_sockets:
|
||||
self._websockets.discard(ws)
|
||||
|
||||
@@ -43,9 +43,10 @@ export const TerminalComponent: React.FC<TerminalProps> = ({
|
||||
const onTerminalReadyRef = useRef(onTerminalReady);
|
||||
onTerminalReadyRef.current = onTerminalReady;
|
||||
const [status, setStatus] = useState<
|
||||
"connecting" | "connected" | "disconnected" | "error"
|
||||
"connecting" | "connected" | "disconnected" | "error" | "resetting"
|
||||
>("connecting");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [showResetConfirm, setShowResetConfirm] = useState(false);
|
||||
const activeModifierRef = useRef(activeModifier);
|
||||
activeModifierRef.current = activeModifier;
|
||||
const [fontSize, setFontSize] = useState(() => {
|
||||
@@ -87,8 +88,13 @@ export const TerminalComponent: React.FC<TerminalProps> = ({
|
||||
} else if (typeof event.data === "string") {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (msg.type === "status" && msg.status === "connected") {
|
||||
setStatus("connected");
|
||||
if (msg.type === "status") {
|
||||
if (msg.status === "connected") {
|
||||
setStatus("connected");
|
||||
setError(null);
|
||||
} else if (msg.status === "resetting") {
|
||||
setStatus("resetting");
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
termRef.current?.write(event.data);
|
||||
@@ -369,7 +375,9 @@ export const TerminalComponent: React.FC<TerminalProps> = ({
|
||||
aria-label={`Terminal status: ${status}`}
|
||||
/>
|
||||
<span className="status-text">
|
||||
{reconnectAttemptsRef.current > 0 && status !== "connected"
|
||||
{status === "resetting"
|
||||
? "Resetting..."
|
||||
: reconnectAttemptsRef.current > 0 && status !== "connected"
|
||||
? `Reconnecting (${reconnectAttemptsRef.current}/${RECONNECT_ATTEMPTS})...`
|
||||
: status}
|
||||
</span>
|
||||
@@ -412,6 +420,14 @@ export const TerminalComponent: React.FC<TerminalProps> = ({
|
||||
>
|
||||
A+
|
||||
</button>
|
||||
<button
|
||||
className="terminal-header-button"
|
||||
onClick={() => setShowResetConfirm(true)}
|
||||
type="button"
|
||||
aria-label="Reset terminal"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
{onClose && (
|
||||
<button className="terminal-close" onClick={onClose} type="button">
|
||||
Close
|
||||
@@ -419,6 +435,34 @@ export const TerminalComponent: React.FC<TerminalProps> = ({
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{showResetConfirm && (
|
||||
<div className="terminal-reset-confirm">
|
||||
<div className="terminal-reset-confirm-content">
|
||||
<p>Reset terminal? This will kill the current shell session and start fresh.</p>
|
||||
<div className="terminal-reset-confirm-buttons">
|
||||
<button
|
||||
className="terminal-reset-confirm-button cancel"
|
||||
onClick={() => setShowResetConfirm(false)}
|
||||
type="button"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
className="terminal-reset-confirm-button confirm"
|
||||
onClick={() => {
|
||||
setShowResetConfirm(false);
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({ type: "reset" }));
|
||||
}
|
||||
}}
|
||||
type="button"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{error && (
|
||||
<div className="terminal-error">
|
||||
{error}
|
||||
|
||||
@@ -3382,6 +3382,57 @@ a.nav-item,
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.terminal-reset-confirm {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.8);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 100;
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-content {
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: 8px;
|
||||
padding: var(--space-4);
|
||||
max-width: 400px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-content p {
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-buttons {
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-button {
|
||||
padding: var(--space-2) var(--space-4);
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-button.cancel {
|
||||
background: var(--bg-elevated);
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.terminal-reset-confirm-button.confirm {
|
||||
background: #cd3131;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.terminal-hidden-input {
|
||||
position: fixed;
|
||||
left: -9999px;
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
schema: spec-driven
|
||||
created: 2026-05-24
|
||||
@@ -0,0 +1,73 @@
|
||||
## Context
|
||||
|
||||
Currently, terminal sessions are ephemeral. Each WebSocket connection to `/ws/tool-instances/{instance_id}/terminal` spawns a new `docker exec` process via `TerminalManager.create_session()`. When the WebSocket disconnects, the session is cleaned up and the `docker exec` process is killed. This means users lose their shell state, running processes, and command history every time they disconnect.
|
||||
|
||||
The current architecture:
|
||||
- `TerminalManager` tracks sessions by `session_id` (UUID) in a dictionary
|
||||
- Each session creates a new PTY and `docker exec` process
|
||||
- WebSocket I/O loops are tied to the session lifecycle
|
||||
- No concept of reconnection or session persistence
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:**
|
||||
- Terminal sessions persist across WebSocket disconnections/reconnections
|
||||
- Users reconnect to the same shell process, maintaining state and history
|
||||
- Add explicit "Reset Terminal" functionality to kill and restart the session
|
||||
- Graceful handling of idle timeouts to clean up abandoned sessions
|
||||
- Buffer recent output for replay on reconnect
|
||||
|
||||
**Non-Goals:**
|
||||
- Multi-user shared terminal sessions (one session per instance, but only one active WebSocket at a time)
|
||||
- Session persistence across container restarts or instance stops
|
||||
- Full terminal scrollback history persistence (only recent buffer)
|
||||
- Automatic session restoration after instance restart
|
||||
|
||||
## Decisions
|
||||
|
||||
**1. Session tracking by instance_id**
|
||||
- Rationale: One persistent session per tool instance is the simplest model
|
||||
- Alternative: Track by user_id + instance_id — rejected because it's overkill; users don't need multiple terminals to the same container
|
||||
- Trade-off: Only one user can have an active terminal at a time per instance
|
||||
|
||||
**2. Detach WebSocket from session on disconnect**
|
||||
- Rationale: Keep the `docker exec` process alive, just remove the WebSocket reference
|
||||
- Implementation: Session stores a list of WebSocket connections (initially just one)
|
||||
- On disconnect: remove WebSocket from session, don't kill process
|
||||
- On reconnect: attach new WebSocket to existing session
|
||||
|
||||
**3. Circular buffer for output replay**
|
||||
- Rationale: Users should see what happened while disconnected
|
||||
- Size: 10KB buffer (configurable) — enough for ~100 lines of typical output
|
||||
- Implementation: Buffer stores raw bytes, replayed on WebSocket attach
|
||||
|
||||
**4. Reset terminal via WebSocket message**
|
||||
- Rationale: Users need a way to kill a stuck or corrupted session
|
||||
- Implementation: JSON control message `{"type": "reset"}` kills process and starts fresh
|
||||
- Alternative: HTTP endpoint — rejected because it's more complex and less intuitive
|
||||
|
||||
**5. Idle timeout cleanup**
|
||||
- Rationale: Prevent resource leaks from abandoned sessions
|
||||
- Timeout: 30 minutes of no WebSocket connections
|
||||
- Implementation: Background task checks last_activity timestamp
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
- **[Risk] Zombie sessions**: Users disconnect and never reconnect, leaving `docker exec` processes running
|
||||
- Mitigation: Idle timeout of 30 minutes cleans up abandoned sessions
|
||||
- **[Risk] Session corruption**: If the shell process crashes, the session is dead but still tracked
|
||||
- Mitigation: Health check on `docker exec` process; auto-reset on next connect if dead
|
||||
- **[Risk] Concurrent connections**: Multiple tabs trying to connect to the same instance
|
||||
- Mitigation: Only allow one active WebSocket per session; new connection closes old one with a message
|
||||
|
||||
## Migration Plan
|
||||
|
||||
1. Deploy updated backend services (TerminalManager, TerminalSession, terminal endpoint)
|
||||
2. Deploy frontend changes (reset button, reconnection handling)
|
||||
3. No database migration needed
|
||||
4. Rollback: Revert to previous code; existing sessions will be killed on disconnect as before
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Should we show a "session resumed" indicator in the UI?
|
||||
- Should we persist the last N commands for command history?
|
||||
@@ -0,0 +1,29 @@
|
||||
## Why
|
||||
|
||||
Currently, each WebSocket connection to a terminal spawns a new `docker exec` process. When the user disconnects (e.g., closes the browser tab, navigates away, or loses network), their shell session is killed and all state is lost. This is frustrating for users who expect their terminal session to persist like a traditional SSH session. We need persistent terminal sessions that survive reconnections.
|
||||
|
||||
## What Changes
|
||||
|
||||
- **Backend**: Refactor `TerminalManager` to track sessions by `instance_id` instead of generating a new session per WebSocket connection
|
||||
- **Backend**: Support reattaching to an existing `docker exec` process when a WebSocket reconnects
|
||||
- **Backend**: Add session reset functionality (kill existing shell and start fresh)
|
||||
- **Backend**: Add idle timeout-based cleanup for abandoned sessions
|
||||
- **Frontend**: Add "Reset Terminal" button in the UI
|
||||
- **Frontend**: Handle reconnection gracefully with buffer replay of recent output
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- `persistent-terminal`: Terminal sessions persist across WebSocket reconnections, maintaining shell state and history
|
||||
|
||||
### Modified Capabilities
|
||||
- `terminal-session-management`: Existing terminal session creation and lifecycle behavior changes to support reconnection instead of always creating new sessions
|
||||
|
||||
## Impact
|
||||
|
||||
- `apps/api/src/services/terminal_manager.py`: Major refactoring to support instance-keyed sessions
|
||||
- `apps/api/src/services/terminal_session.py`: Support multiple/detached WebSocket connections
|
||||
- `apps/api/src/api/terminal.py`: Attach to existing session logic
|
||||
- `apps/web/src/components/terminal.tsx` or related: Add reset button, handle reconnection
|
||||
- `apps/web/src/hooks/use-terminal.ts` or related: Buffer replay on reconnect
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Terminal sessions persist across reconnections
|
||||
The system SHALL maintain a terminal session for a tool instance even when the WebSocket connection is closed. When a new WebSocket connection is established for the same instance, the system SHALL reattach to the existing terminal session instead of creating a new one.
|
||||
|
||||
#### Scenario: Reconnect to existing session
|
||||
- **WHEN** a user disconnects from a terminal session
|
||||
- **THEN** the underlying docker exec process continues running
|
||||
- **AND** when the user reconnects to the same instance
|
||||
- **THEN** they are attached to the same shell process
|
||||
|
||||
#### Scenario: New connection creates session
|
||||
- **WHEN** a user connects to an instance with no existing terminal session
|
||||
- **THEN** a new terminal session is created
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Terminal session output buffer
|
||||
The system SHALL maintain a circular buffer of recent terminal output (minimum 10KB) for each persistent session. When a WebSocket reconnects, the system SHALL replay the buffered output to bring the client up to date.
|
||||
|
||||
#### Scenario: Output replay on reconnect
|
||||
- **WHEN** a user reconnects to an existing terminal session
|
||||
- **THEN** the recent output buffer is sent to the WebSocket
|
||||
- **AND** the user sees the terminal state as it was before disconnect
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Terminal session reset
|
||||
The system SHALL support resetting a terminal session. When a reset is requested, the system SHALL kill the existing docker exec process, clean up the session, and create a new one.
|
||||
|
||||
#### Scenario: Reset terminal session
|
||||
- **WHEN** a user sends a reset command via WebSocket
|
||||
- **THEN** the existing terminal session is terminated
|
||||
- **AND** a new terminal session is created
|
||||
- **AND** the user is connected to the fresh session
|
||||
|
||||
#### Scenario: Reset from API
|
||||
- **WHEN** a user sends a POST request to reset a terminal session
|
||||
- **THEN** the existing terminal session is terminated
|
||||
- **AND** a new terminal session is created
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Terminal session idle timeout
|
||||
The system SHALL automatically clean up terminal sessions that have had no active WebSocket connections for 30 minutes. This prevents resource leaks from abandoned sessions.
|
||||
|
||||
#### Scenario: Idle session cleanup
|
||||
- **WHEN** a terminal session has no WebSocket connections for 30 minutes
|
||||
- **THEN** the session is terminated and cleaned up
|
||||
|
||||
#### Scenario: Active session not cleaned up
|
||||
- **WHEN** a terminal session has an active WebSocket connection
|
||||
- **THEN** it is not cleaned up regardless of duration
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Terminal session creation
|
||||
The system SHALL create a terminal session when a WebSocket connects to a tool instance. The session SHALL be associated with the instance and SHALL persist until explicitly reset, the instance stops, or an idle timeout occurs.
|
||||
|
||||
#### Scenario: Create persistent session
|
||||
- **WHEN** a user connects to a running instance via WebSocket
|
||||
- **THEN** if no session exists for that instance, a new session is created
|
||||
- **AND** if a session already exists, the WebSocket is attached to it
|
||||
- **AND** recent output is replayed
|
||||
|
||||
## REMOVED Requirements
|
||||
|
||||
### Requirement: Terminal session cleanup on disconnect
|
||||
**Reason**: Sessions now persist across disconnections
|
||||
**Migration**: Sessions are cleaned up on idle timeout or explicit reset instead
|
||||
@@ -0,0 +1,64 @@
|
||||
## 1. Backend - TerminalSession Refactoring
|
||||
|
||||
- [x] 1.1 Add circular output buffer to TerminalSession (10KB, stores raw bytes)
|
||||
- [x] 1.2 Add WebSocket connection tracking (support multiple connections, detach without closing)
|
||||
- [x] 1.3 Add last_activity timestamp and idle timeout support
|
||||
- [x] 1.4 Add reset() method to kill process and prepare for restart
|
||||
- [x] 1.5 Add health check for docker exec process
|
||||
- [x] 1.6 Modify read_output to also write to circular buffer
|
||||
|
||||
## 2. Backend - TerminalManager Refactoring
|
||||
|
||||
- [x] 2.1 Change session tracking from session_id to instance_id
|
||||
- [x] 2.2 Add get_or_create_session() method (reattach if exists, create if not)
|
||||
- [x] 2.3 Modify create_session to support reconnection (don't always create new)
|
||||
- [x] 2.4 Add reset_session() method (kill existing, create new)
|
||||
- [x] 2.5 Add attach_websocket() method (add WebSocket to existing session, replay buffer)
|
||||
- [x] 2.6 Add detach_websocket() method (remove WebSocket, keep session alive)
|
||||
- [x] 2.7 Add idle timeout background task (check every minute, cleanup after 30min)
|
||||
- [x] 2.8 Handle concurrent connections (close old WebSocket when new one connects)
|
||||
|
||||
## 3. Backend - Terminal WebSocket Endpoint
|
||||
|
||||
- [x] 3.1 Modify endpoint to check for existing session first
|
||||
- [x] 3.2 Add reconnection logic (attach to existing vs create new)
|
||||
- [x] 3.3 Handle reset command from WebSocket (JSON message type: "reset")
|
||||
- [x] 3.4 Add buffer replay on WebSocket attach
|
||||
- [x] 3.5 Add proper cleanup on WebSocket disconnect (detach, don't kill)
|
||||
|
||||
## 4. Backend - API Reset Endpoint
|
||||
|
||||
- [x] 4.1 Add POST /api/projects/{project_id}/repositories/{repo_id}/instances/{instance_id}/terminal/reset endpoint
|
||||
- [x] 4.2 Add authorization checks
|
||||
- [x] 4.3 Call TerminalManager.reset_session()
|
||||
- [x] 4.4 Return success/error response
|
||||
|
||||
## 5. Frontend - Terminal Component
|
||||
|
||||
- [x] 5.1 Add "Reset Terminal" button to terminal UI
|
||||
- [x] 5.2 Handle WebSocket reconnection gracefully
|
||||
- [x] 5.3 Display "Reconnecting..." indicator
|
||||
- [x] 5.4 Handle reset confirmation dialog
|
||||
- [x] 5.5 Display session status (connected, reconnecting, reset)
|
||||
|
||||
## 6. Frontend - Terminal Hook
|
||||
|
||||
- [x] 6.1 Add reconnection logic with exponential backoff
|
||||
- [x] 6.2 Handle buffer replay on reconnect (process incoming bytes)
|
||||
- [x] 6.3 Add reset function (send WebSocket message or call API)
|
||||
- [ ] 6.4 Add heartbeat/ping to detect disconnections faster
|
||||
|
||||
## 7. Testing
|
||||
|
||||
- [ ] 7.1 Test terminal session persistence across reconnections
|
||||
- [ ] 7.2 Test output buffer replay
|
||||
- [ ] 7.3 Test reset functionality
|
||||
- [ ] 7.4 Test idle timeout cleanup
|
||||
- [ ] 7.5 Test concurrent connection handling
|
||||
- [ ] 7.6 Verify existing functionality still works (create, stop, delete instances)
|
||||
|
||||
## 8. Documentation
|
||||
|
||||
- [ ] 8.1 Update API documentation with new reset endpoint
|
||||
- [ ] 8.2 Update user documentation about persistent terminals
|
||||
- [ ] 8.3 Add troubleshooting guide for terminal issues
|
||||
Reference in New Issue
Block a user