From b40eb3e88ce93c4ebe353e7855e2dda71a49c593 Mon Sep 17 00:00:00 2001 From: Fusion Date: Wed, 20 May 2026 14:25:06 +0200 Subject: [PATCH] feat(cloudflare-tunnel): integrate Cloudflare tunnels for instance access Backend: - Add cloudflare_tunnel.py service for creating/deleting tunnels via Cloudflare API - Add public_url and tunnel_id fields to ToolInstance model - Update start_instance to create Cloudflare tunnel after container starts - Update stop_instance to delete tunnel before stopping container - Update delete_instance to cleanup tunnel before deletion - Update restart_instance to recreate tunnel on restart - Create Alembic migration 0011 for tunnel fields - Add Cloudflare config settings (API token, zone ID, account ID, base domain) Infrastructure: - Add cloudflared service to docker-compose.traefik.yml - Mount shared cloudflared_config volume between API and cloudflared containers - Add Cloudflare env vars to API service Frontend: - Update instance Open button to handle both full URLs and proxy paths The instance URL is now set to the Cloudflare tunnel public URL when available, falling back to the API proxy path if tunnel creation fails. --- .../0011_tool_instance_tunnel_fields.py | 33 +++ apps/api/src/api/tool_instances.py | 85 +++++- apps/api/src/config.py | 7 + apps/api/src/models/tool_instance.py | 6 + apps/api/src/services/cloudflare_tunnel.py | 268 ++++++++++++++++++ apps/web/src/components/instance-list.tsx | 2 +- docker-compose.traefik.yml | 20 ++ .../.openspec.yaml | 2 + .../cloudflare-tunnel-instances/design.md | 142 ++++++++++ .../cloudflare-tunnel-instances/proposal.md | 31 ++ .../cloudflare-tunnel-management/spec.md | 50 ++++ .../cloudflare-tunnel-instances/tasks.md | 45 +++ 12 files changed, 688 insertions(+), 3 deletions(-) create mode 100644 apps/api/alembic/versions/0011_tool_instance_tunnel_fields.py create mode 100644 apps/api/src/services/cloudflare_tunnel.py create mode 100644 openspec/changes/cloudflare-tunnel-instances/.openspec.yaml create mode 100644 openspec/changes/cloudflare-tunnel-instances/design.md create mode 100644 openspec/changes/cloudflare-tunnel-instances/proposal.md create mode 100644 openspec/changes/cloudflare-tunnel-instances/specs/cloudflare-tunnel-management/spec.md create mode 100644 openspec/changes/cloudflare-tunnel-instances/tasks.md diff --git a/apps/api/alembic/versions/0011_tool_instance_tunnel_fields.py b/apps/api/alembic/versions/0011_tool_instance_tunnel_fields.py new file mode 100644 index 0000000..a7ee7c0 --- /dev/null +++ b/apps/api/alembic/versions/0011_tool_instance_tunnel_fields.py @@ -0,0 +1,33 @@ +"""add tunnel fields to tool_instances + +Revision ID: 0011_tool_instance_tunnel_fields +Revises: 0010_tool_type_default_port +Create Date: 2026-05-20 12:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "0011_tool_instance_tunnel_fields" +down_revision: Union[str, None] = "0010_tool_type_default_port" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "tool_instances", + sa.Column("public_url", sa.String(1024), nullable=True) + ) + op.add_column( + "tool_instances", + sa.Column("tunnel_id", sa.String(255), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("tool_instances", "tunnel_id") + op.drop_column("tool_instances", "public_url") diff --git a/apps/api/src/api/tool_instances.py b/apps/api/src/api/tool_instances.py index 7540d91..dbedef5 100644 --- a/apps/api/src/api/tool_instances.py +++ b/apps/api/src/api/tool_instances.py @@ -22,6 +22,7 @@ from src.models.tool_config import ToolConfig from src.models.tool_instance import ToolInstance from src.models.tool_type import ToolType from src.models.user import User +from src.services.cloudflare_tunnel import create_tunnel, delete_tunnel from src.services.docker import ( ensure_instance_directory, execute_compose_command, @@ -399,9 +400,32 @@ async def start_instance( instance.status = "running" instance.last_started_at = datetime.now() - instance.url = f"/instances/{instance.id}/proxy/" await session.commit() + # Create Cloudflare tunnel for public access + try: + tunnel_info = await create_tunnel( + instance_name=instance.name, + instance_id=str(instance.id), + ) + instance.tunnel_id = tunnel_info["tunnel_id"] + instance.public_url = tunnel_info["public_url"] + instance.url = tunnel_info["public_url"] + await session.commit() + logger.info( + "Created tunnel for instance %s: %s", + instance.id, + tunnel_info["public_url"], + ) + except Exception as exc: + logger.warning( + "Failed to create tunnel for instance %s: %s. Falling back to proxy URL.", + instance.id, + exc, + ) + instance.url = f"/instances/{instance.id}/proxy/" + await session.commit() + return {"status": instance.status, "url": instance.url} @@ -438,12 +462,25 @@ async def stop_instance( status_code=status.HTTP_404_NOT_FOUND, detail="instance not found" ) + # Delete Cloudflare tunnel if exists + if instance.tunnel_id: + try: + await delete_tunnel( + tunnel_id=instance.tunnel_id, + subdomain=f"instance-{instance.id}", + ) + logger.info("Deleted tunnel for instance %s", instance.id) + except Exception as exc: + logger.warning("Failed to delete tunnel for instance %s: %s", instance.id, exc) + if instance.compose_path and os.path.exists(instance.compose_path): execute_compose_command(instance.compose_path, "stop") instance.status = "stopped" instance.last_stopped_at = datetime.now() instance.url = None + instance.public_url = None + instance.tunnel_id = None await session.commit() return {"status": instance.status} @@ -482,6 +519,17 @@ async def restart_instance( status_code=status.HTTP_404_NOT_FOUND, detail="instance not found" ) + # Delete old tunnel if exists + if instance.tunnel_id: + try: + await delete_tunnel( + tunnel_id=instance.tunnel_id, + subdomain=f"instance-{instance.id}", + ) + logger.info("Deleted old tunnel for instance %s", instance.id) + except Exception as exc: + logger.warning("Failed to delete old tunnel for instance %s: %s", instance.id, exc) + if instance.compose_path and os.path.exists(instance.compose_path): returncode, stdout, stderr = execute_compose_command( instance.compose_path, "restart" @@ -490,7 +538,29 @@ async def restart_instance( if returncode == 0: instance.status = "running" instance.last_started_at = datetime.now() - instance.url = f"http://localhost:{instance.port}" + + # Create new tunnel + try: + tunnel_info = await create_tunnel( + instance_name=instance.name, + instance_id=str(instance.id), + ) + instance.tunnel_id = tunnel_info["tunnel_id"] + instance.public_url = tunnel_info["public_url"] + instance.url = tunnel_info["public_url"] + logger.info( + "Created new tunnel for instance %s: %s", + instance.id, + tunnel_info["public_url"], + ) + except Exception as exc: + logger.warning( + "Failed to create tunnel for instance %s: %s", + instance.id, + exc, + ) + instance.url = f"/instances/{instance.id}/proxy/" + await session.commit() return {"status": instance.status, "url": instance.url} @@ -532,6 +602,17 @@ async def delete_instance( status_code=status.HTTP_404_NOT_FOUND, detail="instance not found" ) + # Delete Cloudflare tunnel if exists + if instance.tunnel_id: + try: + await delete_tunnel( + tunnel_id=instance.tunnel_id, + subdomain=f"instance-{instance.id}", + ) + logger.info("Deleted tunnel for instance %s", instance.id) + except Exception as exc: + logger.warning("Failed to delete tunnel for instance %s: %s", instance.id, exc) + # Stop and remove container if instance.compose_path and os.path.exists(instance.compose_path): execute_compose_command(instance.compose_path, "down") diff --git a/apps/api/src/config.py b/apps/api/src/config.py index 6c68615..e731c39 100644 --- a/apps/api/src/config.py +++ b/apps/api/src/config.py @@ -53,6 +53,13 @@ class Settings(BaseSettings): # Tool instance storage instance_base_path: str = "/data/instances" + # Cloudflare Tunnel configuration + cloudflare_api_token: str | None = None + cloudflare_zone_id: str | None = None + cloudflare_account_id: str | None = None + cloudflare_base_domain: str | None = None + cloudflared_config_dir: str = "/etc/cloudflared" + model_config = SettingsConfigDict(env_file=".env", extra="ignore", populate_by_name=True) @property diff --git a/apps/api/src/models/tool_instance.py b/apps/api/src/models/tool_instance.py index a493be1..b556bab 100644 --- a/apps/api/src/models/tool_instance.py +++ b/apps/api/src/models/tool_instance.py @@ -47,6 +47,12 @@ class ToolInstance(UUIDPrimaryKeyMixin, TimestampMixin, Base): url: Mapped[str | None] = mapped_column( String(1024), nullable=True ) + public_url: Mapped[str | None] = mapped_column( + String(1024), nullable=True + ) + tunnel_id: Mapped[str | None] = mapped_column( + String(255), nullable=True + ) port: Mapped[int | None] = mapped_column( Integer, nullable=True ) diff --git a/apps/api/src/services/cloudflare_tunnel.py b/apps/api/src/services/cloudflare_tunnel.py new file mode 100644 index 0000000..8d99e83 --- /dev/null +++ b/apps/api/src/services/cloudflare_tunnel.py @@ -0,0 +1,268 @@ +"""Cloudflare Tunnel management service.""" + +import json +import logging +import os +import uuid +from pathlib import Path + +import httpx +from src.config import Settings + +logger = logging.getLogger(__name__) + +CLOUDFLARE_API_BASE = "https://api.cloudflare.com/client/v4" + + +def _get_headers(settings: Settings) -> dict[str, str]: + """Get Cloudflare API headers.""" + return { + "Authorization": f"Bearer {settings.cloudflare_api_token}", + "Content-Type": "application/json", + } + + +async def create_tunnel( + instance_name: str, + instance_id: str, + settings: Settings | None = None, +) -> dict[str, str]: + """Create a Cloudflare tunnel for an instance. + + Args: + instance_name: Name of the instance (used for tunnel name) + instance_id: UUID of the instance + settings: Optional settings override + + Returns: + Dict with tunnel_id and public_url + """ + if settings is None: + settings = Settings() + + if not settings.cloudflare_api_token: + raise ValueError("CLOUDFLARE_API_TOKEN not configured") + + headers = _get_headers(settings) + account_id = settings.cloudflare_account_id + + # Create tunnel + async with httpx.AsyncClient() as client: + response = await client.post( + f"{CLOUDFLARE_API_BASE}/accounts/{account_id}/cfd_tunnel", + headers=headers, + json={ + "name": f"headquarter-{instance_name}", + "config_src": "cloudflare", + }, + ) + response.raise_for_status() + data = response.json() + + if not data.get("success"): + raise ValueError(f"Failed to create tunnel: {data.get('errors')}") + + tunnel = data["result"] + tunnel_id = tunnel["id"] + + # Get tunnel token + token_response = await client.get( + f"{CLOUDFLARE_API_BASE}/accounts/{account_id}/cfd_tunnel/{tunnel_id}/token", + headers=headers, + ) + token_response.raise_for_status() + token_data = token_response.json() + tunnel_token = token_data["result"] + + # Create DNS record for the tunnel + subdomain = f"instance-{instance_id[:8]}" + hostname = f"{subdomain}.{settings.cloudflare_base_domain}" + + dns_response = await client.post( + f"{CLOUDFLARE_API_BASE}/zones/{settings.cloudflare_zone_id}/dns_records", + headers=headers, + json={ + "type": "CNAME", + "name": subdomain, + "content": f"{tunnel_id}.cfargotunnel.com", + "ttl": 1, + "proxied": True, + }, + ) + dns_response.raise_for_status() + + # Update cloudflared config + await update_cloudflared_config( + tunnel_id=tunnel_id, + tunnel_token=tunnel_token, + hostname=hostname, + instance_name=instance_name, + settings=settings, + ) + + return { + "tunnel_id": tunnel_id, + "public_url": f"https://{hostname}", + "subdomain": subdomain, + } + + +async def delete_tunnel( + tunnel_id: str, + subdomain: str, + settings: Settings | None = None, +) -> None: + """Delete a Cloudflare tunnel and its DNS record. + + Args: + tunnel_id: Cloudflare tunnel ID + subdomain: Subdomain to remove DNS record for + settings: Optional settings override + """ + if settings is None: + settings = Settings() + + if not settings.cloudflare_api_token: + raise ValueError("CLOUDFLARE_API_TOKEN not configured") + + headers = _get_headers(settings) + account_id = settings.cloudflare_account_id + + async with httpx.AsyncClient() as client: + # Delete DNS record + # First find the DNS record ID + dns_list = await client.get( + f"{CLOUDFLARE_API_BASE}/zones/{settings.cloudflare_zone_id}/dns_records", + headers=headers, + params={"name": f"{subdomain}.{settings.cloudflare_base_domain}"}, + ) + dns_list.raise_for_status() + dns_data = dns_list.json() + + if dns_data.get("success") and dns_data.get("result"): + for record in dns_data["result"]: + await client.delete( + f"{CLOUDFLARE_API_BASE}/zones/{settings.cloudflare_zone_id}/dns_records/{record['id']}", + headers=headers, + ) + + # Delete tunnel + await client.delete( + f"{CLOUDFLARE_API_BASE}/accounts/{account_id}/cfd_tunnel/{tunnel_id}", + headers=headers, + ) + + # Remove from cloudflared config + await remove_tunnel_from_config(tunnel_id, settings) + + +async def update_cloudflared_config( + tunnel_id: str, + tunnel_token: str, + hostname: str, + instance_name: str, + settings: Settings | None = None, +) -> None: + """Update the cloudflared config.yml with a new tunnel. + + Args: + tunnel_id: Cloudflare tunnel ID + tunnel_token: Tunnel token for authentication + hostname: Public hostname for the tunnel + instance_name: Instance name for the service + settings: Optional settings override + """ + if settings is None: + settings = Settings() + + config_dir = Path(settings.cloudflared_config_dir) + config_file = config_dir / "config.yml" + credentials_file = config_dir / f"{tunnel_id}.json" + + # Ensure config directory exists + config_dir.mkdir(parents=True, exist_ok=True) + + # Write tunnel credentials + credentials = { + "AccountTag": settings.cloudflare_account_id, + "TunnelID": tunnel_id, + "TunnelName": f"headquarter-{instance_name}", + "TunnelSecret": tunnel_token, + } + with open(credentials_file, "w") as f: + json.dump(credentials, f, indent=2) + + # Read existing config or create new one + config: dict = {"tunnel": "", "credentials-file": "", "ingress": []} + if config_file.exists(): + import yaml + with open(config_file, "r") as f: + config = yaml.safe_load(f) or config + + # Update tunnel and credentials-file (should point to latest) + config["tunnel"] = tunnel_id + config["credentials-file"] = str(credentials_file) + + # Add ingress rule for this instance + ingress_rule = { + "hostname": hostname, + "service": f"http://{instance_name}:8080", + } + + # Remove existing rule for this hostname if present + config["ingress"] = [ + rule for rule in config.get("ingress", []) + if rule.get("hostname") != hostname + ] + + # Add new rule and catch-all + config["ingress"].append(ingress_rule) + config["ingress"].append({"service": "http_status:404"}) + + # Write updated config + import yaml + with open(config_file, "w") as f: + yaml.dump(config, f, default_flow_style=False) + + logger.info("Updated cloudflared config for tunnel %s", tunnel_id) + + +async def remove_tunnel_from_config( + tunnel_id: str, + settings: Settings | None = None, +) -> None: + """Remove a tunnel from the cloudflared config. + + Args: + tunnel_id: Cloudflare tunnel ID to remove + settings: Optional settings override + """ + if settings is None: + settings = Settings() + + config_dir = Path(settings.cloudflared_config_dir) + config_file = config_dir / "config.yml" + credentials_file = config_dir / f"{tunnel_id}.json" + + if not config_file.exists(): + return + + import yaml + with open(config_file, "r") as f: + config = yaml.safe_load(f) or {} + + # Remove ingress rules for this tunnel + config["ingress"] = [ + rule for rule in config.get("ingress", []) + if rule.get("hostname") != f"instance-{tunnel_id[:8]}.{settings.cloudflare_base_domain}" + ] + + # Write updated config + with open(config_file, "w") as f: + yaml.dump(config, f, default_flow_style=False) + + # Remove credentials file + if credentials_file.exists(): + credentials_file.unlink() + + logger.info("Removed tunnel %s from cloudflared config", tunnel_id) diff --git a/apps/web/src/components/instance-list.tsx b/apps/web/src/components/instance-list.tsx index e834b3c..3b7d43e 100644 --- a/apps/web/src/components/instance-list.tsx +++ b/apps/web/src/components/instance-list.tsx @@ -149,7 +149,7 @@ export const InstanceList = ({ projectId, repoId, toolTypes }: InstanceListProps
{instance.status === "running" && instance.url && instance.tool_type_interfaces.includes("web") && (