feat(services): backend service registry foundation (encryption, definitions, CRUD)
PR 1 of 4 for the runtime service registry change. - Add Fernet encryption helper (services/secrets.py) with a required MANAGE_ENCRYPTION_KEY; validate it on startup. - Add closed integrations/ registry with Pydantic config + widget-config definitions for grafana, prometheus, jellyfin, nextcloud, and ssh_tasks. - Add services + service_task_runs tables and SettingsStore CRUD with cascade-delete (defensive until widgets carry service_id). - Add /api/services/types and /api/services/instances CRUD (encrypted secrets, secrets_set flags only; never plaintext). - Declare cryptography as a direct dependency. - Require MANAGE_ENCRYPTION_KEY in compose + .env.example + README. - Add 25 backend tests (registry, encryption, CRUD, cascade, task-run history). Verification: ruff clean; pytest 225 passed; frontend lint/build green.
This commit is contained in:
Developer
@@ -0,0 +1 @@
|
||||
"""Closed registry of service integrations."""
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Base classes for service integrations.
|
||||
|
||||
A *service definition* is a closed, compile-time description of an external service
|
||||
the app can talk to (Grafana, Jellyfin, …). Each definition declares:
|
||||
|
||||
* its non-secret ``config_schema`` (derived from a Pydantic model),
|
||||
* the secret fields it accepts (API keys / tokens),
|
||||
* the widget kinds it can contribute to the dashboard (each with its own
|
||||
Pydantic-derived config schema).
|
||||
|
||||
Definitions live in :mod:`media_library_viewer_api.integrations` modules and are
|
||||
assembled into the closed :data:`~media_library_viewer_api.integrations.registry.SERVICE_DEFINITIONS`
|
||||
map. There is no runtime plugin loading.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ServiceConfigBase(BaseModel):
|
||||
"""Base for per-service non-secret config models.
|
||||
|
||||
Subclass this in each integration module and declare the connection fields.
|
||||
The JSON schema is derived via ``model_json_schema()`` and exposed to the UI.
|
||||
"""
|
||||
|
||||
|
||||
class WidgetConfigBase(BaseModel):
|
||||
"""Base for per-widget config models.
|
||||
|
||||
Subclass this for each widget kind a service provides. Widget configs never
|
||||
hold secrets; credentials live on the parent service record.
|
||||
"""
|
||||
|
||||
model_config = {"extra": "forbid"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SecretField:
|
||||
"""A secret field stored encrypted on the service record."""
|
||||
|
||||
key: str
|
||||
label: str
|
||||
required: bool = False
|
||||
helper: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WidgetKind:
|
||||
"""A widget kind contributed by a service definition."""
|
||||
|
||||
kind: str
|
||||
name: str
|
||||
description: str
|
||||
config_schema: dict[str, Any]
|
||||
default_config: dict[str, Any] = field(default_factory=dict)
|
||||
refresh_interval_ms: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServiceDefinition:
|
||||
"""Closed description of an external service type."""
|
||||
|
||||
service_type: str
|
||||
name: str
|
||||
description: str
|
||||
config_model: type[ServiceConfigBase]
|
||||
secret_fields: list[SecretField]
|
||||
widget_kinds: list[WidgetKind]
|
||||
|
||||
@property
|
||||
def config_schema(self) -> dict[str, Any]:
|
||||
"""JSON schema for the service's non-secret config."""
|
||||
return self.config_model.model_json_schema()
|
||||
|
||||
@property
|
||||
def secret_keys(self) -> set[str]:
|
||||
return {sf.key for sf in self.secret_fields}
|
||||
|
||||
def widget_kind(self, kind: str) -> WidgetKind | None:
|
||||
for wk in self.widget_kinds:
|
||||
if wk.kind == kind:
|
||||
return wk
|
||||
return None
|
||||
|
||||
|
||||
def widget_kind(
|
||||
kind: str,
|
||||
name: str,
|
||||
description: str,
|
||||
model_cls: type[WidgetConfigBase],
|
||||
*,
|
||||
default_config: dict[str, Any] | None = None,
|
||||
refresh_interval_ms: int = 0,
|
||||
) -> WidgetKind:
|
||||
"""Build a :class:`WidgetKind` from a Pydantic widget-config model."""
|
||||
schema = model_cls.model_json_schema()
|
||||
# Strip Pydantic's title noise so the exposed schema stays clean.
|
||||
schema.pop("title", None)
|
||||
return WidgetKind(
|
||||
kind=kind,
|
||||
name=name,
|
||||
description=description,
|
||||
config_schema=schema,
|
||||
default_config=dict(default_config or {}),
|
||||
refresh_interval_ms=refresh_interval_ms,
|
||||
)
|
||||
|
||||
|
||||
def validate_config(model_cls: type[BaseModel], config: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Validate a config dict against a Pydantic model and return the cleaned dict."""
|
||||
instance = model_cls.model_validate(config or {})
|
||||
return instance.model_dump(exclude_none=True)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Grafana service definition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import (
|
||||
SecretField,
|
||||
ServiceConfigBase,
|
||||
ServiceDefinition,
|
||||
WidgetConfigBase,
|
||||
widget_kind,
|
||||
)
|
||||
|
||||
|
||||
class GrafanaConfig(ServiceConfigBase):
|
||||
"""Non-secret Grafana connection config."""
|
||||
|
||||
base_url: str
|
||||
timeout_seconds: int = 5
|
||||
|
||||
|
||||
class GrafanaLinkWidgetConfig(WidgetConfigBase):
|
||||
"""Deep-link to a Grafana dashboard or panel."""
|
||||
|
||||
dashboard_uid: str
|
||||
panel_id: int | None = None
|
||||
|
||||
|
||||
DEFINITION = ServiceDefinition(
|
||||
service_type="grafana",
|
||||
name="Grafana",
|
||||
description="Dashboards, metrics, and logs.",
|
||||
config_model=GrafanaConfig,
|
||||
secret_fields=[
|
||||
SecretField(key="api_key", label="API key", helper="Service account token (optional)"),
|
||||
],
|
||||
widget_kinds=[
|
||||
widget_kind(
|
||||
kind="link",
|
||||
name="Dashboard link",
|
||||
description="Deep-link to a Grafana dashboard or panel.",
|
||||
model_cls=GrafanaLinkWidgetConfig,
|
||||
default_config={"dashboard_uid": ""},
|
||||
refresh_interval_ms=0,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Jellyfin service definition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import (
|
||||
SecretField,
|
||||
ServiceConfigBase,
|
||||
ServiceDefinition,
|
||||
WidgetConfigBase,
|
||||
widget_kind,
|
||||
)
|
||||
|
||||
|
||||
class JellyfinConfig(ServiceConfigBase):
|
||||
"""Non-secret Jellyfin connection config."""
|
||||
|
||||
base_url: str
|
||||
user_id: str = ""
|
||||
timeout_seconds: int = 10
|
||||
|
||||
|
||||
class JellyfinActivityWidgetConfig(WidgetConfigBase):
|
||||
"""Live Jellyfin session activity."""
|
||||
|
||||
# No user-overridable fields; the service record carries user_id.
|
||||
pass
|
||||
|
||||
|
||||
DEFINITION = ServiceDefinition(
|
||||
service_type="jellyfin",
|
||||
name="Jellyfin",
|
||||
description="Media server with live session activity.",
|
||||
config_model=JellyfinConfig,
|
||||
secret_fields=[
|
||||
SecretField(key="api_key", label="API key", required=True),
|
||||
],
|
||||
widget_kinds=[
|
||||
widget_kind(
|
||||
kind="activity",
|
||||
name="Activity",
|
||||
description="Live sessions and idle users.",
|
||||
model_cls=JellyfinActivityWidgetConfig,
|
||||
default_config={},
|
||||
refresh_interval_ms=30_000,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Nextcloud service definition.
|
||||
|
||||
Nextcloud is included as a proof-of-concept third-party service. It has no
|
||||
dashboard widgets yet; its service page holds connection config only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import (
|
||||
SecretField,
|
||||
ServiceConfigBase,
|
||||
ServiceDefinition,
|
||||
)
|
||||
|
||||
|
||||
class NextcloudConfig(ServiceConfigBase):
|
||||
"""Non-secret Nextcloud connection config."""
|
||||
|
||||
base_url: str
|
||||
username: str = ""
|
||||
|
||||
|
||||
DEFINITION = ServiceDefinition(
|
||||
service_type="nextcloud",
|
||||
name="Nextcloud",
|
||||
description="Self-hosted files and collaboration.",
|
||||
config_model=NextcloudConfig,
|
||||
secret_fields=[
|
||||
SecretField(key="app_password", label="App password", required=True),
|
||||
],
|
||||
widget_kinds=[],
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Prometheus service definition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import (
|
||||
SecretField,
|
||||
ServiceConfigBase,
|
||||
ServiceDefinition,
|
||||
WidgetConfigBase,
|
||||
widget_kind,
|
||||
)
|
||||
|
||||
|
||||
class PrometheusConfig(ServiceConfigBase):
|
||||
"""Non-secret Prometheus connection config."""
|
||||
|
||||
base_url: str
|
||||
timeout_seconds: int = 10
|
||||
|
||||
|
||||
class PrometheusMetricWidgetConfig(WidgetConfigBase):
|
||||
"""A PromQL instant query rendered as a metric."""
|
||||
|
||||
promql: str
|
||||
|
||||
|
||||
DEFINITION = ServiceDefinition(
|
||||
service_type="prometheus",
|
||||
name="Prometheus",
|
||||
description="Metrics storage and PromQL queries.",
|
||||
config_model=PrometheusConfig,
|
||||
secret_fields=[
|
||||
SecretField(key="api_key", label="API key", helper="Optional bearer token"),
|
||||
],
|
||||
widget_kinds=[
|
||||
widget_kind(
|
||||
kind="metric",
|
||||
name="Metric",
|
||||
description="Instant query result rendered as a metric.",
|
||||
model_cls=PrometheusMetricWidgetConfig,
|
||||
default_config={"promql": ""},
|
||||
refresh_interval_ms=30_000,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Closed registry of service definitions.
|
||||
|
||||
Adding a brand-new service still requires a backend deploy and a module here.
|
||||
There is no runtime plugin loading.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import ServiceDefinition, WidgetKind
|
||||
from media_library_viewer_api.integrations.grafana import DEFINITION as GRAFANA
|
||||
from media_library_viewer_api.integrations.jellyfin import DEFINITION as JELLYFIN
|
||||
from media_library_viewer_api.integrations.nextcloud import DEFINITION as NEXTCLOUD
|
||||
from media_library_viewer_api.integrations.prometheus import DEFINITION as PROMETHEUS
|
||||
from media_library_viewer_api.integrations.ssh_tasks import DEFINITION as SSH_TASKS
|
||||
|
||||
SERVICE_DEFINITIONS: dict[str, ServiceDefinition] = {
|
||||
GRAFANA.service_type: GRAFANA,
|
||||
PROMETHEUS.service_type: PROMETHEUS,
|
||||
JELLYFIN.service_type: JELLYFIN,
|
||||
NEXTCLOUD.service_type: NEXTCLOUD,
|
||||
SSH_TASKS.service_type: SSH_TASKS,
|
||||
}
|
||||
|
||||
|
||||
def list_service_types() -> list[str]:
|
||||
"""Return all registered service type names (sorted for stable output)."""
|
||||
return sorted(SERVICE_DEFINITIONS)
|
||||
|
||||
|
||||
def get_service_definition(service_type: str) -> ServiceDefinition | None:
|
||||
"""Return the definition for a service type, or ``None`` if unknown."""
|
||||
return SERVICE_DEFINITIONS.get(service_type)
|
||||
|
||||
|
||||
def get_widget_kind(service_type: str, widget_kind: str) -> WidgetKind | None:
|
||||
"""Return a widget kind declared by a service definition, or ``None``."""
|
||||
definition = get_service_definition(service_type)
|
||||
if definition is None:
|
||||
return None
|
||||
return definition.widget_kind(widget_kind)
|
||||
|
||||
|
||||
def require_service_definition(service_type: str) -> ServiceDefinition:
|
||||
"""Return the definition or raise ``ValueError`` for an unknown type."""
|
||||
definition = get_service_definition(service_type)
|
||||
if definition is None:
|
||||
raise ValueError(f"Unknown service type: {service_type}")
|
||||
return definition
|
||||
@@ -0,0 +1,60 @@
|
||||
"""SSH task runner service definition.
|
||||
|
||||
An ``ssh_tasks`` instance is an SSH endpoint that can run reusable saved tasks.
|
||||
Tasks themselves stay in the global saved-task registry; the instance only owns
|
||||
transport (host/port/user/key). Every run is recorded in ``service_task_runs``
|
||||
and shown as history on the instance's service page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from media_library_viewer_api.integrations.base import (
|
||||
SecretField,
|
||||
ServiceConfigBase,
|
||||
ServiceDefinition,
|
||||
WidgetConfigBase,
|
||||
widget_kind,
|
||||
)
|
||||
|
||||
|
||||
class SshTasksConfig(ServiceConfigBase):
|
||||
"""Non-secret SSH task runner config.
|
||||
|
||||
The SSH key itself lives in the saved SSH-key registry and is referenced by
|
||||
``ssh_key_id``. An optional ``passphrase`` is stored as a secret.
|
||||
"""
|
||||
|
||||
host: str
|
||||
port: int = 22
|
||||
username: str = ""
|
||||
ssh_key_id: str = ""
|
||||
timeout_seconds: int = 30
|
||||
|
||||
|
||||
class SshTaskOutputWidgetConfig(WidgetConfigBase):
|
||||
"""Output of a saved task run on this instance."""
|
||||
|
||||
task_id: str
|
||||
# service_id is implicit (the widget's service); allow overriding per-widget.
|
||||
service_id: str | None = None
|
||||
|
||||
|
||||
DEFINITION = ServiceDefinition(
|
||||
service_type="ssh_tasks",
|
||||
name="SSH task runner",
|
||||
description="Run reusable saved tasks over SSH and keep run history.",
|
||||
config_model=SshTasksConfig,
|
||||
secret_fields=[
|
||||
SecretField(key="passphrase", label="Key passphrase", helper="Optional"),
|
||||
],
|
||||
widget_kinds=[
|
||||
widget_kind(
|
||||
kind="task_output",
|
||||
name="Task output",
|
||||
description="Output of a saved task run.",
|
||||
model_cls=SshTaskOutputWidgetConfig,
|
||||
default_config={"task_id": ""},
|
||||
refresh_interval_ms=0,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -23,6 +23,7 @@ from media_library_viewer_api.observability import (
|
||||
)
|
||||
from media_library_viewer_api.routers import backups as backups_router
|
||||
from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
|
||||
from media_library_viewer_api.routers import services as services_router
|
||||
from media_library_viewer_api.routers import widgets as widgets_router
|
||||
from media_library_viewer_api.routers.settings import router as settings_router
|
||||
|
||||
@@ -38,6 +39,9 @@ async def lifespan(app: FastAPI):
|
||||
settings = get_settings()
|
||||
configure_logging(settings.log_level, settings.log_format)
|
||||
validate_auth_settings(settings)
|
||||
from media_library_viewer_api.services.secrets import validate_encryption_key
|
||||
|
||||
validate_encryption_key()
|
||||
logger.info("Backend startup complete: %s", describe_settings(settings))
|
||||
logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
|
||||
try:
|
||||
@@ -143,6 +147,7 @@ app.include_router(tasks.router)
|
||||
app.include_router(settings_router)
|
||||
app.include_router(backups_router.router)
|
||||
app.include_router(widgets_router.router)
|
||||
app.include_router(services_router.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pydantic models for the service registry API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
def _validate_config_keys(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Reject credential keys in non-secret service config.
|
||||
|
||||
Secrets are sent in the separate ``secrets`` mapping; the plain ``config``
|
||||
object must never hold them.
|
||||
"""
|
||||
forbidden = {
|
||||
"password",
|
||||
"token",
|
||||
"secret",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"private_key",
|
||||
"passphrase",
|
||||
"credential",
|
||||
}
|
||||
|
||||
def _check(value: Any) -> None:
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
if key.lower() in forbidden:
|
||||
raise ValueError(
|
||||
f"Credential key '{key}' is not allowed in service config"
|
||||
)
|
||||
_check(child)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
_check(item)
|
||||
|
||||
_check(config)
|
||||
return config
|
||||
|
||||
|
||||
class ServiceInstanceInput(BaseModel):
|
||||
"""Payload for creating or updating a service instance."""
|
||||
|
||||
id: str | None = None
|
||||
service_type: str = Field(..., min_length=1)
|
||||
name: str = Field(..., min_length=1)
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
secrets: dict[str, str] = Field(default_factory=dict)
|
||||
enabled: bool = True
|
||||
|
||||
@field_validator("config")
|
||||
@classmethod
|
||||
def reject_credential_keys(cls, value: dict[str, Any]) -> dict[str, Any]:
|
||||
return _validate_config_keys(value or {})
|
||||
|
||||
|
||||
class ServiceInstance(BaseModel):
|
||||
"""Persisted service instance returned by the API (no plaintext secrets)."""
|
||||
|
||||
id: str
|
||||
service_type: str
|
||||
name: str
|
||||
config: dict[str, Any]
|
||||
secrets_set: dict[str, bool]
|
||||
enabled: bool
|
||||
created_at: int
|
||||
updated_at: int
|
||||
|
||||
|
||||
class SecretFieldInfo(BaseModel):
|
||||
key: str
|
||||
label: str
|
||||
required: bool = False
|
||||
helper: str | None = None
|
||||
|
||||
|
||||
class WidgetKindInfo(BaseModel):
|
||||
kind: str
|
||||
name: str
|
||||
description: str
|
||||
config_schema: dict[str, Any]
|
||||
default_config: dict[str, Any]
|
||||
refresh_interval_ms: int
|
||||
|
||||
|
||||
class ServiceTypeInfo(BaseModel):
|
||||
"""Metadata about a registered service type."""
|
||||
|
||||
service_type: str
|
||||
name: str
|
||||
description: str
|
||||
config_schema: dict[str, Any]
|
||||
secret_fields: list[SecretFieldInfo]
|
||||
widget_kinds: list[WidgetKindInfo]
|
||||
@@ -0,0 +1,182 @@
|
||||
"""REST API for the service registry.
|
||||
|
||||
Service instances hold non-secret config and encrypted secrets. Plaintext
|
||||
secrets are never returned; only the boolean ``secrets_set`` map is exposed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from media_library_viewer_api.dependencies import get_settings_store
|
||||
from media_library_viewer_api.integrations.base import validate_config
|
||||
from media_library_viewer_api.integrations.registry import (
|
||||
SERVICE_DEFINITIONS,
|
||||
get_service_definition,
|
||||
require_service_definition,
|
||||
)
|
||||
from media_library_viewer_api.models.services import (
|
||||
SecretFieldInfo,
|
||||
ServiceInstance,
|
||||
ServiceInstanceInput,
|
||||
ServiceTypeInfo,
|
||||
WidgetKindInfo,
|
||||
)
|
||||
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||
|
||||
router = APIRouter(prefix="/api/services", tags=["services"])
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _to_type_info(service_type: str) -> ServiceTypeInfo:
|
||||
definition = require_service_definition(service_type)
|
||||
return ServiceTypeInfo(
|
||||
service_type=definition.service_type,
|
||||
name=definition.name,
|
||||
description=definition.description,
|
||||
config_schema=definition.config_schema,
|
||||
secret_fields=[
|
||||
SecretFieldInfo(
|
||||
key=sf.key,
|
||||
label=sf.label,
|
||||
required=sf.required,
|
||||
helper=sf.helper,
|
||||
)
|
||||
for sf in definition.secret_fields
|
||||
],
|
||||
widget_kinds=[
|
||||
WidgetKindInfo(
|
||||
kind=wk.kind,
|
||||
name=wk.name,
|
||||
description=wk.description,
|
||||
config_schema=wk.config_schema,
|
||||
default_config=wk.default_config,
|
||||
refresh_interval_ms=wk.refresh_interval_ms,
|
||||
)
|
||||
for wk in definition.widget_kinds
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _to_instance(row: dict[str, Any]) -> ServiceInstance:
|
||||
"""Build an API response model, surfacing only secret 'set' flags."""
|
||||
definition = get_service_definition(row["service_type"])
|
||||
known_secrets = definition.secret_keys if definition else set()
|
||||
secrets_blob = row.get("secrets") or {}
|
||||
secrets_set = {key: (key in secrets_blob and bool(secrets_blob[key])) for key in known_secrets}
|
||||
return ServiceInstance(
|
||||
id=row["id"],
|
||||
service_type=row["service_type"],
|
||||
name=row["name"],
|
||||
config=row.get("config") or {},
|
||||
secrets_set=secrets_set,
|
||||
enabled=row["enabled"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
|
||||
|
||||
def _validate_input(body: ServiceInstanceInput) -> None:
|
||||
"""Validate service_type, config, and secret keys against the definition."""
|
||||
definition = get_service_definition(body.service_type)
|
||||
if definition is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=f"Unknown service type: {body.service_type}",
|
||||
)
|
||||
try:
|
||||
validate_config(definition.config_model, body.config)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=f"Invalid service config: {exc}",
|
||||
) from exc
|
||||
unknown_secrets = set(body.secrets) - definition.secret_keys
|
||||
if unknown_secrets:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=f"Unknown secret fields for {body.service_type}: {sorted(unknown_secrets)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/types")
|
||||
def list_types() -> list[ServiceTypeInfo]:
|
||||
"""Return metadata for every registered service type."""
|
||||
return [_to_type_info(service_type) for service_type in sorted(SERVICE_DEFINITIONS)]
|
||||
|
||||
|
||||
@router.get("/instances")
|
||||
def list_instances(
|
||||
service_type: str | None = None,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> list[ServiceInstance]:
|
||||
"""Return all persisted service instances (no plaintext secrets)."""
|
||||
rows = store.list_services(service_type)
|
||||
return [_to_instance(row) for row in rows]
|
||||
|
||||
|
||||
@router.post("/instances", status_code=status.HTTP_201_CREATED)
|
||||
def create_instance(
|
||||
body: ServiceInstanceInput,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> ServiceInstance:
|
||||
"""Create a new service instance."""
|
||||
_validate_input(body)
|
||||
row = store.upsert_service(
|
||||
{
|
||||
"id": body.id,
|
||||
"service_type": body.service_type,
|
||||
"name": body.name,
|
||||
"config": body.config,
|
||||
"enabled": body.enabled,
|
||||
},
|
||||
secret_values=body.secrets,
|
||||
)
|
||||
return _to_instance(row)
|
||||
|
||||
|
||||
@router.put("/instances/{service_id}")
|
||||
def update_instance(
|
||||
service_id: str,
|
||||
body: ServiceInstanceInput,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> ServiceInstance:
|
||||
"""Update an existing service instance."""
|
||||
existing = store.get_service(service_id)
|
||||
if not existing:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Service not found")
|
||||
if body.id is not None and body.id != service_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="ID in path does not match ID in body",
|
||||
)
|
||||
_validate_input(body)
|
||||
row = store.upsert_service(
|
||||
{
|
||||
"id": service_id,
|
||||
"service_type": body.service_type,
|
||||
"name": body.name,
|
||||
"config": body.config,
|
||||
"enabled": body.enabled,
|
||||
},
|
||||
secret_values=body.secrets,
|
||||
service_id=service_id,
|
||||
)
|
||||
return _to_instance(row)
|
||||
|
||||
|
||||
@router.delete("/instances/{service_id}")
|
||||
def delete_instance(
|
||||
service_id: str,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Delete a service instance (cascade-deletes widgets referencing it)."""
|
||||
existing = store.get_service(service_id)
|
||||
if not existing:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Service not found")
|
||||
store.delete_service(service_id)
|
||||
return {"status": "deleted"}
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Encryption-at-rest for service secrets.
|
||||
|
||||
Service API keys / tokens are stored encrypted in the ``services.secrets_json``
|
||||
column. Encryption uses Fernet (symmetric authenticated encryption) with a single
|
||||
master key provided via the ``MANAGE_ENCRYPTION_KEY`` environment variable.
|
||||
|
||||
* The key **must** be a urlsafe base64-encoded 32-byte value (Fernet format).
|
||||
* The key is **always required** — there is no development fallback, so secrets
|
||||
are never accidentally stored in plaintext.
|
||||
* Secrets are encrypted field-by-field; the ``"which secrets are set"`` metadata
|
||||
can be derived from the ciphertext blob without decrypting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
ENCRYPTION_KEY_ENV = "MANAGE_ENCRYPTION_KEY"
|
||||
|
||||
|
||||
class EncryptionKeyError(RuntimeError):
|
||||
"""Raised when the encryption key is missing or invalid."""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_encryption_key() -> bytes:
|
||||
"""Return the raw Fernet key, or raise if missing/invalid.
|
||||
|
||||
The result is cached for the process lifetime. Tests should call
|
||||
:func:`reset_encryption_key_cache` after changing the environment.
|
||||
"""
|
||||
raw = os.environ.get(ENCRYPTION_KEY_ENV)
|
||||
if not raw:
|
||||
raise EncryptionKeyError(
|
||||
f"{ENCRYPTION_KEY_ENV} is required to store service secrets"
|
||||
)
|
||||
key = raw.strip().encode()
|
||||
try:
|
||||
Fernet(key)
|
||||
except (ValueError, TypeError) as exc: # pragma: no cover - validated by tests
|
||||
raise EncryptionKeyError(
|
||||
f"{ENCRYPTION_KEY_ENV} must be a valid Fernet key"
|
||||
) from exc
|
||||
return key
|
||||
|
||||
|
||||
def reset_encryption_key_cache() -> None:
|
||||
"""Drop the cached encryption key (used by tests that swap keys)."""
|
||||
get_encryption_key.cache_clear()
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
return Fernet(get_encryption_key())
|
||||
|
||||
|
||||
def encrypt_value(plaintext: str) -> str:
|
||||
"""Encrypt a single secret value and return the ciphertext string."""
|
||||
return _fernet().encrypt(plaintext.encode()).decode()
|
||||
|
||||
|
||||
def decrypt_value(ciphertext: str) -> str:
|
||||
"""Decrypt a single ciphertext value."""
|
||||
try:
|
||||
return _fernet().decrypt(ciphertext.encode()).decode()
|
||||
except InvalidToken as exc:
|
||||
raise EncryptionKeyError("Service secret could not be decrypted") from exc
|
||||
|
||||
|
||||
def encrypt_secrets(values: dict[str, str]) -> dict[str, str]:
|
||||
"""Encrypt every provided secret value."""
|
||||
fernet = _fernet()
|
||||
return {key: fernet.encrypt(value.encode()).decode() for key, value in values.items()}
|
||||
|
||||
|
||||
def decrypt_secrets(blob: dict[str, str]) -> dict[str, str]:
|
||||
"""Decrypt every secret value in a blob."""
|
||||
fernet = _fernet()
|
||||
result: dict[str, str] = {}
|
||||
for key, ciphertext in blob.items():
|
||||
try:
|
||||
result[key] = fernet.decrypt(ciphertext.encode()).decode()
|
||||
except InvalidToken as exc:
|
||||
raise EncryptionKeyError(f"Service secret '{key}' could not be decrypted") from exc
|
||||
return result
|
||||
|
||||
|
||||
def generate_development_key() -> str:
|
||||
"""Return a freshly generated Fernet key (helper for operators/docs)."""
|
||||
return Fernet.generate_key().decode()
|
||||
|
||||
|
||||
def validate_encryption_key() -> None:
|
||||
"""Eagerly validate that the encryption key is present and well-formed."""
|
||||
get_encryption_key() # raises EncryptionKeyError on failure
|
||||
@@ -226,6 +226,45 @@ class SettingsStore:
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_backup_alerts_job_id ON backup_alerts(job_id)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_backup_alerts_acknowledged ON backup_alerts(acknowledged)")
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS services (
|
||||
id TEXT PRIMARY KEY,
|
||||
service_type TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
config_json TEXT NOT NULL DEFAULT '{}',
|
||||
secrets_json TEXT NOT NULL DEFAULT '{}',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_services_type ON services(service_type)")
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS service_task_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
exit_status INTEGER,
|
||||
duration_ms INTEGER,
|
||||
stdout_tail TEXT NOT NULL DEFAULT '',
|
||||
stderr_tail TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_service_task_runs_service "
|
||||
"ON service_task_runs(service_id, created_at DESC)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_service_task_runs_task "
|
||||
"ON service_task_runs(task_id, created_at DESC)"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_services(value: Any, fallback: list[str] | None = None) -> list[str]:
|
||||
@@ -1476,6 +1515,211 @@ class SettingsStore:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM dashboard_widgets WHERE id = ?", (widget_id,))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Service registry
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _row_to_service(self, row: sqlite3.Row) -> dict[str, Any]:
|
||||
secrets_blob = json.loads(row["secrets_json"] or "{}")
|
||||
return {
|
||||
"id": row["id"],
|
||||
"service_type": row["service_type"],
|
||||
"name": row["name"],
|
||||
"config": json.loads(row["config_json"] or "{}"),
|
||||
"secrets": secrets_blob,
|
||||
"enabled": bool(row["enabled"]),
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
def list_services(self, service_type: str | None = None) -> list[dict[str, Any]]:
|
||||
self.init_schema()
|
||||
with self.connect() as conn:
|
||||
if service_type:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM services WHERE service_type = ? ORDER BY name ASC",
|
||||
(service_type,),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM services ORDER BY name ASC").fetchall()
|
||||
return [self._row_to_service(row) for row in rows]
|
||||
|
||||
def get_service(self, service_id: str) -> dict[str, Any] | None:
|
||||
self.init_schema()
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM services WHERE id = ?", (service_id,)
|
||||
).fetchone()
|
||||
return self._row_to_service(row) if row else None
|
||||
|
||||
def _normalize_service_payload(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
service_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
current = self.get_service(service_id) if service_id else None
|
||||
service_id = (
|
||||
str(payload.get("id") or service_id or uuid.uuid4().hex[:12]).strip()
|
||||
or uuid.uuid4().hex[:12]
|
||||
)
|
||||
service_type = str(
|
||||
payload.get("service_type") or (current or {}).get("service_type", "")
|
||||
).strip()
|
||||
name = str(payload.get("name") or (current or {}).get("name", "") or "").strip()
|
||||
config = payload.get("config", (current or {}).get("config", {}))
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
|
||||
return {
|
||||
"id": service_id,
|
||||
"service_type": service_type,
|
||||
"name": name,
|
||||
"config": config,
|
||||
"enabled": enabled,
|
||||
}
|
||||
|
||||
def upsert_service(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
secret_values: dict[str, str] | None = None,
|
||||
service_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Insert or update a service instance.
|
||||
|
||||
``secret_values`` carries plaintext secrets to encrypt and store. A key
|
||||
absent from ``secret_values`` preserves the existing ciphertext; a key
|
||||
mapped to an empty string clears it.
|
||||
"""
|
||||
self.init_schema()
|
||||
service = self._normalize_service_payload(payload, service_id)
|
||||
now = int(time.time())
|
||||
|
||||
existing = self.get_service(service["id"])
|
||||
secrets_blob: dict[str, str]
|
||||
if existing is not None:
|
||||
secrets_blob = dict(existing["secrets"])
|
||||
else:
|
||||
secrets_blob = {}
|
||||
if secret_values:
|
||||
from media_library_viewer_api.services.secrets import encrypt_value
|
||||
|
||||
for key, value in secret_values.items():
|
||||
if value == "":
|
||||
secrets_blob.pop(key, None)
|
||||
else:
|
||||
secrets_blob[key] = encrypt_value(value)
|
||||
|
||||
with self.connect() as conn:
|
||||
created_at = int(existing["created_at"]) if existing else now
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO services (
|
||||
id, service_type, name, config_json, secrets_json,
|
||||
enabled, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
service_type = excluded.service_type,
|
||||
name = excluded.name,
|
||||
config_json = excluded.config_json,
|
||||
secrets_json = excluded.secrets_json,
|
||||
enabled = excluded.enabled,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
service["id"],
|
||||
service["service_type"],
|
||||
service["name"],
|
||||
json.dumps(service["config"]),
|
||||
json.dumps(secrets_blob),
|
||||
1 if service["enabled"] else 0,
|
||||
created_at,
|
||||
now,
|
||||
),
|
||||
)
|
||||
return self.get_service(service["id"]) or service
|
||||
|
||||
def delete_service(self, service_id: str) -> None:
|
||||
"""Delete a service and cascade-delete widgets referencing it."""
|
||||
self.init_schema()
|
||||
with self.connect() as conn:
|
||||
# The service_id column on dashboard_widgets is added in a later
|
||||
# slice; only cascade when it is present.
|
||||
widget_cols = {row[1] for row in conn.execute("PRAGMA table_info(dashboard_widgets)").fetchall()}
|
||||
if "service_id" in widget_cols:
|
||||
conn.execute(
|
||||
"DELETE FROM dashboard_widgets WHERE service_id = ?",
|
||||
(service_id,),
|
||||
)
|
||||
conn.execute("DELETE FROM services WHERE id = ?", (service_id,))
|
||||
|
||||
def record_service_task_run(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Append a service task run history row."""
|
||||
self.init_schema()
|
||||
run_id = str(payload.get("id") or uuid.uuid4().hex[:12])
|
||||
now = int(time.time())
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO service_task_runs (
|
||||
id, task_id, service_id, status, exit_status, duration_ms,
|
||||
stdout_tail, stderr_tail, error, created_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run_id,
|
||||
str(payload.get("task_id") or ""),
|
||||
str(payload.get("service_id") or ""),
|
||||
str(payload.get("status") or "error"),
|
||||
payload.get("exit_status"),
|
||||
payload.get("duration_ms"),
|
||||
str(payload.get("stdout_tail") or "")[:8000],
|
||||
str(payload.get("stderr_tail") or "")[:8000],
|
||||
str(payload.get("error") or "")[:1000],
|
||||
int(payload.get("created_at") or now),
|
||||
),
|
||||
)
|
||||
return {"id": run_id}
|
||||
|
||||
def list_service_task_runs(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
limit: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
self.init_schema()
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
if service_id:
|
||||
clauses.append("service_id = ?")
|
||||
params.append(service_id)
|
||||
if task_id:
|
||||
clauses.append("task_id = ?")
|
||||
params.append(task_id)
|
||||
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
params.append(int(limit))
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM service_task_runs {where} ORDER BY created_at DESC LIMIT ?",
|
||||
params,
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"task_id": row["task_id"],
|
||||
"service_id": row["service_id"],
|
||||
"status": row["status"],
|
||||
"exit_status": row["exit_status"],
|
||||
"duration_ms": row["duration_ms"],
|
||||
"stdout_tail": row["stdout_tail"],
|
||||
"stderr_tail": row["stderr_tail"],
|
||||
"error": row["error"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
_store: SettingsStore | None = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user