feat(v2): add validated runtime and metadata store
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
from .engine import SchemaNotCurrentError, assert_schema_current, create_engine
|
||||
from .models import Base
|
||||
|
||||
__all__ = ["Base", "SchemaNotCurrentError", "assert_schema_current", "create_engine"]
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
from backup_tool.config import Settings
|
||||
|
||||
|
||||
class SchemaNotCurrentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def create_engine(settings: Settings) -> AsyncEngine:
|
||||
engine = create_async_engine(settings.database_url, pool_pre_ping=True)
|
||||
|
||||
@event.listens_for(engine.sync_engine, "connect")
|
||||
def configure_sqlite(dbapi_connection: Any, _connection_record: Any) -> None:
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.execute(f"PRAGMA busy_timeout={settings.sqlite_busy_timeout_ms}")
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
return engine
|
||||
|
||||
|
||||
async def assert_schema_current(engine: AsyncEngine, alembic_config: Config) -> None:
|
||||
expected = ScriptDirectory.from_config(alembic_config).get_current_head()
|
||||
|
||||
def get_current_revision(connection: Connection) -> str | None:
|
||||
return MigrationContext.configure(connection).get_current_revision()
|
||||
|
||||
async with engine.connect() as connection:
|
||||
current = await connection.run_sync(get_current_revision)
|
||||
if current != expected:
|
||||
raise SchemaNotCurrentError(
|
||||
f"database schema is not current: expected {expected!r}, found {current!r}"
|
||||
)
|
||||
|
||||
|
||||
SchemaCheck = Callable[[AsyncEngine, Config], Any]
|
||||
@@ -0,0 +1,290 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
LargeBinary,
|
||||
MetaData,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
|
||||
|
||||
class IdentityMixin:
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.current_timestamp(),
|
||||
onupdate=func.current_timestamp(),
|
||||
)
|
||||
|
||||
|
||||
class User(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "users"
|
||||
username: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (CheckConstraint("state IN ('active','disabled')", name="state"),)
|
||||
|
||||
|
||||
class ApiToken(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "api_tokens"
|
||||
owner_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
scopes: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
__table_args__ = (Index("ix_api_tokens_owner_id", "owner_id"),)
|
||||
|
||||
|
||||
class Secret(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "secrets"
|
||||
ciphertext: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
key_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
purpose: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
__table_args__ = (CheckConstraint("version > 0", name="version_positive"),)
|
||||
|
||||
|
||||
class Repository(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "repositories"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
root: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
format_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
compression: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
encryption: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (
|
||||
CheckConstraint("format_version > 0", name="format_version_positive"),
|
||||
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||
)
|
||||
|
||||
|
||||
class Source(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "sources"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
public_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
secret_refs: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||
__table_args__ = (
|
||||
CheckConstraint("kind IN ('local','sftp','postgresql','mysql')", name="kind"),
|
||||
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||
)
|
||||
|
||||
|
||||
class Job(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "jobs"
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id", ondelete="RESTRICT"))
|
||||
repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
|
||||
requested_mode: Mapped[str] = mapped_column(String(32), nullable=False, default="incremental")
|
||||
exclusions: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
retention: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
allow_empty: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (
|
||||
CheckConstraint("requested_mode IN ('full','incremental')", name="requested_mode"),
|
||||
CheckConstraint("state IN ('active','archived')", name="state"),
|
||||
Index("ix_jobs_source_id", "source_id"),
|
||||
Index("ix_jobs_repository_id", "repository_id"),
|
||||
)
|
||||
|
||||
|
||||
class Schedule(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "schedules"
|
||||
job_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("jobs.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||
)
|
||||
cron: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
timezone: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
misfire_grace_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=900)
|
||||
overlap_policy: Mapped[str] = mapped_column(String(32), nullable=False, default="prohibit")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
next_nominal_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_enqueue_outcome: Mapped[str | None] = mapped_column(String(64))
|
||||
__table_args__ = (
|
||||
CheckConstraint("misfire_grace_seconds >= 0", name="misfire_nonnegative"),
|
||||
CheckConstraint("overlap_policy IN ('prohibit')", name="overlap_policy"),
|
||||
)
|
||||
|
||||
|
||||
class Execution(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "executions"
|
||||
job_id: Mapped[str] = mapped_column(ForeignKey("jobs.id", ondelete="RESTRICT"))
|
||||
schedule_id: Mapped[str | None] = mapped_column(ForeignKey("schedules.id", ondelete="RESTRICT"))
|
||||
nominal_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
trigger: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(255))
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
progress: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
reason_code: Mapped[str | None] = mapped_column(String(64))
|
||||
operator_message: Mapped[str | None] = mapped_column(Text)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt > 0", name="attempt_positive"),
|
||||
CheckConstraint(
|
||||
"state IN ('queued','preparing','running','verifying','committed',"
|
||||
"'cancelling','cancelled','failed')",
|
||||
name="state",
|
||||
),
|
||||
CheckConstraint("trigger IN ('manual','schedule','retry')", name="trigger"),
|
||||
UniqueConstraint("schedule_id", "nominal_run_at", name="uq_execution_schedule_occurrence"),
|
||||
Index("ix_executions_job_id", "job_id"),
|
||||
Index("ix_executions_state_created", "state", "created_at"),
|
||||
Index(
|
||||
"uq_executions_active_job",
|
||||
"job_id",
|
||||
unique=True,
|
||||
sqlite_where=text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Backup(IdentityMixin, Base):
|
||||
__tablename__ = "backups"
|
||||
execution_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("executions.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||
)
|
||||
parent_backup_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("backups.id", ondelete="RESTRICT")
|
||||
)
|
||||
manifest_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True)
|
||||
manifest_digest: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified")
|
||||
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
tombstoned_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
CheckConstraint("logical_bytes >= 0", name="logical_bytes_nonnegative"),
|
||||
CheckConstraint("stored_bytes >= 0", name="stored_bytes_nonnegative"),
|
||||
CheckConstraint(
|
||||
"integrity IN ('unverified','verified','degraded','corrupt')", name="integrity"
|
||||
),
|
||||
Index("ix_backups_created_at", "created_at"),
|
||||
)
|
||||
|
||||
|
||||
class Restore(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "restores"
|
||||
backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT"))
|
||||
destination: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
selection: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
result: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||
__table_args__ = (
|
||||
CheckConstraint("overwrite_policy IN ('fail','skip','replace')", name="overwrite_policy"),
|
||||
CheckConstraint(
|
||||
"state IN ('queued','running','committed','cancelled','failed')", name="state"
|
||||
),
|
||||
Index("ix_restores_backup_id", "backup_id"),
|
||||
)
|
||||
|
||||
|
||||
class AuditEvent(IdentityMixin, Base):
|
||||
__tablename__ = "audit_events"
|
||||
actor_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
action: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
resource_id: Mapped[str | None] = mapped_column(String(36))
|
||||
outcome: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
request_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
details: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
CheckConstraint("outcome IN ('success','failure','denied')", name="outcome"),
|
||||
Index("ix_audit_events_created_at", "created_at"),
|
||||
Index("ix_audit_events_actor_id", "actor_id"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationSubscription(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "notification_subscriptions"
|
||||
channel: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||
destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||
__table_args__ = (
|
||||
CheckConstraint("channel IN ('webhook','email')", name="channel"),
|
||||
CheckConstraint("state IN ('active','disabled','archived')", name="state"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationDelivery(IdentityMixin, TimestampMixin, Base):
|
||||
__tablename__ = "notification_deliveries"
|
||||
event_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
subscription_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("notification_subscriptions.id", ondelete="RESTRICT")
|
||||
)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
response_class: Mapped[str | None] = mapped_column(String(64))
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt > 0", name="attempt_positive"),
|
||||
CheckConstraint("state IN ('pending','delivered','retry','failed')", name="state"),
|
||||
UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
|
||||
Index("ix_notification_deliveries_state_next", "state", "next_attempt_at"),
|
||||
)
|
||||
|
||||
|
||||
class IdempotencyRecord(IdentityMixin, Base):
|
||||
__tablename__ = "idempotency_records"
|
||||
actor_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||
key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
operation: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_digest: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
response_resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
response_resource_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||
)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("actor_id", "key", "operation", name="uq_idempotency_actor_key_operation"),
|
||||
Index("ix_idempotency_created_at", "created_at"),
|
||||
)
|
||||
Reference in New Issue
Block a user