feat(v2): add validated runtime and metadata store
This commit is contained in:
@@ -16,8 +16,8 @@ test-fast:
|
|||||||
$(PYTHON) -m pytest tests/unit tests/contract -q
|
$(PYTHON) -m pytest tests/unit tests/contract -q
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
$(PYTHON) -m ruff check --config backend/pyproject.toml backend/src tests tools
|
$(PYTHON) -m ruff check --config backend/pyproject.toml backend/src backend/alembic tests tools
|
||||||
$(PYTHON) -m ruff format --check --config backend/pyproject.toml backend/src tests tools
|
$(PYTHON) -m ruff format --check --config backend/pyproject.toml backend/src backend/alembic tests tools
|
||||||
|
|
||||||
typecheck:
|
typecheck:
|
||||||
$(PYTHON) -m mypy --config-file backend/pyproject.toml
|
$(PYTHON) -m mypy --config-file backend/pyproject.toml
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
[alembic]
|
||||||
|
script_location = %(here)s/alembic
|
||||||
|
prepend_sys_path = %(here)s/src
|
||||||
|
path_separator = os
|
||||||
|
sqlalchemy.url = sqlite+aiosqlite:////var/lib/backup-tool/metadata.db
|
||||||
|
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
from sqlalchemy import create_engine, pool
|
||||||
|
from sqlalchemy.engine import URL, make_url
|
||||||
|
|
||||||
|
Base = importlib.import_module("backup_tool.db.models").Base
|
||||||
|
|
||||||
|
config = context.config
|
||||||
|
if config.config_file_name is not None:
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
|
||||||
|
def configured_url() -> str:
|
||||||
|
raw_url = os.environ.get("BACKUP_TOOL_DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||||
|
if raw_url is None:
|
||||||
|
raise RuntimeError("sqlalchemy.url is required")
|
||||||
|
return raw_url
|
||||||
|
|
||||||
|
|
||||||
|
def synchronous_url(raw_url: str) -> URL:
|
||||||
|
url = make_url(raw_url)
|
||||||
|
if url.drivername != "sqlite+aiosqlite":
|
||||||
|
raise RuntimeError("v2 migrations require sqlite+aiosqlite")
|
||||||
|
return url.set(drivername="sqlite")
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline() -> None:
|
||||||
|
url = synchronous_url(configured_url())
|
||||||
|
context.configure(
|
||||||
|
url=url.render_as_string(hide_password=False),
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
literal_binds=True,
|
||||||
|
dialect_opts={"paramstyle": "named"},
|
||||||
|
compare_type=True,
|
||||||
|
)
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online() -> None:
|
||||||
|
engine = create_engine(
|
||||||
|
synchronous_url(configured_url()),
|
||||||
|
poolclass=pool.NullPool,
|
||||||
|
)
|
||||||
|
with engine.connect() as connection:
|
||||||
|
context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
revision: str = ${repr(up_revision)}
|
||||||
|
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||||
|
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
${downgrades if downgrades else "pass"}
|
||||||
@@ -0,0 +1,548 @@
|
|||||||
|
"""v2 baseline
|
||||||
|
|
||||||
|
Revision ID: 0001_v2_baseline
|
||||||
|
Revises:
|
||||||
|
Create Date: 2026-07-27 19:15:52.319338
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0001_v2_baseline"
|
||||||
|
down_revision: str | Sequence[str] | None = None
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table(
|
||||||
|
"repositories",
|
||||||
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("root", sa.Text(), nullable=False),
|
||||||
|
sa.Column("format_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("compression", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("encryption", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('active','archived','unavailable')", name=op.f("ck_repositories_state")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"format_version > 0", name=op.f("ck_repositories_format_version_positive")
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_repositories")),
|
||||||
|
sa.UniqueConstraint("name", name=op.f("uq_repositories_name")),
|
||||||
|
sa.UniqueConstraint("root", name=op.f("uq_repositories_root")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"secrets",
|
||||||
|
sa.Column("ciphertext", sa.LargeBinary(), nullable=False),
|
||||||
|
sa.Column("key_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("purpose", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("version > 0", name=op.f("ck_secrets_version_positive")),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_secrets")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"sources",
|
||||||
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("public_config", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("secret_refs", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("last_probe", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"kind IN ('local','sftp','postgresql','mysql')", name=op.f("ck_sources_kind")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('active','archived','unavailable')", name=op.f("ck_sources_state")
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")),
|
||||||
|
sa.UniqueConstraint("name", name=op.f("uq_sources_name")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"users",
|
||||||
|
sa.Column("username", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("password_hash", sa.Text(), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("state IN ('active','disabled')", name=op.f("ck_users_state")),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
|
||||||
|
sa.UniqueConstraint("username", name=op.f("uq_users_username")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"api_tokens",
|
||||||
|
sa.Column("owner_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("token_hash", sa.Text(), nullable=False),
|
||||||
|
sa.Column("scopes", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["owner_id"],
|
||||||
|
["users.id"],
|
||||||
|
name=op.f("fk_api_tokens_owner_id_users"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_api_tokens")),
|
||||||
|
sa.UniqueConstraint("token_hash", name=op.f("uq_api_tokens_token_hash")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_api_tokens_owner_id", "api_tokens", ["owner_id"], unique=False)
|
||||||
|
op.create_table(
|
||||||
|
"audit_events",
|
||||||
|
sa.Column("actor_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("action", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("resource_type", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("resource_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("outcome", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("request_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("details", sa.JSON(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"outcome IN ('success','failure','denied')", name=op.f("ck_audit_events_outcome")
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["actor_id"],
|
||||||
|
["users.id"],
|
||||||
|
name=op.f("fk_audit_events_actor_id_users"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_events")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_audit_events_actor_id", "audit_events", ["actor_id"], unique=False)
|
||||||
|
op.create_index("ix_audit_events_created_at", "audit_events", ["created_at"], unique=False)
|
||||||
|
op.create_table(
|
||||||
|
"idempotency_records",
|
||||||
|
sa.Column("actor_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("operation", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_digest", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("response_resource_type", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("response_resource_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["actor_id"],
|
||||||
|
["users.id"],
|
||||||
|
name=op.f("fk_idempotency_records_actor_id_users"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_idempotency_records")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"actor_id", "key", "operation", name="uq_idempotency_actor_key_operation"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_idempotency_created_at", "idempotency_records", ["created_at"], unique=False
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"jobs",
|
||||||
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("repository_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("requested_mode", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("exclusions", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("retention", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("allow_empty", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"requested_mode IN ('full','incremental')", name=op.f("ck_jobs_requested_mode")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("state IN ('active','archived')", name=op.f("ck_jobs_state")),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["repository_id"],
|
||||||
|
["repositories.id"],
|
||||||
|
name=op.f("fk_jobs_repository_id_repositories"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"],
|
||||||
|
["sources.id"],
|
||||||
|
name=op.f("fk_jobs_source_id_sources"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_jobs")),
|
||||||
|
sa.UniqueConstraint("name", name=op.f("uq_jobs_name")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_jobs_repository_id", "jobs", ["repository_id"], unique=False)
|
||||||
|
op.create_index("ix_jobs_source_id", "jobs", ["source_id"], unique=False)
|
||||||
|
op.create_table(
|
||||||
|
"notification_subscriptions",
|
||||||
|
sa.Column("channel", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("event_filters", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("destination_config", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("secret_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"channel IN ('webhook','email')", name=op.f("ck_notification_subscriptions_channel")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('active','disabled','archived')",
|
||||||
|
name=op.f("ck_notification_subscriptions_state"),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["secret_id"],
|
||||||
|
["secrets.id"],
|
||||||
|
name=op.f("fk_notification_subscriptions_secret_id_secrets"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_notification_subscriptions")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"notification_deliveries",
|
||||||
|
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("subscription_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("response_class", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('pending','delivered','retry','failed')",
|
||||||
|
name=op.f("ck_notification_deliveries_state"),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("attempt > 0", name=op.f("ck_notification_deliveries_attempt_positive")),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["subscription_id"],
|
||||||
|
["notification_subscriptions.id"],
|
||||||
|
name=op.f("fk_notification_deliveries_subscription_id_notification_subscriptions"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_notification_deliveries")),
|
||||||
|
sa.UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_notification_deliveries_state_next",
|
||||||
|
"notification_deliveries",
|
||||||
|
["state", "next_attempt_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"schedules",
|
||||||
|
sa.Column("job_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("cron", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("timezone", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("misfire_grace_seconds", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("overlap_policy", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("next_nominal_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_enqueue_outcome", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"overlap_policy IN ('prohibit')", name=op.f("ck_schedules_overlap_policy")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"misfire_grace_seconds >= 0", name=op.f("ck_schedules_misfire_nonnegative")
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["job_id"], ["jobs.id"], name=op.f("fk_schedules_job_id_jobs"), ondelete="RESTRICT"
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_schedules")),
|
||||||
|
sa.UniqueConstraint("job_id", name=op.f("uq_schedules_job_id")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"executions",
|
||||||
|
sa.Column("job_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("schedule_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("nominal_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("trigger", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("lease_owner", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("progress", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("reason_code", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("operator_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('queued','preparing','running','verifying','committed',"
|
||||||
|
"'cancelling','cancelled','failed')",
|
||||||
|
name=op.f("ck_executions_state"),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"trigger IN ('manual','schedule','retry')", name=op.f("ck_executions_trigger")
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("attempt > 0", name=op.f("ck_executions_attempt_positive")),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["job_id"], ["jobs.id"], name=op.f("fk_executions_job_id_jobs"), ondelete="RESTRICT"
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["schedule_id"],
|
||||||
|
["schedules.id"],
|
||||||
|
name=op.f("fk_executions_schedule_id_schedules"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_executions")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"schedule_id", "nominal_run_at", name="uq_execution_schedule_occurrence"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_executions_job_id", "executions", ["job_id"], unique=False)
|
||||||
|
op.create_index(
|
||||||
|
"ix_executions_state_created", "executions", ["state", "created_at"], unique=False
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_executions_active_job",
|
||||||
|
"executions",
|
||||||
|
["job_id"],
|
||||||
|
unique=True,
|
||||||
|
sqlite_where=sa.text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"backups",
|
||||||
|
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("parent_backup_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("manifest_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("manifest_digest", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("logical_bytes", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("stored_bytes", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("integrity", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("pinned", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("tombstoned_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"integrity IN ('unverified','verified','degraded','corrupt')",
|
||||||
|
name=op.f("ck_backups_integrity"),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("logical_bytes >= 0", name=op.f("ck_backups_logical_bytes_nonnegative")),
|
||||||
|
sa.CheckConstraint("stored_bytes >= 0", name=op.f("ck_backups_stored_bytes_nonnegative")),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["execution_id"],
|
||||||
|
["executions.id"],
|
||||||
|
name=op.f("fk_backups_execution_id_executions"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["parent_backup_id"],
|
||||||
|
["backups.id"],
|
||||||
|
name=op.f("fk_backups_parent_backup_id_backups"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_backups")),
|
||||||
|
sa.UniqueConstraint("execution_id", name=op.f("uq_backups_execution_id")),
|
||||||
|
sa.UniqueConstraint("manifest_digest", name=op.f("uq_backups_manifest_digest")),
|
||||||
|
sa.UniqueConstraint("manifest_id", name=op.f("uq_backups_manifest_id")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_backups_created_at", "backups", ["created_at"], unique=False)
|
||||||
|
op.create_table(
|
||||||
|
"restores",
|
||||||
|
sa.Column("backup_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("destination", sa.Text(), nullable=False),
|
||||||
|
sa.Column("selection", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("overwrite_policy", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("result", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("(CURRENT_TIMESTAMP)"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"overwrite_policy IN ('fail','skip','replace')",
|
||||||
|
name=op.f("ck_restores_overwrite_policy"),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('queued','running','committed','cancelled','failed')",
|
||||||
|
name=op.f("ck_restores_state"),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["backup_id"],
|
||||||
|
["backups.id"],
|
||||||
|
name=op.f("fk_restores_backup_id_backups"),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_restores")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_restores_backup_id", "restores", ["backup_id"], unique=False)
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index("ix_restores_backup_id", table_name="restores")
|
||||||
|
op.drop_table("restores")
|
||||||
|
op.drop_index("ix_backups_created_at", table_name="backups")
|
||||||
|
op.drop_table("backups")
|
||||||
|
op.drop_index(
|
||||||
|
"uq_executions_active_job",
|
||||||
|
table_name="executions",
|
||||||
|
sqlite_where=sa.text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||||
|
)
|
||||||
|
op.drop_index("ix_executions_state_created", table_name="executions")
|
||||||
|
op.drop_index("ix_executions_job_id", table_name="executions")
|
||||||
|
op.drop_table("executions")
|
||||||
|
op.drop_table("schedules")
|
||||||
|
op.drop_index("ix_notification_deliveries_state_next", table_name="notification_deliveries")
|
||||||
|
op.drop_table("notification_deliveries")
|
||||||
|
op.drop_table("notification_subscriptions")
|
||||||
|
op.drop_index("ix_jobs_source_id", table_name="jobs")
|
||||||
|
op.drop_index("ix_jobs_repository_id", table_name="jobs")
|
||||||
|
op.drop_table("jobs")
|
||||||
|
op.drop_index("ix_idempotency_created_at", table_name="idempotency_records")
|
||||||
|
op.drop_table("idempotency_records")
|
||||||
|
op.drop_index("ix_audit_events_created_at", table_name="audit_events")
|
||||||
|
op.drop_index("ix_audit_events_actor_id", table_name="audit_events")
|
||||||
|
op.drop_table("audit_events")
|
||||||
|
op.drop_index("ix_api_tokens_owner_id", table_name="api_tokens")
|
||||||
|
op.drop_table("api_tokens")
|
||||||
|
op.drop_table("users")
|
||||||
|
op.drop_table("sources")
|
||||||
|
op.drop_table("secrets")
|
||||||
|
op.drop_table("repositories")
|
||||||
|
# ### end Alembic commands ###
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Minimal v2 command entry point; runtime roles arrive in M1."""
|
"""Backup Tool v2 process-role command line."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -7,10 +7,15 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from backup_tool import __version__
|
from backup_tool import __version__
|
||||||
|
|
||||||
|
ROLES = ("web", "scheduler", "worker", "migrate", "admin")
|
||||||
|
|
||||||
|
|
||||||
def build_parser() -> argparse.ArgumentParser:
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
parser = argparse.ArgumentParser(prog="backup-tool")
|
parser = argparse.ArgumentParser(prog="backup-tool")
|
||||||
parser.add_argument("--version", action="version", version=__version__)
|
parser.add_argument("--version", action="version", version=__version__)
|
||||||
|
subparsers = parser.add_subparsers(dest="role", required=True)
|
||||||
|
for role in ROLES:
|
||||||
|
subparsers.add_parser(role, help=f"run the {role} role")
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class Clock(Protocol):
|
||||||
|
def now(self) -> datetime: ...
|
||||||
|
|
||||||
|
|
||||||
|
class SystemClock:
|
||||||
|
def now(self) -> datetime:
|
||||||
|
return datetime.now(UTC)
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Literal, Self
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from pydantic import Field, field_validator, model_validator
|
||||||
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
|
|
||||||
|
|
||||||
|
class Settings(BaseSettings):
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="BACKUP_TOOL_",
|
||||||
|
extra="forbid",
|
||||||
|
case_sensitive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
data_dir: Path = Path("/var/lib/backup-tool")
|
||||||
|
database_url: str = "sqlite+aiosqlite:////var/lib/backup-tool/metadata.db"
|
||||||
|
repository_roots: tuple[Path, ...]
|
||||||
|
local_source_roots: tuple[Path, ...]
|
||||||
|
restore_roots: tuple[Path, ...]
|
||||||
|
master_key_file: Path
|
||||||
|
public_base_url: str = "http://127.0.0.1:8000"
|
||||||
|
cors_origins: tuple[str, ...] = ()
|
||||||
|
worker_concurrency: Literal[1] = 1
|
||||||
|
min_free_bytes: int = Field(default=1_073_741_824, ge=0)
|
||||||
|
min_free_percent: int = Field(default=5, gt=0, lt=100)
|
||||||
|
sqlite_busy_timeout_ms: int = Field(default=5_000, ge=1_000, le=120_000)
|
||||||
|
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
||||||
|
|
||||||
|
@field_validator(
|
||||||
|
"data_dir",
|
||||||
|
"master_key_file",
|
||||||
|
mode="before",
|
||||||
|
)
|
||||||
|
@classmethod
|
||||||
|
def validate_absolute_path(cls, value: object) -> Path:
|
||||||
|
path = Path(str(value)).expanduser()
|
||||||
|
if not path.is_absolute():
|
||||||
|
raise ValueError("path must be absolute")
|
||||||
|
return path.resolve()
|
||||||
|
|
||||||
|
@field_validator("repository_roots", "local_source_roots", "restore_roots", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def validate_roots(cls, value: object) -> tuple[Path, ...]:
|
||||||
|
if isinstance(value, (str, Path)):
|
||||||
|
raw_roots = [value]
|
||||||
|
elif isinstance(value, (list, tuple, set)):
|
||||||
|
raw_roots = list(value)
|
||||||
|
else:
|
||||||
|
raise ValueError("allowlist roots must be a sequence of paths")
|
||||||
|
roots: list[Path] = []
|
||||||
|
for raw_root in raw_roots:
|
||||||
|
root = Path(str(raw_root)).expanduser()
|
||||||
|
if not root.is_absolute():
|
||||||
|
raise ValueError("allowlist roots must be absolute")
|
||||||
|
roots.append(root.resolve())
|
||||||
|
if not roots:
|
||||||
|
raise ValueError("at least one allowlist root is required")
|
||||||
|
if len(set(roots)) != len(roots):
|
||||||
|
raise ValueError("allowlist roots must be unique")
|
||||||
|
return tuple(roots)
|
||||||
|
|
||||||
|
@field_validator("database_url")
|
||||||
|
@classmethod
|
||||||
|
def validate_database_url(cls, value: str) -> str:
|
||||||
|
url = make_url(value)
|
||||||
|
if url.drivername != "sqlite+aiosqlite":
|
||||||
|
raise ValueError("v2 requires sqlite+aiosqlite")
|
||||||
|
if not url.database or not Path(url.database).is_absolute():
|
||||||
|
raise ValueError("SQLite database path must be absolute")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("public_base_url")
|
||||||
|
@classmethod
|
||||||
|
def validate_public_base_url(cls, value: str) -> str:
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||||
|
raise ValueError("public base URL must be an absolute HTTP(S) URL")
|
||||||
|
if parsed.query or parsed.fragment:
|
||||||
|
raise ValueError("public base URL must not contain query or fragment")
|
||||||
|
return value.rstrip("/")
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_master_key(self) -> Self:
|
||||||
|
key = self.master_key_file
|
||||||
|
if not key.is_file():
|
||||||
|
raise ValueError("master key file must exist and be a regular file")
|
||||||
|
key_stat = key.stat()
|
||||||
|
if hasattr(os, "getuid") and key_stat.st_uid != os.getuid():
|
||||||
|
raise ValueError("master key file must be owned by the service user")
|
||||||
|
mode = stat.S_IMODE(key_stat.st_mode)
|
||||||
|
if mode != 0o600:
|
||||||
|
raise ValueError("master key file permissions must be 0600")
|
||||||
|
if key_stat.st_size < 32:
|
||||||
|
raise ValueError("master key file must contain at least 32 bytes")
|
||||||
|
return self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def database_path(self) -> Path:
|
||||||
|
database = make_url(self.database_url).database
|
||||||
|
if database is None: # pragma: no cover - guarded by validation
|
||||||
|
raise RuntimeError("database path is unavailable")
|
||||||
|
return Path(database).resolve()
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
from .engine import SchemaNotCurrentError, assert_schema_current, create_engine
|
||||||
|
from .models import Base
|
||||||
|
|
||||||
|
__all__ = ["Base", "SchemaNotCurrentError", "assert_schema_current", "create_engine"]
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from alembic.config import Config
|
||||||
|
from alembic.migration import MigrationContext
|
||||||
|
from alembic.script import ScriptDirectory
|
||||||
|
from sqlalchemy import event
|
||||||
|
from sqlalchemy.engine import Connection
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||||
|
|
||||||
|
from backup_tool.config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaNotCurrentError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def create_engine(settings: Settings) -> AsyncEngine:
|
||||||
|
engine = create_async_engine(settings.database_url, pool_pre_ping=True)
|
||||||
|
|
||||||
|
@event.listens_for(engine.sync_engine, "connect")
|
||||||
|
def configure_sqlite(dbapi_connection: Any, _connection_record: Any) -> None:
|
||||||
|
cursor = dbapi_connection.cursor()
|
||||||
|
try:
|
||||||
|
cursor.execute("PRAGMA foreign_keys=ON")
|
||||||
|
cursor.execute(f"PRAGMA busy_timeout={settings.sqlite_busy_timeout_ms}")
|
||||||
|
cursor.execute("PRAGMA journal_mode=WAL")
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
return engine
|
||||||
|
|
||||||
|
|
||||||
|
async def assert_schema_current(engine: AsyncEngine, alembic_config: Config) -> None:
|
||||||
|
expected = ScriptDirectory.from_config(alembic_config).get_current_head()
|
||||||
|
|
||||||
|
def get_current_revision(connection: Connection) -> str | None:
|
||||||
|
return MigrationContext.configure(connection).get_current_revision()
|
||||||
|
|
||||||
|
async with engine.connect() as connection:
|
||||||
|
current = await connection.run_sync(get_current_revision)
|
||||||
|
if current != expected:
|
||||||
|
raise SchemaNotCurrentError(
|
||||||
|
f"database schema is not current: expected {expected!r}, found {current!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
SchemaCheck = Callable[[AsyncEngine, Config], Any]
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
Boolean,
|
||||||
|
CheckConstraint,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
LargeBinary,
|
||||||
|
MetaData,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
|
||||||
|
NAMING_CONVENTION = {
|
||||||
|
"ix": "ix_%(column_0_label)s",
|
||||||
|
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||||
|
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||||
|
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||||
|
"pk": "pk_%(table_name)s",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Base(DeclarativeBase):
|
||||||
|
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||||
|
|
||||||
|
|
||||||
|
class IdentityMixin:
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||||
|
|
||||||
|
|
||||||
|
class TimestampMixin:
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=func.current_timestamp(),
|
||||||
|
onupdate=func.current_timestamp(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class User(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "users"
|
||||||
|
username: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
__table_args__ = (CheckConstraint("state IN ('active','disabled')", name="state"),)
|
||||||
|
|
||||||
|
|
||||||
|
class ApiToken(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "api_tokens"
|
||||||
|
owner_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||||
|
token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||||
|
scopes: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
__table_args__ = (Index("ix_api_tokens_owner_id", "owner_id"),)
|
||||||
|
|
||||||
|
|
||||||
|
class Secret(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "secrets"
|
||||||
|
ciphertext: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||||
|
key_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
purpose: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||||
|
__table_args__ = (CheckConstraint("version > 0", name="version_positive"),)
|
||||||
|
|
||||||
|
|
||||||
|
class Repository(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "repositories"
|
||||||
|
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
root: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||||
|
format_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
compression: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
encryption: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("format_version > 0", name="format_version_positive"),
|
||||||
|
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Source(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "sources"
|
||||||
|
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
public_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||||
|
secret_refs: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
last_probe: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("kind IN ('local','sftp','postgresql','mysql')", name="kind"),
|
||||||
|
CheckConstraint("state IN ('active','archived','unavailable')", name="state"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Job(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "jobs"
|
||||||
|
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id", ondelete="RESTRICT"))
|
||||||
|
repository_id: Mapped[str] = mapped_column(ForeignKey("repositories.id", ondelete="RESTRICT"))
|
||||||
|
requested_mode: Mapped[str] = mapped_column(String(32), nullable=False, default="incremental")
|
||||||
|
exclusions: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||||
|
retention: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||||
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
allow_empty: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("requested_mode IN ('full','incremental')", name="requested_mode"),
|
||||||
|
CheckConstraint("state IN ('active','archived')", name="state"),
|
||||||
|
Index("ix_jobs_source_id", "source_id"),
|
||||||
|
Index("ix_jobs_repository_id", "repository_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Schedule(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "schedules"
|
||||||
|
job_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("jobs.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||||
|
)
|
||||||
|
cron: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
timezone: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
misfire_grace_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=900)
|
||||||
|
overlap_policy: Mapped[str] = mapped_column(String(32), nullable=False, default="prohibit")
|
||||||
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
next_nominal_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_enqueue_outcome: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("misfire_grace_seconds >= 0", name="misfire_nonnegative"),
|
||||||
|
CheckConstraint("overlap_policy IN ('prohibit')", name="overlap_policy"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Execution(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "executions"
|
||||||
|
job_id: Mapped[str] = mapped_column(ForeignKey("jobs.id", ondelete="RESTRICT"))
|
||||||
|
schedule_id: Mapped[str | None] = mapped_column(ForeignKey("schedules.id", ondelete="RESTRICT"))
|
||||||
|
nominal_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
trigger: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||||
|
lease_owner: Mapped[str | None] = mapped_column(String(255))
|
||||||
|
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
progress: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||||
|
reason_code: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
operator_message: Mapped[str | None] = mapped_column(Text)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("attempt > 0", name="attempt_positive"),
|
||||||
|
CheckConstraint(
|
||||||
|
"state IN ('queued','preparing','running','verifying','committed',"
|
||||||
|
"'cancelling','cancelled','failed')",
|
||||||
|
name="state",
|
||||||
|
),
|
||||||
|
CheckConstraint("trigger IN ('manual','schedule','retry')", name="trigger"),
|
||||||
|
UniqueConstraint("schedule_id", "nominal_run_at", name="uq_execution_schedule_occurrence"),
|
||||||
|
Index("ix_executions_job_id", "job_id"),
|
||||||
|
Index("ix_executions_state_created", "state", "created_at"),
|
||||||
|
Index(
|
||||||
|
"uq_executions_active_job",
|
||||||
|
"job_id",
|
||||||
|
unique=True,
|
||||||
|
sqlite_where=text("state IN ('queued','preparing','running','verifying','cancelling')"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Backup(IdentityMixin, Base):
|
||||||
|
__tablename__ = "backups"
|
||||||
|
execution_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("executions.id", ondelete="RESTRICT"), nullable=False, unique=True
|
||||||
|
)
|
||||||
|
parent_backup_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("backups.id", ondelete="RESTRICT")
|
||||||
|
)
|
||||||
|
manifest_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True)
|
||||||
|
manifest_digest: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||||
|
logical_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
stored_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
integrity: Mapped[str] = mapped_column(String(32), nullable=False, default="unverified")
|
||||||
|
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
tombstoned_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||||
|
)
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("logical_bytes >= 0", name="logical_bytes_nonnegative"),
|
||||||
|
CheckConstraint("stored_bytes >= 0", name="stored_bytes_nonnegative"),
|
||||||
|
CheckConstraint(
|
||||||
|
"integrity IN ('unverified','verified','degraded','corrupt')", name="integrity"
|
||||||
|
),
|
||||||
|
Index("ix_backups_created_at", "created_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Restore(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "restores"
|
||||||
|
backup_id: Mapped[str] = mapped_column(ForeignKey("backups.id", ondelete="RESTRICT"))
|
||||||
|
destination: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
selection: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||||
|
overwrite_policy: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||||
|
result: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("overwrite_policy IN ('fail','skip','replace')", name="overwrite_policy"),
|
||||||
|
CheckConstraint(
|
||||||
|
"state IN ('queued','running','committed','cancelled','failed')", name="state"
|
||||||
|
),
|
||||||
|
Index("ix_restores_backup_id", "backup_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AuditEvent(IdentityMixin, Base):
|
||||||
|
__tablename__ = "audit_events"
|
||||||
|
actor_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||||
|
action: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
resource_id: Mapped[str | None] = mapped_column(String(36))
|
||||||
|
outcome: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
request_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||||
|
details: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||||
|
)
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("outcome IN ('success','failure','denied')", name="outcome"),
|
||||||
|
Index("ix_audit_events_created_at", "created_at"),
|
||||||
|
Index("ix_audit_events_actor_id", "actor_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationSubscription(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "notification_subscriptions"
|
||||||
|
channel: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
event_filters: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
||||||
|
destination_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||||
|
secret_id: Mapped[str | None] = mapped_column(ForeignKey("secrets.id", ondelete="RESTRICT"))
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("channel IN ('webhook','email')", name="channel"),
|
||||||
|
CheckConstraint("state IN ('active','disabled','archived')", name="state"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationDelivery(IdentityMixin, TimestampMixin, Base):
|
||||||
|
__tablename__ = "notification_deliveries"
|
||||||
|
event_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||||
|
subscription_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("notification_subscriptions.id", ondelete="RESTRICT")
|
||||||
|
)
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||||
|
state: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
response_class: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("attempt > 0", name="attempt_positive"),
|
||||||
|
CheckConstraint("state IN ('pending','delivered','retry','failed')", name="state"),
|
||||||
|
UniqueConstraint("event_id", "subscription_id", "attempt", name="uq_delivery_attempt"),
|
||||||
|
Index("ix_notification_deliveries_state_next", "state", "next_attempt_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IdempotencyRecord(IdentityMixin, Base):
|
||||||
|
__tablename__ = "idempotency_records"
|
||||||
|
actor_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"))
|
||||||
|
key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
operation: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_digest: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
response_resource_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
response_resource_id: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.current_timestamp()
|
||||||
|
)
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("actor_id", "key", "operation", name="uq_idempotency_actor_key_operation"),
|
||||||
|
Index("ix_idempotency_created_at", "created_at"),
|
||||||
|
)
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import secrets
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
_lock = threading.Lock()
|
||||||
|
_last_millisecond = -1
|
||||||
|
_last_random = 0
|
||||||
|
_RANDOM_MASK = (1 << 74) - 1
|
||||||
|
|
||||||
|
|
||||||
|
def new_uuid7() -> UUID:
|
||||||
|
"""Return a process-monotonic RFC 9562 UUIDv7."""
|
||||||
|
global _last_millisecond, _last_random
|
||||||
|
with _lock:
|
||||||
|
millisecond = time.time_ns() // 1_000_000
|
||||||
|
if millisecond > _last_millisecond:
|
||||||
|
_last_millisecond = millisecond
|
||||||
|
_last_random = secrets.randbits(74)
|
||||||
|
else:
|
||||||
|
millisecond = _last_millisecond
|
||||||
|
_last_random = (_last_random + 1) & _RANDOM_MASK
|
||||||
|
if _last_random == 0:
|
||||||
|
_last_millisecond += 1
|
||||||
|
millisecond = _last_millisecond
|
||||||
|
random_a = (_last_random >> 62) & 0xFFF
|
||||||
|
random_b = _last_random & ((1 << 62) - 1)
|
||||||
|
integer = (millisecond & ((1 << 48) - 1)) << 80
|
||||||
|
integer |= 0x7 << 76
|
||||||
|
integer |= random_a << 64
|
||||||
|
integer |= 0b10 << 62
|
||||||
|
integer |= random_b
|
||||||
|
return UUID(int=integer)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# M1 Runtime and Metadata Evidence
|
||||||
|
|
||||||
|
## RED
|
||||||
|
|
||||||
|
Commit: `8f78484 test(v2): define runtime and metadata contracts`
|
||||||
|
|
||||||
|
Command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python -m pytest tests/unit/test_config.py tests/integration/test_migrations.py -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Observed result: exit 2 during collection because `backup_tool.clock` did not exist. This proves the test-only commit preceded runtime implementation.
|
||||||
|
|
||||||
|
## GREEN
|
||||||
|
|
||||||
|
Migration gate executed against a disposable absolute SQLite database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend
|
||||||
|
BACKUP_TOOL_DATABASE_URL=sqlite+aiosqlite:////absolute/path/.m1-gate.db ../.venv/bin/python -m alembic upgrade head
|
||||||
|
BACKUP_TOOL_DATABASE_URL=sqlite+aiosqlite:////absolute/path/.m1-gate.db ../.venv/bin/python -m alembic downgrade base
|
||||||
|
BACKUP_TOOL_DATABASE_URL=sqlite+aiosqlite:////absolute/path/.m1-gate.db ../.venv/bin/python -m alembic upgrade head
|
||||||
|
../.venv/bin/python -m pytest ../tests/unit/test_config.py ../tests/integration/test_migrations.py -q
|
||||||
|
cd ..
|
||||||
|
make check
|
||||||
|
```
|
||||||
|
|
||||||
|
Observed results:
|
||||||
|
|
||||||
|
- Alembic upgrade, downgrade, and second upgrade each exited 0.
|
||||||
|
- Focused M1 suite: `13 passed`.
|
||||||
|
- Fast unit/contract suite: `28 passed`.
|
||||||
|
- Ruff check/format and strict mypy: passed.
|
||||||
|
- Frontend typecheck/build: passed.
|
||||||
|
- Disposable database removed after verification.
|
||||||
|
|
||||||
|
## Hand Review
|
||||||
|
|
||||||
|
- Baseline revision contains explicit `op.create_table`, constraints, foreign keys, and indexes for all 14 v2 entities; it does not call `create_all`.
|
||||||
|
- SQLite connections enable foreign keys, WAL, and a bounded busy timeout.
|
||||||
|
- Startup compares the database Alembic revision with the current script head and rejects unmigrated databases.
|
||||||
|
- Runtime roles are explicit CLI subcommands; no role starts another role in M1.
|
||||||
@@ -51,7 +51,8 @@ def settings_for(tmp_path: Path) -> Any:
|
|||||||
|
|
||||||
|
|
||||||
def alembic_config(database_url: str) -> Config:
|
def alembic_config(database_url: str) -> Config:
|
||||||
config = Config("backend/alembic.ini")
|
project_root = Path(__file__).resolve().parents[2]
|
||||||
|
config = Config(project_root / "backend" / "alembic.ini")
|
||||||
config.set_main_option("sqlalchemy.url", database_url)
|
config.set_main_option("sqlalchemy.url", database_url)
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user