alex/backup-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v2): add local sources and repository jobs

Browse Source
This commit is contained in:
Alex
2026-07-27 21:36:45 +02:00
parent f773a1b771
commit 61189c27d2
3 changed files with 257 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/backup_tool/adapters.py
+58
View File
@@ -0,0 +1,58 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from dataclasses import dataclass
from pathlib import Path
from backup_tool.config import Settings
class SourceError(ValueError):
pass
@dataclass(frozen=True)
class Entry:
path: str
kind: str
size: int | None = None
class LocalAdapter:
def __init__(self, root: Path, settings: Settings):
self.root = root.expanduser().resolve()
self.settings = settings
def validate_config(self) -> None:
if not self.root.is_dir():
raise SourceError("local source root must be an existing directory")
if not any(self.root.is_relative_to(root) for root in self.settings.local_source_roots):
raise SourceError("local source root is outside configured allowlists")
async def probe(self) -> dict[str, int]:
self.validate_config()
count = sum(1 for item in self.root.rglob("*") if item.is_file() and not item.is_symlink())
return {"entry_count": count}
async def enumerate_entries(self) -> AsyncIterator[Entry]:
self.validate_config()
for item in self.root.rglob("*"):
relative = item.relative_to(self.root).as_posix()
if item.is_symlink():
yield Entry(relative, "symlink")
elif item.is_file():
yield Entry(relative, "file", item.stat().st_size)
elif item.is_dir():
yield Entry(relative, "directory")
async def open_content(self, path: str) -> AsyncIterator[bytes]:
candidate = (self.root / path).resolve()
if (
not candidate.is_relative_to(self.root)
or not candidate.is_file()
or candidate.is_symlink()
):
raise SourceError("invalid local source entry")
with candidate.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
yield chunk
backend/src/backup_tool/api/app.py
+129
View File
@@ -14,6 +14,7 @@ from sqlalchemy import desc, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from backup_tool.adapters import LocalAdapter, SourceError
from backup_tool.cli import build_alembic_config
from backup_tool.config import Settings
from backup_tool.db.engine import SchemaNotCurrentError, assert_schema_current, create_engine
@@ -21,9 +22,11 @@ from backup_tool.db.models import (
ApiToken,
AuditEvent,
IdempotencyRecord,
Job,
Repository,
Secret,
Session,
Source,
User,
)
from backup_tool.repository import (
@@ -78,6 +81,23 @@ class LoginInput(BaseModel):
password: str = Field(min_length=1, max_length=1024)
class SourceInput(BaseModel):
name: str = Field(min_length=1, max_length=255)
kind: str
public_config: dict[str, Any]
class JobInput(BaseModel):
name: str = Field(min_length=1, max_length=255)
source_id: str
repository_id: str
requested_mode: str = "incremental"
exclusions: list[str] = Field(default_factory=list)
retention: dict[str, Any] = Field(default_factory=dict)
enabled: bool = True
allow_empty: bool = False
class TokenInput(BaseModel):
scopes: list[str] = Field(min_length=1)
expires_at: datetime | None = None
@@ -621,6 +641,115 @@ def create_app(settings: Settings) -> FastAPI:
raise Problem(409, "repository_policy_immutable", "Repository policy is immutable.")
raise Problem(422, "validation_failed", "No mutable fields supplied.")
@app.post("/api/v2/sources", status_code=201)
async def create_source(
input_: SourceInput,
db: Annotated[AsyncSession, Depends(session)],
_: Annotated[tuple[User, set[str], bool], Depends(require)],
) -> dict[str, Any]:
if input_.kind != "local":
raise Problem(422, "validation_failed", "Only local sources are available.")
root = input_.public_config.get("root")
if not isinstance(root, str):
raise Problem(422, "validation_failed", "Local source root is required.")
try:
LocalAdapter(Path(root), settings).validate_config()
except SourceError as error:
raise Problem(422, "validation_failed", str(error)) from error
source = Source(
name=input_.name, kind="local", public_config={"root": root}, secret_refs=[]
)
db.add(source)
try:
await db.commit()
except IntegrityError as error:
await db.rollback()
raise Problem(409, "resource_conflict", "Source name already exists.") from error
await db.refresh(source)
return {
"id": source.id,
"name": source.name,
"kind": source.kind,
"state": source.state,
"public_config": source.public_config,
}
@app.post("/api/v2/sources/{source_id}/probe")
async def probe_source(
source_id: str,
db: Annotated[AsyncSession, Depends(session)],
_: Annotated[tuple[User, set[str], bool], Depends(require)],
) -> dict[str, Any]:
source = await db.get(Source, source_id)
if source is None:
raise Problem(404, "resource_not_found", "Source was not found.")
if source.state != "active":
raise Problem(409, "source_archived", "Source is archived.")
try:
result = await LocalAdapter(Path(source.public_config["root"]), settings).probe()
except SourceError as error:
raise Problem(409, "source_probe_failed", str(error)) from error
source.last_probe = result
await db.commit()
return result
@app.delete("/api/v2/sources/{source_id}", status_code=204)
async def archive_source(
source_id: str,
db: Annotated[AsyncSession, Depends(session)],
_: Annotated[tuple[User, set[str], bool], Depends(require)],
) -> Response:
source = await db.get(Source, source_id)
if source is None:
raise Problem(404, "resource_not_found", "Source was not found.")
source.state = "archived"
await db.commit()
return Response(status_code=204)
@app.post("/api/v2/jobs", status_code=201)
async def create_job(
input_: JobInput,
db: Annotated[AsyncSession, Depends(session)],
_: Annotated[tuple[User, set[str], bool], Depends(require)],
) -> dict[str, Any]:
if input_.requested_mode not in {"full", "incremental"}:
raise Problem(422, "validation_failed", "Invalid requested mode.")
source = await db.get(Source, input_.source_id)
repository = await db.get(Repository, input_.repository_id)
if source is None or repository is None:
raise Problem(422, "validation_failed", "Source and repository must exist.")
if source.state != "active" or repository.state != "active":
raise Problem(409, "resource_archived", "Source or repository is unavailable.")
job = Job(
name=input_.name,
source_id=source.id,
repository_id=repository.id,
requested_mode=input_.requested_mode,
exclusions=input_.exclusions,
retention=input_.retention,
enabled=input_.enabled,
allow_empty=input_.allow_empty,
)
db.add(job)
try:
await db.commit()
except IntegrityError as error:
await db.rollback()
raise Problem(409, "resource_conflict", "Job name already exists.") from error
await db.refresh(job)
return {
"id": job.id,
"name": job.name,
"source_id": job.source_id,
"repository_id": job.repository_id,
"requested_mode": job.requested_mode,
"exclusions": job.exclusions,
"retention": job.retention,
"enabled": job.enabled,
"allow_empty": job.allow_empty,
"state": job.state,
}
@app.get("/api/v2/audit")
async def list_audit(
db: Annotated[AsyncSession, Depends(session)],
tests/integration/test_sources_jobs.py
+70 -24
View File
@@ -14,37 +14,54 @@ PASSWORD = "correct-horse-battery-staple"
async def login(client: httpx.AsyncClient) -> dict[str, str]:
response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
assert response.status_code == 201
return {"X-CSRF-Token": response.json()["csrf_token"]}
return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
@pytest.mark.asyncio
async def test_local_source_probe_archive_and_repository_targeted_job(tmp_path: Path) -> None:
async def test_local_source_probe_archive_and_repository_targeted_job(
tmp_path: Path,
) -> None:
source_root = tmp_path / "sources"
source_root.mkdir()
(source_root / "data.txt").write_text("contents")
data_dir = tmp_path / "data"
data_dir.mkdir()
key = tmp_path / "master.key"
key.write_bytes(b"x" * 32)
key.chmod(0o600)
repositories = tmp_path / "repositories"
restore = tmp_path / "restore"
repositories.mkdir()
restore.mkdir()
settings = Settings(
data_dir=tmp_path / "data",
database_url=f"sqlite+aiosqlite:///{tmp_path / 'data' / 'db.sqlite'}",
repository_roots=(tmp_path / "repositories",),
data_dir=data_dir,
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
repository_roots=(repositories,),
local_source_roots=(source_root,),
restore_roots=(tmp_path / "restore",),
master_key_file=tmp_path / "master.key",
restore_roots=(restore,),
master_key_file=key,
)
settings.master_key_file.write_bytes(b"x" * 32)
settings.master_key_file.chmod(0o600)
for root in (*settings.repository_roots, *settings.restore_roots):
root.mkdir(parents=True)
app = create_app(settings)
from backup_tool.db.models import Base
async with app.state.engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
async with httpx.AsyncClient(transport=transport, base_url="https://test") as client:
headers = await login(client)
repository = await client.post(
"/api/v2/repositories", json={"name": "repo", "relative_path": "main"}, headers=headers
"/api/v2/repositories",
json={"name": "repo", "relative_path": "main"},
headers=headers,
)
assert repository.status_code == 201
source = await client.post(
"/api/v2/sources",
json={"name": "local", "kind": "local", "public_config": {"root": str(source_root)}},
json={
"name": "local",
"kind": "local",
"public_config": {"root": str(source_root)},
},
headers=headers,
)
assert source.status_code == 201
@@ -54,14 +71,25 @@ async def test_local_source_probe_archive_and_repository_targeted_job(tmp_path:
assert probe.json()["entry_count"] == 1
job = await client.post(
"/api/v2/jobs",
json={"name": "job", "source_id": source_id, "repository_id": repository.json()["id"], "requested_mode": "full", "exclusions": ["*.tmp"], "retention": {}, "enabled": True, "allow_empty": False},
json={
"name": "job",
"source_id": source_id,
"repository_id": repository.json()["id"],
"requested_mode": "full",
"exclusions": ["*.tmp"],
"retention": {},
"enabled": True,
"allow_empty": False,
},
headers=headers,
)
assert job.status_code == 201
assert "destination_path" not in job.json()
archived = await client.delete(f"/api/v2/sources/{source_id}", headers=headers)
assert archived.status_code == 204
assert (await client.post(f"/api/v2/sources/{source_id}/probe", headers=headers)).status_code == 409
assert (
await client.post(f"/api/v2/sources/{source_id}/probe", headers=headers)
).status_code == 409
@pytest.mark.asyncio
@@ -70,21 +98,39 @@ async def test_local_source_rejects_unallowlisted_root(tmp_path: Path) -> None:
allowed.mkdir()
outside = tmp_path / "outside"
outside.mkdir()
data_dir = tmp_path / "data"
data_dir.mkdir()
key = tmp_path / "master.key"
key.write_bytes(b"x" * 32)
key.chmod(0o600)
repositories = tmp_path / "repositories"
restore = tmp_path / "restore"
repositories.mkdir()
restore.mkdir()
settings = Settings(
data_dir=tmp_path / "data",
database_url=f"sqlite+aiosqlite:///{tmp_path / 'data' / 'db.sqlite'}",
repository_roots=(tmp_path / "repositories",),
data_dir=data_dir,
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
repository_roots=(repositories,),
local_source_roots=(allowed,),
restore_roots=(tmp_path / "restore",),
restore_roots=(restore,),
master_key_file=key,
)
settings.repository_roots[0].mkdir()
settings.restore_roots[0].mkdir()
app = create_app(settings)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
from backup_tool.db.models import Base
async with app.state.engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="https://test"
) as client:
headers = await login(client)
response = await client.post("/api/v2/sources", json={"name":"bad","kind":"local","public_config":{"root":str(outside)}}, headers=headers)
response = await client.post(
"/api/v2/sources",
json={
"name": "bad",
"kind": "local",
"public_config": {"root": str(outside)},
},
headers=headers,
)
assert response.status_code == 422