437 lines
17 KiB
Python
437 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, cast
|
|
|
|
import httpx
|
|
import pytest
|
|
from backup_tool.api.app import create_app, execution_events_stream
|
|
from backup_tool.config import Settings
|
|
from backup_tool.db.models import Execution, Repository, Source
|
|
from backup_tool.execution import (
|
|
claim,
|
|
complete_cancellation,
|
|
enqueue,
|
|
heartbeat,
|
|
record_event,
|
|
recover_stale,
|
|
request_cancellation,
|
|
)
|
|
|
|
PASSWORD = "correct-horse-battery-staple"
|
|
|
|
|
|
async def login(client: httpx.AsyncClient) -> dict[str, str]:
|
|
response = await client.post("/api/v2/setup", json={"username": "admin", "password": PASSWORD})
|
|
assert response.status_code == 201
|
|
return {"X-CSRF-Token": client.cookies["backup_tool_csrf"]}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_local_source_probe_archive_and_repository_targeted_job(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
source_root = tmp_path / "sources"
|
|
source_root.mkdir()
|
|
(source_root / "data.txt").write_text("contents")
|
|
data_dir = tmp_path / "data"
|
|
data_dir.mkdir()
|
|
key = tmp_path / "master.key"
|
|
key.write_bytes(b"x" * 32)
|
|
key.chmod(0o600)
|
|
repositories = tmp_path / "repositories"
|
|
restore = tmp_path / "restore"
|
|
repositories.mkdir()
|
|
restore.mkdir()
|
|
settings = Settings(
|
|
data_dir=data_dir,
|
|
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
|
|
repository_roots=(repositories,),
|
|
local_source_roots=(source_root,),
|
|
restore_roots=(restore,),
|
|
master_key_file=key,
|
|
)
|
|
app = create_app(settings)
|
|
from backup_tool.db.models import Base
|
|
|
|
async with app.state.engine.begin() as connection:
|
|
await connection.run_sync(Base.metadata.create_all)
|
|
transport = httpx.ASGITransport(app=cast(Any, app))
|
|
async with httpx.AsyncClient(transport=transport, base_url="https://test") as client:
|
|
headers = await login(client)
|
|
repository = await client.post(
|
|
"/api/v2/repositories",
|
|
json={"name": "repo", "relative_path": "main"},
|
|
headers=headers,
|
|
)
|
|
assert repository.status_code == 201
|
|
source = await client.post(
|
|
"/api/v2/sources",
|
|
json={
|
|
"name": "local",
|
|
"kind": "local",
|
|
"public_config": {"root": str(source_root)},
|
|
},
|
|
headers=headers,
|
|
)
|
|
assert source.status_code == 201
|
|
source_id = source.json()["id"]
|
|
probe = await client.post(f"/api/v2/sources/{source_id}/probe", headers=headers)
|
|
assert probe.status_code == 200
|
|
assert probe.json()["entry_count"] == 1
|
|
job = await client.post(
|
|
"/api/v2/jobs",
|
|
json={
|
|
"name": "job",
|
|
"source_id": source_id,
|
|
"repository_id": repository.json()["id"],
|
|
"requested_mode": "full",
|
|
"exclusions": ["*.tmp"],
|
|
"retention": {},
|
|
"enabled": True,
|
|
"allow_empty": False,
|
|
},
|
|
headers=headers,
|
|
)
|
|
assert job.status_code == 201
|
|
assert "destination_path" not in job.json()
|
|
invalid_patch = await client.patch(
|
|
f"/api/v2/jobs/{job.json()['id']}", json={}, headers=headers
|
|
)
|
|
assert invalid_patch.status_code == 422
|
|
updated_job = await client.patch(
|
|
f"/api/v2/jobs/{job.json()['id']}",
|
|
json={"name": "renamed-job", "exclusions": ["*.cache"]},
|
|
headers=headers,
|
|
)
|
|
assert updated_job.status_code == 200
|
|
assert updated_job.json()["name"] == "renamed-job"
|
|
assert updated_job.json()["exclusions"] == ["*.cache"]
|
|
execution = await client.post(
|
|
f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers
|
|
)
|
|
assert execution.status_code == 202
|
|
assert execution.json()["state"] == "queued"
|
|
duplicate = await client.post(
|
|
f"/api/v2/jobs/{job.json()['id']}/executions", headers=headers
|
|
)
|
|
assert duplicate.status_code == 409
|
|
assert duplicate.json()["code"] == "execution_active"
|
|
execution_id = execution.json()["id"]
|
|
listed_sources = await client.get("/api/v2/sources", headers=headers)
|
|
assert listed_sources.status_code == 200
|
|
assert listed_sources.json()["items"] == [
|
|
{
|
|
"id": source_id,
|
|
"name": "local",
|
|
"kind": "local",
|
|
"state": "active",
|
|
"public_config": {"root": str(source_root)},
|
|
}
|
|
]
|
|
listed_jobs = await client.get("/api/v2/jobs", headers=headers)
|
|
assert listed_jobs.status_code == 200
|
|
assert listed_jobs.json()["items"][0]["id"] == job.json()["id"]
|
|
assert listed_jobs.json()["items"][0]["schedule"] is None
|
|
listed_executions = await client.get("/api/v2/executions", headers=headers)
|
|
assert listed_executions.status_code == 200
|
|
assert listed_executions.json()["items"][0]["id"] == execution_id
|
|
scoped_token = await client.post(
|
|
"/api/v2/auth/tokens",
|
|
json={"scopes": ["audit:read"], "expires_at": None},
|
|
headers={**headers, "Idempotency-Key": "execution-audit-token"},
|
|
)
|
|
assert scoped_token.status_code == 201
|
|
token_headers = {"Authorization": f"Bearer {scoped_token.json()['token']}"}
|
|
assert (
|
|
await client.get(f"/api/v2/executions/{execution_id}", headers=token_headers)
|
|
).status_code == 403
|
|
assert (
|
|
await client.post(f"/api/v2/executions/{execution_id}/cancel", headers=token_headers)
|
|
).status_code == 403
|
|
async with app.state.sessions() as db:
|
|
assert await claim(db, execution_id, "expired-worker") is not None
|
|
stored = await db.get(Execution, execution_id)
|
|
assert stored is not None
|
|
stored.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
await db.commit()
|
|
async with app.state.sessions() as db:
|
|
assert await recover_stale(db) == 1
|
|
assert await claim(db, execution_id, "replacement-worker") is not None
|
|
async with app.state.sessions() as db:
|
|
assert not await heartbeat(db, execution_id, "expired-worker")
|
|
assert await request_cancellation(db, execution_id) is not None
|
|
assert not await complete_cancellation(db, execution_id, "expired-worker")
|
|
assert await complete_cancellation(db, execution_id, "replacement-worker")
|
|
async with app.state.sessions() as db:
|
|
stored = await db.get(Execution, execution_id)
|
|
assert stored is not None
|
|
stored.progress = {
|
|
"source_path": "/private/source",
|
|
"details": {"Token": "nested-secret", "password": "nested-password"},
|
|
}
|
|
await db.commit()
|
|
polled = await client.get(f"/api/v2/executions/{execution_id}", headers=headers)
|
|
assert polled.status_code == 200
|
|
assert polled.json()["state"] == "cancelled"
|
|
assert "/private/source" not in str(polled.json())
|
|
assert "nested-secret" not in str(polled.json())
|
|
assert "nested-password" not in str(polled.json())
|
|
async with app.state.sessions() as db:
|
|
stored = await db.get(Execution, execution.json()["id"])
|
|
assert stored is not None
|
|
stored.state = "failed"
|
|
stored.reason_code = "timeout"
|
|
await db.commit()
|
|
retried = await client.post(
|
|
f"/api/v2/executions/{execution.json()['id']}/retry", headers=headers
|
|
)
|
|
assert retried.status_code == 202
|
|
assert retried.json()["id"] == execution.json()["id"]
|
|
assert retried.json()["attempt"] == 2
|
|
cancelled = await client.post(
|
|
f"/api/v2/executions/{execution.json()['id']}/cancel", headers=headers
|
|
)
|
|
assert cancelled.status_code == 202
|
|
stream = await client.get(
|
|
f"/api/v2/executions/{execution.json()['id']}/events", headers=headers
|
|
)
|
|
assert stream.status_code == 200
|
|
assert "event: execution" in stream.text
|
|
assert "/private/source" not in stream.text
|
|
assert "nested-secret" not in stream.text
|
|
assert "nested-password" not in stream.text
|
|
archived = await client.delete(f"/api/v2/sources/{source_id}", headers=headers)
|
|
assert archived.status_code == 204
|
|
assert (
|
|
await client.post(f"/api/v2/sources/{source_id}/probe", headers=headers)
|
|
).status_code == 409
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execution_sse_replays_later_redacted_revision(tmp_path: Path) -> None:
|
|
source_root = tmp_path / "sources"
|
|
source_root.mkdir()
|
|
data_dir = tmp_path / "data"
|
|
data_dir.mkdir()
|
|
key = tmp_path / "master.key"
|
|
key.write_bytes(b"x" * 32)
|
|
key.chmod(0o600)
|
|
repositories = tmp_path / "repositories"
|
|
restore = tmp_path / "restore"
|
|
repositories.mkdir()
|
|
restore.mkdir()
|
|
settings = Settings(
|
|
data_dir=data_dir,
|
|
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
|
|
repository_roots=(repositories,),
|
|
local_source_roots=(source_root,),
|
|
restore_roots=(restore,),
|
|
master_key_file=key,
|
|
)
|
|
app = create_app(settings)
|
|
from backup_tool.db.models import Base, Job
|
|
|
|
async with app.state.engine.begin() as connection:
|
|
await connection.run_sync(Base.metadata.create_all)
|
|
async with app.state.sessions() as db:
|
|
repository = Repository(
|
|
name="stream-repository",
|
|
root=str(repositories / "stream"),
|
|
format_version=1,
|
|
compression="none",
|
|
encryption="none",
|
|
)
|
|
source = Source(
|
|
name="stream-source",
|
|
kind="local",
|
|
public_config={"root": str(source_root)},
|
|
secret_refs=[],
|
|
)
|
|
db.add_all([repository, source])
|
|
await db.flush()
|
|
job = Job(
|
|
name="stream-job",
|
|
source_id=source.id,
|
|
repository_id=repository.id,
|
|
requested_mode="full",
|
|
exclusions=[],
|
|
retention={},
|
|
)
|
|
db.add(job)
|
|
await db.commit()
|
|
execution = await enqueue(db, job.id)
|
|
execution_id = execution.id
|
|
|
|
stream = execution_events_stream(app.state.sessions, execution_id, None)
|
|
first = await anext(stream)
|
|
first_id = first.split("\n", 1)[0].removeprefix("id: ")
|
|
assert "queued" in first
|
|
async with app.state.sessions() as db:
|
|
assert await request_cancellation(db, execution_id) is not None
|
|
second = await anext(stream)
|
|
assert second.split("\n", 1)[0].removeprefix("id: ") != first_id
|
|
assert "cancelled" in second
|
|
|
|
replay = execution_events_stream(app.state.sessions, execution_id, first_id)
|
|
assert "cancelled" in await anext(replay)
|
|
await stream.aclose()
|
|
await replay.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execution_events_preserve_progress_replay_and_recovery_order(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
source_root = tmp_path / "sources"
|
|
source_root.mkdir()
|
|
data_dir = tmp_path / "data"
|
|
data_dir.mkdir()
|
|
key = tmp_path / "master.key"
|
|
key.write_bytes(b"x" * 32)
|
|
key.chmod(0o600)
|
|
repositories = tmp_path / "repositories"
|
|
restore = tmp_path / "restore"
|
|
repositories.mkdir()
|
|
restore.mkdir()
|
|
settings = Settings(
|
|
data_dir=data_dir,
|
|
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
|
|
repository_roots=(repositories,),
|
|
local_source_roots=(source_root,),
|
|
restore_roots=(restore,),
|
|
master_key_file=key,
|
|
)
|
|
app = create_app(settings)
|
|
from backup_tool.db.models import Base, Job
|
|
|
|
async with app.state.engine.begin() as connection:
|
|
await connection.run_sync(Base.metadata.create_all)
|
|
async with app.state.sessions() as db:
|
|
repository = Repository(
|
|
name="ordered-events-repository",
|
|
root=str(repositories / "ordered"),
|
|
format_version=1,
|
|
compression="none",
|
|
encryption="none",
|
|
)
|
|
source = Source(
|
|
name="ordered-events-source",
|
|
kind="local",
|
|
public_config={"root": str(source_root)},
|
|
secret_refs=[],
|
|
)
|
|
db.add_all([repository, source])
|
|
await db.flush()
|
|
job = Job(
|
|
name="ordered-events-job",
|
|
source_id=source.id,
|
|
repository_id=repository.id,
|
|
requested_mode="full",
|
|
exclusions=[],
|
|
retention={},
|
|
)
|
|
db.add(job)
|
|
await db.commit()
|
|
execution = await enqueue(db, job.id)
|
|
execution_id = execution.id
|
|
|
|
async with app.state.sessions() as db:
|
|
assert await claim(db, execution_id, "ordering-worker") is not None
|
|
execution = await db.get(Execution, execution_id)
|
|
assert execution is not None and execution.state == "preparing"
|
|
execution.state = "running"
|
|
execution.started_at = datetime.now(UTC)
|
|
await record_event(db, execution)
|
|
await db.commit()
|
|
async with app.state.sessions() as db:
|
|
execution = await db.get(Execution, execution_id)
|
|
assert execution is not None and execution.state == "running"
|
|
execution.progress = {"files": 1}
|
|
await record_event(db, execution)
|
|
execution.progress = {"files": 2}
|
|
await record_event(db, execution)
|
|
await db.commit()
|
|
execution.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
await db.commit()
|
|
assert await recover_stale(db) == 1
|
|
execution = await db.get(Execution, execution_id)
|
|
assert execution is not None and execution.state == "queued"
|
|
assert await claim(db, execution_id, "cancelling-worker") is not None
|
|
assert await request_cancellation(db, execution_id) is not None
|
|
assert await complete_cancellation(db, execution_id, "cancelling-worker")
|
|
|
|
def parse_frame(frame: str) -> tuple[int, dict[str, object]]:
|
|
try:
|
|
identifier, _, data = frame.partition("\n")
|
|
revision = int(identifier.removeprefix("id: "))
|
|
payload = json.loads(data.split("data: ", 1)[1].strip())
|
|
except (IndexError, TypeError, ValueError, json.JSONDecodeError) as error:
|
|
pytest.fail(f"Invalid SSE frame: {error}")
|
|
return revision, payload
|
|
|
|
replay = execution_events_stream(app.state.sessions, execution_id, "0")
|
|
payloads: list[dict[str, object]] = []
|
|
revisions: list[int] = []
|
|
async for frame in replay:
|
|
revision, payload = parse_frame(frame)
|
|
revisions.append(revision)
|
|
payloads.append(payload)
|
|
assert revisions == list(range(1, len(revisions) + 1))
|
|
progress_values = [payload["progress"] for payload in payloads if payload["progress"]]
|
|
assert progress_values[:2] == [{"files": 1}, {"files": 2}]
|
|
assert payloads[-1]["state"] == "cancelled"
|
|
|
|
reconnect = execution_events_stream(app.state.sessions, execution_id, str(revisions[-2]))
|
|
replayed = await anext(reconnect)
|
|
replayed_revision, replayed_payload = parse_frame(replayed)
|
|
assert replayed_revision == revisions[-1]
|
|
assert replayed_payload["state"] == "cancelled"
|
|
await reconnect.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_local_source_rejects_unallowlisted_root(tmp_path: Path) -> None:
|
|
allowed = tmp_path / "allowed"
|
|
allowed.mkdir()
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
data_dir = tmp_path / "data"
|
|
data_dir.mkdir()
|
|
key = tmp_path / "master.key"
|
|
key.write_bytes(b"x" * 32)
|
|
key.chmod(0o600)
|
|
repositories = tmp_path / "repositories"
|
|
restore = tmp_path / "restore"
|
|
repositories.mkdir()
|
|
restore.mkdir()
|
|
settings = Settings(
|
|
data_dir=data_dir,
|
|
database_url=f"sqlite+aiosqlite:///{data_dir / 'db.sqlite'}",
|
|
repository_roots=(repositories,),
|
|
local_source_roots=(allowed,),
|
|
restore_roots=(restore,),
|
|
master_key_file=key,
|
|
)
|
|
app = create_app(settings)
|
|
from backup_tool.db.models import Base
|
|
|
|
async with app.state.engine.begin() as connection:
|
|
await connection.run_sync(Base.metadata.create_all)
|
|
async with httpx.AsyncClient(
|
|
transport=httpx.ASGITransport(app=cast(Any, app)), base_url="https://test"
|
|
) as client:
|
|
headers = await login(client)
|
|
response = await client.post(
|
|
"/api/v2/sources",
|
|
json={
|
|
"name": "bad",
|
|
"kind": "local",
|
|
"public_config": {"root": str(outside)},
|
|
},
|
|
headers=headers,
|
|
)
|
|
assert response.status_code == 422
|