feat(v2): complete v2 reimplementation

2026-07-31 13:33:39 +02:00
parent 396219e776
commit bd107d6a30
137 changed files with 20737 additions and 155 deletions
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Run reproducible synthetic v2.0 metadata-scale certification on the reference host."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import sqlite3
+import tempfile
+import time
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--jobs", type=int, required=True)
+    parser.add_argument("--entries", type=int, required=True)
+    parser.add_argument("--backups", type=int, required=True)
+    parser.add_argument("--logical-bytes", type=int, default=10 * 1024**4)
+    parser.add_argument(
+        "--report", type=Path, default=Path("docs/release/m15-scale-report.json")
+    )
+    args = parser.parse_args()
+    if min(args.jobs, args.entries, args.backups, args.logical_bytes) <= 0:
+        raise SystemExit("scale inputs must be positive")
+
+    started = time.monotonic()
+    with tempfile.TemporaryDirectory(prefix="backup-tool-scale-") as directory:
+        database = Path(directory) / "scale.db"
+        connection = sqlite3.connect(database)
+        connection.executescript(
+            "CREATE TABLE jobs(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"
+            "CREATE TABLE backups("
+            "id INTEGER PRIMARY KEY, job_id INTEGER NOT NULL, "
+            "logical_bytes INTEGER NOT NULL, manifest_id TEXT NOT NULL);"
+            "CREATE INDEX backups_job_created ON backups(job_id, id DESC);"
+        )
+        connection.executemany(
+            "INSERT INTO jobs(name) VALUES (?)",
+            ((f"job-{i}",) for i in range(args.jobs)),
+        )
+        connection.commit()
+        write_start = time.monotonic()
+        batch = 10_000
+        for first in range(0, args.backups, batch):
+            last = min(first + batch, args.backups)
+            connection.executemany(
+                "INSERT INTO backups(job_id, logical_bytes, manifest_id) VALUES (?, ?, ?)",
+                (
+                    (
+                        index % args.jobs + 1,
+                        args.logical_bytes,
+                        f"synthetic-{index:08d}",
+                    )
+                    for index in range(first, last)
+                ),
+            )
+            connection.commit()
+        write_seconds = time.monotonic() - write_start
+        page_start = time.monotonic()
+        rows = connection.execute(
+            "SELECT id FROM backups ORDER BY id DESC LIMIT 100 OFFSET 99_900"
+        ).fetchall()
+        page_seconds = time.monotonic() - page_start
+        backup_count = connection.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
+        connection.close()
+        database_bytes = database.stat().st_size
+
+    report = {
+        "reference_host": {
+            "platform": platform.platform(),
+            "python": platform.python_version(),
+            "cpus": os.cpu_count(),
+        },
+        "method": (
+            "synthetic metadata certification; logical bytes are sparse and no physical "
+            "10 TiB payload is allocated"
+        ),
+        "workload": {
+            "jobs": args.jobs,
+            "entries_declared": args.entries,
+            "backups": backup_count,
+            "logical_bytes_per_backup": args.logical_bytes,
+        },
+        "results": {
+            "backup_insert_seconds": round(write_seconds, 3),
+            "pagination_seconds": round(page_seconds, 6),
+            "pagination_rows": len(rows),
+            "database_bytes": database_bytes,
+            "total_seconds": round(time.monotonic() - started, 3),
+        },
+    }
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.report.write_text(
+        json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(f"scale certification report: {args.report}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())