101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Run reproducible synthetic v2.0 metadata-scale certification on the reference host."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import platform
|
|
import sqlite3
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--jobs", type=int, required=True)
|
|
parser.add_argument("--entries", type=int, required=True)
|
|
parser.add_argument("--backups", type=int, required=True)
|
|
parser.add_argument("--logical-bytes", type=int, default=10 * 1024**4)
|
|
parser.add_argument("--report", type=Path, default=Path("docs/release/m15-scale-report.json"))
|
|
args = parser.parse_args()
|
|
if min(args.jobs, args.entries, args.backups, args.logical_bytes) <= 0:
|
|
raise SystemExit("scale inputs must be positive")
|
|
|
|
started = time.monotonic()
|
|
with tempfile.TemporaryDirectory(prefix="backup-tool-scale-") as directory:
|
|
database = Path(directory) / "scale.db"
|
|
connection = sqlite3.connect(database)
|
|
connection.executescript(
|
|
"CREATE TABLE jobs(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"
|
|
"CREATE TABLE backups("
|
|
"id INTEGER PRIMARY KEY, job_id INTEGER NOT NULL, "
|
|
"logical_bytes INTEGER NOT NULL, manifest_id TEXT NOT NULL);"
|
|
"CREATE INDEX backups_job_created ON backups(job_id, id DESC);"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO jobs(name) VALUES (?)",
|
|
((f"job-{i}",) for i in range(args.jobs)),
|
|
)
|
|
connection.commit()
|
|
write_start = time.monotonic()
|
|
batch = 10_000
|
|
for first in range(0, args.backups, batch):
|
|
last = min(first + batch, args.backups)
|
|
connection.executemany(
|
|
"INSERT INTO backups(job_id, logical_bytes, manifest_id) VALUES (?, ?, ?)",
|
|
(
|
|
(
|
|
index % args.jobs + 1,
|
|
args.logical_bytes,
|
|
f"synthetic-{index:08d}",
|
|
)
|
|
for index in range(first, last)
|
|
),
|
|
)
|
|
connection.commit()
|
|
write_seconds = time.monotonic() - write_start
|
|
page_start = time.monotonic()
|
|
rows = connection.execute(
|
|
"SELECT id FROM backups ORDER BY id DESC LIMIT 100 OFFSET 99_900"
|
|
).fetchall()
|
|
page_seconds = time.monotonic() - page_start
|
|
backup_count = connection.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
|
|
connection.close()
|
|
database_bytes = database.stat().st_size
|
|
|
|
report = {
|
|
"reference_host": {
|
|
"platform": platform.platform(),
|
|
"python": platform.python_version(),
|
|
"cpus": os.cpu_count(),
|
|
},
|
|
"method": (
|
|
"synthetic metadata certification; logical bytes are sparse and no physical "
|
|
"10 TiB payload is allocated"
|
|
),
|
|
"workload": {
|
|
"jobs": args.jobs,
|
|
"entries_declared": args.entries,
|
|
"backups": backup_count,
|
|
"logical_bytes_per_backup": args.logical_bytes,
|
|
},
|
|
"results": {
|
|
"backup_insert_seconds": round(write_seconds, 3),
|
|
"pagination_seconds": round(page_seconds, 6),
|
|
"pagination_rows": len(rows),
|
|
"database_bytes": database_bytes,
|
|
"total_seconds": round(time.monotonic() - started, 3),
|
|
},
|
|
}
|
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
|
args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(f"scale certification report: {args.report}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|