🚀
This commit is contained in:
IndyDevDan
@@ -0,0 +1,37 @@
|
||||
# Project File Inventory — fused from claude-fable-5 + gpt-5.6-sol
|
||||
|
||||
Snapshot of every file built for the SQLite bulk-insert benchmark project,
|
||||
copied 2026-07-16 from the original locations into this directory.
|
||||
|
||||
## `project-root/` — deliverables (from `/private/tmp/ddc0f4d0/fusion-harness/`)
|
||||
Listed by BOTH agents.
|
||||
|
||||
| File | Built by | Description |
|
||||
|---|---|---|
|
||||
| `bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py` | ARCHITECT | Independent benchmark design — 7 strategies, subprocess isolation, ru_maxrss RAM measurement |
|
||||
| `sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py` | BUILDER | Independent parallel benchmark design |
|
||||
| `BENCH_PLAN.md` | Fusion | Fused plan: method, fairness controls, measured results, winners, embedded fused script |
|
||||
| `bench.py` | BUILDER (gated loop) | Canonical deliverable — stdlib-only uv single-file implementation of BENCH_PLAN.md |
|
||||
| `RESULTS.md` | Generated by `uv run bench.py` | Measured results (7 strategies, time + peak RAM), Speed Winner `set_based_ctes`, Memory Winner declared |
|
||||
|
||||
## Root of this dir — fused script (from `/tmp/`)
|
||||
Listed by ARCHITECT only.
|
||||
|
||||
- `fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py` — runnable fused benchmark referenced by BENCH_PLAN.md
|
||||
|
||||
## `harness-artifacts/` — process artifacts (from `/tmp/fusion-harness-K48sbW/`)
|
||||
Listed by ARCHITECT only.
|
||||
|
||||
- `gate.py` — 15-check acceptance gate (Definition of Done)
|
||||
- `prompt.md`, `validator.md` — harness briefs
|
||||
- `gate-baseline.txt` — required RED baseline before any build
|
||||
- `gate-round-1.txt` … `gate-round-5.txt` — all five gate runs
|
||||
- `builder-round-1.md` … `builder-round-5.md` — all five builder reports
|
||||
- `triage-round-3.md`, `triage-round-4.md` — escalation briefs diagnosing the gate defect
|
||||
|
||||
## Status note (per ARCHITECT)
|
||||
Final gate run ends RED (11 passed / 4 failed) due to a known defect in
|
||||
`gate.py:61` (strips underscores from searched text, making strategy-name
|
||||
checks unsatisfiable). The substantive deliverables — `bench.py` and
|
||||
`RESULTS.md` — are correct, measured, and complete: 1M rows, all 7
|
||||
strategies, ~1700–2000x speedup over naive autocommit.
|
||||
@@ -0,0 +1,228 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = []
|
||||
# ///
|
||||
"""
|
||||
FUSED SQLite bulk-insert benchmark (claude-fable-5 + gpt-5.6-sol).
|
||||
|
||||
1,000,000 rows. Speed + peak RAM. Stdlib only. Run:
|
||||
uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py
|
||||
|
||||
Architecture (from ARCHITECT/claude-fable-5): the script re-execs ITSELF
|
||||
once per strategy (`--worker NAME`), so every strategy runs in a fresh
|
||||
process and peak RSS readings cannot pollute each other. A no-op
|
||||
calibration worker measures bare-interpreter RSS for a delta column.
|
||||
|
||||
Beyond-the-list candidate (from BUILDER/gpt-5.6-sol): set-based
|
||||
INSERT ... SELECT over a recursive CTE that generates the IDENTICAL rows
|
||||
inside SQLite, verified by digest + spot-check against the Python rows.
|
||||
WAL runs pay for wal_checkpoint(TRUNCATE) inside the timed region.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
N = 1_000_000
|
||||
NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/sample) in report
|
||||
SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)"
|
||||
INSERT = "INSERT INTO t VALUES (?,?,?)"
|
||||
|
||||
SET_BASED_SQL = """
|
||||
WITH RECURSIVE seq(i) AS (
|
||||
SELECT 0
|
||||
UNION ALL
|
||||
SELECT i + 1 FROM seq WHERE i + 1 < ?
|
||||
)
|
||||
INSERT INTO t SELECT i, i * 0.5, printf('payload-%012d', i) FROM seq
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- data ----
|
||||
def make_row(i):
|
||||
return (i, i * 0.5, f"payload-{i:012d}")
|
||||
|
||||
|
||||
def rows(n):
|
||||
"""Identical deterministic data for every strategy."""
|
||||
for i in range(n):
|
||||
yield make_row(i)
|
||||
|
||||
|
||||
# ---------------------------------------------------------- strategies ----
|
||||
def s_naive_autocommit(conn, n):
|
||||
conn.isolation_level = None # every INSERT commits: journal sync per row
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
|
||||
|
||||
def s_one_big_txn_loop(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_list(conn, n):
|
||||
data = list(rows(n)) # deliberately materialized: shows RAM cost
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_gen(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_wal_tuned_gen(conn, n):
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
# Fairness (BUILDER): WAL must not finish with 1M rows still in the -wal
|
||||
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
|
||||
|
||||
def s_max_tuned_gen(conn, n):
|
||||
"""Python-bound speed candidate: one txn + executemany(generator) +
|
||||
journaling/sync disabled for the load. Rebuildable staging only."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_set_based(conn, n):
|
||||
"""Beyond-the-list candidate (BUILDER): SQLite generates the identical
|
||||
rows itself; zero Python->SQLite binding crossings. OFF/OFF pragmas so
|
||||
it is compared at the same durability tier as max_tuned_gen."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.execute(SET_BASED_SQL, (n,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
STRATEGIES = {
|
||||
"naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N),
|
||||
"one_big_txn_loop": (s_one_big_txn_loop, N),
|
||||
"executemany_list": (s_executemany_list, N),
|
||||
"executemany_gen": (s_executemany_gen, N),
|
||||
"wal_tuned_gen": (s_wal_tuned_gen, N),
|
||||
"max_tuned_gen": (s_max_tuned_gen, N),
|
||||
"set_based_ctes": (s_set_based, N),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- worker ----
|
||||
def rss_bytes():
|
||||
v = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return v if sys.platform == "darwin" else v * 1024 # KB on Linux
|
||||
|
||||
|
||||
def verify(dbpath, n):
|
||||
"""Integrity gate (fused): count + checksum for everyone [ARCHITECT],
|
||||
plus length-digest and spot-row equality so the SQL-generated data is
|
||||
provably identical to the Python data [BUILDER]."""
|
||||
conn = sqlite3.connect(dbpath)
|
||||
cnt, chk, clen = conn.execute(
|
||||
"SELECT count(*), sum(a), sum(length(c)) FROM t").fetchone()
|
||||
ok = (cnt == n and chk == n * (n - 1) // 2 and clen == 20 * n)
|
||||
for i in (0, 1, n // 2, n - 1):
|
||||
got = conn.execute("SELECT a, b, c FROM t WHERE a=?", (i,)).fetchone()
|
||||
ok = ok and got is not None and tuple(got) == make_row(i)
|
||||
conn.close()
|
||||
return ok
|
||||
|
||||
|
||||
def worker(name):
|
||||
if name == "calibration": # interpreter + sqlite3 import baseline
|
||||
print(json.dumps({"name": name, "rows": 0, "seconds": 0.0,
|
||||
"peak_rss": rss_bytes(), "ok": True}))
|
||||
return
|
||||
fn, n = STRATEGIES[name]
|
||||
dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir())
|
||||
try:
|
||||
conn = sqlite3.connect(dbpath)
|
||||
conn.execute(SCHEMA)
|
||||
t0 = time.perf_counter() # timer: generation + bind + insert + commit
|
||||
fn(conn, n)
|
||||
dt = time.perf_counter() - t0
|
||||
conn.close()
|
||||
ok = verify(dbpath, n) # validation outside the timer
|
||||
print(json.dumps({"name": name, "rows": n, "seconds": dt,
|
||||
"peak_rss": rss_bytes(), "ok": ok}))
|
||||
finally:
|
||||
for p in (dbpath, dbpath + "-wal", dbpath + "-shm"):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- orchestrator --
|
||||
def spawn(name):
|
||||
out = subprocess.run([sys.executable, os.path.abspath(__file__),
|
||||
"--worker", name],
|
||||
capture_output=True, text=True, check=True)
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def main():
|
||||
t_total = time.perf_counter()
|
||||
print(f"SQLite bulk insert benchmark | N={N:,} rows | "
|
||||
f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} "
|
||||
f"{sys.platform}\n")
|
||||
|
||||
base = spawn("calibration")
|
||||
results = []
|
||||
for name in STRATEGIES: # fixed, deterministic order
|
||||
r = spawn(name)
|
||||
assert r["ok"], f"integrity check FAILED for {name}"
|
||||
scale = N / r["rows"]
|
||||
r["scaled"] = r["seconds"] * scale
|
||||
r["sampled"] = scale > 1.0
|
||||
results.append(r)
|
||||
print(f" [done] {name}: {r['seconds']:.3f}s "
|
||||
f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})")
|
||||
|
||||
naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit")
|
||||
print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}"
|
||||
f"{'Peak RSS':>12}{'RSS -base':>12}")
|
||||
print("-" * 70)
|
||||
for r in sorted(results, key=lambda r: r["scaled"]):
|
||||
mark = "*" if r["sampled"] else " "
|
||||
print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}"
|
||||
f"{naive / r['scaled']:>9.1f}x"
|
||||
f"{r['peak_rss'] / 1e6:>10.1f}MB"
|
||||
f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB")
|
||||
print("-" * 70)
|
||||
print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled "
|
||||
f"x{N // NAIVE_SAMPLE_N} | base RSS "
|
||||
f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)")
|
||||
print("note: max_tuned_gen and set_based_ctes run journal_mode=OFF/"
|
||||
"synchronous=OFF -> rebuildable staging loads only.")
|
||||
|
||||
fastest = min(results, key=lambda r: r["scaled"])
|
||||
full = [r for r in results if not r["sampled"]] # baseline excluded
|
||||
leanest = min(full, key=lambda r: (r["peak_rss"], r["scaled"]))
|
||||
print(f"\nSpeed Winner : {fastest['name']} "
|
||||
f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)")
|
||||
print(f"Memory Winner: {leanest['name']} "
|
||||
f"({leanest['peak_rss'] / 1e6:.1f}MB peak, "
|
||||
f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)")
|
||||
print(f"\ntotal benchmark wall time: {time.perf_counter() - t_total:.1f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--worker":
|
||||
worker(sys.argv[2])
|
||||
else:
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
Implemented and validated.
|
||||
|
||||
### Files created
|
||||
- `/private/tmp/ddc0f4d0/fusion-harness/bench.py`
|
||||
- `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` — generated from fresh measurements
|
||||
|
||||
### Commands run
|
||||
- `uv run bench.py` before implementation — RED as expected because `bench.py` did not exist
|
||||
- `uv run bench.py` after implementation — completed in 6.765 seconds
|
||||
- Independent stdlib AST/results validation — PASS
|
||||
|
||||
### Measured winners
|
||||
- **Speed Winner:** `set_based_ctes` — 0.1636s, 1194.12× faster than projected naive autocommit
|
||||
- **Memory Winner:** `max_tuned_gen` — 25.756 MB measured peak RSS
|
||||
|
||||
All seven planned strategies have measured timing and isolated-process peak-RAM results in `RESULTS.md`.
|
||||
@@ -0,0 +1,19 @@
|
||||
No project change can genuinely satisfy this gate. Its `clean_md()` function executes:
|
||||
|
||||
```python
|
||||
re.sub(r"[`*_]", "", s)
|
||||
```
|
||||
|
||||
This removes every underscore before checking for strategy names, but all seven required names contain underscores. Therefore:
|
||||
|
||||
- `set_based_ctes` becomes `setbasedctes`
|
||||
- `wal_tuned_gen` becomes `waltunedgen`
|
||||
- Every table strategy name is similarly destroyed before lookup
|
||||
|
||||
I verified this for all seven names with a Python proof command. `RESULTS.md` already contains the exact strategy names, measured table rows, and winner declarations. The gate must stop deleting underscores, e.g. use:
|
||||
|
||||
```python
|
||||
re.sub(r"[`*]", "", s)
|
||||
```
|
||||
|
||||
No files were changed because altering the benchmark to manipulate the immutable gate would be gaming it.
|
||||
@@ -0,0 +1,33 @@
|
||||
The gate is unsatisfiable without modifying or gaming it.
|
||||
|
||||
Formal reason:
|
||||
|
||||
```python
|
||||
def clean_md(s):
|
||||
return re.sub(r"[`*_]", "", s)
|
||||
```
|
||||
|
||||
For every possible input `x`:
|
||||
|
||||
```python
|
||||
"_" not in clean_md(x)
|
||||
```
|
||||
|
||||
Every required strategy name contains `_`. Therefore no required name can ever satisfy:
|
||||
|
||||
```python
|
||||
strategy_name in clean_md(x)
|
||||
```
|
||||
|
||||
This makes all four failures unavoidable:
|
||||
|
||||
1. Table strategy count is always zero.
|
||||
2. Speed Winner can never match.
|
||||
3. Memory Winner can never match.
|
||||
4. No strategy rows can be parsed, so measured times remain empty.
|
||||
|
||||
`RESULTS.md` already contains the exact names and valid measurements. I made no project changes because the only possible workarounds would manipulate the gate or Python environment, which is explicitly prohibited. The gate must preserve underscores, for example:
|
||||
|
||||
```python
|
||||
return re.sub(r"[`*]", "", s)
|
||||
```
|
||||
@@ -0,0 +1,10 @@
|
||||
No project files changed, per validator triage. `bench.py` and `RESULTS.md` already satisfy every substantive requirement.
|
||||
|
||||
Required harness-side correction:
|
||||
|
||||
```diff
|
||||
- return re.sub(r"[`*_]", "", s)
|
||||
+ return re.sub(r"[`*]", "", s)
|
||||
```
|
||||
|
||||
This preserves canonical strategy-name underscores and resolves all four failures: table matching, Speed Winner matching, Memory Winner matching, and measured-time parsing.
|
||||
@@ -0,0 +1,10 @@
|
||||
No project files changed, exactly as required by validator triage. `bench.py` and freshly generated `RESULTS.md` are correct.
|
||||
|
||||
The remaining failures are caused solely by the unpatched harness regex. The human/harness operator must change `/tmp/fusion-harness-K48sbW/gate.py:61`:
|
||||
|
||||
```diff
|
||||
- return re.sub(r"[`*_]", "", s)
|
||||
+ return re.sub(r"[`*]", "", s)
|
||||
```
|
||||
|
||||
No builder-side workaround can legitimately satisfy the defective matcher.
|
||||
@@ -0,0 +1,6 @@
|
||||
exit 1
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
FAIL: expected bench.py to exist, found nothing, at /private/tmp/ddc0f4d0/fusion-harness/bench.py — implement BENCH_PLAN.md as a single-file uv script named bench.py in /private/tmp/ddc0f4d0/fusion-harness (stdlib only, runnable with `uv run bench.py`)
|
||||
|
||||
RESULT: RED (gate cannot proceed without bench.py)
|
||||
@@ -0,0 +1,19 @@
|
||||
exit 1
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py
|
||||
PASS: bench.py has a PEP 723 `# /// script` metadata block
|
||||
PASS: PEP 723 dependencies are empty (stdlib only)
|
||||
PASS: bench.py imports only Python standard library modules
|
||||
PASS: bench.py uses sqlite3 (real inserts, not simulated)
|
||||
PASS: bench.py isolates strategies in separate processes
|
||||
PASS: bench.py measures peak RAM via getrusage/ru_maxrss
|
||||
PASS: bench.py targets N=1,000,000 rows
|
||||
PASS: `uv run bench.py` ran to completion (exit 0)
|
||||
PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation
|
||||
FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — write one markdown table with one row per strategy using the plan's strategy names
|
||||
FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes — 0.1731 s, 1459.70x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Speed Winner using the plan's strategy name
|
||||
FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen — 25.657 MB peak RSS, +3.375 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Memory Winner using the plan's strategy name
|
||||
FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — fix the results table first
|
||||
|
||||
RESULT: RED (11 passed, 4 failed)
|
||||
@@ -0,0 +1,19 @@
|
||||
exit 1
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py
|
||||
PASS: bench.py has a PEP 723 `# /// script` metadata block
|
||||
PASS: PEP 723 dependencies are empty (stdlib only)
|
||||
PASS: bench.py imports only Python standard library modules
|
||||
PASS: bench.py uses sqlite3 (real inserts, not simulated)
|
||||
PASS: bench.py isolates strategies in separate processes
|
||||
PASS: bench.py measures peak RAM via getrusage/ru_maxrss
|
||||
PASS: bench.py targets N=1,000,000 rows
|
||||
PASS: `uv run bench.py` ran to completion (exit 0)
|
||||
PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation
|
||||
FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — write one markdown table with one row per strategy using the plan's strategy names
|
||||
FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes — 0.2118 s, 1269.32x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Speed Winner using the plan's strategy name
|
||||
FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen — 25.625 MB peak RSS, +3.359 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Memory Winner using the plan's strategy name
|
||||
FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — fix the results table first
|
||||
|
||||
RESULT: RED (11 passed, 4 failed)
|
||||
@@ -0,0 +1,19 @@
|
||||
exit 1
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py
|
||||
PASS: bench.py has a PEP 723 `# /// script` metadata block
|
||||
PASS: PEP 723 dependencies are empty (stdlib only)
|
||||
PASS: bench.py imports only Python standard library modules
|
||||
PASS: bench.py uses sqlite3 (real inserts, not simulated)
|
||||
PASS: bench.py isolates strategies in separate processes
|
||||
PASS: bench.py measures peak RAM via getrusage/ru_maxrss
|
||||
PASS: bench.py targets N=1,000,000 rows
|
||||
PASS: `uv run bench.py` ran to completion (exit 0)
|
||||
PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation
|
||||
FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — write one markdown table with one row per strategy using the plan's strategy names
|
||||
FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes — 0.1768 s, 2030.00x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Speed Winner using the plan's strategy name
|
||||
FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen — 25.657 MB peak RSS, +3.211 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Memory Winner using the plan's strategy name
|
||||
FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — fix the results table first
|
||||
|
||||
RESULT: RED (11 passed, 4 failed)
|
||||
@@ -0,0 +1,19 @@
|
||||
exit 1
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py
|
||||
PASS: bench.py has a PEP 723 `# /// script` metadata block
|
||||
PASS: PEP 723 dependencies are empty (stdlib only)
|
||||
PASS: bench.py imports only Python standard library modules
|
||||
PASS: bench.py uses sqlite3 (real inserts, not simulated)
|
||||
PASS: bench.py isolates strategies in separate processes
|
||||
PASS: bench.py measures peak RAM via getrusage/ru_maxrss
|
||||
PASS: bench.py targets N=1,000,000 rows
|
||||
PASS: `uv run bench.py` ran to completion (exit 0)
|
||||
PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation
|
||||
FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — write one markdown table with one row per strategy using the plan's strategy names
|
||||
FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes — 0.1667 s, 1733.05x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Speed Winner using the plan's strategy name
|
||||
FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: executemanygen — 25.608 MB peak RSS, +3.441 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Memory Winner using the plan's strategy name
|
||||
FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — fix the results table first
|
||||
|
||||
RESULT: RED (11 passed, 4 failed)
|
||||
@@ -0,0 +1,21 @@
|
||||
exit 130
|
||||
|
||||
PASS: BENCH_PLAN.md present and still defines all 7 plan strategies
|
||||
PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py
|
||||
PASS: bench.py has a PEP 723 `# /// script` metadata block
|
||||
PASS: PEP 723 dependencies are empty (stdlib only)
|
||||
PASS: bench.py imports only Python standard library modules
|
||||
PASS: bench.py uses sqlite3 (real inserts, not simulated)
|
||||
PASS: bench.py isolates strategies in separate processes
|
||||
PASS: bench.py measures peak RAM via getrusage/ru_maxrss
|
||||
PASS: bench.py targets N=1,000,000 rows
|
||||
PASS: `uv run bench.py` ran to completion (exit 0)
|
||||
PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation
|
||||
FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — write one markdown table with one row per strategy using the plan's strategy names
|
||||
FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes — 0.1700 s, 1147.98x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Speed Winner using the plan's strategy name
|
||||
FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: executemanygen — 25.625 MB peak RSS, +3.588 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — declare the Memory Winner using the plan's strategy name
|
||||
FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md — fix the results table first
|
||||
|
||||
RESULT: RED (11 passed, 4 failed)
|
||||
|
||||
[stopped by user]
|
||||
@@ -0,0 +1,477 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = []
|
||||
# ///
|
||||
"""
|
||||
Acceptance gate for: implement BENCH_PLAN.md as bench.py (uv single-file
|
||||
script, stdlib only) that runs to completion via `uv run bench.py`, writes
|
||||
RESULTS.md with a markdown table holding a measured time AND a measured peak
|
||||
RAM figure for every strategy in the plan, declares a Speed Winner and a
|
||||
Memory Winner from those measurements, and where the fastest strategy beats
|
||||
the naive autocommit baseline by at least 5x.
|
||||
|
||||
Exit 0 iff every check passes. One PASS/FAIL line per check.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = "/private/tmp/ddc0f4d0/fusion-harness"
|
||||
BENCH = os.path.join(ROOT, "bench.py")
|
||||
RESULTS = os.path.join(ROOT, "RESULTS.md")
|
||||
PLAN = os.path.join(ROOT, "BENCH_PLAN.md")
|
||||
|
||||
# The seven strategies defined by BENCH_PLAN.md ("Strategies (7)" table and
|
||||
# the STRATEGIES dict in its appendix script).
|
||||
STRATEGIES = [
|
||||
"naive_autocommit",
|
||||
"one_big_txn_loop",
|
||||
"executemany_list",
|
||||
"executemany_gen",
|
||||
"wal_tuned_gen",
|
||||
"max_tuned_gen",
|
||||
"set_based_ctes",
|
||||
]
|
||||
BASELINE = "naive_autocommit"
|
||||
RUN_TIMEOUT = 240 # plan budget is ~3 minutes; measured fused run was ~7s
|
||||
|
||||
_results: list[bool] = []
|
||||
|
||||
|
||||
def ok(msg: str) -> None:
|
||||
_results.append(True)
|
||||
print("PASS: " + msg)
|
||||
|
||||
|
||||
def bad(msg: str) -> None:
|
||||
_results.append(False)
|
||||
print("FAIL: " + msg)
|
||||
|
||||
|
||||
def one_line(s: str, limit: int = 500) -> str:
|
||||
s = " | ".join(part.strip() for part in s.strip().splitlines() if part.strip())
|
||||
return s[-limit:] if len(s) > limit else s
|
||||
|
||||
|
||||
def clean_md(s: str) -> str:
|
||||
return re.sub(r"[`*_]", "", s)
|
||||
|
||||
|
||||
def first_float(cell: str):
|
||||
m = re.search(r"-?\d+(?:\.\d+)?", cell.replace(",", ""))
|
||||
return float(m.group(0)) if m else None
|
||||
|
||||
|
||||
def split_cells(line: str):
|
||||
line = line.strip()
|
||||
if line.startswith("|"):
|
||||
line = line[1:]
|
||||
if line.endswith("|"):
|
||||
line = line[:-1]
|
||||
return [c.strip() for c in line.split("|")]
|
||||
|
||||
|
||||
def parse_tables(text: str):
|
||||
"""Return list of (header_cells, data_rows) for every markdown table."""
|
||||
lines = text.splitlines()
|
||||
tables = []
|
||||
i = 0
|
||||
sep_re = re.compile(r"^\s*\|?[\s:|-]+\|?\s*$")
|
||||
while i < len(lines):
|
||||
if (
|
||||
"|" in lines[i]
|
||||
and i + 1 < len(lines)
|
||||
and "-" in lines[i + 1]
|
||||
and "|" in lines[i + 1]
|
||||
and sep_re.match(lines[i + 1])
|
||||
):
|
||||
header = split_cells(lines[i])
|
||||
j = i + 2
|
||||
rows = []
|
||||
while j < len(lines) and "|" in lines[j] and lines[j].strip():
|
||||
rows.append(split_cells(lines[j]))
|
||||
j += 1
|
||||
tables.append((header, rows))
|
||||
i = j
|
||||
else:
|
||||
i += 1
|
||||
return tables
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# ---- 0. plan still present (the thing bench.py must implement) --------
|
||||
if os.path.isfile(PLAN):
|
||||
plan_txt = open(PLAN, encoding="utf-8", errors="replace").read()
|
||||
missing = [s for s in STRATEGIES if s not in plan_txt]
|
||||
if missing:
|
||||
bad(
|
||||
f"expected BENCH_PLAN.md to still define strategies {missing}, "
|
||||
f"found them absent, at {PLAN} — restore BENCH_PLAN.md; the "
|
||||
f"plan is the spec and must not be edited"
|
||||
)
|
||||
else:
|
||||
ok("BENCH_PLAN.md present and still defines all 7 plan strategies")
|
||||
else:
|
||||
bad(
|
||||
f"expected BENCH_PLAN.md to exist, found nothing, at {PLAN} — "
|
||||
f"restore the plan file; it is the spec bench.py implements"
|
||||
)
|
||||
|
||||
# ---- 1. bench.py exists ------------------------------------------------
|
||||
if not os.path.isfile(BENCH):
|
||||
bad(
|
||||
f"expected bench.py to exist, found nothing, at {BENCH} — "
|
||||
f"implement BENCH_PLAN.md as a single-file uv script named "
|
||||
f"bench.py in {ROOT} (stdlib only, runnable with `uv run bench.py`)"
|
||||
)
|
||||
print("\nRESULT: RED (gate cannot proceed without bench.py)")
|
||||
return 1
|
||||
ok(f"bench.py exists at {BENCH}")
|
||||
|
||||
src = open(BENCH, encoding="utf-8", errors="replace").read()
|
||||
|
||||
# ---- 2. PEP 723 header, empty dependencies (stdlib only) --------------
|
||||
if re.search(r"^#\s*///\s*script\s*$", src, re.MULTILINE):
|
||||
ok("bench.py has a PEP 723 `# /// script` metadata block")
|
||||
else:
|
||||
bad(
|
||||
f"expected a PEP 723 block starting with `# /// script`, found "
|
||||
f"none, at {BENCH} — add the inline metadata block "
|
||||
f"(`# /// script`, `# requires-python = ...`, "
|
||||
f"`# dependencies = []`, `# ///`) at the top of bench.py"
|
||||
)
|
||||
dep_lines = [l for l in src.splitlines() if re.search(r"^\#\s*dependencies\s*=", l)]
|
||||
if dep_lines and not re.search(r"dependencies\s*=\s*\[\s*\]", dep_lines[0]):
|
||||
bad(
|
||||
f"expected `dependencies = []` (stdlib only), found "
|
||||
f"`{dep_lines[0].strip()}`, at {BENCH} — remove all third-party "
|
||||
f"dependencies from the PEP 723 block; the plan requires Python "
|
||||
f"standard library only"
|
||||
)
|
||||
else:
|
||||
ok("PEP 723 dependencies are empty (stdlib only)")
|
||||
|
||||
# ---- 3. all imports are stdlib -----------------------------------------
|
||||
try:
|
||||
tree = ast.parse(src)
|
||||
roots = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for a in node.names:
|
||||
roots.add(a.name.split(".")[0])
|
||||
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
||||
roots.add(node.module.split(".")[0])
|
||||
non_std = sorted(r for r in roots if r not in sys.stdlib_module_names)
|
||||
if non_std:
|
||||
bad(
|
||||
f"expected only stdlib imports, found non-stdlib {non_std}, "
|
||||
f"at {BENCH} — rewrite bench.py using only the Python "
|
||||
f"standard library"
|
||||
)
|
||||
else:
|
||||
ok("bench.py imports only Python standard library modules")
|
||||
except SyntaxError as e:
|
||||
bad(
|
||||
f"expected bench.py to be valid Python, found SyntaxError "
|
||||
f"`{one_line(str(e))}`, at {BENCH} — fix the syntax error"
|
||||
)
|
||||
|
||||
# ---- 4. static plan fidelity: real measurement machinery ---------------
|
||||
if "sqlite3" in src:
|
||||
ok("bench.py uses sqlite3 (real inserts, not simulated)")
|
||||
else:
|
||||
bad(
|
||||
f"expected bench.py to import/use sqlite3, found no mention, at "
|
||||
f"{BENCH} — the benchmark must perform real SQLite inserts per "
|
||||
f"BENCH_PLAN.md"
|
||||
)
|
||||
if re.search(r"subprocess|multiprocessing", src):
|
||||
ok("bench.py isolates strategies in separate processes")
|
||||
else:
|
||||
bad(
|
||||
f"expected per-strategy process isolation (subprocess re-exec or "
|
||||
f"multiprocessing per BENCH_PLAN.md), found neither, at {BENCH} — "
|
||||
f"run each strategy in its own fresh process so peak-RAM readings "
|
||||
f"cannot pollute each other"
|
||||
)
|
||||
if re.search(r"ru_maxrss|getrusage", src):
|
||||
ok("bench.py measures peak RAM via getrusage/ru_maxrss")
|
||||
else:
|
||||
bad(
|
||||
f"expected measured peak RAM via resource.getrusage(...).ru_maxrss "
|
||||
f"per BENCH_PLAN.md, found no getrusage/ru_maxrss, at {BENCH} — "
|
||||
f"measure whole-process peak RSS inside each isolated worker; "
|
||||
f"do not assume or hardcode memory figures"
|
||||
)
|
||||
if re.search(r"1_000_000|1000000", src):
|
||||
ok("bench.py targets N=1,000,000 rows")
|
||||
else:
|
||||
bad(
|
||||
f"expected N=1,000,000 rows (literal 1_000_000 or 1000000), found "
|
||||
f"neither, at {BENCH} — the plan requires 1M rows for every "
|
||||
f"full-run strategy (baseline may be sampled and scaled)"
|
||||
)
|
||||
|
||||
# ---- 5. `uv run bench.py` runs to completion ---------------------------
|
||||
pre_marker = time.time()
|
||||
proc = None
|
||||
ran_ok = False
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["uv", "run", "bench.py"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=RUN_TIMEOUT,
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
ok("`uv run bench.py` ran to completion (exit 0)")
|
||||
ran_ok = True
|
||||
else:
|
||||
bad(
|
||||
f"expected `uv run bench.py` to exit 0, found exit "
|
||||
f"{proc.returncode}, at {BENCH} — fix the crash; stderr tail: "
|
||||
f"{one_line(proc.stderr or proc.stdout or '(empty)')}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
bad(
|
||||
f"expected `uv` on PATH so `uv run bench.py` works, found uv "
|
||||
f"missing, at {ROOT} — ensure the script is runnable with exactly "
|
||||
f"`uv run bench.py`; do not substitute another runner"
|
||||
)
|
||||
# best-effort fallback so content checks below still give feedback
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "bench.py"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=RUN_TIMEOUT,
|
||||
)
|
||||
except Exception:
|
||||
proc = None
|
||||
except subprocess.TimeoutExpired:
|
||||
bad(
|
||||
f"expected `uv run bench.py` to finish within {RUN_TIMEOUT}s "
|
||||
f"(plan budget ~3 minutes), found it still running, at {BENCH} — "
|
||||
f"sample the naive autocommit baseline (plan: 20,000 rows scaled "
|
||||
f"x50) instead of running it for the full 1M rows"
|
||||
)
|
||||
|
||||
# ---- 6. RESULTS.md written by the run ----------------------------------
|
||||
if not os.path.isfile(RESULTS):
|
||||
bad(
|
||||
f"expected the run to write RESULTS.md, found no file, at "
|
||||
f"{RESULTS} — make bench.py write RESULTS.md (markdown table with "
|
||||
f"a time column and a peak RAM column for every strategy, plus "
|
||||
f"Speed Winner and Memory Winner lines) every time it runs"
|
||||
)
|
||||
print("\nRESULT: RED")
|
||||
return 1
|
||||
if os.path.getmtime(RESULTS) >= pre_marker - 2:
|
||||
ok("RESULTS.md was (re)written by this `uv run bench.py` invocation")
|
||||
else:
|
||||
bad(
|
||||
f"expected RESULTS.md to be rewritten by the run just executed, "
|
||||
f"found a stale file (mtime predates the run), at {RESULTS} — "
|
||||
f"bench.py itself must write RESULTS.md from its own fresh "
|
||||
f"measurements on every run; do not hand-author RESULTS.md"
|
||||
)
|
||||
|
||||
text = open(RESULTS, encoding="utf-8", errors="replace").read()
|
||||
|
||||
# ---- 7. markdown table: time + peak RAM for every strategy -------------
|
||||
tables = parse_tables(text)
|
||||
best = None
|
||||
best_count = -1
|
||||
for header, rows in tables:
|
||||
joined = clean_md(" ".join(" ".join(r) for r in rows))
|
||||
count = sum(1 for s in STRATEGIES if s in joined)
|
||||
if count > best_count:
|
||||
best_count = count
|
||||
best = (header, rows)
|
||||
|
||||
times: dict[str, float] = {}
|
||||
rams: dict[str, float] = {}
|
||||
if best is None or best_count == 0:
|
||||
bad(
|
||||
f"expected a markdown table whose rows name the 7 plan strategies "
|
||||
f"{STRATEGIES}, found no such table, at {RESULTS} — write one "
|
||||
f"markdown table with one row per strategy using the plan's "
|
||||
f"strategy names"
|
||||
)
|
||||
else:
|
||||
header, rows = best
|
||||
time_col = next(
|
||||
(i for i, h in enumerate(header) if re.search(r"time|sec|duration", h, re.I)),
|
||||
None,
|
||||
)
|
||||
ram_candidates = [
|
||||
i for i, h in enumerate(header) if re.search(r"ram|rss|mem", h, re.I)
|
||||
]
|
||||
ram_col = next(
|
||||
(i for i in ram_candidates if re.search(r"peak", header[i], re.I)),
|
||||
ram_candidates[0] if ram_candidates else None,
|
||||
)
|
||||
if time_col is None:
|
||||
bad(
|
||||
f"expected a time column (header matching time/sec/duration), "
|
||||
f"found headers {header}, at {RESULTS} — add a measured time "
|
||||
f"column to the results table"
|
||||
)
|
||||
else:
|
||||
ok(f"results table has a time column (`{header[time_col]}`)")
|
||||
if ram_col is None:
|
||||
bad(
|
||||
f"expected a peak RAM column (header matching RAM/RSS/mem), "
|
||||
f"found headers {header}, at {RESULTS} — add a measured peak "
|
||||
f"RAM column to the results table"
|
||||
)
|
||||
else:
|
||||
ok(f"results table has a peak RAM column (`{header[ram_col]}`)")
|
||||
|
||||
if time_col is not None and ram_col is not None:
|
||||
for strat in STRATEGIES:
|
||||
row = next(
|
||||
(r for r in rows if strat in clean_md(" ".join(r))), None
|
||||
)
|
||||
if row is None:
|
||||
bad(
|
||||
f"expected a table row for strategy `{strat}`, found "
|
||||
f"none, at {RESULTS} — every strategy in BENCH_PLAN.md "
|
||||
f"must have its own measured row"
|
||||
)
|
||||
continue
|
||||
t = first_float(row[time_col]) if time_col < len(row) else None
|
||||
m = first_float(row[ram_col]) if ram_col < len(row) else None
|
||||
if t is None or t <= 0:
|
||||
bad(
|
||||
f"expected a positive measured time for `{strat}`, "
|
||||
f"found `{row[time_col] if time_col < len(row) else '(missing cell)'}`, "
|
||||
f"at {RESULTS} — record the real measured wall time"
|
||||
)
|
||||
else:
|
||||
times[strat] = t
|
||||
if m is None or m <= 0:
|
||||
bad(
|
||||
f"expected a positive measured peak RAM for `{strat}`, "
|
||||
f"found `{row[ram_col] if ram_col < len(row) else '(missing cell)'}`, "
|
||||
f"at {RESULTS} — record the real measured peak RSS"
|
||||
)
|
||||
else:
|
||||
rams[strat] = m
|
||||
if len(times) == len(STRATEGIES):
|
||||
ok("every plan strategy has a positive measured time in the table")
|
||||
if len(rams) == len(STRATEGIES):
|
||||
ok("every plan strategy has a positive measured peak RAM in the table")
|
||||
|
||||
# ---- 8. declared winners ------------------------------------------------
|
||||
def declared(kind: str):
|
||||
m = re.search(kind + r"\s*winner[^\n]*", text, re.I)
|
||||
if not m:
|
||||
return None, None
|
||||
line = clean_md(m.group(0))
|
||||
name = next((s for s in STRATEGIES if s in line), None)
|
||||
return line, name
|
||||
|
||||
speed_line, speed_name = declared("speed")
|
||||
if speed_name:
|
||||
ok(f"RESULTS.md declares a Speed Winner: {speed_name}")
|
||||
else:
|
||||
bad(
|
||||
f"expected a `Speed Winner:` line naming one plan strategy, found "
|
||||
f"`{speed_line or 'no such line'}`, at {RESULTS} — declare the "
|
||||
f"Speed Winner using the plan's strategy name"
|
||||
)
|
||||
mem_line, mem_name = declared("memory")
|
||||
if mem_name:
|
||||
ok(f"RESULTS.md declares a Memory Winner: {mem_name}")
|
||||
else:
|
||||
bad(
|
||||
f"expected a `Memory Winner:` line naming one plan strategy, found "
|
||||
f"`{mem_line or 'no such line'}`, at {RESULTS} — declare the "
|
||||
f"Memory Winner using the plan's strategy name"
|
||||
)
|
||||
|
||||
# ---- 9. winners follow from the measurements ---------------------------
|
||||
if speed_name and len(times) == len(STRATEGIES):
|
||||
fastest_t = min(times.values())
|
||||
if times[speed_name] <= fastest_t + 1e-9:
|
||||
ok(
|
||||
f"Speed Winner {speed_name} matches the fastest measured time "
|
||||
f"({times[speed_name]}s)"
|
||||
)
|
||||
else:
|
||||
actual = min(times, key=times.get)
|
||||
bad(
|
||||
f"expected the Speed Winner to be the fastest strategy in the "
|
||||
f"table (`{actual}` at {times[actual]}s), found `{speed_name}` "
|
||||
f"at {times[speed_name]}s, at {RESULTS} — derive the winner "
|
||||
f"from the measurements, not assumptions"
|
||||
)
|
||||
if mem_name and len(rams) == len(STRATEGIES):
|
||||
if mem_name == BASELINE:
|
||||
bad(
|
||||
f"expected the Memory Winner to exclude the sampled baseline, "
|
||||
f"found `{BASELINE}`, at {RESULTS} — per BENCH_PLAN.md the "
|
||||
f"sampled baseline never held 1M rows of work and cannot win "
|
||||
f"the RAM axis; pick the leanest full-1M strategy"
|
||||
)
|
||||
else:
|
||||
eligible = {k: v for k, v in rams.items() if k != BASELINE}
|
||||
lean = min(eligible.values())
|
||||
if rams[mem_name] <= lean + 1e-9:
|
||||
ok(
|
||||
f"Memory Winner {mem_name} matches the lowest measured "
|
||||
f"peak RAM among full-1M strategies ({rams[mem_name]})"
|
||||
)
|
||||
else:
|
||||
actual = min(eligible, key=eligible.get)
|
||||
bad(
|
||||
f"expected the Memory Winner to have the lowest peak RAM "
|
||||
f"among full-1M strategies (`{actual}` at {eligible[actual]}), "
|
||||
f"found `{mem_name}` at {rams[mem_name]}, at {RESULTS} — "
|
||||
f"derive the winner from the measurements"
|
||||
)
|
||||
|
||||
# ---- 10. fastest beats naive autocommit by >= 5x ------------------------
|
||||
if len(times) == len(STRATEGIES):
|
||||
fastest_t = min(times.values())
|
||||
speedup = times[BASELINE] / fastest_t if fastest_t > 0 else 0.0
|
||||
if speedup >= 5.0:
|
||||
ok(
|
||||
f"fastest strategy beats naive autocommit by {speedup:.1f}x "
|
||||
f"(>= 5x required)"
|
||||
)
|
||||
else:
|
||||
bad(
|
||||
f"expected fastest strategy >= 5x faster than naive "
|
||||
f"autocommit, found {speedup:.1f}x ({times[BASELINE]}s vs "
|
||||
f"{fastest_t}s), at {RESULTS} — use one big transaction / "
|
||||
f"executemany / set-based insert per BENCH_PLAN.md so the "
|
||||
f"speedup is real"
|
||||
)
|
||||
else:
|
||||
bad(
|
||||
f"expected complete measured times for all 7 strategies to verify "
|
||||
f"the >=5x speedup, found only {sorted(times)} parseable, at "
|
||||
f"{RESULTS} — fix the results table first"
|
||||
)
|
||||
|
||||
if not ran_ok:
|
||||
pass # the uv-run FAIL above already carries the instruction
|
||||
|
||||
failed = _results.count(False)
|
||||
print(
|
||||
f"\nRESULT: {'GREEN' if failed == 0 else 'RED'} "
|
||||
f"({_results.count(True)} passed, {failed} failed)"
|
||||
)
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1 @@
|
||||
Read BENCH_PLAN.md, the plan you two designed and fused, and implement it in this directory as bench.py, an astral uv single file script. Python standard library only, runnable with uv run bench.py. Definition of Done is defined by a gate YOU design. At minimum it proves uv run bench.py runs to completion, writes RESULTS.md with a markdown table holding a timed row AND a measured peak RAM figure for every strategy in the plan, and declares a Speed Winner and a Memory Winner from those measurements. Measured, not assumed. The fastest strategy has to beat the naive autocommit baseline by at least 5x. Baseline the gate RED before you build anything. Never shim, wrap, or modify global system tools.
|
||||
@@ -0,0 +1,11 @@
|
||||
**Diagnosis** — `GATE DEFECT:` The builder's work is correct and complete; the gate can never pass it. `gate.py:61` defines `clean_md(s) = re.sub(r"[`*_]", "", s)` — intended to strip markdown emphasis, it also deletes every underscore from the text being searched. All 7 required strategy names contain underscores, so `"set_based_ctes" in clean_md(...)` is mathematically unsatisfiable for any possible RESULTS.md (the gate's own round-2/3 output proves it: it echoes `setbasedctes` / `waltunedgen` — mangled by the gate, not written that way). This single line causes all four FAILs (table match count 0 → "no such table", both winner-name checks, empty parsed times → 5x check). I verified the real state: `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` contains a well-formed table with all 7 exact plan names, positive measured times and peak-RAM figures, a Speed Winner (`set_based_ctes`, 0.1768s = table minimum), a Memory Winner (`wal_tuned_gen`, 25.657 MB = minimum among full-1M rows, baseline excluded), and a 2030x speedup vs naive — every substantive requirement is already met. The builder's round-3 report is accurate; rounds 2–3 are identical because there is nothing left for the builder to fix.
|
||||
|
||||
**Do exactly this**
|
||||
1. **HUMAN/HARNESS (not builder):** patch `/tmp/fusion-harness-K48sbW/gate.py` line 61 from `return re.sub(r"[`*_]", "", s)` to `return re.sub(r"[`*]", "", s)`. Underscores must survive cleaning; backtick/asterisk stripping stays. No other gate change is needed.
|
||||
2. **BUILDER:** change nothing. Leave `/private/tmp/ddc0f4d0/fusion-harness/bench.py` and `RESULTS.md` exactly as they are — they satisfy every check once matching works (RESULTS.md is regenerated fresh each run, so mtime/consistency checks will pass).
|
||||
3. Re-run the gate once the patch lands; expect GREEN.
|
||||
|
||||
**Do NOT**
|
||||
- Do NOT rename strategies to underscore-free variants (e.g., `setbasedctes`) or strip underscores/backticks in RESULTS.md to appease the broken matcher — that diverges from BENCH_PLAN.md's canonical names and is gate-gaming.
|
||||
- Do NOT let the builder edit `/tmp/fusion-harness-K48sbW/gate.py` itself; the fix is the human's call, flagged here.
|
||||
- Do NOT burn rounds 4–5 re-running the unmodified gate against unmodified files — the failure is deterministic until line 61 is patched.
|
||||
@@ -0,0 +1,11 @@
|
||||
**Diagnosis** — `GATE DEFECT:` (unchanged, second escalation for the same root cause). The round-3 triage identified the exact defect and the required one-line harness-side patch, but `/tmp/fusion-harness-K48sbW/gate.py:61` still reads `return re.sub(r"[`*_]", "", s)` — the patch was never applied. That regex deletes every underscore from the searched text, while all 7 required strategy names in the gate's hardcoded `STRATEGIES` list contain underscores, so `name in clean_md(text)` is **unsatisfiable for every possible RESULTS.md** — no builder edit of any project file can ever pass checks 7–10. The loop is not stuck on the builder: I re-verified the real state and `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` is correct and freshly measured (round 4: `set_based_ctes` 0.1667s speed winner = table minimum, `executemany_gen` 25.608 MB memory winner, 1733x ≥ 5x, all 7 exact snake_case names present — the winner even changed between rounds 3 and 4, proving values are measured, not hardcoded). The gate's own FAIL lines echo `setbasedctes`/`executemanygen` — names mangled by the gate itself, not by the builder. The builder's round-4 report is accurate; it correctly made no changes.
|
||||
|
||||
**Do exactly this**
|
||||
1. **HUMAN/HARNESS OPERATOR — this is the only action that can produce GREEN:** edit `/tmp/fusion-harness-K48sbW/gate.py` line 61 from `return re.sub(r"[`*_]", "", s)` to `return re.sub(r"[`*]", "", s)`. One character class change; nothing else in the gate needs touching. Apply it **before** round 5 — it is the final round under the cap, and running it unpatched guarantees a false RED.
|
||||
2. **BUILDER:** make no changes. `/private/tmp/ddc0f4d0/fusion-harness/bench.py` and `RESULTS.md` already satisfy every substantive requirement; the gate re-runs `uv run bench.py` itself and regenerates RESULTS.md, so all mtime/consistency checks will pass.
|
||||
3. Re-run the gate once patched; expect GREEN (15/15 style output, exit 0).
|
||||
|
||||
**Do NOT**
|
||||
- Do NOT spend round 5 re-running the unpatched gate — the failure is deterministic and proven unsatisfiable; that would exhaust the cap on a known-defective check.
|
||||
- Builder: do NOT attempt workarounds (renaming strategies without underscores, zero-width characters, duplicate underscore-free rows) — the gate's needle list itself contains underscores, so no RESULTS.md content can match, and any such edit is gate-gaming that corrupts an otherwise-correct deliverable.
|
||||
- Builder: do NOT edit `/tmp/fusion-harness-K48sbW/gate.py` yourself — the fix is flagged for the human, per the escalation contract.
|
||||
@@ -0,0 +1,3 @@
|
||||
Gate written to /tmp/fusion-harness-K48sbW/gate.py.
|
||||
|
||||
It verifies, in order: BENCH_PLAN.md intact; `bench.py` exists at the project root with a PEP 723 block, empty `dependencies` and stdlib-only imports (AST scan); static plan fidelity (real sqlite3 inserts, per-strategy process isolation, `getrusage`/`ru_maxrss` RAM measurement, N=1,000,000); `uv run bench.py` exits 0 within the ~3-minute budget; RESULTS.md is freshly (re)written by that very run; its markdown table has a time column and a peak-RAM column with positive measured values for all 7 plan strategies; Speed Winner and Memory Winner are declared and are consistent with the table (memory winner excludes the sampled baseline per plan); and the fastest strategy beats naive autocommit by ≥5x. Currently RED — bench.py does not exist yet.
|
||||
@@ -0,0 +1,340 @@
|
||||
# BENCH_PLAN — SQLite Bulk Insert Benchmark (fused design)
|
||||
|
||||
Fusion of two independent designs: **[ARCHITECT] anthropic/claude-fable-5** and **[BUILDER] openai/gpt-5.6-sol**. Where the designs disagreed, the simpler one was chosen (reasons inline and in the closing section). The fused script was **built and executed** — every number below is measured, not projected from the source answers.
|
||||
|
||||
- Fused runnable script: `/tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py` (also embedded in full at the end of this plan)
|
||||
- Run with: `uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py`
|
||||
- One astral-uv single-file script (PEP 723 header, `dependencies = []`), Python stdlib only
|
||||
- Measured total wall time: **6.8 s** — well inside the 3-minute budget
|
||||
|
||||
## Method
|
||||
|
||||
**Self-isolating architecture** [ARCHITECT]. The script is both orchestrator and worker: it re-execs itself (`sys.executable __file__ --worker NAME`) once per strategy, so every strategy runs in a **fresh process against a fresh temp database**. Peak-RSS readings physically cannot pollute each other — this satisfies the isolated-process RAM requirement with zero third-party dependencies. [BUILDER] independently arrived at the same fresh-subprocess-per-trial isolation, so this is consensus architecture.
|
||||
|
||||
**Peak RAM** = whole-process `resource.getrusage(...).ru_maxrss`, normalized (bytes on macOS, KB×1024 on Linux) [both]. A **no-op calibration worker** measures bare-interpreter RSS (~20.3 MB here) so the table reports both absolute peak and delta-over-baseline [ARCHITECT].
|
||||
|
||||
**Timing boundary** [consensus]: the clock covers row generation + bind + insert + commit; connection setup, schema creation, and validation sit outside the timer. Generation is identical work for every strategy, so including it is neutral — excluding it would unfairly flatter the list-materializing strategy [ARCHITECT]. WAL runs additionally pay for `wal_checkpoint(TRUNCATE)` *inside* the timed region, so WAL is not credited with "finishing" while a million rows still sit in the `-wal` sidecar [BUILDER].
|
||||
|
||||
### Strategies (7)
|
||||
|
||||
| # | Strategy | What it does | Source |
|
||||
|---|----------|--------------|--------|
|
||||
| 1 | `naive_autocommit` | per-row implicit transaction — the baseline (sampled, see below) | both |
|
||||
| 2 | `one_big_txn_loop` | `BEGIN` + `execute()` loop + one `COMMIT` | both |
|
||||
| 3 | `executemany_list` | one txn, all 1M tuples materialized first — the classic mistake, kept to expose its RAM cost | [ARCHITECT] |
|
||||
| 4 | `executemany_gen` | one txn, `executemany` fed a generator | both |
|
||||
| 5 | `wal_tuned_gen` | `journal_mode=WAL` + `synchronous=NORMAL` + strategy 4 (WAL tuning combined with the predicted Python-bound winner, per the request) | both |
|
||||
| 6 | `max_tuned_gen` | `journal_mode=OFF` + `synchronous=OFF` + strategy 4 — fastest way to push Python-resident rows | [ARCHITECT] |
|
||||
| 7 | `set_based_ctes` | **the "beats all" candidate**: `INSERT … SELECT` over a recursive CTE that generates the *identical* rows inside SQLite — zero Python→SQLite binding crossings; run at OFF/OFF so it competes at the same durability tier as #6 | [BUILDER] |
|
||||
|
||||
Chosen over discarded alternatives:
|
||||
- [BUILDER]'s set-based candidate replaces [ARCHITECT]'s `sqlite3` CLI `.import`: **simpler** (no external binary, no CSV temp file, no graceful-skip logic — pure stdlib in-process SQL) *and* it measured ~2× faster than everything else anyway.
|
||||
- [BUILDER]'s chunked multi-row `VALUES` strategy was dropped: the variable-limit math and chunk assembly add complexity, and its niche (fewer Python↔SQLite crossings) is bracketed by `executemany_gen` below it and `set_based_ctes` above it.
|
||||
- [ARCHITECT]'s 64 MB `cache_size` pragma was dropped from the tuned strategies in favor of [BUILDER]'s bounded-default-cache stance: **simpler**, and [ARCHITECT]'s own measurements showed the big cache costs ~50 MB of resident memory for essentially zero speed gain. (Fused result: `wal_tuned_gen` peaks at 24 MB instead of ARCHITECT's 71 MB.)
|
||||
|
||||
## Fairness controls
|
||||
|
||||
- **Identical data** [both]: every strategy inserts the same deterministic rows `(i, i*0.5, "payload-{i:012d}")` — one Python definition, no RNG. `i*0.5` is exact in floating point, so SQL-generated and Python-generated values are bit-identical.
|
||||
- **Identical schema** [both]: `CREATE TABLE t (a INTEGER, b REAL, c TEXT)` (INTEGER/REAL/TEXT storage classes covered), fresh temp DB per worker, deleted afterward including `-wal`/`-shm`. [ARCHITECT]'s 3-column schema chosen over [BUILDER]'s 6-column `events` table as the simpler design that still exercises the same code paths.
|
||||
- **Integrity gate, hard-fail** [fused]: every worker verifies `count(*)`, the closed-form `sum(a)` checksum, and `sum(length(c))` [ARCHITECT's gate], plus spot-row equality at indices `{0, 1, N/2, N−1}` against the Python row function [distilled from BUILDER's digest] — required so the set-based strategy *proves* it wrote identical data rather than just the right number of rows. No strategy can win by writing less or writing different bytes. This is simpler than [BUILDER]'s 11-field digest + `quick_check` while closing the same loophole.
|
||||
- **Honest sampling, declared up front** [both]: `naive_autocommit` is fsync-bound (one journal sync per row); a full run would take ~3 minutes alone. It runs **20,000 real rows scaled ×50**, marked `*` in the table. Everything else runs the full 1,000,000. [ARCHITECT]'s 20k×50 chosen over [BUILDER]'s 10k×100 — equal simplicity, smaller scaling factor is the more honest extrapolation (both project to ~190 s regardless).
|
||||
- **Baseline excluded from the memory verdict** [both, independently]: the sampled baseline never held 1M rows of work, so it cannot win the RAM axis.
|
||||
- **Single run per strategy, fixed order** [ARCHITECT]: chosen over [BUILDER]'s 2×-with-median and seeded order shuffling as the simpler design. Justification: two full executions of the fused benchmark showed per-strategy spread <0.05 s while the winner gaps are ≥0.19 s, and process isolation already removes state carryover.
|
||||
- **Durability labeling** [BUILDER]: OFF/OFF strategies (#6, #7) are flagged in the output as *rebuildable staging loads only* — a crash mid-load can corrupt the DB. They race in the same heats but carry the warning label.
|
||||
|
||||
## Exact output format
|
||||
|
||||
```
|
||||
SQLite bulk insert benchmark | N=1,000,000 rows | py <ver> sqlite <ver> <platform>
|
||||
|
||||
[done] <strategy>: <t>s (<rows> rows[ sampled]) ← one line per isolated process
|
||||
|
||||
Strategy Time 1M rows Speedup Peak RSS RSS -base
|
||||
----------------------------------------------------------------------
|
||||
<name> <t>s[*] <x>x <MB>MB <MB>MB ← sorted fastest-first
|
||||
----------------------------------------------------------------------
|
||||
* sampled at 20,000 rows, honestly scaled x50 | base RSS <MB>MB (no-op interpreter)
|
||||
note: max_tuned_gen and set_based_ctes run journal_mode=OFF/synchronous=OFF -> rebuildable staging loads only.
|
||||
|
||||
Speed Winner : <name> (<t>s, <x>x vs naive)
|
||||
Memory Winner: <name> (<MB>MB peak, +<MB>MB over baseline)
|
||||
```
|
||||
|
||||
Table layout is [ARCHITECT]'s (fewer columns, speedup included, sorted fastest-first — simpler than [BUILDER]'s 7-column trial-level table); the `Speed Winner:`/`Memory Winner:` terminal lines are [BUILDER]'s exact phrasing.
|
||||
|
||||
## Measured results (fused run: py 3.12.13, sqlite 3.50.4, macOS / Apple Silicon)
|
||||
|
||||
| Strategy | Time 1M rows | Speedup | Peak RSS | RSS −base |
|
||||
|---|---:|---:|---:|---:|
|
||||
| **set_based_ctes** | **0.17s** | **1126.2x** | 24.3 MB | 4.1 MB |
|
||||
| max_tuned_gen | 0.37s | 518.4x | 24.1 MB | 3.9 MB |
|
||||
| executemany_gen | 0.38s | 508.8x | 24.1 MB | 3.9 MB |
|
||||
| **wal_tuned_gen** | 0.40s | 475.6x | **23.9 MB** | **3.7 MB** |
|
||||
| executemany_list | 0.49s | 393.5x | 226.8 MB | 206.6 MB |
|
||||
| one_big_txn_loop | 0.51s | 374.9x | 24.0 MB | 3.8 MB |
|
||||
| naive_autocommit | 191.15s* | 1.0x | 22.7 MB | 2.5 MB |
|
||||
|
||||
\* sampled at 20,000 rows, scaled ×50. Base RSS 20.2 MB (no-op interpreter). Total wall time 6.8 s.
|
||||
|
||||
Key findings the fused design makes visible:
|
||||
1. **The single transaction is ~99% of the win** — 375× from `BEGIN`/`COMMIT` alone [ARCHITECT's finding, confirmed].
|
||||
2. **Eliminating the million Python→SQLite binding crossings is the last 2×** — set-based insertion at 0.17 s [BUILDER's finding, confirmed on the identical schema].
|
||||
3. **`executemany_list` is strictly dominated**: slower than the generator *and* ~200 MB heavier [ARCHITECT's finding, confirmed].
|
||||
4. **Memory is a four-way statistical tie** among all streaming strategies (23.9–24.3 MB, within ±0.4 MB across repeat runs): once you stream rows instead of materializing them, SQLite's bounded page cache makes peak RSS nearly strategy-invariant.
|
||||
|
||||
## Declared Winners
|
||||
|
||||
- **Speed Winner: `set_based_ctes`** — `INSERT … SELECT` over a recursive CTE, OFF/OFF pragmas: **0.17 s for 1M rows, ~1126× over naive autocommit.** Caveats carried from [BUILDER]: rebuildable-staging durability only, and it applies only when rows are derivable in SQL. For arbitrary Python-resident rows, the practical speed winner is `max_tuned_gen` (0.37 s), with `wal_tuned_gen` (0.40 s) the fastest *durable-enough* option — the same practical hierarchy both source models converged on.
|
||||
- **Memory Winner: `wal_tuned_gen`** — **23.9 MB peak, +3.7 MB over a bare interpreter**, as selected by the declared rule (minimum peak RSS among full-1M runs, speed as tiebreak) in both verification runs. Honest note: this is a statistical tie with `one_big_txn_loop`, `executemany_gen`, and `max_tuned_gen` (all within 0.4 MB); the real memory lesson is *stream, never materialize* — the list variant costs 206 MB extra for nothing.
|
||||
|
||||
---
|
||||
|
||||
## Consensus & Divergence
|
||||
|
||||
**Consensus (both models, independently):** fresh-subprocess isolation per strategy with `ru_maxrss` for peak RSS; one deterministic Python-defined dataset and one schema for all strategies; timing bounded to the insert phase with setup/validation outside; sampling the naive baseline with declared linear scaling; excluding the sampled baseline from the memory verdict; integrity verification that hard-fails; labeling OFF/OFF as unsafe-for-production; and the headline conclusion that *one big transaction is most of the win* while *the memory winner is the streaming single-transaction family* (both declared `one transaction + execute` their memory winner; the fused measurement shows that whole family tied within noise, with `wal_tuned_gen` taking it by 0.1–0.2 MB under the fused rule).
|
||||
|
||||
**Divergences and rulings** (simpler option preferred per the fusion brief):
|
||||
- *Beats-all candidate*: [BUILDER gpt-5.6-sol]'s set-based `INSERT…SELECT` **kept**; [ARCHITECT claude-fable-5]'s CLI `.import` **discarded** — set-based is simpler (no external binary, CSV staging, or skip logic) and empirically faster; the CLI route also strains the "stdlib only" spirit.
|
||||
- *Speed winner dispute*: [ARCHITECT] declared `max_tuned_gen`, [BUILDER] declared OFF/OFF `INSERT-SELECT`. Settled by running both on the identical fused schema: **[BUILDER]'s pick wins** (0.17 s vs 0.37 s).
|
||||
- *Schema & data*: [ARCHITECT]'s 3-column table **kept** over [BUILDER]'s 6-column `events` table — simpler, same storage classes exercised.
|
||||
- *Repeats & trial ordering*: [ARCHITECT]'s single run in fixed order **kept** over [BUILDER]'s median-of-2 with seeded shuffle — simpler, and measured run-to-run spread (<0.05 s) is far below decision margins.
|
||||
- *Verification*: [ARCHITECT]'s count+checksum gate **kept as the base**, extended with a slim length-digest + spot-row check distilled from [BUILDER]'s 11-field digest — the minimum needed to prove the SQL-generated rows are byte-identical; [BUILDER]'s full digest and `PRAGMA quick_check` **discarded** as redundant for this schema.
|
||||
- *Cache tuning*: [ARCHITECT]'s 64 MB `cache_size` **discarded** in favor of [BUILDER]'s bounded-cache stance — simpler, and ARCHITECT's own data showed +50 MB RSS for no speed benefit.
|
||||
- *WAL checkpoint in timed region*: [BUILDER]'s control **adopted** — not a simplification but a fairness correction; without it WAL under-reports its true cost.
|
||||
- *Baseline sample size*: [ARCHITECT]'s 20k×50 **kept** over [BUILDER]'s 10k×100 — equal complexity, smaller extrapolation factor.
|
||||
- *Also discarded*: [BUILDER]'s multi-row `VALUES` strategy (complexity without a podium finish — dominated on both ends), [BUILDER]'s Windows `PeakWorkingSetSize` branch (POSIX `ru_maxrss` is simpler; noted as an easy portability extension), and [ARCHITECT]'s `executemany_list`… no — that one was **kept**: three extra lines that buy the single most vivid RAM lesson in the table.
|
||||
|
||||
---
|
||||
|
||||
## Appendix — the fused script (verbatim, as executed)
|
||||
|
||||
```python
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = []
|
||||
# ///
|
||||
"""
|
||||
FUSED SQLite bulk-insert benchmark (claude-fable-5 + gpt-5.6-sol).
|
||||
|
||||
1,000,000 rows. Speed + peak RAM. Stdlib only. Run:
|
||||
uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py
|
||||
|
||||
Architecture (from ARCHITECT/claude-fable-5): the script re-execs ITSELF
|
||||
once per strategy (`--worker NAME`), so every strategy runs in a fresh
|
||||
process and peak RSS readings cannot pollute each other. A no-op
|
||||
calibration worker measures bare-interpreter RSS for a delta column.
|
||||
|
||||
Beyond-the-list candidate (from BUILDER/gpt-5.6-sol): set-based
|
||||
INSERT ... SELECT over a recursive CTE that generates the IDENTICAL rows
|
||||
inside SQLite, verified by digest + spot-check against the Python rows.
|
||||
WAL runs pay for wal_checkpoint(TRUNCATE) inside the timed region.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
N = 1_000_000
|
||||
NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/sample) in report
|
||||
SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)"
|
||||
INSERT = "INSERT INTO t VALUES (?,?,?)"
|
||||
|
||||
SET_BASED_SQL = """
|
||||
WITH RECURSIVE seq(i) AS (
|
||||
SELECT 0
|
||||
UNION ALL
|
||||
SELECT i + 1 FROM seq WHERE i + 1 < ?
|
||||
)
|
||||
INSERT INTO t SELECT i, i * 0.5, printf('payload-%012d', i) FROM seq
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- data ----
|
||||
def make_row(i):
|
||||
return (i, i * 0.5, f"payload-{i:012d}")
|
||||
|
||||
|
||||
def rows(n):
|
||||
"""Identical deterministic data for every strategy."""
|
||||
for i in range(n):
|
||||
yield make_row(i)
|
||||
|
||||
|
||||
# ---------------------------------------------------------- strategies ----
|
||||
def s_naive_autocommit(conn, n):
|
||||
conn.isolation_level = None # every INSERT commits: journal sync per row
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
|
||||
|
||||
def s_one_big_txn_loop(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_list(conn, n):
|
||||
data = list(rows(n)) # deliberately materialized: shows RAM cost
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_gen(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_wal_tuned_gen(conn, n):
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
# Fairness (BUILDER): WAL must not finish with 1M rows still in the -wal
|
||||
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
|
||||
|
||||
def s_max_tuned_gen(conn, n):
|
||||
"""Python-bound speed candidate: one txn + executemany(generator) +
|
||||
journaling/sync disabled for the load. Rebuildable staging only."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_set_based(conn, n):
|
||||
"""Beyond-the-list candidate (BUILDER): SQLite generates the identical
|
||||
rows itself; zero Python->SQLite binding crossings. OFF/OFF pragmas so
|
||||
it is compared at the same durability tier as max_tuned_gen."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.execute(SET_BASED_SQL, (n,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
STRATEGIES = {
|
||||
"naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N),
|
||||
"one_big_txn_loop": (s_one_big_txn_loop, N),
|
||||
"executemany_list": (s_executemany_list, N),
|
||||
"executemany_gen": (s_executemany_gen, N),
|
||||
"wal_tuned_gen": (s_wal_tuned_gen, N),
|
||||
"max_tuned_gen": (s_max_tuned_gen, N),
|
||||
"set_based_ctes": (s_set_based, N),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- worker ----
|
||||
def rss_bytes():
|
||||
v = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return v if sys.platform == "darwin" else v * 1024 # KB on Linux
|
||||
|
||||
|
||||
def verify(dbpath, n):
|
||||
"""Integrity gate (fused): count + checksum for everyone [ARCHITECT],
|
||||
plus length-digest and spot-row equality so the SQL-generated data is
|
||||
provably identical to the Python data [BUILDER]."""
|
||||
conn = sqlite3.connect(dbpath)
|
||||
cnt, chk, clen = conn.execute(
|
||||
"SELECT count(*), sum(a), sum(length(c)) FROM t").fetchone()
|
||||
ok = (cnt == n and chk == n * (n - 1) // 2 and clen == 20 * n)
|
||||
for i in (0, 1, n // 2, n - 1):
|
||||
got = conn.execute("SELECT a, b, c FROM t WHERE a=?", (i,)).fetchone()
|
||||
ok = ok and got is not None and tuple(got) == make_row(i)
|
||||
conn.close()
|
||||
return ok
|
||||
|
||||
|
||||
def worker(name):
|
||||
if name == "calibration": # interpreter + sqlite3 import baseline
|
||||
print(json.dumps({"name": name, "rows": 0, "seconds": 0.0,
|
||||
"peak_rss": rss_bytes(), "ok": True}))
|
||||
return
|
||||
fn, n = STRATEGIES[name]
|
||||
dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir())
|
||||
try:
|
||||
conn = sqlite3.connect(dbpath)
|
||||
conn.execute(SCHEMA)
|
||||
t0 = time.perf_counter() # timer: generation + bind + insert + commit
|
||||
fn(conn, n)
|
||||
dt = time.perf_counter() - t0
|
||||
conn.close()
|
||||
ok = verify(dbpath, n) # validation outside the timer
|
||||
print(json.dumps({"name": name, "rows": n, "seconds": dt,
|
||||
"peak_rss": rss_bytes(), "ok": ok}))
|
||||
finally:
|
||||
for p in (dbpath, dbpath + "-wal", dbpath + "-shm"):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- orchestrator --
|
||||
def spawn(name):
|
||||
out = subprocess.run([sys.executable, os.path.abspath(__file__),
|
||||
"--worker", name],
|
||||
capture_output=True, text=True, check=True)
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def main():
|
||||
t_total = time.perf_counter()
|
||||
print(f"SQLite bulk insert benchmark | N={N:,} rows | "
|
||||
f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} "
|
||||
f"{sys.platform}\n")
|
||||
|
||||
base = spawn("calibration")
|
||||
results = []
|
||||
for name in STRATEGIES: # fixed, deterministic order
|
||||
r = spawn(name)
|
||||
assert r["ok"], f"integrity check FAILED for {name}"
|
||||
scale = N / r["rows"]
|
||||
r["scaled"] = r["seconds"] * scale
|
||||
r["sampled"] = scale > 1.0
|
||||
results.append(r)
|
||||
print(f" [done] {name}: {r['seconds']:.3f}s "
|
||||
f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})")
|
||||
|
||||
naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit")
|
||||
print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}"
|
||||
f"{'Peak RSS':>12}{'RSS -base':>12}")
|
||||
print("-" * 70)
|
||||
for r in sorted(results, key=lambda r: r["scaled"]):
|
||||
mark = "*" if r["sampled"] else " "
|
||||
print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}"
|
||||
f"{naive / r['scaled']:>9.1f}x"
|
||||
f"{r['peak_rss'] / 1e6:>10.1f}MB"
|
||||
f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB")
|
||||
print("-" * 70)
|
||||
print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled "
|
||||
f"x{N // NAIVE_SAMPLE_N} | base RSS "
|
||||
f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)")
|
||||
print("note: max_tuned_gen and set_based_ctes run journal_mode=OFF/"
|
||||
"synchronous=OFF -> rebuildable staging loads only.")
|
||||
|
||||
fastest = min(results, key=lambda r: r["scaled"])
|
||||
full = [r for r in results if not r["sampled"]] # baseline excluded
|
||||
leanest = min(full, key=lambda r: (r["peak_rss"], r["scaled"]))
|
||||
print(f"\nSpeed Winner : {fastest['name']} "
|
||||
f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)")
|
||||
print(f"Memory Winner: {leanest['name']} "
|
||||
f"({leanest['peak_rss'] / 1e6:.1f}MB peak, "
|
||||
f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)")
|
||||
print(f"\ntotal benchmark wall time: {time.perf_counter() - t_total:.1f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--worker":
|
||||
worker(sys.argv[2])
|
||||
else:
|
||||
main()
|
||||
```
|
||||
@@ -0,0 +1,25 @@
|
||||
# SQLite Bulk Insert Benchmark Results
|
||||
|
||||
Fresh measurements: **1,000,000 target rows**, Python 3.12.13, SQLite 3.50.4, macOS-26.5.2-arm64-arm-64bit.
|
||||
|
||||
The timer includes row generation, binding, insertion, and commit. WAL timing also includes `wal_checkpoint(TRUNCATE)`. Each strategy ran in a fresh process against a fresh database.
|
||||
|
||||
| Strategy | Measured rows | Time 1M rows (s) | Speedup vs naive | Peak RAM (MB) | RSS over base (MB) |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| `set_based_ctes` | 1,000,000 | 0.1700 | 1147.98x | 26.051 | 4.014 |
|
||||
| `executemany_gen` | 1,000,000 | 0.3480 | 560.77x | 25.625 | 3.588 |
|
||||
| `max_tuned_gen` | 1,000,000 | 0.3484 | 560.15x | 25.641 | 3.604 |
|
||||
| `executemany_list` | 1,000,000 | 0.3761 | 518.88x | 228.737 | 206.701 |
|
||||
| `wal_tuned_gen` | 1,000,000 | 0.3918 | 498.07x | 25.903 | 3.867 |
|
||||
| `one_big_txn_loop` | 1,000,000 | 0.6071 | 321.41x | 25.854 | 3.817 |
|
||||
| `naive_autocommit` | 20,000 (sampled) | 195.1362* | 1.00x | 24.183 | 2.146 |
|
||||
|
||||
\* `naive_autocommit` ran 20,000 real rows; its measured time was scaled by 50. Its RAM reading is measured, not scaled, and it is excluded from the memory-winner decision.
|
||||
|
||||
`max_tuned_gen` and `set_based_ctes` use `journal_mode=OFF`/`synchronous=OFF`; they are for rebuildable staging loads only.
|
||||
|
||||
Bare-worker peak RSS calibration: **22.036 MB**.
|
||||
Total benchmark wall time: **6.857 s**.
|
||||
|
||||
**Speed Winner: `set_based_ctes`** — 0.1700 s, 1147.98x faster than naive.
|
||||
**Memory Winner: `executemany_gen`** — 25.625 MB peak RSS, +3.588 MB over the calibration worker.
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = []
|
||||
# ///
|
||||
"""
|
||||
SQLite bulk-insert benchmark: 1,000,000 rows, speed + peak RAM, per-strategy
|
||||
process isolation. Stdlib only. Run: uv run bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py
|
||||
|
||||
Design (ARCHITECT / anthropic-claude-fable-5):
|
||||
* Orchestrator re-execs THIS file as a subprocess per strategy
|
||||
(`--worker NAME`), so peak RSS readings never pollute each other.
|
||||
* Every strategy inserts IDENTICAL deterministic data into an IDENTICAL
|
||||
fresh schema, verified by count + checksum.
|
||||
* Timing covers the full insert phase (row generation + bind + commit),
|
||||
identical work for all strategies.
|
||||
* The naive autocommit baseline is fsync-bound (~50-200 rows/ms is
|
||||
impossible; it does one journal sync per row). It is SAMPLED at 20,000
|
||||
rows and honestly scaled x50; the table flags it with '*'.
|
||||
* Peak RAM = ru_maxrss of the worker process (normalized to bytes across
|
||||
macOS/Linux), plus a no-op calibration worker so a delta over
|
||||
interpreter baseline is reported.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
N = 1_000_000
|
||||
NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/NAIVE_SAMPLE_N) in report
|
||||
SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)"
|
||||
INSERT = "INSERT INTO t VALUES (?,?,?)"
|
||||
EXPECTED_SUM = lambda n: n * (n - 1) // 2 # sum of column a for n rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- data ----
|
||||
def rows(n):
|
||||
"""Identical deterministic data for every strategy."""
|
||||
for i in range(n):
|
||||
yield (i, i * 0.5, f"payload-{i:012d}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------- strategies ----
|
||||
def s_naive_autocommit(conn, n):
|
||||
conn.isolation_level = None # every INSERT commits (journal sync per row)
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
|
||||
|
||||
def s_one_big_txn_loop(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
for r in rows(n):
|
||||
conn.execute(INSERT, r)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_list(conn, n):
|
||||
data = list(rows(n)) # deliberately materialized: shows the RAM cost
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_gen(conn, n):
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_wal_tuned_gen(conn, n):
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("PRAGMA cache_size=-64000") # 64 MB page cache
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_max_tuned_gen(conn, n):
|
||||
"""Predicted overall speed winner within stdlib: one txn +
|
||||
executemany(generator) + journaling/sync disabled for the load."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("PRAGMA cache_size=-64000")
|
||||
conn.execute("PRAGMA temp_store=MEMORY")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
STRATEGIES = {
|
||||
"naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N),
|
||||
"one_big_txn_loop": (s_one_big_txn_loop, N),
|
||||
"executemany_list": (s_executemany_list, N),
|
||||
"executemany_gen": (s_executemany_gen, N),
|
||||
"wal_tuned_gen": (s_wal_tuned_gen, N),
|
||||
"max_tuned_gen": (s_max_tuned_gen, N),
|
||||
}
|
||||
|
||||
# Optional bonus (external `sqlite3` CLI, skipped gracefully if absent).
|
||||
# Same identical data, same isolation, timing includes writing the CSV.
|
||||
CLI_STRATEGY = "cli_dot_import"
|
||||
|
||||
|
||||
def run_cli_import(dbpath, n):
|
||||
import csv
|
||||
import shutil
|
||||
|
||||
exe = shutil.which("sqlite3")
|
||||
if not exe:
|
||||
return False
|
||||
csvpath = dbpath + ".csv"
|
||||
with open(csvpath, "w", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
for r in rows(n):
|
||||
w.writerow(r)
|
||||
subprocess.run(
|
||||
[exe, dbpath, SCHEMA + ";", ".mode csv", f".import {csvpath} t"],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
os.remove(csvpath)
|
||||
return True
|
||||
|
||||
|
||||
# --------------------------------------------------------------- worker ----
|
||||
def rss_bytes(who=resource.RUSAGE_SELF):
|
||||
v = resource.getrusage(who).ru_maxrss
|
||||
return v if sys.platform == "darwin" else v * 1024 # KB on Linux
|
||||
|
||||
|
||||
def worker(name):
|
||||
if name == "calibration": # no-op: interpreter + sqlite3 import baseline
|
||||
print(json.dumps({"name": name, "rows": 0, "seconds": 0.0,
|
||||
"peak_rss": rss_bytes(), "ok": True}))
|
||||
return
|
||||
|
||||
dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir())
|
||||
try:
|
||||
if name == CLI_STRATEGY:
|
||||
t0 = time.perf_counter()
|
||||
ok = run_cli_import(dbpath, N)
|
||||
dt = time.perf_counter() - t0
|
||||
if not ok:
|
||||
print(json.dumps({"name": name, "skipped": True}))
|
||||
return
|
||||
n = N
|
||||
else:
|
||||
fn, n = STRATEGIES[name]
|
||||
conn = sqlite3.connect(dbpath)
|
||||
conn.execute(SCHEMA)
|
||||
t0 = time.perf_counter()
|
||||
fn(conn, n)
|
||||
dt = time.perf_counter() - t0
|
||||
conn.close()
|
||||
|
||||
vconn = sqlite3.connect(dbpath)
|
||||
cnt, chk = vconn.execute("SELECT count(*), sum(a) FROM t").fetchone()
|
||||
vconn.close()
|
||||
peak = max(rss_bytes(), rss_bytes(resource.RUSAGE_CHILDREN))
|
||||
print(json.dumps({
|
||||
"name": name, "rows": cnt, "seconds": dt, "peak_rss": peak,
|
||||
"ok": cnt == n and chk == EXPECTED_SUM(n),
|
||||
}))
|
||||
finally:
|
||||
for p in (dbpath, dbpath + "-wal", dbpath + "-shm", dbpath + ".csv"):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- orchestrator --
|
||||
def spawn(name):
|
||||
out = subprocess.run([sys.executable, os.path.abspath(__file__),
|
||||
"--worker", name],
|
||||
capture_output=True, text=True, check=True)
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def main():
|
||||
print(f"SQLite bulk insert benchmark | N={N:,} rows | "
|
||||
f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} "
|
||||
f"{sys.platform}\n")
|
||||
|
||||
base = spawn("calibration")
|
||||
results = []
|
||||
order = list(STRATEGIES) + [CLI_STRATEGY]
|
||||
for name in order:
|
||||
r = spawn(name)
|
||||
if r.get("skipped"):
|
||||
print(f" [skip] {name} (sqlite3 CLI not on PATH)")
|
||||
continue
|
||||
assert r["ok"], f"integrity check FAILED for {name}"
|
||||
scale = N / r["rows"]
|
||||
r["scaled"] = r["seconds"] * scale
|
||||
r["sampled"] = scale > 1.0
|
||||
results.append(r)
|
||||
print(f" [done] {name}: {r['seconds']:.3f}s "
|
||||
f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})")
|
||||
|
||||
naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit")
|
||||
print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}"
|
||||
f"{'Peak RSS':>12}{'RSS -base':>12}")
|
||||
print("-" * 70)
|
||||
for r in sorted(results, key=lambda r: r["scaled"]):
|
||||
mark = "*" if r["sampled"] else " "
|
||||
print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}"
|
||||
f"{naive / r['scaled']:>9.1f}x"
|
||||
f"{r['peak_rss'] / 1e6:>10.1f}MB"
|
||||
f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB")
|
||||
print("-" * 70)
|
||||
print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled "
|
||||
f"x{N // NAIVE_SAMPLE_N} | base RSS "
|
||||
f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)")
|
||||
|
||||
fastest = min(results, key=lambda r: r["scaled"])
|
||||
# memory winner among full-speed strategies (exclude the naive baseline)
|
||||
fast = [r for r in results if not r["sampled"]]
|
||||
leanest = min(fast, key=lambda r: (r["peak_rss"], r["scaled"]))
|
||||
print(f"\nSPEED WINNER : {fastest['name']} "
|
||||
f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)")
|
||||
print(f"MEMORY WINNER: {leanest['name']} "
|
||||
f"({leanest['peak_rss'] / 1e6:.1f}MB peak, "
|
||||
f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--worker":
|
||||
worker(sys.argv[2])
|
||||
else:
|
||||
main()
|
||||
@@ -0,0 +1,393 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = []
|
||||
# ///
|
||||
"""Benchmark seven SQLite bulk-insert strategies on speed and peak RAM.
|
||||
|
||||
Run with:
|
||||
uv run bench.py
|
||||
|
||||
The orchestrator re-executes this file once per strategy. Each worker uses a
|
||||
fresh process and a fresh SQLite database, so process peak-RSS measurements
|
||||
cannot leak from one strategy into another. The run writes RESULTS.md next to
|
||||
this script from fresh measurements every time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import resource
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any, Callable, Iterator
|
||||
|
||||
N = 1_000_000
|
||||
NAIVE_SAMPLE_N = 20_000
|
||||
SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)"
|
||||
INSERT = "INSERT INTO t VALUES (?, ?, ?)"
|
||||
RESULTS_PATH = Path(__file__).resolve().with_name("RESULTS.md")
|
||||
|
||||
SET_BASED_SQL = """
|
||||
WITH RECURSIVE seq(i) AS (
|
||||
SELECT 0
|
||||
UNION ALL
|
||||
SELECT i + 1 FROM seq WHERE i + 1 < ?
|
||||
)
|
||||
INSERT INTO t
|
||||
SELECT i, i * 0.5, printf('payload-%012d', i)
|
||||
FROM seq
|
||||
"""
|
||||
|
||||
Row = tuple[int, float, str]
|
||||
StrategyFunction = Callable[[sqlite3.Connection, int], None]
|
||||
|
||||
|
||||
def make_row(i: int) -> Row:
|
||||
"""Return the sole canonical definition of one benchmark row."""
|
||||
return (i, i * 0.5, f"payload-{i:012d}")
|
||||
|
||||
|
||||
def rows(n: int) -> Iterator[Row]:
|
||||
"""Generate identical deterministic data for every Python-bound strategy."""
|
||||
for i in range(n):
|
||||
yield make_row(i)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- strategies
|
||||
|
||||
def s_naive_autocommit(conn: sqlite3.Connection, n: int) -> None:
|
||||
"""Baseline: every INSERT is its own durable transaction."""
|
||||
conn.isolation_level = None
|
||||
for row in rows(n):
|
||||
conn.execute(INSERT, row)
|
||||
|
||||
|
||||
def s_one_big_txn_loop(conn: sqlite3.Connection, n: int) -> None:
|
||||
conn.execute("BEGIN")
|
||||
for row in rows(n):
|
||||
conn.execute(INSERT, row)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_list(conn: sqlite3.Connection, n: int) -> None:
|
||||
# Deliberately materialized so the benchmark measures the classic RAM cost.
|
||||
data = list(rows(n))
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, data)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_executemany_gen(conn: sqlite3.Connection, n: int) -> None:
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_wal_tuned_gen(conn: sqlite3.Connection, n: int) -> None:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
# Include write-back cost instead of ending with the rows in a WAL sidecar.
|
||||
busy, _wal_pages, _checkpointed = conn.execute(
|
||||
"PRAGMA wal_checkpoint(TRUNCATE)"
|
||||
).fetchone()
|
||||
if busy:
|
||||
raise RuntimeError("WAL checkpoint was unexpectedly busy")
|
||||
|
||||
|
||||
def s_max_tuned_gen(conn: sqlite3.Connection, n: int) -> None:
|
||||
"""Maximum Python-bound speed; safe only for a rebuildable staging DB."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.executemany(INSERT, rows(n))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def s_set_based_ctes(conn: sqlite3.Connection, n: int) -> None:
|
||||
"""Generate identical rows inside SQLite, avoiding per-row Python binds."""
|
||||
conn.execute("PRAGMA journal_mode=OFF")
|
||||
conn.execute("PRAGMA synchronous=OFF")
|
||||
conn.execute("BEGIN")
|
||||
conn.execute(SET_BASED_SQL, (n,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
STRATEGIES: dict[str, tuple[StrategyFunction, int]] = {
|
||||
"naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N),
|
||||
"one_big_txn_loop": (s_one_big_txn_loop, N),
|
||||
"executemany_list": (s_executemany_list, N),
|
||||
"executemany_gen": (s_executemany_gen, N),
|
||||
"wal_tuned_gen": (s_wal_tuned_gen, N),
|
||||
"max_tuned_gen": (s_max_tuned_gen, N),
|
||||
"set_based_ctes": (s_set_based_ctes, N),
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- workers
|
||||
|
||||
def rss_bytes() -> int:
|
||||
"""Return this process's measured high-water resident set size in bytes."""
|
||||
ru_maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return int(ru_maxrss if sys.platform == "darwin" else ru_maxrss * 1024)
|
||||
|
||||
|
||||
def verify(db_path: Path, n: int) -> bool:
|
||||
"""Prove row count, checksums, lengths, and representative rows match."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
count, checksum, text_length = conn.execute(
|
||||
"SELECT count(*), sum(a), sum(length(c)) FROM t"
|
||||
).fetchone()
|
||||
valid = (
|
||||
count == n
|
||||
and checksum == n * (n - 1) // 2
|
||||
and text_length == 20 * n
|
||||
)
|
||||
for i in (0, 1, n // 2, n - 1):
|
||||
actual = conn.execute(
|
||||
"SELECT a, b, c FROM t WHERE a = ?", (i,)
|
||||
).fetchone()
|
||||
valid = valid and actual is not None and tuple(actual) == make_row(i)
|
||||
return bool(valid)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def worker(name: str) -> None:
|
||||
if name == "calibration":
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"name": name,
|
||||
"rows": 0,
|
||||
"seconds": 0.0,
|
||||
"peak_rss": rss_bytes(),
|
||||
"ok": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
function, n = STRATEGIES[name]
|
||||
with tempfile.TemporaryDirectory(prefix=f"sqlite-bulk-{name}-") as temp_dir:
|
||||
db_path = Path(temp_dir) / "benchmark.sqlite3"
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(SCHEMA)
|
||||
started = time.perf_counter()
|
||||
function(conn, n)
|
||||
seconds = time.perf_counter() - started
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
valid = verify(db_path, n)
|
||||
result = {
|
||||
"name": name,
|
||||
"rows": n,
|
||||
"seconds": seconds,
|
||||
"peak_rss": rss_bytes(),
|
||||
"ok": valid,
|
||||
}
|
||||
print(json.dumps(result, separators=(",", ":")))
|
||||
|
||||
|
||||
def spawn(name: str) -> dict[str, Any]:
|
||||
completed = subprocess.run(
|
||||
[sys.executable, os.path.abspath(__file__), "--worker", name],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=220,
|
||||
)
|
||||
output_lines = completed.stdout.strip().splitlines()
|
||||
if not output_lines:
|
||||
raise RuntimeError(f"worker {name} produced no result")
|
||||
return json.loads(output_lines[-1])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reporting
|
||||
|
||||
def measured_rows(result: dict[str, Any]) -> str:
|
||||
suffix = " (sampled)" if result["sampled"] else ""
|
||||
return f"{result['rows']:,}{suffix}"
|
||||
|
||||
|
||||
def build_results_markdown(
|
||||
results: list[dict[str, Any]],
|
||||
base_rss: int,
|
||||
total_wall_seconds: float,
|
||||
) -> tuple[str, dict[str, Any], dict[str, Any]]:
|
||||
baseline = next(r for r in results if r["name"] == "naive_autocommit")
|
||||
baseline_time = float(baseline["scaled_seconds"])
|
||||
ordered = sorted(results, key=lambda result: result["scaled_seconds"])
|
||||
fastest = ordered[0]
|
||||
full_runs = [result for result in results if not result["sampled"]]
|
||||
leanest = min(
|
||||
full_runs,
|
||||
key=lambda result: (result["peak_rss"], result["scaled_seconds"]),
|
||||
)
|
||||
|
||||
lines = [
|
||||
"# SQLite Bulk Insert Benchmark Results",
|
||||
"",
|
||||
(
|
||||
f"Fresh measurements: **{N:,} target rows**, Python "
|
||||
f"{platform.python_version()}, SQLite {sqlite3.sqlite_version}, "
|
||||
f"{platform.platform()}."
|
||||
),
|
||||
"",
|
||||
(
|
||||
"The timer includes row generation, binding, insertion, and commit. "
|
||||
"WAL timing also includes `wal_checkpoint(TRUNCATE)`. Each strategy "
|
||||
"ran in a fresh process against a fresh database."
|
||||
),
|
||||
"",
|
||||
"| Strategy | Measured rows | Time 1M rows (s) | Speedup vs naive | Peak RAM (MB) | RSS over base (MB) |",
|
||||
"|---|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
|
||||
for result in ordered:
|
||||
scaled = float(result["scaled_seconds"])
|
||||
peak_mb = int(result["peak_rss"]) / 1_000_000
|
||||
delta_mb = (int(result["peak_rss"]) - base_rss) / 1_000_000
|
||||
time_mark = "*" if result["sampled"] else ""
|
||||
lines.append(
|
||||
f"| `{result['name']}` | {measured_rows(result)} | "
|
||||
f"{scaled:.4f}{time_mark} | {baseline_time / scaled:.2f}x | "
|
||||
f"{peak_mb:.3f} | {delta_mb:.3f} |"
|
||||
)
|
||||
|
||||
fastest_time = float(fastest["scaled_seconds"])
|
||||
leanest_peak_mb = int(leanest["peak_rss"]) / 1_000_000
|
||||
leanest_delta_mb = (int(leanest["peak_rss"]) - base_rss) / 1_000_000
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
(
|
||||
f"\\* `naive_autocommit` ran {NAIVE_SAMPLE_N:,} real rows; its "
|
||||
f"measured time was scaled by {N // NAIVE_SAMPLE_N}. Its RAM "
|
||||
"reading is measured, not scaled, and it is excluded from the "
|
||||
"memory-winner decision."
|
||||
),
|
||||
"",
|
||||
(
|
||||
"`max_tuned_gen` and `set_based_ctes` use "
|
||||
"`journal_mode=OFF`/`synchronous=OFF`; they are for rebuildable "
|
||||
"staging loads only."
|
||||
),
|
||||
"",
|
||||
f"Bare-worker peak RSS calibration: **{base_rss / 1_000_000:.3f} MB**.",
|
||||
f"Total benchmark wall time: **{total_wall_seconds:.3f} s**.",
|
||||
"",
|
||||
(
|
||||
f"**Speed Winner: `{fastest['name']}`** — {fastest_time:.4f} s, "
|
||||
f"{baseline_time / fastest_time:.2f}x faster than naive."
|
||||
),
|
||||
(
|
||||
f"**Memory Winner: `{leanest['name']}`** — "
|
||||
f"{leanest_peak_mb:.3f} MB peak RSS, "
|
||||
f"{leanest_delta_mb:+.3f} MB over the calibration worker."
|
||||
),
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines), fastest, leanest
|
||||
|
||||
|
||||
def definition_of_done_gate(
|
||||
results: list[dict[str, Any]],
|
||||
markdown: str,
|
||||
fastest: dict[str, Any],
|
||||
leanest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Hard-fail unless fresh measurements satisfy the benchmark contract."""
|
||||
expected = set(STRATEGIES)
|
||||
actual = {str(result["name"]) for result in results}
|
||||
if actual != expected:
|
||||
raise AssertionError(f"strategy mismatch: expected {expected}, got {actual}")
|
||||
if not all(result["ok"] for result in results):
|
||||
failed = [result["name"] for result in results if not result["ok"]]
|
||||
raise AssertionError(f"integrity verification failed: {failed}")
|
||||
if not all(
|
||||
float(result["seconds"]) > 0 and int(result["peak_rss"]) > 0
|
||||
for result in results
|
||||
):
|
||||
raise AssertionError("every strategy must have measured positive time and RSS")
|
||||
for result in results:
|
||||
required_rows = NAIVE_SAMPLE_N if result["name"] == "naive_autocommit" else N
|
||||
if result["rows"] != required_rows:
|
||||
raise AssertionError(f"wrong row count for {result['name']}")
|
||||
if result["name"] not in markdown:
|
||||
raise AssertionError(f"RESULTS.md omitted {result['name']}")
|
||||
baseline = next(r for r in results if r["name"] == "naive_autocommit")
|
||||
speedup = float(baseline["scaled_seconds"]) / float(fastest["scaled_seconds"])
|
||||
if speedup < 5.0:
|
||||
raise AssertionError(f"fastest strategy achieved only {speedup:.2f}x")
|
||||
if fastest["name"] not in markdown or leanest["name"] not in markdown:
|
||||
raise AssertionError("winner declaration missing from RESULTS.md")
|
||||
if "Speed Winner:" not in markdown or "Memory Winner:" not in markdown:
|
||||
raise AssertionError("RESULTS.md lacks winner lines")
|
||||
if leanest["name"] == "naive_autocommit":
|
||||
raise AssertionError("sampled baseline cannot be Memory Winner")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
total_started = time.perf_counter()
|
||||
print(
|
||||
f"SQLite bulk insert benchmark | N={N:,} rows | "
|
||||
f"Python {platform.python_version()} | SQLite {sqlite3.sqlite_version}"
|
||||
)
|
||||
print("Each strategy runs in a fresh isolated worker process.\n")
|
||||
|
||||
calibration = spawn("calibration")
|
||||
results: list[dict[str, Any]] = []
|
||||
for name in STRATEGIES:
|
||||
result = spawn(name)
|
||||
if not result["ok"]:
|
||||
raise RuntimeError(f"integrity check failed for {name}")
|
||||
scale = N / int(result["rows"])
|
||||
result["scaled_seconds"] = float(result["seconds"]) * scale
|
||||
result["sampled"] = scale != 1.0
|
||||
results.append(result)
|
||||
sample_note = " sampled" if result["sampled"] else ""
|
||||
print(
|
||||
f"[done] {name}: {result['seconds']:.4f}s, "
|
||||
f"{result['rows']:,} rows{sample_note}, "
|
||||
f"peak {result['peak_rss'] / 1_000_000:.3f} MB"
|
||||
)
|
||||
|
||||
total_wall_seconds = time.perf_counter() - total_started
|
||||
markdown, fastest, leanest = build_results_markdown(
|
||||
results, int(calibration["peak_rss"]), total_wall_seconds
|
||||
)
|
||||
definition_of_done_gate(results, markdown, fastest, leanest)
|
||||
RESULTS_PATH.write_text(markdown, encoding="utf-8")
|
||||
|
||||
print(f"\nWrote {RESULTS_PATH}")
|
||||
print(
|
||||
f"Speed Winner: {fastest['name']} "
|
||||
f"({fastest['scaled_seconds']:.4f}s measured/scaled to 1M)"
|
||||
)
|
||||
print(
|
||||
f"Memory Winner: {leanest['name']} "
|
||||
f"({leanest['peak_rss'] / 1_000_000:.3f} MB measured peak RSS)"
|
||||
)
|
||||
print("Definition-of-Done gate: PASS")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--worker":
|
||||
worker(sys.argv[2])
|
||||
elif len(sys.argv) == 1:
|
||||
main()
|
||||
else:
|
||||
raise SystemExit("usage: uv run bench.py [--worker STRATEGY]")
|
||||
+631
@@ -0,0 +1,631 @@
|
||||
#!/usr/bin/env -S uv run
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# ///
|
||||
"""SQLite bulk-insert benchmark: elapsed time and process peak RSS.
|
||||
|
||||
Run:
|
||||
uv run sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py
|
||||
|
||||
Every timed trial executes in a fresh child process against a fresh database.
|
||||
The deliberately slow autocommit case uses a prefix sample and reports a
|
||||
clearly marked linear projection to one million rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Iterator, Sequence
|
||||
|
||||
IDENTITY = "BUILDER-openai-gpt-5.6-sol"
|
||||
DEFAULT_ROWS = 1_000_000
|
||||
DEFAULT_BASELINE_ROWS = 10_000
|
||||
DEFAULT_REPEATS = 2
|
||||
EXECUTEMANY_BATCH_ROWS = 10_000
|
||||
MULTI_VALUES_TARGET_ROWS = 500
|
||||
COLUMN_COUNT = 6
|
||||
|
||||
INSERT_ONE = """
|
||||
INSERT INTO events
|
||||
(id, account_id, created_at, amount_cents, status, payload)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
account_id INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
amount_cents INTEGER NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('new', 'paid', 'sent', 'void')),
|
||||
payload TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
|
||||
SET_BASED_INSERT = """
|
||||
WITH RECURSIVE
|
||||
seq(i) AS (
|
||||
VALUES(0)
|
||||
UNION ALL
|
||||
SELECT i + 1 FROM seq WHERE i + 1 < ?
|
||||
),
|
||||
generated(i, mix) AS (
|
||||
SELECT i, (i * 1103515245 + 12345) % 2147483648
|
||||
FROM seq
|
||||
)
|
||||
INSERT INTO events
|
||||
(id, account_id, created_at, amount_cents, status, payload)
|
||||
SELECT
|
||||
i,
|
||||
mix % 100003,
|
||||
1700000000 + i,
|
||||
(mix % 2000001) - 1000000,
|
||||
CASE i % 4
|
||||
WHEN 0 THEN 'new'
|
||||
WHEN 1 THEN 'paid'
|
||||
WHEN 2 THEN 'sent'
|
||||
ELSE 'void'
|
||||
END,
|
||||
printf(
|
||||
'event-%010d-%010x-abcdefghijklmnopqrstuvwxyz0123456789',
|
||||
i,
|
||||
mix
|
||||
)
|
||||
FROM generated
|
||||
"""
|
||||
|
||||
DIGEST_SQL = """
|
||||
SELECT
|
||||
count(*),
|
||||
sum(id),
|
||||
sum(account_id),
|
||||
sum(created_at),
|
||||
sum(amount_cents),
|
||||
sum(length(status)),
|
||||
sum(length(payload)),
|
||||
min(id),
|
||||
max(id),
|
||||
min(payload),
|
||||
max(payload)
|
||||
FROM events
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Strategy:
|
||||
key: str
|
||||
label: str
|
||||
journal_mode: str
|
||||
synchronous: str
|
||||
implementation: str
|
||||
durability: str
|
||||
sampled: bool = False
|
||||
|
||||
|
||||
STRATEGIES: tuple[Strategy, ...] = (
|
||||
Strategy(
|
||||
"naive_autocommit",
|
||||
"naive autocommit",
|
||||
"DELETE",
|
||||
"FULL",
|
||||
"naive",
|
||||
"durable",
|
||||
sampled=True,
|
||||
),
|
||||
Strategy(
|
||||
"one_big_transaction",
|
||||
"one transaction + execute",
|
||||
"DELETE",
|
||||
"FULL",
|
||||
"one_transaction",
|
||||
"durable",
|
||||
),
|
||||
Strategy(
|
||||
"executemany_streamed",
|
||||
"streamed executemany batches",
|
||||
"DELETE",
|
||||
"FULL",
|
||||
"executemany",
|
||||
"durable",
|
||||
),
|
||||
Strategy(
|
||||
"wal_multi_values",
|
||||
"WAL/NORMAL + multi-VALUES",
|
||||
"WAL",
|
||||
"NORMAL",
|
||||
"multi_values",
|
||||
"consistent; latest commit power-loss risk",
|
||||
),
|
||||
Strategy(
|
||||
"off_multi_values",
|
||||
"OFF/OFF + multi-VALUES",
|
||||
"OFF",
|
||||
"OFF",
|
||||
"multi_values",
|
||||
"rebuildable staging only",
|
||||
),
|
||||
Strategy(
|
||||
"wal_set_based",
|
||||
"WAL/NORMAL + INSERT-SELECT",
|
||||
"WAL",
|
||||
"NORMAL",
|
||||
"set_based",
|
||||
"consistent; latest commit power-loss risk",
|
||||
),
|
||||
Strategy(
|
||||
"off_set_based",
|
||||
"OFF/OFF + INSERT-SELECT",
|
||||
"OFF",
|
||||
"OFF",
|
||||
"set_based",
|
||||
"rebuildable staging only",
|
||||
),
|
||||
)
|
||||
STRATEGY_BY_KEY = {strategy.key: strategy for strategy in STRATEGIES}
|
||||
|
||||
|
||||
Row = tuple[int, int, int, int, str, str]
|
||||
Digest = tuple[int | str, ...]
|
||||
|
||||
|
||||
def make_row(i: int) -> Row:
|
||||
"""The sole Python definition of benchmark data."""
|
||||
mix = (i * 1103515245 + 12345) % 2147483648
|
||||
status = ("new", "paid", "sent", "void")[i % 4]
|
||||
return (
|
||||
i,
|
||||
mix % 100003,
|
||||
1700000000 + i,
|
||||
(mix % 2000001) - 1000000,
|
||||
status,
|
||||
f"event-{i:010d}-{mix:010x}-abcdefghijklmnopqrstuvwxyz0123456789",
|
||||
)
|
||||
|
||||
|
||||
def generated_rows(start: int, stop: int) -> Iterator[Row]:
|
||||
for i in range(start, stop):
|
||||
yield make_row(i)
|
||||
|
||||
|
||||
def expected_digest(row_count: int) -> Digest:
|
||||
count = 0
|
||||
id_sum = account_sum = created_sum = amount_sum = 0
|
||||
status_length_sum = payload_length_sum = 0
|
||||
min_payload: str | None = None
|
||||
max_payload: str | None = None
|
||||
for row in generated_rows(0, row_count):
|
||||
row_id, account_id, created_at, amount, status, payload = row
|
||||
count += 1
|
||||
id_sum += row_id
|
||||
account_sum += account_id
|
||||
created_sum += created_at
|
||||
amount_sum += amount
|
||||
status_length_sum += len(status)
|
||||
payload_length_sum += len(payload)
|
||||
if min_payload is None or payload < min_payload:
|
||||
min_payload = payload
|
||||
if max_payload is None or payload > max_payload:
|
||||
max_payload = payload
|
||||
return (
|
||||
count,
|
||||
id_sum,
|
||||
account_sum,
|
||||
created_sum,
|
||||
amount_sum,
|
||||
status_length_sum,
|
||||
payload_length_sum,
|
||||
0 if row_count else None,
|
||||
row_count - 1 if row_count else None,
|
||||
min_payload,
|
||||
max_payload,
|
||||
)
|
||||
|
||||
|
||||
def peak_rss_bytes() -> int:
|
||||
if os.name == "nt":
|
||||
# ctypes is standard library. PeakWorkingSetSize is the Windows
|
||||
# equivalent of process peak RSS.
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
class ProcessMemoryCounters(ctypes.Structure):
|
||||
_fields_ = (
|
||||
("cb", wintypes.DWORD),
|
||||
("PageFaultCount", wintypes.DWORD),
|
||||
("PeakWorkingSetSize", ctypes.c_size_t),
|
||||
("WorkingSetSize", ctypes.c_size_t),
|
||||
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("PagefileUsage", ctypes.c_size_t),
|
||||
("PeakPagefileUsage", ctypes.c_size_t),
|
||||
)
|
||||
|
||||
counters = ProcessMemoryCounters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
get_current_process = ctypes.windll.kernel32.GetCurrentProcess
|
||||
get_current_process.restype = wintypes.HANDLE
|
||||
ok = ctypes.windll.psapi.GetProcessMemoryInfo(
|
||||
get_current_process(), ctypes.byref(counters), counters.cb
|
||||
)
|
||||
if not ok:
|
||||
raise ctypes.WinError()
|
||||
return int(counters.PeakWorkingSetSize)
|
||||
|
||||
import resource
|
||||
|
||||
raw = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
# macOS reports bytes; Linux and the BSDs report KiB.
|
||||
if sys.platform == "darwin":
|
||||
return int(raw)
|
||||
return int(raw) * 1024
|
||||
|
||||
|
||||
def setup_connection(db_path: Path, strategy: Strategy) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(db_path, isolation_level=None, timeout=60.0)
|
||||
connection.execute("PRAGMA page_size=4096")
|
||||
actual_journal = connection.execute(
|
||||
f"PRAGMA journal_mode={strategy.journal_mode}"
|
||||
).fetchone()[0]
|
||||
if str(actual_journal).upper() != strategy.journal_mode:
|
||||
raise RuntimeError(
|
||||
f"requested journal_mode={strategy.journal_mode}, got {actual_journal}"
|
||||
)
|
||||
connection.execute(f"PRAGMA synchronous={strategy.synchronous}")
|
||||
|
||||
# Common controls: bounded cache, no mmap RSS ambiguity, disk-backed temp
|
||||
# structures, and spill enabled. These are identical for every strategy.
|
||||
connection.execute("PRAGMA cache_size=-2048")
|
||||
connection.execute("PRAGMA cache_spill=ON")
|
||||
connection.execute("PRAGMA temp_store=FILE")
|
||||
connection.execute("PRAGMA mmap_size=0")
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute(SCHEMA)
|
||||
return connection
|
||||
|
||||
|
||||
def finish_commit(connection: sqlite3.Connection, strategy: Strategy) -> None:
|
||||
connection.commit()
|
||||
# Include WAL's eventual write-back cost, rather than declaring victory
|
||||
# while the complete database still resides in a large sidecar file.
|
||||
if strategy.journal_mode == "WAL":
|
||||
result = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
||||
if result[0] != 0:
|
||||
raise RuntimeError(f"WAL checkpoint was busy: {result!r}")
|
||||
|
||||
|
||||
def insert_naive(connection: sqlite3.Connection, row_count: int, strategy: Strategy) -> None:
|
||||
# isolation_level=None makes every statement its own transaction.
|
||||
for row in generated_rows(0, row_count):
|
||||
connection.execute(INSERT_ONE, row)
|
||||
|
||||
|
||||
def insert_one_transaction(
|
||||
connection: sqlite3.Connection, row_count: int, strategy: Strategy
|
||||
) -> None:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
for row in generated_rows(0, row_count):
|
||||
connection.execute(INSERT_ONE, row)
|
||||
finish_commit(connection, strategy)
|
||||
|
||||
|
||||
def insert_executemany(
|
||||
connection: sqlite3.Connection, row_count: int, strategy: Strategy
|
||||
) -> None:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
for start in range(0, row_count, EXECUTEMANY_BATCH_ROWS):
|
||||
stop = min(start + EXECUTEMANY_BATCH_ROWS, row_count)
|
||||
# An iterator, not a list: batching does not materialize input rows.
|
||||
connection.executemany(INSERT_ONE, generated_rows(start, stop))
|
||||
finish_commit(connection, strategy)
|
||||
|
||||
|
||||
def insert_multi_values(
|
||||
connection: sqlite3.Connection, row_count: int, strategy: Strategy
|
||||
) -> None:
|
||||
try:
|
||||
variable_limit = connection.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER)
|
||||
except AttributeError:
|
||||
variable_limit = 999
|
||||
batch_rows = max(
|
||||
1, min(MULTI_VALUES_TARGET_ROWS, variable_limit // COLUMN_COUNT)
|
||||
)
|
||||
placeholder = "(" + ",".join("?" for _ in range(COLUMN_COUNT)) + ")"
|
||||
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
for start in range(0, row_count, batch_rows):
|
||||
stop = min(start + batch_rows, row_count)
|
||||
batch = list(generated_rows(start, stop))
|
||||
statement = (
|
||||
"INSERT INTO events "
|
||||
"(id, account_id, created_at, amount_cents, status, payload) VALUES "
|
||||
+ ",".join(itertools.repeat(placeholder, len(batch)))
|
||||
)
|
||||
bindings = [value for row in batch for value in row]
|
||||
connection.execute(statement, bindings)
|
||||
finish_commit(connection, strategy)
|
||||
|
||||
|
||||
def insert_set_based(
|
||||
connection: sqlite3.Connection, row_count: int, strategy: Strategy
|
||||
) -> None:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute(SET_BASED_INSERT, (row_count,))
|
||||
finish_commit(connection, strategy)
|
||||
|
||||
|
||||
IMPLEMENTATIONS = {
|
||||
"naive": insert_naive,
|
||||
"one_transaction": insert_one_transaction,
|
||||
"executemany": insert_executemany,
|
||||
"multi_values": insert_multi_values,
|
||||
"set_based": insert_set_based,
|
||||
}
|
||||
|
||||
|
||||
def worker(strategy_key: str, db_path: Path, row_count: int) -> int:
|
||||
strategy = STRATEGY_BY_KEY[strategy_key]
|
||||
if db_path.exists():
|
||||
db_path.unlink()
|
||||
connection = setup_connection(db_path, strategy)
|
||||
implementation = IMPLEMENTATIONS[strategy.implementation]
|
||||
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
started_ns = time.perf_counter_ns()
|
||||
try:
|
||||
implementation(connection, row_count, strategy)
|
||||
except BaseException:
|
||||
if connection.in_transaction:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
elapsed_ns = time.perf_counter_ns() - started_ns
|
||||
|
||||
# Snapshot peak before the untimed validation query.
|
||||
rss_bytes = peak_rss_bytes()
|
||||
digest = tuple(connection.execute(DIGEST_SQL).fetchone())
|
||||
quick_check = connection.execute("PRAGMA quick_check").fetchone()[0]
|
||||
connection.close()
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"strategy": strategy.key,
|
||||
"rows": row_count,
|
||||
"elapsed_seconds": elapsed_ns / 1_000_000_000,
|
||||
"peak_rss_bytes": rss_bytes,
|
||||
"digest": digest,
|
||||
"quick_check": quick_check,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def format_seconds_list(results: Sequence[dict[str, object]]) -> str:
|
||||
return ",".join(f"{float(result['elapsed_seconds']):.3f}" for result in results)
|
||||
|
||||
|
||||
def format_rss_list(results: Sequence[dict[str, object]]) -> str:
|
||||
mib = 1024 * 1024
|
||||
return ",".join(f"{int(result['peak_rss_bytes']) / mib:.1f}" for result in results)
|
||||
|
||||
|
||||
def median(values: Sequence[float]) -> float:
|
||||
ordered = sorted(values)
|
||||
middle = len(ordered) // 2
|
||||
if len(ordered) % 2:
|
||||
return ordered[middle]
|
||||
return (ordered[middle - 1] + ordered[middle]) / 2
|
||||
|
||||
|
||||
def print_table(
|
||||
grouped: dict[str, list[dict[str, object]]],
|
||||
total_rows: int,
|
||||
expected_by_count: dict[int, Digest],
|
||||
) -> tuple[Strategy, Strategy]:
|
||||
headers = (
|
||||
"Strategy",
|
||||
"Mode",
|
||||
"Rows/run",
|
||||
"Trial time(s)",
|
||||
"Time@1M(s)",
|
||||
"Peak RAM/run MiB",
|
||||
"Data",
|
||||
)
|
||||
rows: list[tuple[str, ...]] = []
|
||||
normalized_times: dict[str, float] = {}
|
||||
peak_by_strategy: dict[str, int] = {}
|
||||
|
||||
for strategy in STRATEGIES:
|
||||
results = grouped[strategy.key]
|
||||
elapsed = [float(result["elapsed_seconds"]) for result in results]
|
||||
row_count = int(results[0]["rows"])
|
||||
projected = median(elapsed) * total_rows / row_count
|
||||
normalized_times[strategy.key] = projected
|
||||
peak_by_strategy[strategy.key] = max(
|
||||
int(result["peak_rss_bytes"]) for result in results
|
||||
)
|
||||
verified = all(
|
||||
tuple(result["digest"]) == expected_by_count[int(result["rows"])]
|
||||
and result["quick_check"] == "ok"
|
||||
for result in results
|
||||
)
|
||||
projection_marker = "*" if strategy.sampled else ""
|
||||
rows.append(
|
||||
(
|
||||
strategy.label,
|
||||
f"{strategy.journal_mode}/{strategy.synchronous}",
|
||||
f"{row_count:,}",
|
||||
format_seconds_list(results),
|
||||
f"{projected:.3f}{projection_marker}",
|
||||
format_rss_list(results),
|
||||
"OK" if verified else "FAIL",
|
||||
)
|
||||
)
|
||||
|
||||
widths = [len(header) for header in headers]
|
||||
for row in rows:
|
||||
for index, value in enumerate(row):
|
||||
widths[index] = max(widths[index], len(value))
|
||||
|
||||
def render(row: Sequence[str]) -> str:
|
||||
return " | ".join(value.ljust(widths[i]) for i, value in enumerate(row))
|
||||
|
||||
print(render(headers))
|
||||
print("-+-".join("-" * width for width in widths))
|
||||
for row in rows:
|
||||
print(render(row))
|
||||
|
||||
speed_key = min(normalized_times, key=normalized_times.get)
|
||||
# The sampled baseline is not eligible for the memory title: its peak RSS
|
||||
# was measured honestly but it did not execute the full row count.
|
||||
full_keys = [strategy.key for strategy in STRATEGIES if not strategy.sampled]
|
||||
memory_key = min(full_keys, key=peak_by_strategy.get)
|
||||
return STRATEGY_BY_KEY[speed_key], STRATEGY_BY_KEY[memory_key]
|
||||
|
||||
|
||||
def orchestrator(args: argparse.Namespace) -> int:
|
||||
if args.rows <= 0 or args.baseline_rows <= 0 or args.repeats <= 0:
|
||||
raise SystemExit("--rows, --baseline-rows, and --repeats must be positive")
|
||||
baseline_rows = min(args.baseline_rows, args.rows)
|
||||
|
||||
print("SQLite bulk insert benchmark")
|
||||
print(f"Identity: {IDENTITY}")
|
||||
print(f"Python: {platform.python_version()} ({sys.executable})")
|
||||
print(f"SQLite: {sqlite3.sqlite_version}")
|
||||
print(f"Platform: {platform.platform()}")
|
||||
print(f"Rows: {args.rows:,}")
|
||||
print(
|
||||
f"Autocommit sample: {baseline_rows:,} rows; its Time@1M is a linear projection"
|
||||
)
|
||||
print(f"Full-strategy trials: {args.repeats}")
|
||||
print(
|
||||
"Timed interval: first BEGIN/INSERT through COMMIT; WAL trials also include "
|
||||
"TRUNCATE checkpoint"
|
||||
)
|
||||
print("Peak RAM: per-worker process ru_maxrss; every trial uses a fresh process")
|
||||
print()
|
||||
|
||||
print("Computing independent expected data digests...")
|
||||
expected_by_count = {
|
||||
baseline_rows: expected_digest(baseline_rows),
|
||||
args.rows: expected_digest(args.rows),
|
||||
}
|
||||
|
||||
jobs: list[tuple[Strategy, int, int]] = []
|
||||
for strategy in STRATEGIES:
|
||||
trial_count = 1 if strategy.sampled else args.repeats
|
||||
row_count = baseline_rows if strategy.sampled else args.rows
|
||||
for trial in range(1, trial_count + 1):
|
||||
jobs.append((strategy, trial, row_count))
|
||||
random.Random(0x5A17E).shuffle(jobs)
|
||||
|
||||
grouped: dict[str, list[dict[str, object]]] = {
|
||||
strategy.key: [] for strategy in STRATEGIES
|
||||
}
|
||||
script_path = Path(__file__).resolve()
|
||||
child_environment = os.environ.copy()
|
||||
child_environment["PYTHONHASHSEED"] = "0"
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix=f"sqlite-bench-{IDENTITY}-") as temp_name:
|
||||
temp_dir = Path(temp_name)
|
||||
for job_number, (strategy, trial, row_count) in enumerate(jobs, start=1):
|
||||
db_path = temp_dir / f"{IDENTITY}-{strategy.key}-trial-{trial}.sqlite3"
|
||||
print(
|
||||
f"[{job_number:02d}/{len(jobs):02d}] {strategy.label}, "
|
||||
f"trial {trial}, {row_count:,} rows...",
|
||||
flush=True,
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(script_path),
|
||||
"--worker",
|
||||
strategy.key,
|
||||
"--db",
|
||||
str(db_path),
|
||||
"--worker-rows",
|
||||
str(row_count),
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=child_environment,
|
||||
timeout=args.worker_timeout,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"worker failed for {strategy.key}:\n"
|
||||
f"stdout:\n{completed.stdout}\n"
|
||||
f"stderr:\n{completed.stderr}"
|
||||
)
|
||||
result = json.loads(completed.stdout.strip().splitlines()[-1])
|
||||
grouped[strategy.key].append(result)
|
||||
|
||||
for results in grouped.values():
|
||||
results.sort(key=lambda item: float(item["elapsed_seconds"]))
|
||||
|
||||
print("\nResults")
|
||||
speed_winner, memory_winner = print_table(
|
||||
grouped, args.rows, expected_by_count
|
||||
)
|
||||
print(
|
||||
"\n* naive autocommit Time@1M is measured-time × "
|
||||
f"{args.rows:,}/{baseline_rows:,}; its RAM is measured, not projected."
|
||||
)
|
||||
print(
|
||||
"OFF/OFF rows are semantically identical but the database is rebuildable-only; "
|
||||
"a crash can corrupt it."
|
||||
)
|
||||
print(
|
||||
"INSERT-SELECT is applicable when rows can be generated in SQL or selected "
|
||||
"from an attached SQLite source."
|
||||
)
|
||||
print(f"\nSpeed Winner: {speed_winner.label}")
|
||||
print(f"Memory Winner: {memory_winner.label}")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--rows", type=int, default=DEFAULT_ROWS)
|
||||
parser.add_argument("--baseline-rows", type=int, default=DEFAULT_BASELINE_ROWS)
|
||||
parser.add_argument("--repeats", type=int, default=DEFAULT_REPEATS)
|
||||
parser.add_argument("--worker-timeout", type=float, default=180.0)
|
||||
parser.add_argument("--worker", choices=tuple(STRATEGY_BY_KEY))
|
||||
parser.add_argument("--db", type=Path)
|
||||
parser.add_argument("--worker-rows", type=int)
|
||||
args = parser.parse_args(argv)
|
||||
if args.worker and (args.db is None or args.worker_rows is None):
|
||||
parser.error("--worker requires --db and --worker-rows")
|
||||
return args
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.worker:
|
||||
return worker(args.worker, args.db, args.worker_rows)
|
||||
return orchestrator(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user