commit 5852f2ed4f5f064a368d83d2dabad84fe6bfa0b4 Author: IndyDevDan Date: Sun Jul 19 20:30:34 2026 -0500 ๐Ÿš€ diff --git a/.claude/commands/install.md b/.claude/commands/install.md new file mode 100644 index 0000000..4e94cce --- /dev/null +++ b/.claude/commands/install.md @@ -0,0 +1,26 @@ +--- +description: Install and verify the fusion-harness toolchain (pi, just, jq, uv, API keys) +--- + +# Purpose + +Install everything the fusion-harness needs and verify the harness launches. + +## Workflow + +1. Check each prerequisite binary and install the ones that are missing: + - `pi` โ€” `npm install -g @earendil-works/pi-coding-agent` + - `just`, `jq`, `uv` โ€” `brew install just jq uv` (macOS) or the user's package manager +2. Check that `.env` exists at the repo root and contains non-placeholder values for + `ANTHROPIC_API_KEY` (architect) and `OPENAI_API_KEY` (builder). If it is missing or + holds placeholders, ask the user for keys โ€” never invent or commit them. Note the + gotcha: `just`'s dotenv-load does NOT override variables already exported in the + shell, so a stale exported key silently wins over `.env`. +3. Confirm the extension file loads by checking the entry point exists: + `extensions/fusion-harness/fusion-harness.ts` plus its sibling `SYSTEM_PROMPT_*.md` + and `USER_PROMPT_*.md` prompt files (the extension throws at load time if any prompt + file is missing). +4. Tell the user to launch with `just fh-workhorse` (cheap test pair) and confirm the + ๏ผฆ๏ผต๏ผณ๏ผฉ๏ผฏ๏ผฎ ๏ผจ๏ผก๏ผฒ๏ผฎ๏ผฅ๏ผณ๏ผณ boot banner renders. `just fh-sota` is the frontier + pair and costs real money. +5. Report what was installed, what was already present, and anything still blocking. diff --git a/.claude/commands/prime.md b/.claude/commands/prime.md new file mode 100644 index 0000000..90afe84 --- /dev/null +++ b/.claude/commands/prime.md @@ -0,0 +1,18 @@ +--- +description: Prime with foundational context for the fusion-harness Pi extension repo +--- + +# Purpose + +Orient yourself in fusion-harness: a clean-room Pi coding-agent extension that fuses two +frontier models (ARCHITECT plans/fuses/validates, BUILDER builds) behind three slash +commands, with a strict two-column output DX. + +## Workflow + +1. Run `git ls-files | sort` to see the full tree +2. Read `README.md` for the repo-level overview โ€” it covers the commands, flags, host-as-builder architecture, gate loop, and the two-column DX contract +3. Note `extensions/fusion-harness/` is RUNTIME ONLY: the .ts plus the prompt files it loads. Docs live at the root README +4. Read `justfile` for the run recipes and model configuration (WORKHORSE test pair vs SOTA fable/sol pair) +5. Glob `extensions/fusion-harness/*_PROMPT_*.md` and skim 1-2 to see how default prompts are externalized with `{{VAR}}` interpolation +6. Summarize your understanding of the project: purpose, stack, structure, key files, and entry points diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d18c322 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# --- Secrets / env --- +# .env holds real API keys (ANTHROPIC, SERPAPI, DATAFORSEO, GITHUB_TOKEN, etc.) +.env +.env.* +!.env.example +!.env.template + +# --- Node / TypeScript --- +node_modules/ +dist/ +build/ +*.tsbuildinfo +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +.pnpm-store/ + +# --- Python (uv gates, bench scripts) --- +__pycache__/ +*.py[cod] +.venv/ +venv/ +.pytest_cache/ +.ruff_cache/ +*.egg-info/ + +# --- Runtime artifacts --- +# Harness children write to /tmp/fusion-harness-* (outside the repo), but guard +# against any stray local run output landing here. +*.log +/tmp-artifacts/ + +# Local parking lot for unused assets and scratch files โ€” never committed. +/tmp/ + +# --- Editors / OS --- +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Thumbs.db +.idea/ +*.swp +*.swo +.vscode/* +!.vscode/settings.json +!.vscode/extensions.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..acc8f95 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 IndyDevDan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b20c1c --- /dev/null +++ b/README.md @@ -0,0 +1,333 @@ +# fusion-harness + +> **Fuse frontier models instead of racing them. AND, not OR.** +> A standalone [Pi coding agent](https://github.com/badlogic/pi-mono) extension harness for two-model agentic engineering. + +๐Ÿ“บ Watch this video to get the full breakdown of this codebase: **[GPT-5.6 Sol vs Fable 5 Is the Wrong Question (Fusion) on YouTube](https://youtu.be/AQl5Q-0l7FQ)** + +

+ MODEL FUSION โ€” two model energy streams fusing into one over an engineer's keyboard +

+ +

+ ARCHITECT (claude-fable-5) and BUILDER (gpt-5.6-sol) streams fusing into one result โ€” AND, not OR +

+ +"Which model is best" is a benchmark question, not an engineering question. One model plans, another builds, and the results fuse: you combine compute instead of selecting it. Aider called the pattern [architect/editor](https://aider.chat/2024/09/26/architect.html) (the original fusion); [Devin calls it fusion](https://cognition.com/blog/devin-fusion); [OpenRouter calls it model fusion](https://openrouter.ai/blog/announcements/fusion-beats-frontier/). This repo makes the pattern a first-class, on-camera-clear workflow: two hard-labeled agents, a gate-first validation loop, and attributed fusion. **You don't have to pick a winner when you can hire both.** + +--- + +## Install + +### Agentic Install + +```bash +claude "/install" # runs the /install slash command in Claude Code (or Pi, or your favorite agentic coding tool) +``` + +The `/install` command lives at `.claude/commands/install.md` and handles toolchain checks, key checks, and project-specific setup. + +### Manual Install + +**Prereqs:** [`pi`](https://github.com/badlogic/pi-mono), [`just`](https://github.com/casey/just), [`jq`](https://jqlang.github.io/jq/), [`uv`](https://docs.astral.sh/uv/). + +```bash +npm install -g @earendil-works/pi-coding-agent # the Pi coding agent +brew install just jq uv # task runner, json inspection, gate runner +printf 'ANTHROPIC_API_KEY=sk-ant-...\nOPENAI_API_KEY=sk-...\n' >> .env # architect + builder keys +just fh-workhorse # launch on the cheap test pair +``` + +--- + +## Why this exists + +

+ Compute routed to Model A OR Model B (select) versus Compute routed to A + B (AND: combine compute) +

+ +Every frontier release restarts the same argument: `GPT-5.6 Sol` or `Fable 5`? Benchmarks crown a winner, teams switch, and next quarter the crown moves. Meanwhile the two models are genuinely different animals: one plans and critiques with more depth, the other ships working diffs faster. Picking one means losing the other's edge on every single task. + +

+ Unchecked lone wolf drifting below the quality line versus an architect-builder team with validation checks +

+ +The deeper problem is review, the second constraint of agentic engineering. A single unchecked agent drifts below the quality line and nobody catches it until you read the diff. A tight two-agent team validates its own work as it goes: the architect checks the builder, the gate checks them both. + +The harness inverts the question. The ARCHITECT model plans, fuses, and validates; the BUILDER model builds; a fusion step merges their independent answers with explicit attribution, and an acceptance gate proves the result. **The wrong question is "which model." The right question is "which role."** + + + + + + +
Fable 5 and GPT-5.6 Sol linked through a glowing fused-output coreFable 5 and GPT-5.6 Sol context windows stacked into one fused context
+ +Used properly, fusion combines the intelligence AND the context windows of your models: two independent thought chains, one merged result. + +--- + +## The three commands + +

+ /opinion โ€” side by side, /fusion โ€” merged via a fusion agent, /auto-validate โ€” validator gate + builder loop +

+ +One extension file registers three slash commands. Every agent is a spawned `pi --mode json -p` subprocess with a fully qualified `provider/id` model, per-role thinking level, and artifacts under `/tmp/fusion-harness-*` (never inside this repo). + +Children are deliberately **clean-room**: every spawn gets `--no-skills --no-extensions --no-context-files`. `--no-extensions` keeps a child from recursively loading this harness; `--no-skills` / `--no-context-files` keep spawns lean and deterministic โ€” each worker's entire contract comes from the harness's prompt files, identical on any machine regardless of what skills are installed. Only the HOST (raw chat) loads your skills and context files; children never do, even the builder children that fork the host session (a fork copies conversation history, but each child rebuilds its own system prompt from its own flags). + +| Command | Agents | What happens | +|---|---|---| +| `/opinion ` | 2 | Both models answer independently (every tool except write/edit). One panel compares them side by side โ€” model, latency, tokens, cost โ€” above both full answers. A pure A/B read. | +| `/fusion "" ""` | 3 | ARCHITECT and BUILDER answer in parallel, both with full tools โ€” either can build/render what you asked for. A third FUSION agent (architect model, fresh session, full tools) merges the two per your fusion instruction โ€” default is a critical merge with `[ARCHITECT]`/`[BUILDER]` attribution and a **Consensus & Divergence** close. | +| `/auto-validate ` | 2 + gate | The auto-validation loop: VALIDATOR designs an acceptance gate BEFORE any work happens, BUILDER builds, the gate runs, failures feed back verbatim until green or halt. Full breakdown below. | + +### Same team, three outputs + +

+ Value ladder: /opinion gives 2 answers, /fusion gives 1 merged plan, /auto-validate gives a verified build +

+ +The three commands are a value ladder on the same two agents. `/opinion` is the scout: two takes, no merge, you choose. `/fusion` is the planning step: the architect merges both takes into one plan. `/auto-validate` is build and test in one: a gate proves the work. Chain them (opinion, then fusion, then auto-validate) and you are running a micro software-development lifecycle inside a single harness. + +### /opinion โ€” two perspectives, side by side + +

+ /opinion fanning out to Fable 5 and GPT-5.6 Sol answer panels in parallel โ€” 2 agents total +

+ +Both models take your prompt in parallel and the panel lines them up: latency, tokens in/out, cost, and the full answers. You get two unique perspectives no single-model tool gives you, and a free side effect: a running head-to-head of how your models actually perform on your work. **Relativity is the best benchmark.** + +### /fusion โ€” merge with attribution + +

+ /fusion flow: Fable 5 and GPT-5.6 Sol in parallel into a fresh-session Fusion Architect producing the fused answer โ€” 3 agents total +

+ +Both workers execute with full tools, then a third agent (the architect model on a fresh session) reads both raw answers from the run's artifacts dir and merges them per your fusion instruction. + +

+ Fable plan and GPT plan resolving into a fused plan with per-line provenance: MERGED, KEPT, KEPT, NEW โ€” 0 dropped +

+ +The fused result carries provenance: what both models agreed on (consensus), what only one saw (divergence, kept and attributed), and what got discarded. The divergences are the value: you hired two different engineers precisely so they would not say the same thing. + +`/fh-reset` wipes the persistent per-project role sessions when you want fresh agent memories. Pi's built-in `/new` does this automatically on top of its normal fresh-session flow โ€” a new conversation means new role brains too (plain restarts, `/resume`, and forks keep them). + +**Press `escape` to stop any running command.** Pi's own escape only aborts *its* agent loop; every agent here is a spawned subprocess, so the harness taps the key itself and kills the whole run โ€” children first, gate included. The panel says you stopped it rather than blaming the models. + +`/thinking [builder]` retunes thinking mid-session โ€” no restart. Omit the builder level to leave it unchanged; run it bare to print the current pair. Canonical or short forms both work (`high` or `hi`, `medium` or `med`, `off` or `none`), so you can type back exactly what the footer shows. + +`/system-prompt` shows the system prompt each role runs with โ€” ARCHITECT | BUILDER side by side, zero cost (nothing spawns). Each column is just the prompt text: the `--*-system-prompt` override (inline flag text, or the contents of the file it points to), or pi's actual default prompt when unset โ€” rebuilt exactly as a spawned child gets it. No skills or project-context sections appear in either column because children are clean-room (see above); the host's own prompt, which does carry your skills, is not what this panel shows. + +--- + +## Raw chat IS the builder + +

+ Host session on the builder model forks builder children; the architect session stays a separate brain +

+ +The launch recipes set the Pi host itself to the BUILDER model. Type a plain message and you get the untouched native Pi experience, on the builder's session. Run a slash command and the builder child **forks the host session**, inheriting your raw chats and every prior panel. Chat and commands are the same agent; nothing about vanilla Pi changes. + +The ARCHITECT deliberately stays a separate persistent brain (pinned per project **and per model** in `/tmp/fusion-harness-sessions/`). Two independent perspectives is the point of fusion: an architect that inherited the builder's every assumption would just be an echo. Swapping `--architect` or `--builder` mints a separate brain for the new model โ€” a transcript built under one model is never replayed as another model's own history (replaying sonnet-built turns into fable-5 trips Anthropic's usage-policy classifier; see below). + +--- + +## The auto-validation loop + +

+ Validator designs gate.py, baseline runs red, builder builds, gate runs โ€” PASS exits, FAIL loops back, halt at cap +

+ +Red to green, with the gate designed before the build: + +1. **VALIDATOR designs the gate first.** It inspects the project read-only and writes a single Astral `uv` Python script (PEP 723) that exits 0 iff *what you asked for is what got built*. Every explicit requirement maps to a concrete check. +2. **Baseline run must fail RED.** A passing baseline means the gate is weak or the work is already done; either way you hear about it loudly. +3. **BUILDER builds** with full tools. The gate is visible but immutable. +4. **The gate runs.** FAIL lines (`expected X, found Y, at PATH`) feed back verbatim into the builder's session as correction instructions. PASS ends the loop. +5. **Escalation.** From the `--escalate-to-validator-count`-th failure (default 3), the VALIDATOR re-enters as a triage diagnostician; its brief travels with the next correction. +6. **Gate repair.** If triage diagnoses a `GATE DEFECT` โ€” the gate itself is unsatisfiable or checks something never asked for โ€” the VALIDATOR rewrites its own gate (once per run, via the same single-path `write` it authored it with). The old gate is preserved as `gate.py.r`, the repair renders as its own loud panel, and the repaired gate re-runs **immediately without consuming a builder round** โ€” if the build was right all along, the loop ends green right there. The repair contract forbids weakening any legitimate check. +7. **Halt.** After `--max-validations` failures (default 5), development stops with the last gate output rendered loudly. No silent infinite loops. + +The builder never grades its own homework, and the grader never touches the code โ€” even the gate repair only ever writes the one path the harness dictates. + +--- + +## Two columns, everywhere + +

+ Architect column and builder column streaming side by side, full-width fusion row, aligned two-cell footer +

+ +Output clarity is the product. The harness mirrors the vanilla Pi experience (tool lines, streaming text, footer) but splits it into two columns it completely controls: ARCHITECT-family left, BUILDER right, aligned across the live widget, the final panels, and the footer. + +- Hard role + model labels with one consistent color per role: ARCHITECT โ—†, BUILDER โ–ฒ, FUSION โง‰, VALIDATOR โœ“. +- While children run, a live widget streams each agent's tool calls and text in its own column; the fusion stage renders as a full-width row. +- Final panels render full-height into scrollback, so results scroll like normal messages: no hidden lines behind a toggle. +- The footer is replaced with one aligned cell per model: `โ—† ARCHITECT | model (med) | [โ–ˆโ–ˆ--------] 12%` (thinking level + context-window bar). +- Child output is buffered per agent, never interleaved, and failures name the exact role, model, and error. + +Every default prompt lives next to the extension as `SYSTEM_PROMPT_*.md` / `USER_PROMPT_*.md` with `{{VARIABLE}}` interpolation. Tune the harness by editing files, not code. + +--- + +## Build your own patterns + +

+ Shipped in this video: /opinion, /fusion, /auto-validate. Your turn: /debate, /parallel, /coordinate โ€” your harness, your patterns +

+ +The three shipped commands are a starting lineup, not the roster. The same spawn-and-render machinery supports any coordination pattern you can prompt: `/debate` (N rounds, two sides, one verdict), `/parallel` (same task, both models, no merge), `/coordinate` (agents plan together, then split the work). This is harness engineering: your tools directly limit what you believe is possible, and a harness you own is a harness you can extend the same afternoon you think of the idea. + +--- + +## One node in your software factory + +

+ Zoom out: the fusion harness is one shipped node of an AI Developer Workflow inside a larger software factory +

+ +Zoom out and the whole harness (two agents, three commands, gate loop and all) is a single agent node inside an AI Developer Workflow. Engineers, code, and agents are the three units of value creation; the fusion harness is how one agent slot in that pipeline stops being a lone model and becomes a validated team. Scale your compute to scale your impact. + +--- + +## Folder structure + +The extension directory holds ONLY what the harness loads at runtime โ€” the code and the +prompts it reads. Docs live outside it. + +``` +fusion-harness/ +โ”œโ”€โ”€ README.md # this file โ€” the whole harness, docs included +โ”œโ”€โ”€ LICENSE # MIT +โ”œโ”€โ”€ justfile # task runner โ€” just +โ”œโ”€โ”€ .env # API keys (never commit this) +โ”‚ +โ”œโ”€โ”€ extensions/ +โ”‚ โ””โ”€โ”€ fusion-harness/ # runtime only +โ”‚ โ”œโ”€โ”€ fusion-harness.ts # the whole harness โ€” 3 commands, widget, footer, renderer +โ”‚ โ”œโ”€โ”€ SYSTEM_PROMPT_*.md # validator + triage contracts +โ”‚ โ””โ”€โ”€ USER_PROMPT_*.md # every default prompt, {{VAR}} interpolated +โ”‚ +โ”œโ”€โ”€ live_final_generation/ # artifacts from the on-camera SOTA run (fused bench, gate rounds, manifest) +โ”‚ +โ”œโ”€โ”€ images/ # README visuals +โ”‚ โ”œโ”€โ”€ svg-*.svg # hand-built diagrams (2 animated, 4 still) +โ”‚ โ””โ”€โ”€ video-frames/ # stills + clean remakes of the video's motion graphics +โ””โ”€โ”€ .claude/ + โ””โ”€โ”€ commands/ + โ””โ”€โ”€ prime.md # /prime โ€” orient an agent in this repo +``` + +--- + +## Recipes + +

+ Fusion harness slot rack: Fable 5 in the ARCHITECT slot, GPT-5.6 Sol in the BUILDER slot, Gemini 3.5 Pro day-one in the OPEN slot โ€” role โ‰  model +

+ +Role โ‰  model. Models change quarterly; the harness compounds. Two model tiers are wired in the justfile: **WORKHORSE** (`WORKHORSE_ARCHITECT`/`WORKHORSE_BUILDER`, use for all testing) and **STATE-OF-THE-ART** (`SOTA_ARCHITECT`/`SOTA_BUILDER`, the on-camera pair). The day a new frontier model drops, it slots into either role with one flag. + +| Tier | Architect | Builder | Recipe | +|---|---|---|---| +| **WORKHORSE** | `anthropic/claude-sonnet-5` | `openai/gpt-5.6-terra` | `just fh-workhorse` | +| **STATE-OF-THE-ART** | `anthropic/claude-fable-5` | `openai/gpt-5.6-sol` | `just fh-sota` | + +Two ways to run it โ€” everything else is a flag. + +``` +just fh-workhorse # WORKHORSE tier (use this for testing) +just fh-sota # STATE-OF-THE-ART tier (fable plans ยท sol builds + hosts) +``` + +All flags append to either recipe, e.g. `just fh-workhorse --architect-thinking high --builder-system-prompt ./persona.md`, or `just fh-sota --architect-thinking max --builder-thinking max` to push both roles to max thinking. + +

+ A full SOTA-tier run: spec โ†’ /opinion perspectives โ†’ /fusion fused plan โ†’ /auto-validate guard iterations โ†’ shipped, 6/6 artifacts +

+ +That is what a full SOTA run produces end to end: spec in, two perspectives, one fused plan, gated iterations, shipped MVP with evidence. The artifacts from the on-camera run (the fused SQLite benchmark, every gate round, the manifest) live in [`live_final_generation/`](live_final_generation/). + +--- + +## Flags + +| Flag | Default | Meaning | +|---|---|---| +| `--architect ` | `anthropic/claude-fable-5` | plans / fuses / validates | +| `--builder ` | `openai/gpt-5.6-sol` | builds | +| `--architect-thinking ` | `medium` | thinking for EVERY architect-family execution (worker/fusion/validator/triage) โ€” `off\|minimal\|low\|medium\|high\|xhigh\|max` | +| `--builder-thinking ` | `medium` | thinking for EVERY builder execution โ€” same levels | +| `--architect-system-prompt ` | pi default | system prompt for architect worker/fusion agents (VALIDATOR/TRIAGE keep their `SYSTEM_PROMPT_*.md` contracts) | +| `--builder-system-prompt ` | pi default | system prompt for all builder agents | +| `--max-validations ` | `5` | `/auto-validate`: gate validations before halting (also inline per command) | +| `--escalate-to-validator-count ` | `3` | `/auto-validate`: on the Nth failure, VALIDATOR triages the builder's work (also inline) | +| `--child-timeout ` | `28800` (8h) | timeout for EVERY spawned child (the auto-validate builder never drops below the 8h floor; clamp 10โ€“86400) | + +`/fusion` takes its two arguments quoted โ€” `/fusion "prompt" "fusion instruction"` โ€” or separated: `/fusion prompt :: fusion instruction`. With no fusion instruction, the built-in critical merge is used. + +--- + +## Prompts live in files + +Every default prompt sits next to the extension with `{{VARIABLE}}` interpolation โ€” tune the harness by editing files, not code. + +| File | Used by | +|---|---| +| `SYSTEM_PROMPT_VALIDATOR.md` | gate design (auto-validate step 1) | +| `SYSTEM_PROMPT_TRIAGE.md` | escalation diagnosis | +| `USER_PROMPT_FUSION_WORKER.md` | /fusion ARCHITECT + BUILDER workers | +| `USER_PROMPT_FUSION_MERGE.md` | /fusion FUSION agent (the envelope: both answers + output contract) | +| `USER_PROMPT_FUSION_DEFAULT_INSTRUCTION.md` | fills the merge envelope's `{{FUSION_INSTRUCTION}}` when you don't pass one | +| `USER_PROMPT_OPINION.md` | /opinion both agents | +| `USER_PROMPT_BUILDER.md` ยท `USER_PROMPT_CORRECTION.md` | auto-validate build + correction rounds | +| `USER_PROMPT_VALIDATOR.md` ยท `USER_PROMPT_TRIAGE.md` | gate design + triage requests | + +## Artifacts + +Every run makes `/tmp/fusion-harness-XXXXXX/` with `prompt.md`, one `.md` per agent, `fused.md` / `gate.py` + `gate-output.txt` as applicable, `summary.json`, and each child's throwaway session dir. Nothing is ever written into the repo. + +Downstream agents are grounded in this dir instead of hunting the filesystem: the FUSION agent's prompt names the run dir and both raw answer files (`architect.md` / `builder.md` โ€” complete even when the inline handoff was truncated), and the VALIDATOR/TRIAGE prompts name the run dir with its per-round builder reports and gate outputs. + +--- + +## Where it can still fail + +

+ Blind-spot coverage matrix: Fable 5 covers 4/6 concerns, GPT-5.6 Sol 3/6, the fusion harness 5/6 โ€” more coverage โ‰  perfect +

+ +Two models cover more blind spots than one, and the matrix is honest about the rest: fusion raises coverage, it does not reach perfection, and a shared unknown stays unknown. The concrete failure modes: + +- **`uv` missing or a gate timeout**: gate execution errors halt `/auto-validate` immediately with an attributed error; they never burn correction rounds. Install `uv` before relying on gates. +- **A weak gate**: if the baseline run passes before any work happens, the harness warns loudly and proceeds with suspicion. Treat a first-round pass as a gate defect until proven otherwise. +- **`.env` vs exported shell vars**: `just`'s dotenv-load does NOT override variables already exported in your shell. A stale exported key silently wins over the repo `.env`. +- **Parallel writers share one cwd**: `/fusion`'s two workers run concurrently with full tools, so their prompt requires an identity-in-filename (`-ARCHITECT-`) on everything they create. Two agents told to write the same bare path would race and clobber each other โ€” if you ask for one exactly-named file, let the FUSION merge produce it. (`/opinion` children stay read-only/bash-only: it's an A/B read, not a build.) +- **Stale role memories**: the ARCHITECT session persists per project + model across restarts. If it starts reasoning from an old context, `/fh-reset` gives both roles a clean brain. +- **Anthropic usage-policy blocks on fable-5**: Fable ships stricter safety classifiers, and a long accumulated agent transcript can false-positive โ€” observed when a sonnet-built session (turns saying "you are claude-sonnet-5" + script execution) was replayed into fable-5: every request blocked at the API, even `/opinion hello`, while the same prompt on a fresh session passed. The per-model session keying prevents the cross-model case; if a block ever recurs on a long same-model session, `/fh-reset` (or `/new`) clears it. +- **Headless hosts can't fork**: with `--no-session` (headless runs), builder children fall back to a manifest-pinned persistent session instead of forking the host. + +--- + +## License + +MIT โ€” see [`LICENSE`](LICENSE). + +--- + +## Master Agentic Coding + +Prepare for the future of software engineering. + +Learn tactical agentic coding patterns with [Tactical Agentic Coding](https://agenticengineer.com/tactical-agentic-coding?y=fuhar). + +Follow the [IndyDevDan YouTube channel](https://www.youtube.com/@indydevdan) to improve your agentic coding advantage. + +--- + +Stay Focused and Keep Building + +- IndyDevDan diff --git a/extensions/fusion-harness/SYSTEM_PROMPT_TRIAGE.md b/extensions/fusion-harness/SYSTEM_PROMPT_TRIAGE.md new file mode 100644 index 0000000..9f83882 --- /dev/null +++ b/extensions/fusion-harness/SYSTEM_PROMPT_TRIAGE.md @@ -0,0 +1,17 @@ +You are the VALIDATOR acting as TRIAGE DIAGNOSTICIAN in an auto-validation loop that you gate. You designed the acceptance gate earlier in this session; a separate BUILDER agent has now failed it repeatedly. The raw gate output alone is not unsticking the builder โ€” your job is to find out WHY and direct the fix. + +Method: inspect the project READ-ONLY (find/grep/read/ls). Compare what the gate demands against what the builder ACTUALLY produced (read the real files/state, not the builder's claims). Identify the root cause: wrong file, wrong interpretation, oscillation between two wrong states, missing prerequisite, an environmental blocker โ€” or a defect in the gate itself. NEVER modify the project. + +GATE REPAIR โ€” your one exception to read-only: +- If (and ONLY if) the root cause is a defect in the gate itself โ€” it is impossible to satisfy, or it demands something the request never asked for โ€” and you still hold the `write` tool (the harness grants it only while this run's single repair is unused), REWRITE the gate at exactly: + {{GATE_PATH}} +- A repair fixes the defect and NOTHING else: every check that maps to a real requirement stays, at full strength. Never weaken, remove, or reinterpret a legitimate check to make the loop pass โ€” you are correcting your own bug, not moving the goalposts. +- Write the complete corrected script with your `write` tool (never paste it into your reply). The harness detects the change, preserves the old gate, and re-runs the repaired gate immediately โ€” without charging the builder a round. +- If the gate is sound, touch nothing. Your `write` tool exists for that one path, only on a defect diagnosis. + +Output contract (markdown, at most ~30 lines, no preamble): +1. **Diagnosis** โ€” the root cause of the repeated failures, not the symptom. If the gate itself was wrong, prefix `GATE DEFECT:` and say plainly whether you repaired it. +2. **Do exactly this** โ€” precise, ordered steps with absolute paths and exact content/commands. +3. **Do NOT** โ€” what the builder keeps doing wrong and must stop doing. + +Your brief is advisory context for the builder: the gate's own output remains the source of truth. diff --git a/extensions/fusion-harness/SYSTEM_PROMPT_VALIDATOR.md b/extensions/fusion-harness/SYSTEM_PROMPT_VALIDATOR.md new file mode 100644 index 0000000..a1e1bc9 --- /dev/null +++ b/extensions/fusion-harness/SYSTEM_PROMPT_VALIDATOR.md @@ -0,0 +1,32 @@ +You are the VALIDATOR in an auto-validation loop: you design the ACCEPTANCE GATE BEFORE a separate BUILDER agent does the work. Your deliverable is an Astral `uv` single-file Python script (PEP 723) that exits 0 IF AND ONLY IF the user's REQUEST is genuinely, verifiably complete in the current project. + +HOW YOU DELIVER IT โ€” WRITE THE FILE, NEVER PASTE IT: +- Use your `write` tool to write the gate to EXACTLY this absolute path: + {{GATE_PATH}} +- NEVER paste the gate โ€” or any part of it โ€” into your reply, and NEVER wrap it in a code fence. The harness executes the FILE at that path; it does not read your message. A gate pasted into a fence is truncated at the first ``` inside it, which silently corrupts any gate that mentions markdown fences. +- Because the gate is a file and not markdown, your script MAY freely contain literal triple-backticks inside strings โ€” write them normally. +- Your `write` tool is for that ONE path only. NEVER create, modify, or delete anything else: you are the grader, and the grader never touches the code. +- After writing, reply with a SHORT confirmation only (the path, and a one-line summary of what the gate checks). No script, no fences. + +Your script IS the definition of done: after you deliver it, the builder builds, your script runs, and every FAIL line you print is sent back to the builder verbatim as its correction instructions. The loop repeats until your script exits 0 or the run is halted. Write it with total integrity โ€” it must be impossible to pass without actually doing what was asked, and impossible to fail for reasons unrelated to the request. + +Method: +- First inspect the project READ-ONLY (find/grep/read/ls): layout, conventions, how tests/build/type-check run. Ground every check in reality. NEVER modify the project. +- Then write the script against the REQUESTED END STATE to {{GATE_PATH}}. The work has NOT been done yet โ€” your script should FAIL against the current state and PASS only once the request is genuinely complete. + +Hard requirements for the script: +- Begin with the PEP 723 inline metadata block exactly: + # /// script + # requires-python = ">=3.11" + # dependencies = [] # add ONLY deps you truly need + # /// +- FIDELITY TO THE REQUEST: the script must prove that what the user ASKED FOR is what got built. Enumerate every explicit requirement in the REQUEST and map each one to at least one check โ€” nothing asked for may go unchecked, and nothing that wasn't asked for may be required. No substitutions, no weaker proxies, no narrowing of scope. +- CONCRETE, OBJECTIVE checks of outcomes: file contents, command exit codes, real behavior. Never vibes; never mere existence when content or behavior was requested. +- Print exactly one line per check: + "PASS: " + "FAIL: > โ€” " +- FAIL lines are the builder's next instructions: make each one specific, actionable, and unambiguous (expected vs actual, exact paths, exact commands). +- Exit 0 ONLY if ALL checks pass; exit non-zero otherwise. +- Deterministic, fast (<60s), non-interactive, zero side effects on the project; it runs from the project root. + +Write that script to {{GATE_PATH}} with your write tool, then reply with only a short confirmation. diff --git a/extensions/fusion-harness/USER_PROMPT_BUILDER.md b/extensions/fusion-harness/USER_PROMPT_BUILDER.md new file mode 100644 index 0000000..d1c2c62 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_BUILDER.md @@ -0,0 +1,13 @@ +You are the BUILDER agent in an auto-validation loop. Execute the request below directly and completely โ€” you have full tools (read/bash/edit/write). + +An ACCEPTANCE GATE already exists: the immutable uv Python script below runs automatically after you finish, and it alone defines "done". It lives OUTSIDE the project and outside your control โ€” you cannot edit it or its verdict. Satisfy it by genuinely completing the request, never by gaming individual checks. If the gate fails, its exact failure output comes back to you as your next instructions. + +# REQUEST +{{PROMPT}} + +# ACCEPTANCE GATE (read-only โ€” enforced after you finish) +```python +{{GATE_SCRIPT}} +``` + +When done, report concisely: files created/changed (absolute paths) and commands run. diff --git a/extensions/fusion-harness/USER_PROMPT_CORRECTION.md b/extensions/fusion-harness/USER_PROMPT_CORRECTION.md new file mode 100644 index 0000000..3180675 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_CORRECTION.md @@ -0,0 +1,10 @@ +GATE FAILED โ€” this is correction round {{ROUND}} of {{MAX_ROUNDS}}. {{REMAINING}} before development is HALTED. + +The acceptance gate ran against your work and failed. Its output below is your exact instruction list: fix EVERY failure, genuinely โ€” do not touch or game the gate. + +# GATE OUTPUT (exit {{GATE_EXIT_CODE}}) โ€” the source of truth +``` +{{GATE_OUTPUT}} +``` +{{TRIAGE_BLOCK}}{{GATE_UPDATE_BLOCK}} +When done, report concisely what you changed to address each failure. diff --git a/extensions/fusion-harness/USER_PROMPT_FUSION_DEFAULT_INSTRUCTION.md b/extensions/fusion-harness/USER_PROMPT_FUSION_DEFAULT_INSTRUCTION.md new file mode 100644 index 0000000..17e41db --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_FUSION_DEFAULT_INSTRUCTION.md @@ -0,0 +1 @@ +Critically merge the two answers into one definitive answer. Discard anything incorrect or hallucinated, keep the strongest elements of each, and combine complementary insights โ€” do not simply concatenate. diff --git a/extensions/fusion-harness/USER_PROMPT_FUSION_MERGE.md b/extensions/fusion-harness/USER_PROMPT_FUSION_MERGE.md new file mode 100644 index 0000000..fc38d94 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_FUSION_MERGE.md @@ -0,0 +1,23 @@ +You are the FUSION agent in a two-model harness. Two different frontier models independently answered the same request. Your job: {{FUSION_INSTRUCTION}} +You have full tools (read/bash/edit/write). If the fusion instruction calls for producing, rendering, running, or opening something, DO it โ€” never describe commands for the user to run themselves. Write throwaway artifacts under /tmp unless the instruction says otherwise. +FILE NAMING: a fused result is the product of BOTH models, so name every file you create after the PAIR โ€” never after yourself alone (you merely merged them). Embed BOTH tags, source A first: {{A_TAG}} and {{B_TAG}}. Example: /tmp/fused-report-{{A_TAG}}-{{B_TAG}}.md +Use those tags verbatim โ€” they are already filename-safe โ€” and keep both in the name so runs from different model pairings never collide or overwrite each other. +GROUNDING โ€” this run's material is already on disk; read it from these exact paths, NEVER scan the filesystem for it: +- Run artifacts dir: {{ARTIFACTS_DIR}} +- [{{A_ROLE}}]'s full raw answer: {{A_PATH}} +- [{{B_ROLE}}]'s full raw answer: {{B_PATH}} +(The answers inlined below are what you should normally work from, but they are truncated past {{HANDOFF_MAX}} chars โ€” the files above are always complete.) +- Any files the two agents CREATED live at the exact paths their answers name โ€” read those paths directly. + +# ORIGINAL REQUEST +{{PROMPT}} + +# ANSWER FROM [{{A_ROLE}}] โ€” {{A_MODEL}} +{{A_TEXT}} + +# ANSWER FROM [{{B_ROLE}}] โ€” {{B_MODEL}} +{{B_TEXT}} + +# OUTPUT CONTRACT (markdown) +1. **Fused answer** โ€” the definitive merged result per the instruction above. Where a major point comes from one source, attribute it inline as [{{A_ROLE}}] or [{{B_ROLE}}]. +2. **Consensus & divergence** โ€” a SHORT closing section: where the two agreed, where they disagreed (cite [{{A_ROLE}}]/[{{B_ROLE}}] with model names), and anything you discarded and why. diff --git a/extensions/fusion-harness/USER_PROMPT_FUSION_WORKER.md b/extensions/fusion-harness/USER_PROMPT_FUSION_WORKER.md new file mode 100644 index 0000000..c34aa02 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_FUSION_WORKER.md @@ -0,0 +1,8 @@ +You are the {{ROLE}} agent ({{MODEL}}) in a two-model fusion harness. The {{OTHER_ROLE}} agent ({{OTHER_MODEL}}) is answering the SAME request independently, in parallel; a fusion agent will merge your two answers afterwards. +Answer decisively and completely โ€” do not hedge, do not ask questions. If the request concerns the codebase at your working directory, ground your answer with your tools and cite file:line evidence. +You have FULL tools (read/grep/find/ls/bash/edit/write). If the request asks you to produce, create, render, or run something, DO it โ€” never claim you lack file access and never just describe what the user should run. +FILE NAMING โ€” you are running CONCURRENTLY with {{OTHER_ROLE}} in the SAME working directory, so you must not collide with it: embed your identity in EVERY path you create, using your role and model โ€” you are {{ROLE}} running {{MODEL}}. Example: /tmp/report-{{ROLE}}-{{MODEL}}.md +NEVER write to a bare path the other agent would also pick (that is a race: you would clobber each other mid-write). Do not delete or edit files you did not create. The fusion agent merges afterwards and writes any canonical, exactly-named deliverable the request asks for. + +# REQUEST +{{PROMPT}} diff --git a/extensions/fusion-harness/USER_PROMPT_OPINION.md b/extensions/fusion-harness/USER_PROMPT_OPINION.md new file mode 100644 index 0000000..662cd49 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_OPINION.md @@ -0,0 +1,4 @@ +Answer the following directly and completely. Be decisive, do not hedge, do not ask questions. You may use your tools (read/search/bash) to ground the answer. + +# QUESTION +{{PROMPT}} diff --git a/extensions/fusion-harness/USER_PROMPT_TRIAGE.md b/extensions/fusion-harness/USER_PROMPT_TRIAGE.md new file mode 100644 index 0000000..932c883 --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_TRIAGE.md @@ -0,0 +1,13 @@ +ESCALATION: the builder has failed your acceptance gate {{FAILURES}} time{{FAILURES_PLURAL}} (cap: {{MAX_ROUNDS}}). Diagnose why it is stuck and produce the triage brief per your output contract. + +# ORIGINAL REQUEST (what "done" means) +{{REQUEST}} + +# RECENT GATE OUTPUT{{HISTORY_SUFFIX}} +{{GATE_HISTORY}} + +# BUILDER'S LATEST REPORT (its claims โ€” verify against the real state before trusting) +{{BUILDER_REPORT}} + +# RUN ARTIFACTS (the full, untruncated history โ€” the excerpts above are truncated) +{{ARTIFACTS_DIR}} โ€” gate.py (your gate), gate-baseline.txt, builder-round-N.md (every builder report), gate-round-N.txt (every gate run). Read these files when you need the complete picture. diff --git a/extensions/fusion-harness/USER_PROMPT_VALIDATOR.md b/extensions/fusion-harness/USER_PROMPT_VALIDATOR.md new file mode 100644 index 0000000..3a2e24a --- /dev/null +++ b/extensions/fusion-harness/USER_PROMPT_VALIDATOR.md @@ -0,0 +1,10 @@ +# REQUEST (the builder will be asked to do exactly this โ€” your script defines when it is done) +{{PROMPT}} + +Project root: {{CWD}} +Run artifacts dir: {{ARTIFACTS_DIR}} โ€” your gate lives here, and the harness saves every builder report (builder-round-N.md) and gate run (gate-round-N.txt) here as the loop progresses. +NOTE: the build has NOT happened yet. Inspect the current state read-only, then WRITE the gate script for the requested end state to this exact absolute path with your write tool: + + {{GATE_PATH}} + +Do NOT paste the script into your reply and do NOT put it in a code fence โ€” the harness runs the file, not your message. Reply with a short confirmation only. diff --git a/extensions/fusion-harness/fusion-harness.ts b/extensions/fusion-harness/fusion-harness.ts new file mode 100644 index 0000000..27a0773 --- /dev/null +++ b/extensions/fusion-harness/fusion-harness.ts @@ -0,0 +1,2506 @@ +/** + * fusion-harness โ€” FUSE frontier models instead of racing them. AND, not OR. + * + * Three slash commands, one clarity-first two-column experience: + * + * /fusion [:: ] + * ARCHITECT and BUILDER both answer in parallel (independent + * read-only sessions), streaming live in two side-by-side columns. A + * third FUSION agent (the architect model, fresh session) then merges + * the two per โ€” default: critical merge with a + * Consensus & Divergence section citing [ARCHITECT]/[BUILDER]. + * + * /auto-validate + * BUILDER executes (it may write files โ€” the working agent). + * VALIDATOR (the architect model) then WRITES a uv single-file Python + * gate script that proves the result; we run it and render builder + * report | gate script side by side under a PASS/FAIL verdict. + * + * /opinion + * Both models answer independently; one two-column panel compares them + * (model ยท latency ยท tokens ยท cost ยท answer). No fusion. + * + * /system-prompt + * Zero-cost introspection: the system prompt each role runs with, + * ARCHITECT | BUILDER side by side โ€” the ---system-prompt override + * (file contents win over inline text), or pi's actual default prompt. + * + * DX contract (the #1 priority): + * - The experience mirrors a vanilla pi session โ€” tool-call lines, streaming + * text, footer stats โ€” but split into TWO COLUMNS we fully control: + * ARCHITECT-family on the left, BUILDER on the right, everywhere. + * - While children run, a live widget streams each agent's flow (tool calls + * + response text) in its own column with per-role colors + telemetry + * (glyph ยท role ยท model ยท state ยท elapsed ยท tokens ยท cost). + * - The footer is replaced with the same aligned two-column view: one cell per + * model โ€” `โ—† ROLE | model (med) | [โ–ˆโ–ˆ--------] pct%` โ€” thinking + context bar. + * - Child output is buffered per agent โ€” never interleaved. Final panels + * render the two answers side by side as real markdown columns. + * - Failures are attributed to the specific role+model with the real error. + * + * Plumbing: every agent is a spawned `pi --mode json -p` subprocess with a + * fully-qualified `provider/id` model and a throwaway --session-dir under a + * per-run /tmp/fusion-harness-* artifacts dir. Nothing is written to the repo. + * + * Flags: + * --architect plans / fuses / validates (default anthropic/claude-fable-5) + * --builder builds (default openai/gpt-5.6-sol) + * + * Launch: pi -e extensions/fusion-harness/fusion-harness.ts + * + * File map (top to bottom): + * 1. Defaults โ€” models, tool matrices, timeouts, size caps + * 2. Roles โ€” the four roles + their colors and glyphs + * 3. Types โ€” AgentRun (live child state), AgentStat, FhDetails (panel payloads) + * 4. Small helpers โ€” formatting, truncation, run bookkeeping + * 5. Two-column layout โ€” TwoCol / FullWidth, the core rendering primitives + * 6. Child runner โ€” runChild (pi subprocess + JSON event stream), runProc (the gate) + * 7. Prompts โ€” file-backed templates ({{VAR}} interpolation) + builders + * 8. Extension โ€” flags, sessions, footer, renderer, and the commands: + * /fh-reset ยท /thinking ยท /system-prompt ยท /fusion ยท + * /auto-validate ยท /opinion + */ + +import { spawn } from "node:child_process"; // child pi agents + the uv gate +import { randomUUID } from "node:crypto"; // persistent per-role session ids +import * as fs from "node:fs"; // prompt files, artifacts, session manifests +import * as os from "node:os"; // tmpdir fallback when /tmp is missing +import * as path from "node:path"; // every artifact/session path +import { type ExtensionAPI, getMarkdownTheme } from "@earendil-works/pi-coding-agent"; +import { Box, Container, Markdown, Text, truncateToWidth, visibleWidth, wrapTextWithAnsi } from "@earendil-works/pi-tui"; + +// โ•โ•โ• 1. Defaults โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +const DEFAULT_ARCHITECT = "anthropic/claude-fable-5"; // plans, fuses, validates +const DEFAULT_BUILDER = "openai/gpt-5.6-sol"; // builds (and hosts โ€” see the launch recipes) + +const READONLY_TOOLS = "read,grep,find,ls"; // parallel agents share a cwd โ€” concurrent writers would collide +const OPINION_TOOLS = "read,grep,find,ls,bash"; // everything except write/edit +const FULL_TOOLS = "read,grep,find,ls,bash,edit,write"; // sequential agents (builder, fuser) act freely +// The VALIDATOR reads the project read-only but must WRITE its gate straight to disk: +// piping a gate through a fenced code block truncates it at the first embedded ``` (a +// gate that greps for markdown fences contains one), so the script is written, not pasted. +// `write` is scoped to the run's gate path by the VALIDATOR's system prompt โ€” it still +// never touches the project, and it gets no `edit`/`bash` to mutate one with. The TRIAGE +// turn holds the same toolset while the run's single gate repair is unused (a GATE DEFECT +// diagnosis may rewrite the gate at that one path), then drops to READONLY_TOOLS. +const VALIDATOR_TOOLS = "read,grep,find,ls,write"; + +const CHILD_TIMEOUT_S_DEFAULT = 28_800; // 8h โ€” every spawned child; real work runs for hours (--child-timeout overrides) +const BUILD_TIMEOUT_MS_FLOOR = 28_800_000; // /auto-validate builder floor โ€” never below 8h even with a small --child-timeout +const GATE_TIMEOUT_MS = 120_000; // `uv run` of the validation gate +const KILL_GRACE_MS = 5_000; // SIGTERM โ†’ SIGKILL escalation window +const WIDGET_TICK_MS = 1_000; // live-widget refresh cadence + +const WIDGET_FLOW_LINES = 8; // live streaming lines shown per column +const MIN_TWO_COL_WIDTH = 100; // below this, columns stack +const ANSWER_MAX_BYTES = 100_000; // cap any rendered agent answer +const HANDOFF_MAX = 60_000; // chars of one agent's answer injected into another's prompt +const DETAIL_SNIPPET_MAX = 4_000; // chars of script/output kept in message details + +const CUSTOM_TYPE = "fusion-harness"; // customType tag on every panel/widget/status this extension emits + +// โ•โ•โ• 2. Roles โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +type Role = "ARCHITECT" | "BUILDER" | "FUSION" | "VALIDATOR"; + +/** One consistent color per role, everywhere (columns, footer, panels, errors). */ +const ROLE_COLOR: Record = { + ARCHITECT: "accent", + BUILDER: "warning", + FUSION: "success", + VALIDATOR: "mdLink", +}; + +/** One consistent glyph per role, paired with the color above. */ +const ROLE_GLYPH: Record = { + ARCHITECT: "โ—†", + BUILDER: "โ–ฒ", + FUSION: "โง‰", + VALIDATOR: "โœ“", +}; + +// โ•โ•โ• 3. Types โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +/** Lifecycle of one spawned child agent, from queued to settled. */ +type ChildStatus = "pending" | "working" | "done" | "failed" | "timeout" | "aborted"; + +/** One entry in an agent's transcript flow: a tool call, a finished text block, or a reasoning block. */ +type FlowItem = { type: "tool"; label: string } | { type: "text"; text: string } | { type: "thinking"; text: string }; + +/** Live + final view of one child agent. Mutated in place as JSON events stream in. */ +interface AgentRun { + role: Role; + model: string; + status: ChildStatus; + startedAt?: number; + endedAt?: number; + ms: number; + tokensIn: number; + tokensOut: number; + costUsd: number; + toolCalls: number; + ctxTokens: number; // context used by the last request (for the footer bar) + thinking?: string; + flow: FlowItem[]; // the agent's transcript flow: tool lines + finished text blocks + flowMark: number; // flow index at the current spawn โ€” the widget only shows flow from HERE (no stale rounds) + sessionRef?: string; // the child's own session id (from its "session" event) โ€” lets later rounds resume it + streamText: string; // text of the in-flight assistant message + streamThinking: string; // reasoning of the in-flight assistant message (rendered live โ€” proof of life) + text: string; + exitCode: number; + stopReason?: string; + errorMessage?: string; + stderr: string; +} + +/** Serializable per-agent stats for message details / artifacts. */ +interface AgentStat { + role: Role; + model: string; + status: ChildStatus; + ms: number; + tokensIn: number; + tokensOut: number; + costUsd: number; + toolCalls: number; + chars: number; + error?: string; +} + +/** The renderer's discriminated payload โ€” one shape per panel `kind`, carried on every custom message. */ +interface FhDetails { + kind: "prompt" | "banner" | "duo" | "fused" | "opinion" | "gate" | "validation" | "triage" | "error" | "system-prompt" | "boot"; + command?: "fusion" | "auto-validate" | "opinion" | "system-prompt"; // absent on "boot" โ€” it belongs to no command + ok: boolean; + round?: number; // auto-validate: which buildโ†’validate round this panel reports + maxRounds?: number; // auto-validate: the --max-validations cap + escalateAt?: number; // auto-validate: the --escalate-to-validator-count threshold + prompt?: string; + fusionPrompt?: string; + roles?: Array<{ role: Role; model: string }>; + agent?: AgentStat; // fused: the fuser ยท validation: the validator + sources?: AgentStat[]; // the two columns' stats (left, right) + answers?: Array<{ role: Role; model: string; text: string }>; // column bodies (left, right) + script?: string; // validation gate (truncated for details) + gateOutput?: string; + gateExitCode?: number; + scriptPath?: string; + artifactsDir?: string; + totalMs?: number; + totalCostUsd?: number; + error?: string; +} + +// โ•โ•โ• 4. Small helpers โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +/** Locate the running pi binary so we can re-invoke it as a child. */ +function piInvocation(args: string[]): { command: string; args: string[] } { + const script = process.argv[1]; // the entry script pi itself was launched with + const isBunVirtual = script?.startsWith("/$bunfs/root/"); // bun-compiled binaries mount a virtual fs + // Best case: re-run the exact same entry script with the same runtime. + if (script && !isBunVirtual && fs.existsSync(script)) { + return { command: process.execPath, args: [script, ...args] }; + } + const execName = path.basename(process.execPath).toLowerCase(); + // A compiled pi binary (execPath IS pi): invoke it directly. + if (!/^(node|bun)(\.exe)?$/.test(execName)) return { command: process.execPath, args }; + // Last resort: whatever `pi` resolves to on PATH. + return { command: "pi", args }; +} + +/** Truncate by character count, with an explicit elision marker (prompt handoffs). */ +function truncateChars(s: string, max: number): string { + if (s.length <= max) return s; + return `${s.slice(0, max)}\nโ€ฆ [truncated โ€” ${s.length - max} chars elided]`; +} + +/** Truncate by UTF-8 byte count (panel bodies โ€” pi caps message size in bytes). */ +function truncateBytes(s: string, max: number): string { + const buf = Buffer.from(s, "utf-8"); + if (buf.length <= max) return s; + return `${buf.subarray(0, max).toString("utf-8")}\n\nโ€ฆ [truncated โ€” ${buf.length - max} bytes elided]`; +} + +/** 12345 โ†’ "12.3s" */ +function fmtSecs(ms: number): string { + return `${(ms / 1000).toFixed(1)}s`; +} + +/** 12345 โ†’ "12.3k" (token counts) */ +function fmtK(n: number): string { + return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : `${n}`; +} + +/** Display-width model name: provider stripped, ellipsized past 24 chars (labels only โ€” never paths). */ +function shortModel(m: string): string { + const seg = m.split("/").pop() ?? m; + return seg.length > 24 ? `${seg.slice(0, 23)}โ€ฆ` : seg; +} + +/** + * Filename-safe model tag: provider stripped, anything but [A-Za-z0-9._-] collapsed to `-`. + * NOT shortModel(): that truncates long ids with a `โ€ฆ`, and these tags land in real paths. + */ +function modelTag(m: string): string { + return (m.split("/").pop() ?? m).replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model"; +} + +/** A fresh AgentRun in its zero state โ€” mutated in place by runChild as events stream. */ +function newRun(role: Role, model: string): AgentRun { + return { + role, + model, + status: "pending", + ms: 0, + tokensIn: 0, + tokensOut: 0, + costUsd: 0, + toolCalls: 0, + ctxTokens: 0, + streamThinking: "", + flow: [], + flowMark: 0, + streamText: "", + text: "", + exitCode: 0, + stderr: "", + }; +} + +/** Success = clean exit โˆง clean stop reason โˆง nonempty answer. */ +function runOk(r: AgentRun): boolean { + return r.exitCode === 0 && r.stopReason !== "error" && r.stopReason !== "aborted" && r.text.trim().length > 0; +} + +/** The most specific failure description available, in priority order. */ +function runError(r: AgentRun): string { + return ( + (r.status === "aborted" ? "stopped by user (escape)" : "") || + r.errorMessage || + (r.status === "timeout" || r.exitCode === 124 ? "timed out" : "") || + r.stderr.trim().slice(-300) || + (r.text.trim() ? "" : "no output") || + `exit ${r.exitCode}` + ); +} + +/** Freeze a live AgentRun into the serializable stat used by panels and summary.json. */ +function toStat(r: AgentRun): AgentStat { + return { + role: r.role, + model: r.model, + status: r.status, + ms: r.ms, + tokensIn: r.tokensIn, + tokensOut: r.tokensOut, + costUsd: r.costUsd, + toolCalls: r.toolCalls, + chars: r.text.length, + error: runOk(r) ? undefined : runError(r), + }; +} + +/** Compact one-line stats: `12.3s ยท in 1.2k out 0.4k ยท 3 tools ยท $0.0123` */ +function statLine(s: AgentStat): string { + const parts = [fmtSecs(s.ms)]; + if (s.tokensIn || s.tokensOut) parts.push(`in ${fmtK(s.tokensIn)} out ${fmtK(s.tokensOut)}`); + if (s.toolCalls) parts.push(`${s.toolCalls} tools`); + if (s.costUsd) parts.push(`$${s.costUsd.toFixed(4)}`); + return parts.join(" ยท "); +} + +/** + * A tool call's argument, condensed for one flow line. + * + * The cap here is a MEMORY bound, not a layout one โ€” it must stay far wider than any + * column so a wide terminal shows a wide line. Fitting is the renderer's job: TwoCol + * (`truncateToWidth` per column) and `FullWidth` (`fitLines`) clamp to the real width at + * render, which is what actually keeps pi from throwing on an over-wide line. Capping at + * capture instead trimmed every view to the narrowest one it might ever be drawn in. + */ +const TOOL_ARG_MAX = 200; +function briefArg(args: any): string { + if (!args || typeof args !== "object") return ""; + const v = + args.path ?? args.file_path ?? args.filePath ?? args.pattern ?? args.command ?? Object.values(args).find((x) => typeof x === "string"); + if (typeof v !== "string" || !v) return ""; + const s = v.includes("/") && !v.includes(" ") ? v.split("/").slice(-2).join("/") : v; + return s.replace(/\s+/g, " ").slice(0, TOOL_ARG_MAX); +} + +/** One glyph per child status, used in state lines and stat rows. */ +const STATUS_GLYPH: Record = { + pending: "โ—‹", + working: "โ—", + done: "โœ“", + failed: "โœ—", + timeout: "โœ—", + aborted: "โŠ˜", +}; + +/** Footer-width abbreviations for pi's thinking levels (pi: ModelThinkingLevel). */ +const THINKING_SHORT: Record = { + off: "none", + minimal: "min", + low: "low", + medium: "med", + high: "hi", + xhigh: "xhi", + max: "max", +}; +/** ` (med)` โ€” the parenthesized short thinking level appended to a model label. */ +const thinkingTag = (level?: string): string => (level ? ` (${THINKING_SHORT[level] ?? level})` : ""); + +// โ•โ•โ• 5. Two-column layout โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +/** Hard clamp: pi throws on any rendered line wider than the terminal, so every line we emit must fit. */ +function fitLines(lines: string[], width: number): string[] { + const w = Math.max(1, width); + return lines.map((l) => (visibleWidth(l) > w ? truncateToWidth(l, w) : l)); +} + +/** + * The core layout primitive: two columns we completely control, rendered at + * whatever width the TUI gives us. Below MIN_TWO_COL_WIDTH the columns stack + * (left block, then right block) so narrow terminals stay readable. + */ +class TwoCol { + constructor( + private build: (colW: number, stacked: boolean) => { left: string[]; right: string[] }, + private gutter: string = " ", + ) {} + render(width: number): string[] { + // Narrow terminal: stack left block over right block instead of squeezing columns. + if (width < MIN_TWO_COL_WIDTH) { + const { left, right } = this.build(Math.max(20, width), true); + return fitLines([...left, "", ...right], width); + } + const gw = visibleWidth(this.gutter); + const colW = Math.floor((width - gw) / 2); // equal halves of what's left after the gutter + const { left, right } = this.build(colW, false); + const out: string[] = []; + const n = Math.max(left.length, right.length); + // Zip the two columns row by row: clamp left, pad it to the column edge, gutter, clamp right. + for (let i = 0; i < n; i++) { + const l = truncateToWidth(left[i] ?? "", colW); + const pad = " ".repeat(Math.max(0, colW - visibleWidth(l))); + out.push(l + pad + this.gutter + truncateToWidth(right[i] ?? "", colW)); + } + return fitLines(out, width); + } + invalidate() {} // pi-tui Component contract โ€” nothing cached to invalidate +} + +/** + * A full-width row (the FUSION merge stage), rendered at the width the TUI actually gives + * us โ€” the same clamp discipline as TwoCol, one column instead of two. It exists so the + * span row can't be pinned to a guessed width: hardcoding one trims a 200-col terminal to + * the guess, and guessing high would emit lines wider than a narrow terminal (which pi + * throws on). Ask for the width, then fit to it. + */ +class FullWidth { + constructor(private build: (w: number) => string[]) {} + render(width: number): string[] { + const inner = Math.max(20, width - 2); // leave room for the 1-col pad on each side + return fitLines( + this.build(inner).map((l) => ` ${l}`), + width, + ); + } + invalidate() {} // pi-tui Component contract โ€” nothing cached to invalidate +} + +/** Wrap possibly-styled text to a column width, defensively. */ +function wrapCol(text: string, colW: number): string[] { + try { + return wrapTextWithAnsi(text, Math.max(10, colW)); + } catch { + return text.split("\n"); + } +} + +/** Render markdown to styled lines at a column width (the "same output as pi" body). */ +function mdLines(text: string, colW: number): string[] { + try { + return new Markdown(text || "(no output)", 0, 0, getMarkdownTheme()).render(Math.max(10, colW)); + } catch { + return wrapCol(text, colW); + } +} + +// โ•โ•โ• 6. Child runner โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +/** + * Spawn one `pi --mode json -p` child agent and stream its JSON events into `run`. + * Final answer = last assistant text part. The child writes its session into a + * throwaway --session-dir under the run's /tmp artifacts dir. + */ +function runChild(opts: { + run: AgentRun; // mutated live + prompt: string; + systemPrompt?: string; + tools: string | "none"; + thinking: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; + sessionDir: string; + sessionId?: string; // stable per-role session โ€” the agent keeps its context across commands + fork?: string; // fork this session FILE (copy-on-write) โ€” the child inherits the host's full context + resume?: string; // resume this session id inside sessionDir (later auto-validate rounds re-enter the fork) + cwd: string; + timeoutMs: number; + signal?: AbortSignal; // escape key โ€” kill this child and settle it as "aborted" +}): Promise { + const run = opts.run; + run.thinking = opts.thinking; + // Clean-room spawn: children never load skills, extensions (recursion guard), or + // context files โ€” their entire contract comes from the harness's prompt files. + const args: string[] = [ + "--mode", + "json", + "-p", + "--session-dir", + opts.sessionDir, + "--no-skills", + "--no-extensions", + "--no-context-files", + "--thinking", + opts.thinking, + "--model", + run.model, + ]; + // Session identity, in precedence order: fork the host > resume an earlier fork > pinned per-role id. + if (opts.fork) args.push("--fork", opts.fork); + else if (opts.resume) args.push("--session", opts.resume); + else if (opts.sessionId) args.push("--session-id", opts.sessionId); + if (opts.systemPrompt) args.push("--system-prompt", opts.systemPrompt); + if (opts.tools === "none") args.push("--no-tools"); + else args.push("--tools", opts.tools); + args.push(opts.prompt); + + return new Promise((resolve) => { + const started = Date.now(); + let buffer = ""; + let timedOut = false; + let aborted = false; + // Already stopped before this stage began (e.g. escape during the previous agent): + // settle without spawning, so an abort never starts new model work. + if (opts.signal?.aborted) { + run.status = "aborted"; + run.startedAt = started; + run.endedAt = started; + run.ms = 0; + run.exitCode = 130; + resolve(run); + return; + } + run.status = "working"; + run.startedAt = started; + run.flowMark = run.flow.length; + + // One line of the child's JSON event stream โ†’ the relevant AgentRun mutation. + const processLine = (line: string) => { + if (!line.trim()) return; + let event: any; + try { + event = JSON.parse(line); + } catch { + return; // non-JSON noise on stdout โ€” ignore + } + if (event.type === "session" && typeof event.id === "string") { + run.sessionRef = event.id; // remember the child's session so later rounds can resume it + } else if (event.type === "message_end" && event.message?.role === "assistant") { + const msg = event.message; + for (const part of msg.content ?? []) { + // A turn's reasoning arrives as `thinking` parts (pi-ai ThinkingContent) โ€” a + // different shape from `text`, which is why it was invisible before. + if (part.type === "thinking" && part.thinking?.trim()) { + run.flow.push({ type: "thinking", text: part.thinking }); + } + if (part.type === "text" && part.text?.trim()) { + run.text = part.text; + run.flow.push({ type: "text", text: part.text }); + } + } + run.streamText = ""; + run.streamThinking = ""; + if (msg.stopReason) run.stopReason = msg.stopReason; + if (msg.errorMessage) run.errorMessage = msg.errorMessage; + if (msg.usage) { + // Prompt tokens = input + cacheRead + cacheWrite (pi's own definition, see + // core/cache-stats.ts). cacheWrite is NOT optional accounting: on a cold + // cache the WHOLE prompt is billed as a write and `input` is only the few + // uncached tokens โ€” dropping it renders a real 10k-token prompt as "in 3". + run.tokensIn += (msg.usage.input || 0) + (msg.usage.cacheRead || 0) + (msg.usage.cacheWrite || 0); + run.tokensOut += msg.usage.output || 0; + if (msg.usage.cost?.total) run.costUsd += msg.usage.cost.total; + // Matches pi's calculateContextTokens: `totalTokens || input+output+read+write` + // (|| not ??, so a provider reporting 0 falls through to the sum). + const ctxTokens = + msg.usage.totalTokens || (msg.usage.input || 0) + (msg.usage.cacheRead || 0) + (msg.usage.cacheWrite || 0) + (msg.usage.output || 0); + // Children emit an opening message_end whose usage fields are all null; this is + // an assignment, not a sum, so counting one would clobber a real reading with 0. + if (ctxTokens > 0) run.ctxTokens = ctxTokens; + } + } else if (event.type === "tool_execution_start") { + run.toolCalls++; + const name: string = event.toolName ?? "?"; + const arg = briefArg(event.args); + run.flow.push({ type: "tool", label: arg ? `${name} ${arg}` : name }); + } else if (event.type === "message_update" && event.message?.role === "assistant") { + let t = ""; + let think = ""; + for (const part of event.message.content ?? []) { + if (part.type === "text" && part.text) t += part.text; + else if (part.type === "thinking" && part.thinking) think += part.thinking; + } + if (t) run.streamText = t; + // Streaming the reasoning is what makes a long opening turn look ALIVE: an agent + // at xhigh on a big session can think for minutes before its first token of text, + // and rendering nothing made it read as hung. + if (think) run.streamThinking = think; + } + }; + + const settle = () => { + run.endedAt = Date.now(); + run.ms = run.endedAt - started; + // abort wins over runOk: a killed child may still have emitted usable text, but the + // user asked it to stop โ€” reporting "done" would silently accept a partial answer. + run.status = aborted ? "aborted" : runOk(run) ? "done" : timedOut ? "timeout" : "failed"; + run.streamText = ""; + }; + + const invocation = piInvocation(args); + const proc = spawn(invocation.command, invocation.args, { + cwd: opts.cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + // Children still make their real model API calls โ€” this only skips startup chores. + env: { ...process.env, PI_OFFLINE: "1", PI_SKIP_VERSION_CHECK: "1" }, + }); + + // Line-buffer stdout: events arrive one JSON object per line, possibly split across chunks. + proc.stdout?.on("data", (data: Buffer) => { + buffer += data.toString(); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; // keep the trailing partial line for the next chunk + for (const line of lines) processLine(line); + }); + proc.stderr?.on("data", (data: Buffer) => { + run.stderr += data.toString(); + }); + // SIGTERM, then SIGKILL after the grace period โ€” same escalation as the timeout path. + const killChild = () => { + try { + proc.kill("SIGTERM"); + } catch { + /* already gone */ + } + setTimeout(() => { + if (!proc.killed) { + try { + proc.kill("SIGKILL"); + } catch { + /* already gone */ + } + } + }, KILL_GRACE_MS); + }; + const onAbort = () => { + aborted = true; + killChild(); + }; + opts.signal?.addEventListener("abort", onAbort, { once: true }); + const cleanup = () => { + clearTimeout(timer); + opts.signal?.removeEventListener("abort", onAbort); + }; + + proc.on("close", (code) => { + if (buffer.trim()) processLine(buffer); // flush a final unterminated line + run.exitCode = aborted ? 130 : timedOut ? 124 : (code ?? 0); + cleanup(); + settle(); + resolve(run); + }); + proc.on("error", (err) => { + run.stderr += `\nspawn error: ${String(err)}`; + run.exitCode = 1; + cleanup(); + settle(); + resolve(run); + }); + + // Wall-clock timeout: same SIGTERM โ†’ SIGKILL escalation as the abort path. + const timer = setTimeout(() => { + timedOut = true; + try { + proc.kill("SIGTERM"); + } catch { + /* ignore */ + } + setTimeout(() => { + if (!proc.killed) { + try { + proc.kill("SIGKILL"); + } catch { + /* ignore */ + } + } + }, KILL_GRACE_MS); + }, opts.timeoutMs); + }); +} + +/** Run a plain subprocess (the validation gate) and capture combined output. */ +function runProc( + command: string, + args: string[], + cwd: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise<{ code: number; output: string; aborted?: boolean }> { + return new Promise((resolve) => { + let output = ""; + let timedOut = false; + let aborted = false; + // A gate can burn the full 120s timeout; escape must cut it short like any child. + if (signal?.aborted) { + resolve({ code: 130, output: "[stopped by user before the gate ran]", aborted: true }); + return; + } + let proc: ReturnType; + try { + proc = spawn(command, args, { cwd, shell: false, stdio: ["ignore", "pipe", "pipe"] }); + } catch (err) { + resolve({ code: 127, output: `failed to spawn ${command}: ${String(err)}` }); + return; + } + const onAbort = () => { + aborted = true; + try { + proc.kill("SIGKILL"); + } catch { + /* already gone */ + } + }; + signal?.addEventListener("abort", onAbort, { once: true }); + proc.stdout?.on("data", (d: Buffer) => { + output += d.toString(); + }); + proc.stderr?.on("data", (d: Buffer) => { + output += d.toString(); + }); + const cleanup = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }; + proc.on("close", (code) => { + cleanup(); + if (aborted) { + resolve({ code: 130, output: `${output}\n[stopped by user]`, aborted: true }); + return; + } + resolve({ code: timedOut ? 124 : (code ?? 0), output: timedOut ? `${output}\n[gate timed out]` : output }); + }); + proc.on("error", (err) => { + cleanup(); + resolve({ code: 127, output: `${output}\nspawn error: ${String(err)}` }); + }); + const timer = setTimeout(() => { + timedOut = true; + try { + proc.kill("SIGKILL"); + } catch { + /* ignore */ + } + }, timeoutMs); + }); +} + +// โ•โ•โ• 7. Prompts โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +// Every default prompt lives in its own file next to this extension โ€” +// SYSTEM_PROMPT_*.md and USER_PROMPT_*.md โ€” with {{VARIABLE}} interpolation. +// Edit those files to tune the harness without touching code. + +// Directory holding this extension file (and therefore its prompt files) โ€” +// __dirname under CJS transpilation, import.meta.url under ESM. +const EXT_DIR: string = + typeof __dirname !== "undefined" && __dirname + ? __dirname + : path.dirname(new URL(import.meta.url).pathname); + +const promptCache = new Map(); // each template file is read once per process +/** Load a prompt template from disk (cached). A missing file is a loud load-time error. */ +function promptTemplate(file: string): string { + let tpl = promptCache.get(file); + if (tpl === undefined) { + try { + tpl = fs.readFileSync(path.join(EXT_DIR, file), "utf-8").trim(); + } catch (err) { + throw new Error(`fusion-harness: missing prompt file ${file} (expected next to the extension): ${String(err)}`); + } + promptCache.set(file, tpl); + } + return tpl; +} + +/** Interpolate a template: every {{KEY}} is replaced from vars (missing keys โ†’ empty). */ +function fill(file: string, vars: Record): string { + return promptTemplate(file).replace(/\{\{(\w+)\}\}/g, (_m, k) => vars[k] ?? ""); +} + +/** The /fusion parallel-worker prompt โ€” each worker knows its own role AND its counterpart. */ +function workerPrompt(role: Role, model: string, otherRole: Role, otherModel: string, prompt: string): string { + return fill("USER_PROMPT_FUSION_WORKER.md", { ROLE: role, MODEL: model, OTHER_ROLE: otherRole, OTHER_MODEL: otherModel, PROMPT: prompt }); +} + +/** The built-in critical-merge instruction, used when /fusion gets no explicit fusion prompt. */ +const defaultFusionPrompt = (): string => promptTemplate("USER_PROMPT_FUSION_DEFAULT_INSTRUCTION.md"); + +/** + * /fusion argument parsing โ€” two forms: + * Quoted: /fusion "prompt to both agents" "fusion instruction" + * Separator: /fusion prompt to both agents :: fusion instruction + * A lone prompt (either form) uses the default critical-merge instruction. + */ +function parseFusionArgs(input: string): { prompt: string; fusion?: string } { + // Quoted form: first quoted string is the prompt, the (optionally quoted) rest is the instruction. + const quoted = input.match(/^\s*(["'])([\s\S]*?)\1\s*([\s\S]*)$/); + if (quoted?.[2]?.trim()) { + let rest = quoted[3].trim(); + const restQuoted = rest.match(/^(["'])([\s\S]*?)\1\s*$/); + if (restQuoted) rest = restQuoted[2].trim(); + return { prompt: quoted[2].trim(), fusion: rest || undefined }; + } + // Separator form: split on the first " :: ". + const sep = input.indexOf(" :: "); + if (sep !== -1) return { prompt: input.slice(0, sep).trim(), fusion: input.slice(sep + 4).trim() || undefined }; + // Lone prompt โ€” caller substitutes the default fusion instruction. + return { prompt: input.trim() }; +} + +/** The FUSION merge envelope: both answers, both file paths, and the output contract. */ +function fuserPrompt( + fusionInstruction: string, + prompt: string, + a: { role: Role; model: string; text: string }, + b: { role: Role; model: string; text: string }, + fuserModel: string, + fuserThinking: string, + artifactsDir: string, +): string { + return fill("USER_PROMPT_FUSION_MERGE.md", { + FUSION_INSTRUCTION: fusionInstruction, + MODEL: shortModel(fuserModel), + THINKING: fuserThinking, + PROMPT: prompt, + A_ROLE: a.role, + A_MODEL: a.model, + A_TEXT: truncateChars(a.text, HANDOFF_MAX), + B_ROLE: b.role, + B_MODEL: b.model, + B_TEXT: truncateChars(b.text, HANDOFF_MAX), + // Filename-safe tags for BOTH source models: a fused artifact is the product of the + // pair, so it's named after the pair โ€” not after whichever model happened to merge. + A_TAG: modelTag(a.model), + B_TAG: modelTag(b.model), + // Grounding: the fuser has full tools but no idea where this run's material lives โ€” + // without these paths it resorts to scanning the filesystem for files the workers + // mentioned. The .md files are saved BEFORE the fuser spawns and are complete + // even when the inline texts above were truncated at HANDOFF_MAX. + ARTIFACTS_DIR: artifactsDir, + A_PATH: path.join(artifactsDir, `${a.role.toLowerCase()}.md`), + B_PATH: path.join(artifactsDir, `${b.role.toLowerCase()}.md`), + HANDOFF_MAX: String(HANDOFF_MAX), + }); +} + +/** Round 1 of /auto-validate: the user's request plus the full (immutable) gate script. */ +function builderPrompt(prompt: string, gateScript: string): string { + return fill("USER_PROMPT_BUILDER.md", { PROMPT: prompt, GATE_SCRIPT: gateScript }); +} + +/** Rounds 2+: the verbatim gate failure, plus optional triage brief and repaired-gate update. */ +function correctionPrompt( + round: number, + maxRounds: number, + gateExitCode: number, + gateOutput: string, + triageBrief?: string, + repairedGate?: string, +): string { + const remaining = maxRounds - round; + return fill("USER_PROMPT_CORRECTION.md", { + ROUND: String(round), + MAX_ROUNDS: String(maxRounds), + REMAINING: `${remaining} attempt${remaining === 1 ? "" : "s"} remain`, + GATE_EXIT_CODE: String(gateExitCode), + GATE_OUTPUT: truncateChars(gateOutput.trim() || "(no output)", 8_000), + TRIAGE_BLOCK: triageBrief + ? `\n# VALIDATOR'S TRIAGE (advisory diagnosis from the agent that designed the gate โ€” follow it, but the gate output above remains the source of truth)\n${truncateChars(triageBrief.trim(), 8_000)}\n` + : "", + // A repaired gate makes the builder's round-1 copy STALE โ€” hand it the script that + // now actually judges its work, or it reasons against checks that no longer exist. + GATE_UPDATE_BLOCK: repairedGate + ? `\n# GATE REPAIRED โ€” the VALIDATOR fixed a defect in the acceptance gate. The copy from your first prompt is STALE; THIS is the gate that now judges your work (still immutable to you):\n\`\`\`python\n${truncateChars(repairedGate.trim(), 20_000)}\n\`\`\`\n` + : "", + }); +} + +// Escalation triage: after N failures the VALIDATOR stops being a silent gate and +// diagnoses WHY the builder is stuck โ€” with read-only eyes on the actual state, plus a +// one-shot mandate to rewrite the gate at GATE_PATH if the gate itself is the defect. +const triageSystem = (gatePath: string): string => fill("SYSTEM_PROMPT_TRIAGE.md", { GATE_PATH: gatePath }); + +/** The triage request: what was asked, how many failures, the recent gate history. */ +function triagePrompt( + request: string, + failures: number, + maxRounds: number, + builderReport: string, + gateOutputs: Array<{ round: number; code: number; output: string }>, + artifactsDir: string, +): string { + const history = gateOutputs + .slice(-2) + .map((g) => `## Gate run โ€” round ${g.round} (exit ${g.code})\n\`\`\`\n${truncateChars(g.output.trim() || "(no output)", 6_000)}\n\`\`\``) + .join("\n\n"); + return fill("USER_PROMPT_TRIAGE.md", { + FAILURES: String(failures), + FAILURES_PLURAL: failures === 1 ? "" : "s", + MAX_ROUNDS: String(maxRounds), + REQUEST: request, + HISTORY_SUFFIX: gateOutputs.length > 1 ? "S (note what changed โ€” or didn't โ€” between rounds)" : "", + GATE_HISTORY: history, + BUILDER_REPORT: truncateChars(builderReport, 12_000), + // TRIAGE is read-only but sighted: the run dir holds every full builder report and + // gate output, so it can read past the truncated excerpts above. + ARTIFACTS_DIR: artifactsDir, + }); +} + +// The VALIDATOR designs the gate BEFORE the builder does any work (red โ†’ green): +// its script is the definition of done, and its FAIL lines become the builder's +// correction instructions โ€” so clarity and integrity are hard requirements. +// GATE_PATH is the harness-dictated absolute path the validator must WRITE its gate to โ€” +// the transport is a file, never a code fence (see extractGateScript). +const validatorSystem = (gatePath: string): string => fill("SYSTEM_PROMPT_VALIDATOR.md", { GATE_PATH: gatePath }); + +/** The gate-design request: the user's prompt, the project cwd, and the dictated gate path. */ +function validatorPrompt(prompt: string, cwd: string, gatePath: string): string { + // The gate always lives at /gate.py, so the run dir is its dirname. + return fill("USER_PROMPT_VALIDATOR.md", { PROMPT: prompt, CWD: cwd, GATE_PATH: gatePath, ARTIFACTS_DIR: path.dirname(gatePath) }); +} + +/** The /opinion prompt โ€” answer directly and decisively, tools allowed, no hedging. */ +function opinionPrompt(prompt: string): string { + return fill("USER_PROMPT_OPINION.md", { PROMPT: prompt }); +} + +/** A gate is a PEP 723 uv script; inject the metadata block when the author omitted it. */ +function ensureGateMetadata(script: string): string | undefined { + const s = script.trim(); + if (!s) return undefined; + const withMeta = s.includes("# /// script") ? s : `# /// script\n# requires-python = ">=3.11"\n# dependencies = []\n# ///\n${s}`; + return `${withMeta}\n`; +} + +/** + * LEGACY FALLBACK ONLY โ€” the validator now writes gate.py to disk itself. + * + * Pulling the gate out of a fenced block is lossy by construction: the closing fence is + * whatever ``` comes first, so any gate whose own source contains a literal triple-backtick + * (e.g. asserting raw markdown fences are absent from rendered HTML) is silently truncated + * mid-token. That produced a real 43.6KB gate cut to 33.3KB at `and "```" not in text`, + * failing every round with a SyntaxError while the build under test was fine. Kept only for + * validators that ignore the write instruction and paste the script anyway. + */ +function extractGateScript(text: string): string | undefined { + const fence = text.match(/```(?:python|py|uv)?\s*\n([\s\S]*?)```/); + const script = fence ? fence[1] : text.trim().startsWith("# /// script") ? text.trim() : undefined; + if (!script?.trim()) return undefined; + return ensureGateMetadata(script); +} + +// โ•โ•โ• 8. Extension โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +export default function (pi: ExtensionAPI) { + // โ”€โ”€ 8.1 Flags โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + pi.registerFlag("architect", { + type: "string", + description: `ARCHITECT model (provider/id) โ€” plans, fuses, validates. Default ${DEFAULT_ARCHITECT}.`, + }); + pi.registerFlag("builder", { + type: "string", + description: `BUILDER model (provider/id) โ€” builds. Default ${DEFAULT_BUILDER}.`, + }); + pi.registerFlag("max-validations", { + type: "string", + description: "Max gate validations (build attempts) for /auto-validate before development halts. Default 5. Also overridable inline: /auto-validate --max-validations 3 .", + }); + pi.registerFlag("escalate-to-validator-count", { + type: "string", + description: + "On the Nth gate failure, escalate: the VALIDATOR inspects the builder's work and writes a directed triage brief that accompanies the raw gate output. Default 3. Inline-overridable per command.", + }); + pi.registerFlag("architect-system-prompt", { + type: "string", + description: + "Override the system prompt for ARCHITECT-family worker/fusion agents (inline text, or a path to a file). VALIDATOR/TRIAGE keep their SYSTEM_PROMPT_*.md contracts โ€” edit those files to tune them.", + }); + pi.registerFlag("builder-system-prompt", { + type: "string", + description: "Override the system prompt for all BUILDER agents (inline text, or a path to a file).", + }); + pi.registerFlag("architect-thinking", { + type: "string", + description: "Thinking level for EVERY architect-family execution (worker/fusion/validator/triage): off|minimal|low|medium|high|xhigh|max. Default medium.", + }); + pi.registerFlag("builder-thinking", { + type: "string", + description: "Thinking level for EVERY builder execution: off|minimal|low|medium|high|xhigh|max. Default medium.", + }); + pi.registerFlag("child-timeout", { + type: "string", + description: + "Timeout in SECONDS for every spawned child agent (/opinion + /fusion workers, the FUSION merge, /auto-validate builder rounds and validator). Default 28800 (8h), clamp 10-86400 (24h); the /auto-validate builder never drops below the 8h floor. Real work runs for hours โ€” don't starve it.", + }); + + // โ”€โ”€ 8.2 Flag readers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + /** A string flag's trimmed value, or "" when unset. */ + const flagStr = (name: string): string => { + const v = pi.getFlag(name); + return typeof v === "string" ? v.trim() : ""; + }; + const architectModel = () => flagStr("architect") || DEFAULT_ARCHITECT; // resolved per call โ€” flags are static, but cheap to re-read + const builderModel = () => flagStr("builder") || DEFAULT_BUILDER; + + /** ---system-prompt: inline text, or a file path (file contents win if it exists). */ + const roleSystemPrompt = (role: "architect" | "builder"): string | undefined => { + const v = flagStr(`${role}-system-prompt`); + if (!v) return undefined; + try { + if (fs.existsSync(v) && fs.statSync(v).isFile()) return fs.readFileSync(v, "utf-8"); + } catch { + /* treat as inline text */ + } + return v; + }; + + /** + * pi's own buildSystemPrompt(), for /system-prompt: when a role has no override, the + * prompt its children actually run with is pi's DEFAULT โ€” which the package builds at + * spawn time and does not re-export from its main entry. Import it straight from the + * running pi installation's dist (a file URL bypasses the "exports" map); a bun-compiled + * binary has no real dist on disk, so this resolves undefined and the caller falls back. + */ + let buildSystemPromptLoad: Promise<((o: Record) => string) | undefined> | undefined; + const loadBuildSystemPrompt = (): Promise<((o: Record) => string) | undefined> => { + buildSystemPromptLoad ??= (async () => { + try { + const script = process.argv[1]; + if (!script || script.startsWith("/$bunfs/")) return undefined; + const real = await fs.promises.realpath(script); + const mod = await import(new URL(`file://${path.join(path.dirname(real), "core", "system-prompt.js")}`).href); + return typeof mod.buildSystemPrompt === "function" ? mod.buildSystemPrompt : undefined; + } catch { + return undefined; + } + })(); + return buildSystemPromptLoad; + }; + + /** --child-timeout: seconds before ANY spawned child agent is killed. Default 28800 (8h), clamp 10-86400 (24h). */ + const childTimeoutMs = (): number => { + const v = Number.parseInt(flagStr("child-timeout"), 10); + const s = Number.isFinite(v) && v > 0 ? Math.max(10, Math.min(v, 86_400)) : CHILD_TIMEOUT_S_DEFAULT; + return s * 1000; + }; + /** The /auto-validate builder does real work โ€” never below the 8h floor, even with a small --child-timeout. */ + const buildTimeoutMs = (): number => Math.max(childTimeoutMs(), BUILD_TIMEOUT_MS_FLOOR); + + /** ---thinking: one thinking level for EVERY execution of that model. Default medium. */ + type Thinking = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; + const THINKING_LEVELS: Thinking[] = ["off", "minimal", "low", "medium", "high", "xhigh", "max"]; + /** + * Accept BOTH the canonical level and the short form the footer prints (`high` and `hi`, + * `medium` and `med`, `off` and `none`, โ€ฆ). The footer only ever shows the short form, so + * refusing it would mean rejecting the exact word the UI just displayed. + */ + const THINKING_ALIAS: Record = {}; + for (const level of THINKING_LEVELS) { + THINKING_ALIAS[level] = level; + const short = THINKING_SHORT[level]; + if (short) THINKING_ALIAS[short] = level; + } + const resolveThinking = (raw: string): Thinking | undefined => THINKING_ALIAS[raw.trim().toLowerCase()]; + /** Human help: `off|none, minimal|min, low, medium|med, high|hi, xhigh|xhi, max`. */ + const THINKING_HELP = THINKING_LEVELS.map((l) => (THINKING_SHORT[l] && THINKING_SHORT[l] !== l ? `${l}|${THINKING_SHORT[l]}` : l)).join(", "); + + /** /thinking overrides the boot flag for the rest of the session (per role, in memory). */ + const thinkingOverride: Partial> = {}; + const roleThinking = (role: "architect" | "builder"): Thinking => { + const override = thinkingOverride[role]; + if (override) return override; + // The boot flags take the same aliases, so `--architect-thinking hi` works too. + return resolveThinking(flagStr(`${role}-thinking`)) ?? "medium"; + }; + + // โ”€โ”€ 8.3 Shared live state (widget + footer read this) โ”€โ”€โ”€โ”€โ”€โ”€ + // Left cell = ARCHITECT-family (ARCHITECT/FUSION/VALIDATOR), right cell = BUILDER. + let liveRuns: AgentRun[] = []; // whatever the current command is running (empty when idle) + const sideOf = (r: AgentRun): "left" | "right" => (r.role === "BUILDER" ? "right" : "left"); + // Last finished run per side โ€” keeps the footer's context bar alive between commands. + const sideLast: { left?: AgentRun; right?: AgentRun } = {}; + const absorbTotals = (runs: AgentRun[]) => { + for (const r of runs) { + // FUSION is a FRESH throwaway session by design and runs LAST in /fusion โ€” letting + // it become sideLast would pin the left cell to a session that no longer exists and + // overwrite the persistent ARCHITECT brain's real context with the merge's ~2%. + // ARCHITECT/VALIDATOR/TRIAGE all share the one persistent architect session, so + // only they may speak for the left cell. + if (r.role === "FUSION") continue; + if (r.ctxTokens || r.status !== "pending") sideLast[sideOf(r)] = r; + } + }; + + // โ”€โ”€ 8.4 Persistent per-role sessions โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // The SAME session id is reused for a role across EVERY execution in this project โ€” + // including across pi restarts โ€” via a manifest at + // /tmp/fusion-harness-sessions//manifest.json. ARCHITECT and BUILDER keep + // their context until /fh-reset. (FUSION stays fresh โ€” the merge must judge the two + // answers without prior contamination.) + const projectSlug = (cwd: string): string => + cwd + .replace(/[^a-zA-Z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(-60) || "root"; + const sessionsRootFor = (cwd: string): string => path.join(ARTIFACT_ROOT, "fusion-harness-sessions", projectSlug(cwd)); + // Keyed per role AND model: a transcript built under one model must never be replayed + // as another model's own history. Observed live: a sonnet-5-built architect session + // (full of "You are the ARCHITECT agent (anthropic/claude-sonnet-5)" turns) replayed + // into claude-fable-5 tripped Anthropic's usage-policy classifier โ€” every request + // BLOCKED at the API, even "/opinion hello" โ€” while the identical prompt on a fresh + // fable session passed. Swapping --architect/--builder now simply mints a separate + // brain for that model; switching back resumes the old one. + const roleSessions: Record = {}; + const roleModel = (side: "architect" | "builder"): string => (side === "architect" ? architectModel() : builderModel()); + const roleKey = (side: "architect" | "builder"): string => `${side}:${modelTag(roleModel(side))}`; + const roleSession = (side: "architect" | "builder", cwd: string): { id: string; dir: string } => { + const key = roleKey(side); + const cached = roleSessions[key]; + if (cached) return cached; + const root = sessionsRootFor(cwd); + fs.mkdirSync(root, { recursive: true }); + const manifestPath = path.join(root, "manifest.json"); + let manifest: Record = {}; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); + } catch { + /* first run for this project */ + } + // Pre-model-keying manifests held bare "architect"/"builder" ids โ€” ignored on + // purpose: those sessions carry another model's identity, the exact poison this + // keying exists to prevent. They start fresh once and never come back. + if (!manifest[key]) { + manifest[key] = randomUUID(); + try { + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + } catch { + /* non-fatal โ€” ids just won't survive a restart */ + } + } + const dir = path.join(root, side); + fs.mkdirSync(dir, { recursive: true }); + roleSessions[key] = { id: manifest[key], dir }; + return roleSessions[key]; + }; + /** Session id for summaries โ€” cache-only, never mints a session. */ + const cachedRoleId = (side: "architect" | "builder"): string | undefined => roleSessions[roleKey(side)]?.id; + + /** Wipe this project's persistent role sessions (disk + in-memory, all models) โ€” shared by /fh-reset and /new. */ + const resetRoleSessions = async (cwd: string): Promise => { + const root = sessionsRootFor(cwd); + await fs.promises.rm(root, { recursive: true, force: true }).catch(() => {}); + for (const key of Object.keys(roleSessions)) delete roleSessions[key]; + sideLast.left = undefined; + sideLast.right = undefined; + return root; + }; + + // โ”€โ”€ /new resets the role brains too โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // Pi's built-in /new gives the HOST a fresh session, but the ARCHITECT (and the + // headless-fallback BUILDER) children resume their persistent per-project sessions โ€” + // without this hook they'd drag the old context straight into the "new" conversation. + // So a /new also does the /fh-reset work. `reason` distinguishes the user's /new from + // startup/reload/resume/fork, where persisting across restarts is the whole design. + pi.on("session_start", async (ev: any, ctx: any) => { + if (ev?.reason !== "new") return; + // Silent by design (user preference): a fresh session resetting the role brains is + // the expected behavior, not news. /fh-reset keeps its notify โ€” it's an explicit ask. + await resetRoleSessions(ctx.cwd); + }); + + /** + * The BUILDER is the HOST's agent: launch recipes set the host --model to the builder + * model, so raw (non-slash) input IS the builder, natively. Builder children therefore + * FORK the host session โ€” inheriting every raw chat turn and every panel โ€” instead of + * keeping a separate brain. + * + * The builder must ALWAYS land in a brain that persists across commands. Pi only + * flushes the host session file on the host's first ASSISTANT message (session-manager + * `_persist`) โ€” appended panels don't trigger it โ€” so a session driven purely by slash + * commands has a session PATH but no file to fork. In that window the only way to give + * the builder a memory is the manifest-pinned persistent session; handing it a fresh + * throwaway instead makes it amnesiac on every command (a visible ~1k cold-start prompt + * each time, while the ARCHITECT accumulates in its own persistent session). + */ + const builderSpawn = (ctx: any, artifactsDir: string): { fork?: string; sessionDir: string; sessionId?: string } => { + let hostFile: string | undefined; + try { + hostFile = ctx.sessionManager.getSessionFile?.(); + } catch { + /* treat as sessionless */ + } + // Host session on disk โ†’ fork it: that IS the shared brain (raw chat + every panel). + if (hostFile) { + let flushed = false; + try { + flushed = fs.existsSync(hostFile) && fs.statSync(hostFile).size > 0; + } catch { + /* not flushed yet */ + } + if (flushed) return { fork: hostFile, sessionDir: path.join(artifactsDir, "builder") }; + } + // No host file yet (slash-commands-only session) or no host session at all + // (--no-session / headless): fall back to the persistent builder session so the + // builder still remembers across commands. Once the host does flush, builder + // children move to forking it โ€” a promotion to the intended shared brain, whose + // transcript already carries the panels from these earlier commands; only the + // child's own verbose turns (throwaway by design) are left behind. + const s = roleSession("builder", ctx.cwd); + return { sessionDir: s.dir, sessionId: s.id }; + }; + + /** `โ—† ARCHITECT ยท model` โ€” the role-colored label that opens every column and cell. */ + const roleLabelStr = (theme: any, role: Role, model: string, bold = true, sep = " ยท ") => { + const label = `${ROLE_GLYPH[role]} ${role}`; + return ( + theme.fg(ROLE_COLOR[role], bold ? theme.bold(label) : label) + theme.fg("dim", sep) + theme.fg(ROLE_COLOR[role], shortModel(model)) + ); + }; + + /** + * One footer cell: `โ—† ARCHITECT | model (med) | [โ–ˆโ–ˆ--------] 12%`. + * EVERY content segment carries the role's own color โ€” glyph, role, model, thinking + * level and context bar alike โ€” so a cell reads as one ARCHITECT-blue / BUILDER-orange + * line and the role stays identifiable from any part of it. Only the `|` separators are + * theme-colored (`dim`), which some themes render as a bright hue rather than a muted + * one โ€” so keep separators the ONLY non-role segment, or the cell loses its identity. + */ + const cellStr = (theme: any, role: Role, model: string, thinking: string | undefined, barStr: string): string => { + const sep = theme.fg("dim", " | "); + return ( + roleLabelStr(theme, role, model, false, " | ") + theme.fg(ROLE_COLOR[role], thinkingTag(thinking)) + sep + theme.fg(ROLE_COLOR[role], barStr) + ); + }; + + /** One agent's live column: label, state line, then its flow tail (tools + streaming text). */ + const liveColumn = (theme: any, r: AgentRun | undefined, colW: number): string[] => { + if (!r) return []; + const now = Date.now(); + const elapsed = r.startedAt ? (r.endedAt ?? now) - r.startedAt : 0; + const state = + r.status === "pending" ? "waiting" : r.status === "working" ? `working ${Math.floor(elapsed / 1000)}s` : `${r.status} ${fmtSecs(elapsed)}`; + const stateColor = r.status === "done" ? "success" : r.status === "working" ? ROLE_COLOR[r.role] : r.status === "pending" ? "dim" : "error"; + const bits = [`${STATUS_GLYPH[r.status]} ${state}`]; + if (r.tokensIn || r.tokensOut) bits.push(`in ${fmtK(r.tokensIn)} out ${fmtK(r.tokensOut)}`); + if (r.costUsd) bits.push(`$${r.costUsd.toFixed(4)}`); + const lines: string[] = [roleLabelStr(theme, r.role, r.model), theme.fg(stateColor, bits.join(" ยท "))]; + + // DONE agents collapse to their summary line โ€” the full output lives in the + // transcript panel; re-streaming it here would duplicate what's already shown. + if (r.status === "done") return lines; + if (r.status === "failed" || r.status === "timeout") { + lines.push(theme.fg("error", `โœ— ${runError(r)}`)); + return lines; + } + + // WORKING agents stream only the CURRENT spawn's flow (from flowMark) โ€” never + // stale text from earlier rounds โ€” plus the in-flight message text. + // Three visually distinct flows, same right-facing-triangle family: + // โ–ธ solid + toolTitle โ†’ tool calls (what it DID) + // โ–น hollow + thinkingText italic โ†’ reasoning (what it's THINKING) โ€” pi's own thinking + // color/italic, so it tracks the theme instead of a hardcoded purple + // plain muted/text โ†’ its answer + const thinkLines = (text: string): string[] => + wrapCol(text, colW).map((l, i) => theme.italic(theme.fg("thinkingText", i === 0 ? `โ–น ${l}` : ` ${l}`))); + const flowLines: string[] = []; + for (const item of r.flow.slice(r.flowMark).slice(-6)) { + if (item.type === "tool") flowLines.push(theme.fg("toolTitle", `โ–ธ ${item.label}`)); + else if (item.type === "thinking") flowLines.push(...thinkLines(item.text)); + else for (const l of wrapCol(item.text, colW)) flowLines.push(theme.fg("muted", l)); + } + // Reasoning stays on screen for the WHOLE turn, above the answer it produced โ€” the same + // order the model emits them. (Hiding it as soon as text starts made it near-invisible: + // a turn can stream its whole reasoning between two 1s widget ticks.) + if (r.streamThinking) flowLines.push(...thinkLines(r.streamThinking)); + if (r.streamText) for (const l of wrapCol(r.streamText, colW)) flowLines.push(theme.fg("text", l)); + lines.push(...flowLines.slice(-WIDGET_FLOW_LINES)); + return lines; + }; + + // โ”€โ”€ 8.5 The FOOTER: one aligned cell per model, replacing pi's default entirely โ”€โ”€ + // Per model: `โ—† ROLE | model (med) | [โ–ˆโ–ˆ--------] 12%` โ€” thinking level + context bar. + // No run state (the live widget already shows who's working), no middle divider. + pi.on("session_start", async (_ev: any, ctx: any) => { + if (ctx.mode !== "tui") return; + const contextWindow = (model: string): number => { + try { + const slash = model.indexOf("/"); + const found = ctx.modelRegistry.find(model.slice(0, slash), model.slice(slash + 1)); + if (found?.contextWindow) return found.contextWindow; + } catch { + /* fall through */ + } + return 1_000_000; + }; + const bar = (used: number, window: number): string => { + const pct = Math.max(0, Math.min(1, window > 0 ? used / window : 0)); + const filled = Math.round(pct * 10); + return `[${"โ–ˆ".repeat(filled)}${"-".repeat(10 - filled)}] ${Math.round(pct * 100)}%`; + }; + try { + ctx.ui.setFooter((_tui: any, theme: any) => ({ + invalidate() {}, + render(width: number): string[] { + const cell = (side: "left" | "right"): string => { + const live = liveRuns.filter((r) => sideOf(r) === side); + const active = live.find((r) => r.status === "working") ?? live[live.length - 1] ?? sideLast[side]; + const role: Role = side === "left" ? (active?.role ?? "ARCHITECT") : "BUILDER"; + // The BUILDER cell is the HOST when no child is running โ€” raw input IS the + // builder (recipes launch the host on the builder model), so show the host's + // live model + context usage rather than a stale child snapshot. + if (side === "right" && !live.length) { + const hostModel = ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : builderModel(); + const usage = ctx.getContextUsage?.(); + // The host only HAS usage once it has answered a turn itself. A session + // driven purely by slash commands never does, leaving the cell at 0% while + // builder children forked that session and did real work โ€” so fall back to + // the last builder child's context, which is the host's plus its own turns. + // MAX, not "host first": both readings are partial views of one brain. The host + // only reports usage for turns IT answered, so a slash-command-only session + // reports ~0 even after a child forked it and burned 68k โ€” and the child's own + // reading is discarded with its fork, so it can't be the sole source either. + // Taking the larger keeps the bar from collapsing to 0% right after real work. + const used = Math.max(usage?.tokens ?? 0, sideLast.right?.ctxTokens ?? 0); + const window = usage?.contextWindow ?? contextWindow(hostModel); + // --builder-thinking, i.e. the level every builder CHILD runs at. The host's + // own live level would be truer for this cell, but ExtensionContext exposes + // no getThinkingLevel (it's command-context only) and emits no + // thinking_level_changed event, so it isn't reachable from a footer. + return cellStr(theme, "BUILDER", hostModel, roleThinking("builder"), bar(used, window)); + } + const model = active?.model ?? (side === "left" ? architectModel() : builderModel()); + const thinking = active?.thinking ?? roleThinking(side === "left" ? "architect" : "builder"); + // ctxTokens only exists once a child reports its FIRST message_end. An agent that + // is still on its opening turn (a big persistent session at xhigh can think for + // minutes before emitting anything) would otherwise read 0% โ€” looking dead, not + // busy. Hold the side's last known reading until the new one lands. + const ctxTokens = active?.ctxTokens || sideLast[side]?.ctxTokens || 0; + return cellStr(theme, role, model, thinking, bar(ctxTokens, contextWindow(model))); + }; + const twoCol = new TwoCol(() => ({ left: [cell("left")], right: [cell("right")] }), " "); + return twoCol.render(width); + }, + })); + } catch { + /* footer is progressive enhancement โ€” never break the session over it */ + } + }); + + // โ”€โ”€ 8.6 The transcript renderer โ€” final results, two-column โ”€โ”€ + // Results render FULL-HEIGHT (like normal pi assistant messages) so they land in the + // terminal scrollback and scroll naturally โ€” no hidden lines behind an expand toggle. + pi.registerMessageRenderer(CUSTOM_TYPE, (message, _opts, theme) => { + const d = (message.details ?? {}) as FhDetails; + const content = + typeof message.content === "string" ? message.content : message.content.map((c: any) => (c.type === "text" ? c.text : "")).join(""); + + // The echoed prompt: styled exactly like a normal pi user message. + if (d.kind === "prompt") { + const box = new Box(1, 1, (t: string) => theme.bg("userMessageBg", t)); + box.addChild(new Text(theme.fg("userMessageText", content), 1, 0)); + return box; + } + + const inner = new Container(); + const add = (c: any) => inner.addChild(c); + const blank = () => add(new Text("", 0, 0)); + + /** One finished agent as a column: label + stats + full markdown body. */ + const finalColumn = (s: AgentStat | undefined, body: string, colW: number): string[] => { + if (!s) return []; + const lines: string[] = [ + roleLabelStr(theme, s.role, s.model), + theme.fg(s.error ? "error" : "dim", `${STATUS_GLYPH[s.status]} ${statLine(s)}`), + "", + ]; + if (s.error) lines.push(theme.fg("error", theme.bold(`โœ— FAILED โ€” ${s.error}`))); + lines.push(...mdLines(body, colW)); + return lines; + }; + + const duoBody = () => { + const [ls, rs] = [d.sources?.[0], d.sources?.[1]]; + const [la, ra] = [d.answers?.[0], d.answers?.[1]]; + add( + new TwoCol( + (colW) => ({ left: finalColumn(ls, la?.text ?? "", colW), right: finalColumn(rs, ra?.text ?? "", colW) }), + theme.fg("dim", " โ”‚ "), + ), + ); + }; + + const md = (body: string) => { + add(new Markdown(body || "(no output)", 1, 0, getMarkdownTheme())); + }; + + switch (d.kind) { + case "boot": { + // The boot banner: raw text, sized up the only way a terminal sizes text up โ€” + // fullwidth glyphs (2 cells per letter), bold, centered, floating bare (no + // background box, no rules). Falls back to ASCII if even that can't fit. + add( + new FullWidth((w) => { + const center = (l: string) => " ".repeat(Math.max(0, Math.floor((w - visibleWidth(l)) / 2))) + l; + const big = "FUSION HARNESS".replace(/[A-Z]/g, (c) => String.fromCharCode(c.charCodeAt(0) + 0xfee0)).replace(/ /g, "ใ€€"); + const title = visibleWidth(big) <= w ? big : "FUSION HARNESS"; + // The fusion mark: ARCHITECT circle + BUILDER circle, in their role colors. + const mark = + theme.fg(ROLE_COLOR.ARCHITECT, "โ—") + theme.fg("customMessageLabel", " + ") + theme.fg(ROLE_COLOR.BUILDER, "โ—"); + // One blank line between every element โ€” equal vertical rhythm. + return [ + "", + theme.fg("customMessageLabel", theme.bold(center(title))), + "", + theme.fg("customMessageLabel", center("Combine Your Compute")), + "", + center(mark), + "", + ]; + }), + ); + break; + } + case "banner": { + add(new Text(theme.fg("customMessageLabel", theme.bold(`FUSION HARNESS ยท /${d.command}`)), 1, 0)); + for (const r of d.roles ?? []) add(new Text(` ${roleLabelStr(theme, r.role, r.model)}`, 1, 0)); + if (d.prompt) add(new Text(theme.fg("muted", ` prompt: ${d.prompt.replace(/\s+/g, " ").slice(0, 100)}`), 1, 0)); + if (d.fusionPrompt) add(new Text(theme.fg("muted", ` fusion: ${d.fusionPrompt.replace(/\s+/g, " ").slice(0, 100)}`), 1, 0)); + if (d.maxRounds) + add( + new Text( + theme.fg("muted", ` max validations: ${d.maxRounds}${d.escalateAt ? ` ยท validator triage from failure ${d.escalateAt}` : ""}`), + 1, + 0, + ), + ); + break; + } + case "duo": + case "opinion": { + const title = d.kind === "opinion" ? "โ—† OPINION โ€” side by side" : `FUSION HARNESS ยท /${d.command} โ€” both agents`; + add(new Text(theme.fg("customMessageLabel", theme.bold(title)), 1, 0)); + blank(); + duoBody(); + break; + } + case "system-prompt": { + // Same two-column discipline as every other panel: ARCHITECT-family left, BUILDER + // right. No stats row โ€” nothing ran; these are the prompts the NEXT spawns get. + add(new Text(theme.fg("customMessageLabel", theme.bold("FUSION HARNESS ยท /system-prompt โ€” what each role runs with")), 1, 0)); + blank(); + const spCol = (a: { role: Role; model: string; text: string } | undefined, colW: number): string[] => + a ? [roleLabelStr(theme, a.role, a.model), "", ...mdLines(a.text, colW)] : []; + add(new TwoCol((colW) => ({ left: spCol(d.answers?.[0], colW), right: spCol(d.answers?.[1], colW) }), theme.fg("dim", " โ”‚ "))); + break; + } + case "fused": { + const src = d.sources ?? []; + const srcLabel = src.map((s) => theme.fg(ROLE_COLOR[s.role], `${s.role}(${shortModel(s.model)})`)).join(theme.fg("dim", " โŠ• ")); + add( + new Text( + theme.fg("success", theme.bold(`โง‰ FUSED`)) + + theme.fg("dim", " โ† ") + + srcLabel + + (d.agent ? theme.fg("dim", ` ${STATUS_GLYPH[d.agent.status]} ${statLine(d.agent)}`) : ""), + 1, + 0, + ), + ); + if (d.agent) add(new Text(theme.fg("dim", ` fused by ${d.agent.role} model ${d.agent.model} (fresh session)`), 1, 0)); + blank(); + md(content); + break; + } + case "gate": { + add( + new Text( + theme.fg("customMessageLabel", theme.bold(`FUSION HARNESS ยท /auto-validate โ€” `)) + + theme.fg("mdLink", theme.bold(d.round ? `GATE REPAIRED โš’ (after round ${d.round})` : "GATE DESIGNED โ›จ")) + + (d.agent ? theme.fg("dim", ` ${roleLabelStr(theme, d.agent.role, d.agent.model, false)}${theme.fg("dim", ` ยท ${statLine(d.agent)}`)}`) : ""), + 1, + 0, + ), + ); + if (d.scriptPath) add(new Text(theme.fg("dim", ` gate: ${d.scriptPath} ยท runs after every build round ยท max ${d.maxRounds ?? "?"} validations`), 1, 0)); + blank(); + md(content); + break; + } + case "triage": { + add( + new Text( + theme.fg("customMessageLabel", theme.bold(`FUSION HARNESS ยท /auto-validate โ€” `)) + + theme.fg("warning", theme.bold(`โšก VALIDATOR TRIAGE`)) + + theme.fg("dim", ` ยท escalated after ${d.round ?? "?"} failed validation${(d.round ?? 0) === 1 ? "" : "s"} (threshold ${d.escalateAt ?? "?"})`) + + (d.agent ? theme.fg("dim", ` ${statLine(d.agent)}`) : ""), + 1, + 0, + ), + ); + if (d.agent) add(new Text(` ${roleLabelStr(theme, d.agent.role, d.agent.model)}` + theme.fg("dim", " ยท read-only diagnosis โ€” brief travels with the next correction"), 1, 0)); + blank(); + md(content); + break; + } + case "validation": { + const verdict = d.ok + ? theme.fg("success", theme.bold("GATE PASS โœ“")) + : theme.fg("error", theme.bold(`GATE FAIL โœ— (exit ${d.gateExitCode ?? "?"})`)); + const roundTag = d.round ? theme.fg("dim", ` ยท validation ${d.round}/${d.maxRounds ?? "?"}`) : ""; + add( + new Text(theme.fg("customMessageLabel", theme.bold(`FUSION HARNESS ยท /auto-validate โ€” `)) + verdict + roundTag, 1, 0), + ); + if (d.scriptPath) add(new Text(theme.fg("dim", ` gate: ${d.scriptPath}`), 1, 0)); + blank(); + duoBody(); + break; + } + default: { + // "error" and anything else: attributed failure, loud and specific. + add(new Text(theme.fg("error", theme.bold(`โœ— FUSION HARNESS ยท /${d.command ?? "?"} FAILED`)), 1, 0)); + if (d.agent?.error) add(new Text(theme.fg("error", ` ${d.agent.role} ยท ${d.agent.model} โ€” ${d.agent.error}`), 1, 0)); + for (const s of d.sources ?? []) { + if (s.error) add(new Text(theme.fg("error", ` ${s.role} ยท ${s.model} โ€” ${s.error}`), 1, 0)); + } + if (content.trim()) { + blank(); + md(content); + } + break; + } + } + + if (d.kind !== "banner" && (d.totalMs || d.artifactsDir)) { + blank(); + const bits = [ + d.totalMs ? `run ${fmtSecs(d.totalMs)}` : "", + d.totalCostUsd ? `~$${d.totalCostUsd.toFixed(4)}` : "", + d.artifactsDir ? `artifacts: ${d.artifactsDir}` : "", + ].filter(Boolean); + add(new Text(theme.fg("dim", ` ${bits.join(" ยท ")}`), 1, 0)); + } + + // The boot banner floats bare on the terminal background โ€” every other panel gets + // the custom-message background block. + if (d.kind === "boot") return inner; + const box = new Box(1, 1, (t: string) => theme.bg("customMessageBg", t)); + box.addChild(inner); + return box; + }); + + // โ”€โ”€ 8.7 Shared command machinery โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + const panel = (details: FhDetails, content: string) => { + pi.sendMessage({ + customType: CUSTOM_TYPE, + content: truncateBytes(content, ANSWER_MAX_BYTES), + display: true, + details, + }); + }; + + /** + * The panel for an escape-stopped run. Renders as an `error` panel (no renderer change) + * but says plainly that the user stopped it โ€” an aborted child is !runOk, so without + * this a stop would surface as "the agents failed", blaming the models for the user. + */ + const stoppedPanel = (command: string, runs: AgentRun[], artifactsDir: string, startedAt: number, what: string) => { + panel( + { + kind: "error", + command, + ok: false, + sources: runs.map(toStat), + artifactsDir, + totalMs: Date.now() - startedAt, + totalCostUsd: runs.reduce((s, r) => s + r.costUsd, 0), + }, + `โŠ˜ STOPPED โ€” escape pressed. ${what}\nEverything produced up to this point is in ${artifactsDir}.`, + ); + }; + + /** + * Live two-column widget while children run: left agent | right agent, each + * streaming its own flow (tool lines + response text), plus an optional + * full-width span row (the FUSION merge stage). Re-set every tick. + */ + const startWidget = ( + ctx: any, + command: string, + cols: [AgentRun, AgentRun], + span: AgentRun | undefined, + startedAt: number, + ) => { + liveRuns = span ? [...cols, span] : [...cols]; + const render = () => { + try { + ctx.ui.setWidget( + CUSTOM_TYPE, + (_tui: any, theme: any) => { + const c = new Container(); + const all = span ? [...cols, span] : [...cols]; + const cost = all.reduce((s, r) => s + r.costUsd, 0); + c.addChild( + new Text( + theme.fg("customMessageLabel", theme.bold(`FUSION HARNESS ยท /${command}`)) + + theme.fg("dim", ` ยท ${fmtSecs(Date.now() - startedAt)} ยท ~$${cost.toFixed(4)}`), + 1, + 0, + ), + ); + c.addChild(new TwoCol((colW) => ({ left: liveColumn(theme, cols[0], colW), right: liveColumn(theme, cols[1], colW) }), theme.fg("dim", " โ”‚ "))); + if (span && span.status !== "pending") { + c.addChild(new Text("", 0, 0)); + // Real width, not a guess โ€” the FUSION row spans the whole terminal. + c.addChild(new FullWidth((w) => liveColumn(theme, span, w))); + } + return c; + }, + { placement: "aboveEditor" }, + ); + } catch { + /* widget is best-effort; no-op outside the TUI */ + } + }; + render(); + const ticker = setInterval(render, WIDGET_TICK_MS); + return () => { + clearInterval(ticker); + const all = span ? [...cols, span] : [...cols]; + absorbTotals(all); + liveRuns = []; + try { + ctx.ui.setWidget(CUSTOM_TYPE, undefined); + } catch { + /* ignore */ + } + }; + }; + + /** + * ESCAPE = stop. Pi's own escape only aborts ITS agent loop; a slash command's children + * are our subprocesses, so nothing cancels them unless we listen ourselves. While + * children run we tap raw terminal input and abort the run's controller on Escape. + * + * A bare "\x1b" IS the Escape key; "\x1b[A"/"\x1bOโ€ฆ" are arrow/function-key SEQUENCES + * that merely start with the same byte โ€” matching a prefix would swallow those keys. + * Only Escape is consumed; every other key (incl. ctrl-c, which pi handles) passes through. + * Returns an unsubscribe โ€” always call it, or the tap outlives the command. + */ + const onEscape = (ctx: any, stop: () => void): (() => void) => { + try { + return ( + ctx.ui.onTerminalInput?.((data: string) => { + if (data === "\x1b") { + stop(); + return { consume: true }; + } + return undefined; + }) ?? (() => {}) + ); + } catch { + return () => {}; // headless / no TUI โ€” nothing to tap + } + }; + + /** One abort controller per command run + the Escape tap that trips it. */ + const startStoppable = (ctx: any, command: string): { signal: AbortSignal; stopped: () => boolean; release: () => void } => { + const ctl = new AbortController(); + const release = onEscape(ctx, () => { + if (ctl.signal.aborted) return; + ctl.abort(); + try { + ctx.ui.setStatus(CUSTOM_TYPE, `${command}: stoppingโ€ฆ`); + ctx.ui.notify(`fusion-harness: stopping /${command} โ€” escape pressed`, "warning"); + } catch { + /* best effort */ + } + }); + return { signal: ctl.signal, stopped: () => ctl.signal.aborted, release }; + }; + + // Per-run artifacts land under /tmp/fusion-harness-* (the spec'd, inspectable location โ€” + // note os.tmpdir() on macOS is /var/folders/โ€ฆ, so we pin /tmp explicitly). + const ARTIFACT_ROOT = fs.existsSync("/tmp") ? "/tmp" : os.tmpdir(); + const mkArtifacts = async (): Promise => fs.promises.mkdtemp(path.join(ARTIFACT_ROOT, "fusion-harness-")); + const save = (dir: string, name: string, body: string) => + fs.promises.writeFile(path.join(dir, name), body, "utf-8").catch(() => {}); + const totals = (runs: AgentRun[], startedAt: number) => ({ + totalMs: Date.now() - startedAt, + totalCostUsd: runs.reduce((s, r) => s + r.costUsd, 0), + }); + + // โ”€โ”€ 8.8 Boot banner โ€” big centered "FUSION HARNESS" when the harness starts โ”€โ”€ + // TUI + fresh startup only: no banner noise in headless JSON streams, and no repeat + // banner on /new, /resume, forks, or extension reloads. + pi.on("session_start", async (ev: any, ctx: any) => { + if (ctx.mode !== "tui" || ev?.reason !== "startup") return; + panel({ kind: "boot", ok: true }, "FUSION HARNESS"); + }); + + // โ”€โ”€ 8.9 /fh-reset โ€” wipe the persistent role sessions for this project โ”€โ”€ + // (/new triggers the same reset via the session_start hook above, on top of pi's own fresh session.) + pi.registerCommand("fh-reset", { + description: "Reset the persistent ARCHITECT/BUILDER sessions for this project โ€” both agents start with fresh memories on the next command", + handler: async (_args, ctx) => { + const root = await resetRoleSessions(ctx.cwd); + ctx.ui.notify(`fusion-harness: role sessions reset (${root}) โ€” fresh memories on the next command`, "info"); + }, + }); + + // โ”€โ”€ 8.10 /thinking [builder] โ€” retune thinking mid-session โ”€โ”€ + // Overrides --architect-thinking/--builder-thinking without a restart. Applies to the + // NEXT command: children are spawned per command and read roleThinking() at spawn time. + pi.registerCommand("thinking", { + description: `Set thinking levels: /thinking [builder] (${THINKING_HELP}). Builder is optional โ€” omit it to leave it unchanged. No args shows the current levels.`, + handler: async (args, ctx) => { + const current = () => `ARCHITECT ${roleThinking("architect")} ยท BUILDER ${roleThinking("builder")}`; + const parts = args.trim().split(/\s+/).filter(Boolean); + if (!parts.length) { + ctx.ui.notify(`fusion-harness: thinking โ€” ${current()} (levels: ${THINKING_HELP})`, "info"); + return; + } + if (parts.length > 2) { + ctx.ui.notify(`fusion-harness: /thinking takes at most 2 levels โ€” [builder]. Got ${parts.length}.`, "error"); + return; + } + const bad = parts.filter((p) => !resolveThinking(p)); + if (bad.length) { + ctx.ui.notify(`fusion-harness: invalid thinking level: ${bad.join(", ")}. Valid: ${THINKING_HELP}`, "error"); + return; + } + // Both forms normalize to the canonical level before it reaches a child's --thinking. + thinkingOverride.architect = resolveThinking(parts[0]); + if (parts[1]) thinkingOverride.builder = resolveThinking(parts[1]); + ctx.ui.notify(`fusion-harness: thinking โ†’ ${current()}${parts[1] ? "" : " (builder unchanged)"} โ€” applies to the next command`, "info"); + }, + }); + + // โ”€โ”€ 8.11 /system-prompt โ€” the system prompt each role runs with, two columns โ”€โ”€ + // Zero-cost introspection: no children spawn, nothing hits a model API. Each column is + // just the prompt text: the ---system-prompt override when set, otherwise pi's + // actual default โ€” rebuilt exactly as a spawned child gets it (the host's base prompt + // options minus what the spawn flags strip: no skills, no context files, no custom or + // appended prompt, FULL_TOOLS allowlist). + pi.registerCommand("system-prompt", { + description: "Show the system prompt each role runs with โ€” ARCHITECT | BUILDER side by side (the ---system-prompt override, or pi's default)", + handler: async (_args, ctx) => { + const childDefaultPrompt = async (): Promise => { + const build = await loadBuildSystemPrompt(); + const hostOpts = ctx.getSystemPromptOptions?.(); + if (build && hostOpts) { + return build({ + ...hostOpts, + customPrompt: undefined, + appendSystemPrompt: undefined, + contextFiles: [], + skills: [], + selectedTools: FULL_TOOLS.split(","), + cwd: ctx.cwd, + }); + } + // Last resort: the host's effective prompt โ€” still pi's real prompt, just not + // stripped down to the child's spawn flags. + return ctx.getSystemPrompt?.() ?? "(pi default โ€” could not be resolved from this pi installation)"; + }; + const aOverride = roleSystemPrompt("architect"); + const bOverride = roleSystemPrompt("builder"); + const dflt = aOverride && bOverride ? "" : await childDefaultPrompt(); + const answers: NonNullable = [ + { role: "ARCHITECT", model: architectModel(), text: (aOverride ?? dflt).trim() }, + { role: "BUILDER", model: builderModel(), text: (bOverride ?? dflt).trim() }, + ]; + panel( + { kind: "system-prompt", command: "system-prompt", ok: true, answers }, + [`## ARCHITECT ยท ${answers[0].model}`, answers[0].text, ``, `## BUILDER ยท ${answers[1].model}`, answers[1].text].join("\n"), + ); + }, + }); + + // โ”€โ”€ 8.12 /fusion [:: ] โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + pi.registerCommand("fusion", { + description: 'ARCHITECT + BUILDER answer in parallel (two live columns), then a fusion agent merges them โ€” /fusion "prompt" "fusion-prompt" (or `prompt :: fusion-prompt`)', + handler: async (raw, ctx) => { + const input = (raw ?? "").trim(); + if (!input) { + ctx.ui.notify('Usage: /fusion "" "" (or: /fusion :: )', "warning"); + return; + } + const parsed = parseFusionArgs(input); + const prompt = parsed.prompt; + const fusionInstruction = parsed.fusion || defaultFusionPrompt(); + + const aModel = architectModel(); + const bModel = builderModel(); + const startedAt = Date.now(); + const artifactsDir = await mkArtifacts(); + await save(artifactsDir, "prompt.md", `${prompt}\n\nFUSION INSTRUCTION:\n${fusionInstruction}`); + + // Echo what was asked as a normal user-style message โ€” the transcript stays readable. + panel({ kind: "prompt", command: "fusion", ok: true }, `/fusion ${input}`); + panel( + { + kind: "banner", + command: "fusion", + ok: true, + prompt, + fusionPrompt: fusionInstruction, + roles: [ + { role: "ARCHITECT", model: aModel }, + { role: "BUILDER", model: bModel }, + { role: "FUSION", model: aModel }, + ], + artifactsDir, + }, + "", + ); + + const architect = newRun("ARCHITECT", aModel); + const builder = newRun("BUILDER", bModel); + const fuser = newRun("FUSION", aModel); + const stopper = startStoppable(ctx, "fusion"); + const stopWidget = startWidget(ctx, "fusion", [architect, builder], fuser, startedAt); + ctx.ui.setStatus(CUSTOM_TYPE, "fusion: agents runningโ€ฆ"); + + try { + // โ”€โ”€ Stage 1: ARCHITECT + BUILDER answer in parallel (full tools each) โ”€โ”€ + await Promise.all([ + runChild({ + run: architect, + prompt: workerPrompt("ARCHITECT", aModel, "BUILDER", bModel, prompt), + systemPrompt: roleSystemPrompt("architect"), + tools: FULL_TOOLS, + thinking: roleThinking("architect"), + sessionDir: roleSession("architect", ctx.cwd).dir, + sessionId: roleSession("architect", ctx.cwd).id, + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }), + runChild({ + run: builder, + prompt: workerPrompt("BUILDER", bModel, "ARCHITECT", aModel, prompt), + systemPrompt: roleSystemPrompt("builder"), + tools: FULL_TOOLS, + thinking: roleThinking("builder"), + ...builderSpawn(ctx, artifactsDir), + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }), + ]); + + if (stopper.stopped()) { + stoppedPanel("fusion", [architect, builder], artifactsDir, startedAt, "The two agents were killed; no fusion ran."); + return; + } + + for (const r of [architect, builder]) { + await save(artifactsDir, `${r.role.toLowerCase()}.md`, runOk(r) ? r.text : `FAILED: ${runError(r)}`); + } + + // Both answers, side by side, buffered โ€” never interleaved. + const duoContent = [ + `## ARCHITECT ยท ${aModel}`, + runOk(architect) ? architect.text : `FAILED: ${runError(architect)}`, + ``, + `## BUILDER ยท ${bModel}`, + runOk(builder) ? builder.text : `FAILED: ${runError(builder)}`, + ].join("\n"); + panel( + { + kind: "duo", + command: "fusion", + ok: runOk(architect) && runOk(builder), + sources: [toStat(architect), toStat(builder)], + answers: [ + { role: "ARCHITECT", model: aModel, text: runOk(architect) ? architect.text : "" }, + { role: "BUILDER", model: bModel, text: runOk(builder) ? builder.text : "" }, + ], + artifactsDir, + }, + duoContent, + ); + + if (!runOk(architect) || !runOk(builder)) { + fuser.status = "failed"; + fuser.errorMessage = "skipped โ€” needs both inputs"; + const t = totals([architect, builder], startedAt); + panel( + { kind: "error", command: "fusion", ok: false, sources: [toStat(architect), toStat(builder)], artifactsDir, ...t }, + "Fusion skipped: both agents must succeed to fuse. The failure above is attributed to the specific role + model.", + ); + return; + } + + // โ”€โ”€ Stage 2: the FUSION agent (architect model, fresh session) merges both โ”€โ”€ + ctx.ui.setStatus(CUSTOM_TYPE, "fusion: fusingโ€ฆ"); + await runChild({ + run: fuser, + prompt: fuserPrompt( + fusionInstruction, + prompt, + { role: "ARCHITECT", model: aModel, text: architect.text }, + { role: "BUILDER", model: bModel, text: builder.text }, + aModel, + roleThinking("architect"), + artifactsDir, + ), + systemPrompt: roleSystemPrompt("architect"), + tools: FULL_TOOLS, + thinking: roleThinking("architect"), + sessionDir: path.join(artifactsDir, "fusion"), + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }); + await save(artifactsDir, "fused.md", runOk(fuser) ? fuser.text : `FAILED: ${runError(fuser)}`); + + const t = totals([architect, builder, fuser], startedAt); + if (runOk(fuser)) { + panel( + { + kind: "fused", + command: "fusion", + ok: true, + agent: toStat(fuser), + sources: [toStat(architect), toStat(builder)], + artifactsDir, + ...t, + }, + fuser.text, + ); + } else { + panel( + { + kind: "error", + command: "fusion", + ok: false, + agent: toStat(fuser), + sources: [toStat(architect), toStat(builder)], + artifactsDir, + ...t, + }, + "The two side-by-side answers above are still valid โ€” only the fusion step failed.", + ); + } + await save( + artifactsDir, + "summary.json", + JSON.stringify( + { command: "fusion", ok: runOk(fuser), agents: [toStat(architect), toStat(builder), toStat(fuser)], sessions: { architect: cachedRoleId("architect"), builder: cachedRoleId("builder") }, ...t }, + null, + 2, + ), + ); + } finally { + stopper.release(); // never leave the escape tap installed past the command + stopWidget(); + ctx.ui.setStatus(CUSTOM_TYPE, undefined); + } + }, + }); + + // โ”€โ”€ 8.13 /auto-validate [--max-validations N] โ”€โ”€โ”€โ”€โ”€ + // Gate-first validation loop (red โ†’ green): + // 1. VALIDATOR designs the acceptance gate (uv script) BEFORE any work happens. + // 2. Baseline gate run โ€” expected FAIL (integrity check on the gate itself). + // 3. BUILDER builds against the visible, immutable gate. + // 4. Gate runs. FAIL โ†’ its output feeds back into the builder's persistent + // session as correction instructions. PASS โ†’ done. + // 5. After --max-validations failed validations, development HALTS loudly. + const MAX_VALIDATIONS_DEFAULT = 5; + const ESCALATE_DEFAULT = 3; + const clampCount = (n: number, fallback: number): number => (Number.isFinite(n) && n >= 1 ? Math.min(20, Math.floor(n)) : fallback); + const clampValidations = (n: number): number => clampCount(n, MAX_VALIDATIONS_DEFAULT); + /** A gate result that means "the gate itself could not run" โ€” never the builder's fault. */ + const gateHarnessError = (g: { code: number; output: string }): string | undefined => { + if (g.code === 124 || g.output.includes("[gate timed out]")) return "the gate timed out (gates must finish in <60s)"; + if (g.code === 127 || /failed to spawn|spawn error/.test(g.output)) return "the gate could not be executed โ€” is `uv` installed and on PATH?"; + return undefined; + }; + + pi.registerCommand("auto-validate", { + description: + "Auto-validation loop: VALIDATOR designs a uv acceptance gate FIRST, BUILDER builds, the gate runs, failures feed back to the builder โ€” until pass or --max-validations (default 5)", + handler: async (raw, ctx) => { + let input = (raw ?? "").trim(); + // Inline overrides of the startup flags: /auto-validate --max-validations 3 --escalate-to-validator-count 2 + let maxV = clampValidations(Number.parseInt(flagStr("max-validations"), 10)); + let escalateAt = clampCount(Number.parseInt(flagStr("escalate-to-validator-count"), 10), ESCALATE_DEFAULT); + input = input + .replace(/--max-validations[=\s]+(\d+)\s*/g, (_m, n) => { + maxV = clampValidations(Number.parseInt(n, 10)); + return ""; + }) + .replace(/--escalate-to-validator-count[=\s]+(\d+)\s*/g, (_m, n) => { + escalateAt = clampCount(Number.parseInt(n, 10), ESCALATE_DEFAULT); + return ""; + }) + .trim(); + if (!input) { + ctx.ui.notify("Usage: /auto-validate [--max-validations N] [--escalate-to-validator-count N] ", "warning"); + return; + } + const prompt = input; + const aModel = architectModel(); + const bModel = builderModel(); + const startedAt = Date.now(); + const artifactsDir = await mkArtifacts(); + await save(artifactsDir, "prompt.md", prompt); + + panel({ kind: "prompt", command: "auto-validate", ok: true }, `/auto-validate ${(raw ?? "").trim()}`); + panel( + { + kind: "banner", + command: "auto-validate", + ok: true, + prompt, + maxRounds: maxV, + escalateAt, + roles: [ + { role: "VALIDATOR", model: aModel }, + { role: "BUILDER", model: bModel }, + ], + artifactsDir, + }, + "", + ); + + const validator = newRun("VALIDATOR", aModel); + const builder = newRun("BUILDER", bModel); + // Columns match the footer: VALIDATOR (architect-family) left, BUILDER right. + // One builder AgentRun is reused across correction rounds โ€” same persistent + // session, cumulative tokens/cost, one accumulating flow column. + const stopper = startStoppable(ctx, "auto-validate"); + const stopWidget = startWidget(ctx, "auto-validate", [validator, builder], undefined, startedAt); + const fail = (agentStat: AgentStat, body: string, extra: Partial = {}) => { + const t = totals([validator, builder], startedAt); + panel({ kind: "error", command: "auto-validate", ok: false, agent: agentStat, artifactsDir, maxRounds: maxV, ...t, ...extra }, body); + }; + + try { + // โ”€โ”€ 1. VALIDATOR designs the gate (before any build) โ”€โ”€ + // The gate's transport is the FILESYSTEM: the harness dictates an absolute path and + // the validator writes gate.py there with its own write tool. Nothing is parsed out + // of the reply, so a gate whose own source contains ``` survives intact. + const scriptPath = path.join(artifactsDir, "gate.py"); + ctx.ui.setStatus(CUSTOM_TYPE, "auto-validate: validator designing the gateโ€ฆ"); + await runChild({ + run: validator, + prompt: validatorPrompt(prompt, ctx.cwd, scriptPath), + systemPrompt: validatorSystem(scriptPath), + tools: VALIDATOR_TOOLS, + thinking: roleThinking("architect"), + sessionDir: roleSession("architect", ctx.cwd).dir, + sessionId: roleSession("architect", ctx.cwd).id, + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }); + await save(artifactsDir, "validator.md", runOk(validator) ? validator.text : `FAILED: ${runError(validator)}`); + // Prefer the file the validator wrote. Fence extraction is the legacy fallback, + // used only when it pasted the gate inline instead (lossy โ€” see extractGateScript). + let script: string | undefined; + let gateVia = "written to disk by the validator"; + if (runOk(validator)) { + try { + script = ensureGateMetadata(await fs.promises.readFile(scriptPath, "utf-8")); + } catch { + /* validator didn't write the file โ€” fall back to the fence */ + } + if (!script) { + script = extractGateScript(validator.text); + if (script) gateVia = "recovered from a code fence (legacy โ€” truncates at an embedded ```)"; + } + } + if (stopper.stopped()) { + stoppedPanel("auto-validate", [validator, builder], artifactsDir, startedAt, "The validator was killed while designing the gate; nothing was built."); + return; + } + if (!script) { + const stat = toStat(validator); + if (!stat.error) stat.error = `did not write a uv gate script to ${scriptPath}`; + fail( + stat, + `โœ— VALIDATOR (${aModel}) failed to design the acceptance gate โ€” nothing was built.\nExpected the gate at ${scriptPath}; no file was written and no fenced script was found in its reply.\n\n${validator.text || ""}`, + ); + return; + } + // Only rewrite when the content differs (fence fallback, or injected metadata), so a + // gate the validator wrote itself executes byte-for-byte as authored. + let onDisk: string | undefined; + try { + onDisk = await fs.promises.readFile(scriptPath, "utf-8"); + } catch { + /* not written yet */ + } + if (onDisk !== script) await save(artifactsDir, "gate.py", script); + validator.flow.push({ type: "tool", label: `gate.py โ€” ${gateVia} (${script.length} bytes)` }); + + // โ”€โ”€ 2. Baseline gate run โ€” must FAIL before the build (red) โ”€โ”€ + ctx.ui.setStatus(CUSTOM_TYPE, "auto-validate: baseline gate run (expected FAIL)โ€ฆ"); + const baseline = await runProc("uv", ["run", scriptPath], ctx.cwd, GATE_TIMEOUT_MS, stopper.signal); + await save(artifactsDir, "gate-baseline.txt", `exit ${baseline.code}\n\n${baseline.output}`); + validator.flow.push({ type: "tool", label: `uv run gate.py (baseline) โ†’ exit ${baseline.code}` }); + if (stopper.stopped()) { + stoppedPanel("auto-validate", [validator, builder], artifactsDir, startedAt, "Stopped at the baseline gate run; nothing was built."); + return; + } + const baselineHarnessErr = gateHarnessError(baseline); + if (baselineHarnessErr) { + const stat = toStat(validator); + stat.error = `gate execution error: ${baselineHarnessErr}`; + fail(stat, `โœ— GATE ERROR โ€” ${baselineHarnessErr}\n\nNothing was built. Gate output:\n\`\`\`\n${truncateChars(baseline.output.trim(), DETAIL_SNIPPET_MAX)}\n\`\`\``); + return; + } + const baselineNote = + baseline.code === 0 + ? `### โš  BASELINE WARNING\nThe gate already PASSES before any work was done โ€” either the request is already satisfied or the gate is too weak. Proceeding to build anyway; treat a first-round pass with suspicion.` + : `### Baseline run โ€” RED โœ“ (exit ${baseline.code}, expected)\nThe gate correctly fails against the current state โ€” the loop is live.\n\`\`\`\n${truncateChars(baseline.output.trim() || "(no output)", DETAIL_SNIPPET_MAX)}\n\`\`\``; + panel( + { + kind: "gate", + command: "auto-validate", + ok: true, + agent: toStat(validator), + maxRounds: maxV, + script: truncateChars(script, DETAIL_SNIPPET_MAX), + gateExitCode: baseline.code, + scriptPath, + artifactsDir, + }, + [`### Acceptance gate (designed by VALIDATOR before the build; immutable)`, "```python", script.trim(), "```", baselineNote].join("\n"), + ); + + // โ”€โ”€ 3. Build โ†’ validate loop โ”€โ”€ + // Round 1 forks the host session (the builder IS the host's agent lineage); + // later rounds resume that same fork so the loop keeps its working memory. + let lastGate: { code: number; output: string } | undefined; + const gateHistory: Array<{ round: number; code: number; output: string }> = []; + let pendingTriage: string | undefined; + let pendingGateUpdate: string | undefined; // repaired gate โ†’ next correction prompt (round-1 copy is stale) + let gateRepairUsed = false; // ONE repair per run โ€” the grader never gets to keep moving goalposts + const firstSpawn = builderSpawn(ctx, artifactsDir); + for (let round = 1; round <= maxV; round++) { + const triageBrief = pendingTriage; + pendingTriage = undefined; + const gateUpdate = pendingGateUpdate; + pendingGateUpdate = undefined; + const spawn = + round === 1 + ? firstSpawn + : builder.sessionRef + ? { sessionDir: firstSpawn.sessionDir, resume: builder.sessionRef } + : firstSpawn; + ctx.ui.setStatus(CUSTOM_TYPE, `auto-validate: builder โ€” round ${round}/${maxV}โ€ฆ`); + await runChild({ + run: builder, + prompt: round === 1 ? builderPrompt(prompt, script) : correctionPrompt(round, maxV, lastGate!.code, lastGate!.output, triageBrief, gateUpdate), + systemPrompt: roleSystemPrompt("builder"), + tools: FULL_TOOLS, + thinking: roleThinking("builder"), + ...spawn, + cwd: ctx.cwd, + timeoutMs: buildTimeoutMs(), + signal: stopper.signal, + }); + await save(artifactsDir, `builder-round-${round}.md`, runOk(builder) ? builder.text : `FAILED: ${runError(builder)}`); + // Check the stop BEFORE blaming the builder: an escape-killed child is !runOk, + // and reporting "BUILDER failed" for a user-initiated stop is a lie. + if (stopper.stopped()) { + stoppedPanel("auto-validate", [validator, builder], artifactsDir, startedAt, `Stopped during build round ${round}/${maxV}; the gate was not re-run.`); + return; + } + if (!runOk(builder)) { + fail( + toStat(builder), + `โœ— BUILDER (${bModel}) failed during round ${round}/${maxV} โ€” the loop cannot continue.\n\n${builder.text || ""}`, + { round, sources: [toStat(validator)] }, + ); + return; + } + + ctx.ui.setStatus(CUSTOM_TYPE, `auto-validate: gate โ€” validation ${round}/${maxV}โ€ฆ`); + lastGate = await runProc("uv", ["run", scriptPath], ctx.cwd, GATE_TIMEOUT_MS, stopper.signal); + await save(artifactsDir, `gate-round-${round}.txt`, `exit ${lastGate.code}\n\n${lastGate.output}`); + validator.flow.push({ type: "tool", label: `uv run gate.py (round ${round}) โ†’ exit ${lastGate.code}` }); + if (stopper.stopped()) { + stoppedPanel("auto-validate", [validator, builder], artifactsDir, startedAt, `Stopped at the gate run for round ${round}/${maxV}.`); + return; + } + const harnessErr = gateHarnessError(lastGate); + if (harnessErr) { + const stat = toStat(validator); + stat.error = `gate execution error: ${harnessErr}`; + fail(stat, `โœ— GATE ERROR during validation ${round}/${maxV} โ€” ${harnessErr}\n\nGate output:\n\`\`\`\n${truncateChars(lastGate.output.trim(), DETAIL_SNIPPET_MAX)}\n\`\`\``, { round }); + return; + } + + const ok = lastGate.code === 0; + const t = totals([validator, builder], startedAt); + const gateBody = [ + `### Gate run โ€” ${ok ? "PASS (exit 0)" : `FAIL (exit ${lastGate.code})`}`, + "```", + truncateChars(lastGate.output.trim() || "(no output)", DETAIL_SNIPPET_MAX * 2), + "```", + ok && baseline.code === 0 ? `โš  Note: the gate also passed at baseline โ€” verify the result yourself.` : "", + ].join("\n"); + const builderBody = `### Builder report โ€” round ${round}\n${builder.text}`; + panel( + { + kind: "validation", + command: "auto-validate", + ok, + round, + maxRounds: maxV, + agent: toStat(validator), + sources: [toStat(validator), toStat(builder)], + answers: [ + { role: "VALIDATOR", model: aModel, text: gateBody }, + { role: "BUILDER", model: bModel, text: builderBody }, + ], + gateOutput: truncateChars(lastGate.output, DETAIL_SNIPPET_MAX), + gateExitCode: lastGate.code, + scriptPath, + artifactsDir, + ...t, + }, + `${builderBody}\n\n${gateBody}`, + ); + if (ok) { + await save( + artifactsDir, + "summary.json", + JSON.stringify( + { command: "auto-validate", ok: true, rounds: round, maxValidations: maxV, escalateAt, gateExitCode: 0, agents: [toStat(validator), toStat(builder)], sessions: { architect: cachedRoleId("architect"), builder: cachedRoleId("builder") }, ...t }, + null, + 2, + ), + ); + return; + } + + // โ”€โ”€ Escalation: on the Nth failure, the VALIDATOR diagnoses why the builder is stuck โ”€โ”€ + gateHistory.push({ round, code: lastGate.code, output: lastGate.output }); + if (round >= escalateAt && round < maxV) { + ctx.ui.setStatus(CUSTOM_TYPE, `auto-validate: โšก validator triage (failure ${round}/${maxV})โ€ฆ`); + // Snapshot the gate as it sits on disk BEFORE triage โ€” the repair detector + // compares content, not the brief's wording. + let gateBefore = script; + try { + gateBefore = await fs.promises.readFile(scriptPath, "utf-8"); + } catch { + /* keep the in-memory copy */ + } + await runChild({ + run: validator, + prompt: triagePrompt(prompt, round, maxV, builder.text, gateHistory, artifactsDir), + systemPrompt: triageSystem(scriptPath), + // Repair power is enforced by TOOLS, not trust: while the run's single + // repair is unused, triage holds the validator's write (one dictated + // path); once spent, it drops back to strictly read-only eyes. + tools: gateRepairUsed ? READONLY_TOOLS : VALIDATOR_TOOLS, + thinking: roleThinking("architect"), + sessionDir: roleSession("architect", ctx.cwd).dir, + sessionId: roleSession("architect", ctx.cwd).id, + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }); + await save(artifactsDir, `triage-round-${round}.md`, runOk(validator) ? validator.text : `FAILED: ${runError(validator)}`); + if (runOk(validator)) { + pendingTriage = validator.text; + panel( + { + kind: "triage", + command: "auto-validate", + ok: true, + round, + maxRounds: maxV, + escalateAt, + agent: toStat(validator), + artifactsDir, + }, + validator.text, + ); + + // โ”€โ”€ Gate repair: triage rewrote a defective gate (once per run) โ”€โ”€ + if (!gateRepairUsed) { + let gateAfter: string | undefined; + try { + gateAfter = await fs.promises.readFile(scriptPath, "utf-8"); + } catch { + /* unreadable โ€” treat as unchanged */ + } + if (gateAfter?.trim() && gateAfter !== gateBefore) { + gateRepairUsed = true; + await save(artifactsDir, `gate.py.r${round}`, gateBefore); // the defective gate, preserved for audit + script = ensureGateMetadata(gateAfter) ?? gateAfter; + if (script !== gateAfter) await save(artifactsDir, "gate.py", script); + pendingGateUpdate = script; + validator.flow.push({ type: "tool", label: `gate.py REPAIRED (defect) โ€” old gate saved as gate.py.r${round}` }); + + // The repaired gate re-runs IMMEDIATELY, on the house: a gate defect + // was never the builder's failure, so it costs no correction round. + ctx.ui.setStatus(CUSTOM_TYPE, "auto-validate: gate repaired โ€” free re-runโ€ฆ"); + const rerun = await runProc("uv", ["run", scriptPath], ctx.cwd, GATE_TIMEOUT_MS, stopper.signal); + await save(artifactsDir, `gate-repair-round-${round}.txt`, `exit ${rerun.code}\n\n${rerun.output}`); + validator.flow.push({ type: "tool", label: `uv run gate.py (post-repair) โ†’ exit ${rerun.code}` }); + if (stopper.stopped()) { + stoppedPanel("auto-validate", [validator, builder], artifactsDir, startedAt, `Stopped at the post-repair gate run (round ${round}/${maxV}).`); + return; + } + const rerunHarnessErr = gateHarnessError(rerun); + if (rerunHarnessErr) { + const stat = toStat(validator); + stat.error = `gate execution error: ${rerunHarnessErr}`; + fail(stat, `โœ— GATE ERROR on the post-repair run โ€” ${rerunHarnessErr}\n\nGate output:\n\`\`\`\n${truncateChars(rerun.output.trim(), DETAIL_SNIPPET_MAX)}\n\`\`\``, { round }); + return; + } + panel( + { + kind: "gate", + command: "auto-validate", + ok: rerun.code === 0, + round, + maxRounds: maxV, + agent: toStat(validator), + script: truncateChars(script, DETAIL_SNIPPET_MAX), + gateExitCode: rerun.code, + scriptPath, + artifactsDir, + }, + [ + `### โš’ Gate REPAIRED by VALIDATOR โ€” defect fixed after round ${round} (old gate: gate.py.r${round} ยท one repair per run)`, + "```python", + script.trim(), + "```", + rerun.code === 0 + ? `### Post-repair run โ€” GREEN โœ“ (exit 0, no builder round consumed)\n\`\`\`\n${truncateChars(rerun.output.trim() || "(no output)", DETAIL_SNIPPET_MAX)}\n\`\`\`` + : `### Post-repair run โ€” still RED (exit ${rerun.code}) โ€” these are now the REAL failures\n\`\`\`\n${truncateChars(rerun.output.trim() || "(no output)", DETAIL_SNIPPET_MAX)}\n\`\`\``, + ].join("\n"), + ); + if (rerun.code === 0) { + // The build was right all along โ€” the gate was the bug. End green. + const t = totals([validator, builder], startedAt); + const gateBody = `### Gate run โ€” PASS (exit 0, post-repair)\n\`\`\`\n${truncateChars(rerun.output.trim() || "(no output)", DETAIL_SNIPPET_MAX * 2)}\n\`\`\``; + const builderBody = `### Builder report โ€” round ${round}\n${builder.text}`; + panel( + { + kind: "validation", + command: "auto-validate", + ok: true, + round, + maxRounds: maxV, + agent: toStat(validator), + sources: [toStat(validator), toStat(builder)], + answers: [ + { role: "VALIDATOR", model: aModel, text: gateBody }, + { role: "BUILDER", model: bModel, text: builderBody }, + ], + gateOutput: truncateChars(rerun.output, DETAIL_SNIPPET_MAX), + gateExitCode: 0, + scriptPath, + artifactsDir, + ...t, + }, + `${builderBody}\n\n${gateBody}`, + ); + await save( + artifactsDir, + "summary.json", + JSON.stringify( + { command: "auto-validate", ok: true, rounds: round, gateRepaired: true, maxValidations: maxV, escalateAt, gateExitCode: 0, agents: [toStat(validator), toStat(builder)], sessions: { architect: cachedRoleId("architect"), builder: cachedRoleId("builder") }, ...t }, + null, + 2, + ), + ); + return; + } + // Still red on a now-sound gate: those failures are real โ€” hand them + // to the next correction round. + lastGate = rerun; + gateHistory.push({ round, code: rerun.code, output: rerun.output }); + } + } + } else { + // Triage is an enhancement โ€” a failed triage never blocks the loop. + validator.flow.push({ type: "tool", label: `triage failed (${runError(validator)}) โ€” continuing with raw gate output` }); + } + } + } + + // โ”€โ”€ 4. Max validations exhausted โ€” halt loudly โ”€โ”€ + const stat = toStat(builder); + stat.error = `gate still failing after ${maxV}/${maxV} validations`; + fail( + stat, + [ + `## โœ— HALTED โ€” development stopped after ${maxV}/${maxV} validations`, + `The acceptance gate is still failing. No further corrections will be attempted.`, + ``, + `### Last gate output (exit ${lastGate?.code ?? "?"})`, + "```", + truncateChars(lastGate?.output.trim() || "(no output)", DETAIL_SNIPPET_MAX), + "```", + ``, + `Raise the cap with \`--max-validations N\` (startup flag or inline) or inspect the artifacts: ${artifactsDir}`, + ].join("\n"), + { round: maxV }, + ); + await save( + artifactsDir, + "summary.json", + JSON.stringify( + { command: "auto-validate", ok: false, halted: true, rounds: maxV, maxValidations: maxV, escalateAt, gateExitCode: lastGate?.code, agents: [toStat(validator), toStat(builder)], sessions: { architect: cachedRoleId("architect"), builder: cachedRoleId("builder") }, ...totals([validator, builder], startedAt) }, + null, + 2, + ), + ); + } finally { + stopper.release(); // never leave the escape tap installed past the command + stopWidget(); + ctx.ui.setStatus(CUSTOM_TYPE, undefined); + } + }, + }); + + // โ”€โ”€ 8.14 /opinion โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + pi.registerCommand("opinion", { + description: "Both models answer independently โ€” side-by-side two-column panel (model ยท latency ยท tokens ยท cost). No fusion.", + handler: async (raw, ctx) => { + const prompt = (raw ?? "").trim(); + if (!prompt) { + ctx.ui.notify("Usage: /opinion ", "warning"); + return; + } + const aModel = architectModel(); + const bModel = builderModel(); + const startedAt = Date.now(); + const artifactsDir = await mkArtifacts(); + await save(artifactsDir, "prompt.md", prompt); + + panel({ kind: "prompt", command: "opinion", ok: true }, `/opinion ${prompt}`); + const architect = newRun("ARCHITECT", aModel); + const builder = newRun("BUILDER", bModel); + const stopper = startStoppable(ctx, "opinion"); + const stopWidget = startWidget(ctx, "opinion", [architect, builder], undefined, startedAt); + ctx.ui.setStatus(CUSTOM_TYPE, "opinion: both models answeringโ€ฆ"); + + try { + // Both agents answer the same prompt in parallel โ€” read/bash tools only (an A/B read, not a build). + await Promise.all([ + runChild({ + run: architect, + prompt: opinionPrompt(prompt), + systemPrompt: roleSystemPrompt("architect"), + tools: OPINION_TOOLS, + thinking: roleThinking("architect"), + sessionDir: roleSession("architect", ctx.cwd).dir, + sessionId: roleSession("architect", ctx.cwd).id, + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }), + runChild({ + run: builder, + prompt: opinionPrompt(prompt), + systemPrompt: roleSystemPrompt("builder"), + tools: OPINION_TOOLS, + thinking: roleThinking("builder"), + ...builderSpawn(ctx, artifactsDir), + cwd: ctx.cwd, + timeoutMs: childTimeoutMs(), + signal: stopper.signal, + }), + ]); + + if (stopper.stopped()) { + stoppedPanel("opinion", [architect, builder], artifactsDir, startedAt, "Both agents were killed; no comparison was rendered."); + return; + } + + for (const r of [architect, builder]) { + await save(artifactsDir, `${r.role.toLowerCase()}.md`, runOk(r) ? r.text : `FAILED: ${runError(r)}`); + } + + const ok = runOk(architect) && runOk(builder); + const t = totals([architect, builder], startedAt); + const fallback = [ + `# OPINION โ€” ${prompt.replace(/\s+/g, " ").slice(0, 80)}`, + ``, + `## ARCHITECT ยท ${aModel}`, + runOk(architect) ? architect.text : `FAILED: ${runError(architect)}`, + ``, + `## BUILDER ยท ${bModel}`, + runOk(builder) ? builder.text : `FAILED: ${runError(builder)}`, + ].join("\n"); + panel( + { + kind: "opinion", + command: "opinion", + ok, + prompt, + sources: [toStat(architect), toStat(builder)], + answers: [ + { role: "ARCHITECT", model: aModel, text: runOk(architect) ? architect.text : "" }, + { role: "BUILDER", model: bModel, text: runOk(builder) ? builder.text : "" }, + ], + artifactsDir, + ...t, + }, + fallback, + ); + await save( + artifactsDir, + "summary.json", + JSON.stringify( + { command: "opinion", ok, agents: [toStat(architect), toStat(builder)], sessions: { architect: cachedRoleId("architect"), builder: cachedRoleId("builder") }, ...t }, + null, + 2, + ), + ); + } finally { + stopper.release(); // never leave the escape tap installed past the command + stopWidget(); + ctx.ui.setStatus(CUSTOM_TYPE, undefined); + } + }, + }); +} diff --git a/images/hero.png b/images/hero.png new file mode 100644 index 0000000..ee75e51 Binary files /dev/null and b/images/hero.png differ diff --git a/images/svg-01-fusion-hero-animated.svg b/images/svg-01-fusion-hero-animated.svg new file mode 100644 index 0000000..d431f1c --- /dev/null +++ b/images/svg-01-fusion-hero-animated.svg @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + FUSE FRONTIER MODELS + AND, not OR โ€” combine compute instead of selecting it + + + + + ◆ ARCHITECT + anthropic/claude-fable-5 + plans · fuses · validates + + + + + + ▲ BUILDER + openai/gpt-5.6-sol + builds · ships working diffs + + + + + + + + + + + ⧉ FUSED RESULT + [ARCHITECT]/[BUILDER] attribution · consensus & divergence + + + + fusion-harness · a Pi extension for two-model agentic engineering + diff --git a/images/svg-03-three-commands.svg b/images/svg-03-three-commands.svg new file mode 100644 index 0000000..3d603bb --- /dev/null +++ b/images/svg-03-three-commands.svg @@ -0,0 +1,71 @@ + + + + + ONE EXTENSION · THREE COMMANDS + + + + + /opinion + + + ◆ + answer + + ▲ + answer + + + model · latency · tokens · cost + + 2 agents · independent answers + a pure side-by-side A/B read + + + + + + /fusion + + + ◆ + + ▲ + + + + + + ⧉ FUSION + fresh session · full tools + + 3 agents · critical merge + [ARCHITECT]/[BUILDER] attribution + + + + + + /auto-validate + + + ✓ GATE + designed first + + + + ▲ BUILD + gate immutable + + + + FAIL feeds back verbatim + + red → green, or a loud halt + the builder never grades + its own homework + + + every agent: a clean-room pi subprocess · artifacts in /tmp/fusion-harness-* · never inside the repo + diff --git a/images/svg-04-host-as-builder.svg b/images/svg-04-host-as-builder.svg new file mode 100644 index 0000000..9cb7da8 --- /dev/null +++ b/images/svg-04-host-as-builder.svg @@ -0,0 +1,59 @@ + + + + + RAW CHAT IS THE BUILDER + + + + + ▲ PI HOST · BUILDER MODEL + openai/gpt-5.6-sol · your skills & context load here + + > plain messages = the untouched native pi experience + raw chat and slash commands are the same agent + + + + + + + + fork + + + + + ▲ builder child + inherits raw chat + panels + clean-room spawn + + + + ▲ builder child + auto-validate rounds + resume the same fork + + + + + + + + + ◆ ARCHITECT + anthropic/claude-fable-5 + + + separate persistent brain + pinned per project + per model + /tmp/fusion-harness-sessions/ + restart-proof · /fh-reset wipes + + never forks the host: + an architect that inherited every + builder assumption is just an echo + + + two independent perspectives is the point of fusion + diff --git a/images/svg-05-gate-first-loop-animated.svg b/images/svg-05-gate-first-loop-animated.svg new file mode 100644 index 0000000..76d8830 --- /dev/null +++ b/images/svg-05-gate-first-loop-animated.svg @@ -0,0 +1,86 @@ + + + + + + + THE AUTO-VALIDATION LOOP + the gate is designed BEFORE the build · red → green + + + + + 1 ✓ VALIDATOR + designs gate.py + read-only · uv / PEP 723 + every requirement → a check + + + + + + + + + 2 BASELINE RUN + must fail RED + a passing baseline means + a weak gate โ€” said loudly + + + + + + + + 3 ▲ BUILDER + builds with full tools + gate visible but immutable + persistent session + + + + + + + + 4 GATE RUNS + uv run gate.py + exit 0 iff what you + asked for got built + + + + + + + + + PASS · loop ends ✓ + + + + + + FAIL lines feed back verbatim + expected X · found Y · at PATH · fix + + + + escalation ladder + • failure N (default 3): VALIDATOR triages the work โ€” + its brief rides with the next correction + • GATE DEFECT diagnosed: one gate repair, free re-run + • --max-validations (default 5): loud halt, never silent + + the builder never grades its own homework · the grader never touches the code + diff --git a/images/svg-06-two-column-dx.svg b/images/svg-06-two-column-dx.svg new file mode 100644 index 0000000..3843659 --- /dev/null +++ b/images/svg-06-two-column-dx.svg @@ -0,0 +1,61 @@ + + + + + + + + + + pi · fusion-harness + + + ◆ ARCHITECT · claude-fable-5 + ▲ BUILDER · gpt-5.6-sol + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ⧉ FUSION · full-width row + + + + + panels render full-height into scrollback — no lines hidden behind a toggle + + + + + ◆ ARCHITECT | fable-5 (med) | [██--------] 12% + ▲ BUILDER | gpt-5.6-sol (med) | [███-------] 31% + + ARCHITECT-family left · BUILDER right — live widget, final panels, footer, all aligned + diff --git a/images/video-frames/and-not-or.png b/images/video-frames/and-not-or.png new file mode 100644 index 0000000..8f03f11 Binary files /dev/null and b/images/video-frames/and-not-or.png differ diff --git a/images/video-frames/blind-spot-coverage-matrix.png b/images/video-frames/blind-spot-coverage-matrix.png new file mode 100644 index 0000000..d889ff9 Binary files /dev/null and b/images/video-frames/blind-spot-coverage-matrix.png differ diff --git a/images/video-frames/combined-context-window.png b/images/video-frames/combined-context-window.png new file mode 100644 index 0000000..ac566e0 Binary files /dev/null and b/images/video-frames/combined-context-window.png differ diff --git a/images/video-frames/fusion-command-flow.png b/images/video-frames/fusion-command-flow.png new file mode 100644 index 0000000..77f992d Binary files /dev/null and b/images/video-frames/fusion-command-flow.png differ diff --git a/images/video-frames/fusion-core.png b/images/video-frames/fusion-core.png new file mode 100644 index 0000000..2e23fe8 Binary files /dev/null and b/images/video-frames/fusion-core.png differ diff --git a/images/video-frames/fusion-patterns-gallery.png b/images/video-frames/fusion-patterns-gallery.png new file mode 100644 index 0000000..7036679 Binary files /dev/null and b/images/video-frames/fusion-patterns-gallery.png differ diff --git a/images/video-frames/fusion-provenance-resolver.png b/images/video-frames/fusion-provenance-resolver.png new file mode 100644 index 0000000..bf2160b Binary files /dev/null and b/images/video-frames/fusion-provenance-resolver.png differ diff --git a/images/video-frames/fusion-value-ladder-switchboard.png b/images/video-frames/fusion-value-ladder-switchboard.png new file mode 100644 index 0000000..f08bfc9 Binary files /dev/null and b/images/video-frames/fusion-value-ladder-switchboard.png differ diff --git a/images/video-frames/harness-to-factory-zoom.png b/images/video-frames/harness-to-factory-zoom.png new file mode 100644 index 0000000..c5c905a Binary files /dev/null and b/images/video-frames/harness-to-factory-zoom.png differ diff --git a/images/video-frames/model-slot-rack.png b/images/video-frames/model-slot-rack.png new file mode 100644 index 0000000..07ed421 Binary files /dev/null and b/images/video-frames/model-slot-rack.png differ diff --git a/images/video-frames/opinion-flow.png b/images/video-frames/opinion-flow.png new file mode 100644 index 0000000..6115d87 Binary files /dev/null and b/images/video-frames/opinion-flow.png differ diff --git a/images/video-frames/sota-max-artifact-pipeline.png b/images/video-frames/sota-max-artifact-pipeline.png new file mode 100644 index 0000000..34fb404 Binary files /dev/null and b/images/video-frames/sota-max-artifact-pipeline.png differ diff --git a/images/video-frames/team-vs-lone-wolf.png b/images/video-frames/team-vs-lone-wolf.png new file mode 100644 index 0000000..aac547b Binary files /dev/null and b/images/video-frames/team-vs-lone-wolf.png differ diff --git a/justfile b/justfile new file mode 100644 index 0000000..9f7fdd9 --- /dev/null +++ b/justfile @@ -0,0 +1,55 @@ +set dotenv-load := true + +# โ”€โ”€ fusion-harness โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# /fusion ยท /auto-validate ยท /opinion โ€” fuse two frontier models (AND, not OR). +# The HOST runs on the BUILDER model: raw (non-slash) input IS the builder agent. +# +# Two launch recipes, two tiers โ€” everything else is a flag: +# +# just fh-workhorse cheap pair (sonnet-5 plans ยท terra builds + hosts) โ€” use for testing +# just fh-sota frontier pair (fable-5 plans ยท sol builds + hosts) โ€” the on-camera run +# +# Configuration flags (all optional, appendable to either recipe): +# --architect plans/fuses/validates +# --builder builds +# --architect-thinking EVERY architect-family execution +# --builder-thinking EVERY builder execution +# (levels: off|minimal|low|medium|high|xhigh|max) +# --architect-system-prompt override architect worker/fusion system prompt +# --builder-system-prompt override builder system prompt +# --max-validations /auto-validate halt cap default 5 +# --escalate-to-validator-count validator triage from Nth failure default 3 +# --child-timeout kill any child agent after N sec default 28800 = 8h (max 86400) +# +# e.g. just fh-workhorse --architect-thinking high --builder-system-prompt ./persona.md +# just fh-sota --architect-thinking max --builder-thinking max +# +# Default prompts live in extensions/fusion-harness/{SYSTEM,USER}_PROMPT_*.md โ€” edit to tune. +# Sessions persist per project (/tmp/fusion-harness-sessions) โ€” /fh-reset for fresh memories. + +# WORKHORSE tier โ€” the cheap pair (sonnet-5 plans ยท terra builds + hosts). Use for testing. +WORKHORSE_ARCHITECT := "anthropic/claude-sonnet-5" +WORKHORSE_BUILDER := "openai/gpt-5.6-terra" + +# STATE-OF-THE-ART tier โ€” the frontier, on-camera pair (fable 5 plans ยท sol builds + hosts). +SOTA_ARCHITECT := "anthropic/claude-fable-5" +SOTA_BUILDER := "openai/gpt-5.6-sol" + +default: + @just --list + +# WORKHORSE tier โ€” cheap pair at medium thinking. Use this for testing. +fh-workhorse *ARGS: + pi -e extensions/fusion-harness/fusion-harness.ts \ + --model {{WORKHORSE_BUILDER}} \ + --architect {{WORKHORSE_ARCHITECT}} --builder {{WORKHORSE_BUILDER}} \ + --architect-thinking medium --builder-thinking medium \ + {{ARGS}} + +# STATE-OF-THE-ART tier โ€” frontier pair at xhigh thinking. The on-camera run. +fh-sota *ARGS: + pi -e extensions/fusion-harness/fusion-harness.ts \ + --model {{SOTA_BUILDER}} \ + --architect {{SOTA_ARCHITECT}} --builder {{SOTA_BUILDER}} \ + --architect-thinking xhigh --builder-thinking xhigh \ + {{ARGS}} diff --git a/live_final_generation/MANIFEST-claude-fable-5-gpt-5.6-sol.md b/live_final_generation/MANIFEST-claude-fable-5-gpt-5.6-sol.md new file mode 100644 index 0000000..1d55ca4 --- /dev/null +++ b/live_final_generation/MANIFEST-claude-fable-5-gpt-5.6-sol.md @@ -0,0 +1,37 @@ +# Project File Inventory โ€” fused from claude-fable-5 + gpt-5.6-sol + +Snapshot of every file built for the SQLite bulk-insert benchmark project, +copied 2026-07-16 from the original locations into this directory. + +## `project-root/` โ€” deliverables (from `/private/tmp/ddc0f4d0/fusion-harness/`) +Listed by BOTH agents. + +| File | Built by | Description | +|---|---|---| +| `bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py` | ARCHITECT | Independent benchmark design โ€” 7 strategies, subprocess isolation, ru_maxrss RAM measurement | +| `sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py` | BUILDER | Independent parallel benchmark design | +| `BENCH_PLAN.md` | Fusion | Fused plan: method, fairness controls, measured results, winners, embedded fused script | +| `bench.py` | BUILDER (gated loop) | Canonical deliverable โ€” stdlib-only uv single-file implementation of BENCH_PLAN.md | +| `RESULTS.md` | Generated by `uv run bench.py` | Measured results (7 strategies, time + peak RAM), Speed Winner `set_based_ctes`, Memory Winner declared | + +## Root of this dir โ€” fused script (from `/tmp/`) +Listed by ARCHITECT only. + +- `fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py` โ€” runnable fused benchmark referenced by BENCH_PLAN.md + +## `harness-artifacts/` โ€” process artifacts (from `/tmp/fusion-harness-K48sbW/`) +Listed by ARCHITECT only. + +- `gate.py` โ€” 15-check acceptance gate (Definition of Done) +- `prompt.md`, `validator.md` โ€” harness briefs +- `gate-baseline.txt` โ€” required RED baseline before any build +- `gate-round-1.txt` โ€ฆ `gate-round-5.txt` โ€” all five gate runs +- `builder-round-1.md` โ€ฆ `builder-round-5.md` โ€” all five builder reports +- `triage-round-3.md`, `triage-round-4.md` โ€” escalation briefs diagnosing the gate defect + +## Status note (per ARCHITECT) +Final gate run ends RED (11 passed / 4 failed) due to a known defect in +`gate.py:61` (strips underscores from searched text, making strategy-name +checks unsatisfiable). The substantive deliverables โ€” `bench.py` and +`RESULTS.md` โ€” are correct, measured, and complete: 1M rows, all 7 +strategies, ~1700โ€“2000x speedup over naive autocommit. diff --git a/live_final_generation/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py b/live_final_generation/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py new file mode 100644 index 0000000..0f922dc --- /dev/null +++ b/live_final_generation/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py @@ -0,0 +1,228 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +""" +FUSED SQLite bulk-insert benchmark (claude-fable-5 + gpt-5.6-sol). + +1,000,000 rows. Speed + peak RAM. Stdlib only. Run: + uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py + +Architecture (from ARCHITECT/claude-fable-5): the script re-execs ITSELF +once per strategy (`--worker NAME`), so every strategy runs in a fresh +process and peak RSS readings cannot pollute each other. A no-op +calibration worker measures bare-interpreter RSS for a delta column. + +Beyond-the-list candidate (from BUILDER/gpt-5.6-sol): set-based +INSERT ... SELECT over a recursive CTE that generates the IDENTICAL rows +inside SQLite, verified by digest + spot-check against the Python rows. +WAL runs pay for wal_checkpoint(TRUNCATE) inside the timed region. +""" + +import json +import os +import resource +import sqlite3 +import subprocess +import sys +import tempfile +import time + +N = 1_000_000 +NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/sample) in report +SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)" +INSERT = "INSERT INTO t VALUES (?,?,?)" + +SET_BASED_SQL = """ +WITH RECURSIVE seq(i) AS ( + SELECT 0 + UNION ALL + SELECT i + 1 FROM seq WHERE i + 1 < ? +) +INSERT INTO t SELECT i, i * 0.5, printf('payload-%012d', i) FROM seq +""" + + +# ---------------------------------------------------------------- data ---- +def make_row(i): + return (i, i * 0.5, f"payload-{i:012d}") + + +def rows(n): + """Identical deterministic data for every strategy.""" + for i in range(n): + yield make_row(i) + + +# ---------------------------------------------------------- strategies ---- +def s_naive_autocommit(conn, n): + conn.isolation_level = None # every INSERT commits: journal sync per row + for r in rows(n): + conn.execute(INSERT, r) + + +def s_one_big_txn_loop(conn, n): + conn.execute("BEGIN") + for r in rows(n): + conn.execute(INSERT, r) + conn.commit() + + +def s_executemany_list(conn, n): + data = list(rows(n)) # deliberately materialized: shows RAM cost + conn.execute("BEGIN") + conn.executemany(INSERT, data) + conn.commit() + + +def s_executemany_gen(conn, n): + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_wal_tuned_gen(conn, n): + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + # Fairness (BUILDER): WAL must not finish with 1M rows still in the -wal + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + + +def s_max_tuned_gen(conn, n): + """Python-bound speed candidate: one txn + executemany(generator) + + journaling/sync disabled for the load. Rebuildable staging only.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_set_based(conn, n): + """Beyond-the-list candidate (BUILDER): SQLite generates the identical + rows itself; zero Python->SQLite binding crossings. OFF/OFF pragmas so + it is compared at the same durability tier as max_tuned_gen.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.execute(SET_BASED_SQL, (n,)) + conn.commit() + + +STRATEGIES = { + "naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N), + "one_big_txn_loop": (s_one_big_txn_loop, N), + "executemany_list": (s_executemany_list, N), + "executemany_gen": (s_executemany_gen, N), + "wal_tuned_gen": (s_wal_tuned_gen, N), + "max_tuned_gen": (s_max_tuned_gen, N), + "set_based_ctes": (s_set_based, N), +} + + +# --------------------------------------------------------------- worker ---- +def rss_bytes(): + v = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return v if sys.platform == "darwin" else v * 1024 # KB on Linux + + +def verify(dbpath, n): + """Integrity gate (fused): count + checksum for everyone [ARCHITECT], + plus length-digest and spot-row equality so the SQL-generated data is + provably identical to the Python data [BUILDER].""" + conn = sqlite3.connect(dbpath) + cnt, chk, clen = conn.execute( + "SELECT count(*), sum(a), sum(length(c)) FROM t").fetchone() + ok = (cnt == n and chk == n * (n - 1) // 2 and clen == 20 * n) + for i in (0, 1, n // 2, n - 1): + got = conn.execute("SELECT a, b, c FROM t WHERE a=?", (i,)).fetchone() + ok = ok and got is not None and tuple(got) == make_row(i) + conn.close() + return ok + + +def worker(name): + if name == "calibration": # interpreter + sqlite3 import baseline + print(json.dumps({"name": name, "rows": 0, "seconds": 0.0, + "peak_rss": rss_bytes(), "ok": True})) + return + fn, n = STRATEGIES[name] + dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir()) + try: + conn = sqlite3.connect(dbpath) + conn.execute(SCHEMA) + t0 = time.perf_counter() # timer: generation + bind + insert + commit + fn(conn, n) + dt = time.perf_counter() - t0 + conn.close() + ok = verify(dbpath, n) # validation outside the timer + print(json.dumps({"name": name, "rows": n, "seconds": dt, + "peak_rss": rss_bytes(), "ok": ok})) + finally: + for p in (dbpath, dbpath + "-wal", dbpath + "-shm"): + if os.path.exists(p): + os.remove(p) + + +# ----------------------------------------------------------- orchestrator -- +def spawn(name): + out = subprocess.run([sys.executable, os.path.abspath(__file__), + "--worker", name], + capture_output=True, text=True, check=True) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def main(): + t_total = time.perf_counter() + print(f"SQLite bulk insert benchmark | N={N:,} rows | " + f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} " + f"{sys.platform}\n") + + base = spawn("calibration") + results = [] + for name in STRATEGIES: # fixed, deterministic order + r = spawn(name) + assert r["ok"], f"integrity check FAILED for {name}" + scale = N / r["rows"] + r["scaled"] = r["seconds"] * scale + r["sampled"] = scale > 1.0 + results.append(r) + print(f" [done] {name}: {r['seconds']:.3f}s " + f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})") + + naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit") + print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}" + f"{'Peak RSS':>12}{'RSS -base':>12}") + print("-" * 70) + for r in sorted(results, key=lambda r: r["scaled"]): + mark = "*" if r["sampled"] else " " + print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}" + f"{naive / r['scaled']:>9.1f}x" + f"{r['peak_rss'] / 1e6:>10.1f}MB" + f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB") + print("-" * 70) + print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled " + f"x{N // NAIVE_SAMPLE_N} | base RSS " + f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)") + print("note: max_tuned_gen and set_based_ctes run journal_mode=OFF/" + "synchronous=OFF -> rebuildable staging loads only.") + + fastest = min(results, key=lambda r: r["scaled"]) + full = [r for r in results if not r["sampled"]] # baseline excluded + leanest = min(full, key=lambda r: (r["peak_rss"], r["scaled"])) + print(f"\nSpeed Winner : {fastest['name']} " + f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)") + print(f"Memory Winner: {leanest['name']} " + f"({leanest['peak_rss'] / 1e6:.1f}MB peak, " + f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)") + print(f"\ntotal benchmark wall time: {time.perf_counter() - t_total:.1f}s") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--worker": + worker(sys.argv[2]) + else: + main() diff --git a/live_final_generation/harness-artifacts/builder-round-1.md b/live_final_generation/harness-artifacts/builder-round-1.md new file mode 100644 index 0000000..a44c2bc --- /dev/null +++ b/live_final_generation/harness-artifacts/builder-round-1.md @@ -0,0 +1,16 @@ +Implemented and validated. + +### Files created +- `/private/tmp/ddc0f4d0/fusion-harness/bench.py` +- `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` โ€” generated from fresh measurements + +### Commands run +- `uv run bench.py` before implementation โ€” RED as expected because `bench.py` did not exist +- `uv run bench.py` after implementation โ€” completed in 6.765 seconds +- Independent stdlib AST/results validation โ€” PASS + +### Measured winners +- **Speed Winner:** `set_based_ctes` โ€” 0.1636s, 1194.12ร— faster than projected naive autocommit +- **Memory Winner:** `max_tuned_gen` โ€” 25.756 MB measured peak RSS + +All seven planned strategies have measured timing and isolated-process peak-RAM results in `RESULTS.md`. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/builder-round-2.md b/live_final_generation/harness-artifacts/builder-round-2.md new file mode 100644 index 0000000..5291ada --- /dev/null +++ b/live_final_generation/harness-artifacts/builder-round-2.md @@ -0,0 +1,19 @@ +No project change can genuinely satisfy this gate. Its `clean_md()` function executes: + +```python +re.sub(r"[`*_]", "", s) +``` + +This removes every underscore before checking for strategy names, but all seven required names contain underscores. Therefore: + +- `set_based_ctes` becomes `setbasedctes` +- `wal_tuned_gen` becomes `waltunedgen` +- Every table strategy name is similarly destroyed before lookup + +I verified this for all seven names with a Python proof command. `RESULTS.md` already contains the exact strategy names, measured table rows, and winner declarations. The gate must stop deleting underscores, e.g. use: + +```python +re.sub(r"[`*]", "", s) +``` + +No files were changed because altering the benchmark to manipulate the immutable gate would be gaming it. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/builder-round-3.md b/live_final_generation/harness-artifacts/builder-round-3.md new file mode 100644 index 0000000..0daa6c7 --- /dev/null +++ b/live_final_generation/harness-artifacts/builder-round-3.md @@ -0,0 +1,33 @@ +The gate is unsatisfiable without modifying or gaming it. + +Formal reason: + +```python +def clean_md(s): + return re.sub(r"[`*_]", "", s) +``` + +For every possible input `x`: + +```python +"_" not in clean_md(x) +``` + +Every required strategy name contains `_`. Therefore no required name can ever satisfy: + +```python +strategy_name in clean_md(x) +``` + +This makes all four failures unavoidable: + +1. Table strategy count is always zero. +2. Speed Winner can never match. +3. Memory Winner can never match. +4. No strategy rows can be parsed, so measured times remain empty. + +`RESULTS.md` already contains the exact names and valid measurements. I made no project changes because the only possible workarounds would manipulate the gate or Python environment, which is explicitly prohibited. The gate must preserve underscores, for example: + +```python +return re.sub(r"[`*]", "", s) +``` \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/builder-round-4.md b/live_final_generation/harness-artifacts/builder-round-4.md new file mode 100644 index 0000000..7ac8ecc --- /dev/null +++ b/live_final_generation/harness-artifacts/builder-round-4.md @@ -0,0 +1,10 @@ +No project files changed, per validator triage. `bench.py` and `RESULTS.md` already satisfy every substantive requirement. + +Required harness-side correction: + +```diff +- return re.sub(r"[`*_]", "", s) ++ return re.sub(r"[`*]", "", s) +``` + +This preserves canonical strategy-name underscores and resolves all four failures: table matching, Speed Winner matching, Memory Winner matching, and measured-time parsing. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/builder-round-5.md b/live_final_generation/harness-artifacts/builder-round-5.md new file mode 100644 index 0000000..d44392d --- /dev/null +++ b/live_final_generation/harness-artifacts/builder-round-5.md @@ -0,0 +1,10 @@ +No project files changed, exactly as required by validator triage. `bench.py` and freshly generated `RESULTS.md` are correct. + +The remaining failures are caused solely by the unpatched harness regex. The human/harness operator must change `/tmp/fusion-harness-K48sbW/gate.py:61`: + +```diff +- return re.sub(r"[`*_]", "", s) ++ return re.sub(r"[`*]", "", s) +``` + +No builder-side workaround can legitimately satisfy the defective matcher. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/gate-baseline.txt b/live_final_generation/harness-artifacts/gate-baseline.txt new file mode 100644 index 0000000..4a5d9e6 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-baseline.txt @@ -0,0 +1,6 @@ +exit 1 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +FAIL: expected bench.py to exist, found nothing, at /private/tmp/ddc0f4d0/fusion-harness/bench.py โ€” implement BENCH_PLAN.md as a single-file uv script named bench.py in /private/tmp/ddc0f4d0/fusion-harness (stdlib only, runnable with `uv run bench.py`) + +RESULT: RED (gate cannot proceed without bench.py) diff --git a/live_final_generation/harness-artifacts/gate-round-1.txt b/live_final_generation/harness-artifacts/gate-round-1.txt new file mode 100644 index 0000000..0065f05 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-round-1.txt @@ -0,0 +1,19 @@ +exit 1 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py +PASS: bench.py has a PEP 723 `# /// script` metadata block +PASS: PEP 723 dependencies are empty (stdlib only) +PASS: bench.py imports only Python standard library modules +PASS: bench.py uses sqlite3 (real inserts, not simulated) +PASS: bench.py isolates strategies in separate processes +PASS: bench.py measures peak RAM via getrusage/ru_maxrss +PASS: bench.py targets N=1,000,000 rows +PASS: `uv run bench.py` ran to completion (exit 0) +PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation +FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” write one markdown table with one row per strategy using the plan's strategy names +FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes โ€” 0.1731 s, 1459.70x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Speed Winner using the plan's strategy name +FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen โ€” 25.657 MB peak RSS, +3.375 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Memory Winner using the plan's strategy name +FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” fix the results table first + +RESULT: RED (11 passed, 4 failed) diff --git a/live_final_generation/harness-artifacts/gate-round-2.txt b/live_final_generation/harness-artifacts/gate-round-2.txt new file mode 100644 index 0000000..38df2c6 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-round-2.txt @@ -0,0 +1,19 @@ +exit 1 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py +PASS: bench.py has a PEP 723 `# /// script` metadata block +PASS: PEP 723 dependencies are empty (stdlib only) +PASS: bench.py imports only Python standard library modules +PASS: bench.py uses sqlite3 (real inserts, not simulated) +PASS: bench.py isolates strategies in separate processes +PASS: bench.py measures peak RAM via getrusage/ru_maxrss +PASS: bench.py targets N=1,000,000 rows +PASS: `uv run bench.py` ran to completion (exit 0) +PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation +FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” write one markdown table with one row per strategy using the plan's strategy names +FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes โ€” 0.2118 s, 1269.32x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Speed Winner using the plan's strategy name +FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen โ€” 25.625 MB peak RSS, +3.359 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Memory Winner using the plan's strategy name +FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” fix the results table first + +RESULT: RED (11 passed, 4 failed) diff --git a/live_final_generation/harness-artifacts/gate-round-3.txt b/live_final_generation/harness-artifacts/gate-round-3.txt new file mode 100644 index 0000000..e4748db --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-round-3.txt @@ -0,0 +1,19 @@ +exit 1 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py +PASS: bench.py has a PEP 723 `# /// script` metadata block +PASS: PEP 723 dependencies are empty (stdlib only) +PASS: bench.py imports only Python standard library modules +PASS: bench.py uses sqlite3 (real inserts, not simulated) +PASS: bench.py isolates strategies in separate processes +PASS: bench.py measures peak RAM via getrusage/ru_maxrss +PASS: bench.py targets N=1,000,000 rows +PASS: `uv run bench.py` ran to completion (exit 0) +PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation +FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” write one markdown table with one row per strategy using the plan's strategy names +FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes โ€” 0.1768 s, 2030.00x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Speed Winner using the plan's strategy name +FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: waltunedgen โ€” 25.657 MB peak RSS, +3.211 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Memory Winner using the plan's strategy name +FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” fix the results table first + +RESULT: RED (11 passed, 4 failed) diff --git a/live_final_generation/harness-artifacts/gate-round-4.txt b/live_final_generation/harness-artifacts/gate-round-4.txt new file mode 100644 index 0000000..a36a582 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-round-4.txt @@ -0,0 +1,19 @@ +exit 1 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py +PASS: bench.py has a PEP 723 `# /// script` metadata block +PASS: PEP 723 dependencies are empty (stdlib only) +PASS: bench.py imports only Python standard library modules +PASS: bench.py uses sqlite3 (real inserts, not simulated) +PASS: bench.py isolates strategies in separate processes +PASS: bench.py measures peak RAM via getrusage/ru_maxrss +PASS: bench.py targets N=1,000,000 rows +PASS: `uv run bench.py` ran to completion (exit 0) +PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation +FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” write one markdown table with one row per strategy using the plan's strategy names +FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes โ€” 0.1667 s, 1733.05x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Speed Winner using the plan's strategy name +FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: executemanygen โ€” 25.608 MB peak RSS, +3.441 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Memory Winner using the plan's strategy name +FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” fix the results table first + +RESULT: RED (11 passed, 4 failed) diff --git a/live_final_generation/harness-artifacts/gate-round-5.txt b/live_final_generation/harness-artifacts/gate-round-5.txt new file mode 100644 index 0000000..2dddec1 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate-round-5.txt @@ -0,0 +1,21 @@ +exit 130 + +PASS: BENCH_PLAN.md present and still defines all 7 plan strategies +PASS: bench.py exists at /private/tmp/ddc0f4d0/fusion-harness/bench.py +PASS: bench.py has a PEP 723 `# /// script` metadata block +PASS: PEP 723 dependencies are empty (stdlib only) +PASS: bench.py imports only Python standard library modules +PASS: bench.py uses sqlite3 (real inserts, not simulated) +PASS: bench.py isolates strategies in separate processes +PASS: bench.py measures peak RAM via getrusage/ru_maxrss +PASS: bench.py targets N=1,000,000 rows +PASS: `uv run bench.py` ran to completion (exit 0) +PASS: RESULTS.md was (re)written by this `uv run bench.py` invocation +FAIL: expected a markdown table whose rows name the 7 plan strategies ['naive_autocommit', 'one_big_txn_loop', 'executemany_list', 'executemany_gen', 'wal_tuned_gen', 'max_tuned_gen', 'set_based_ctes'], found no such table, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” write one markdown table with one row per strategy using the plan's strategy names +FAIL: expected a `Speed Winner:` line naming one plan strategy, found `Speed Winner: setbasedctes โ€” 0.1700 s, 1147.98x faster than naive.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Speed Winner using the plan's strategy name +FAIL: expected a `Memory Winner:` line naming one plan strategy, found `Memory Winner: executemanygen โ€” 25.625 MB peak RSS, +3.588 MB over the calibration worker.`, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” declare the Memory Winner using the plan's strategy name +FAIL: expected complete measured times for all 7 strategies to verify the >=5x speedup, found only [] parseable, at /private/tmp/ddc0f4d0/fusion-harness/RESULTS.md โ€” fix the results table first + +RESULT: RED (11 passed, 4 failed) + +[stopped by user] \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/gate.py b/live_final_generation/harness-artifacts/gate.py new file mode 100644 index 0000000..86b66b9 --- /dev/null +++ b/live_final_generation/harness-artifacts/gate.py @@ -0,0 +1,477 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +""" +Acceptance gate for: implement BENCH_PLAN.md as bench.py (uv single-file +script, stdlib only) that runs to completion via `uv run bench.py`, writes +RESULTS.md with a markdown table holding a measured time AND a measured peak +RAM figure for every strategy in the plan, declares a Speed Winner and a +Memory Winner from those measurements, and where the fastest strategy beats +the naive autocommit baseline by at least 5x. + +Exit 0 iff every check passes. One PASS/FAIL line per check. +""" + +import ast +import os +import re +import subprocess +import sys +import time + +ROOT = "/private/tmp/ddc0f4d0/fusion-harness" +BENCH = os.path.join(ROOT, "bench.py") +RESULTS = os.path.join(ROOT, "RESULTS.md") +PLAN = os.path.join(ROOT, "BENCH_PLAN.md") + +# The seven strategies defined by BENCH_PLAN.md ("Strategies (7)" table and +# the STRATEGIES dict in its appendix script). +STRATEGIES = [ + "naive_autocommit", + "one_big_txn_loop", + "executemany_list", + "executemany_gen", + "wal_tuned_gen", + "max_tuned_gen", + "set_based_ctes", +] +BASELINE = "naive_autocommit" +RUN_TIMEOUT = 240 # plan budget is ~3 minutes; measured fused run was ~7s + +_results: list[bool] = [] + + +def ok(msg: str) -> None: + _results.append(True) + print("PASS: " + msg) + + +def bad(msg: str) -> None: + _results.append(False) + print("FAIL: " + msg) + + +def one_line(s: str, limit: int = 500) -> str: + s = " | ".join(part.strip() for part in s.strip().splitlines() if part.strip()) + return s[-limit:] if len(s) > limit else s + + +def clean_md(s: str) -> str: + return re.sub(r"[`*_]", "", s) + + +def first_float(cell: str): + m = re.search(r"-?\d+(?:\.\d+)?", cell.replace(",", "")) + return float(m.group(0)) if m else None + + +def split_cells(line: str): + line = line.strip() + if line.startswith("|"): + line = line[1:] + if line.endswith("|"): + line = line[:-1] + return [c.strip() for c in line.split("|")] + + +def parse_tables(text: str): + """Return list of (header_cells, data_rows) for every markdown table.""" + lines = text.splitlines() + tables = [] + i = 0 + sep_re = re.compile(r"^\s*\|?[\s:|-]+\|?\s*$") + while i < len(lines): + if ( + "|" in lines[i] + and i + 1 < len(lines) + and "-" in lines[i + 1] + and "|" in lines[i + 1] + and sep_re.match(lines[i + 1]) + ): + header = split_cells(lines[i]) + j = i + 2 + rows = [] + while j < len(lines) and "|" in lines[j] and lines[j].strip(): + rows.append(split_cells(lines[j])) + j += 1 + tables.append((header, rows)) + i = j + else: + i += 1 + return tables + + +def main() -> int: + # ---- 0. plan still present (the thing bench.py must implement) -------- + if os.path.isfile(PLAN): + plan_txt = open(PLAN, encoding="utf-8", errors="replace").read() + missing = [s for s in STRATEGIES if s not in plan_txt] + if missing: + bad( + f"expected BENCH_PLAN.md to still define strategies {missing}, " + f"found them absent, at {PLAN} โ€” restore BENCH_PLAN.md; the " + f"plan is the spec and must not be edited" + ) + else: + ok("BENCH_PLAN.md present and still defines all 7 plan strategies") + else: + bad( + f"expected BENCH_PLAN.md to exist, found nothing, at {PLAN} โ€” " + f"restore the plan file; it is the spec bench.py implements" + ) + + # ---- 1. bench.py exists ------------------------------------------------ + if not os.path.isfile(BENCH): + bad( + f"expected bench.py to exist, found nothing, at {BENCH} โ€” " + f"implement BENCH_PLAN.md as a single-file uv script named " + f"bench.py in {ROOT} (stdlib only, runnable with `uv run bench.py`)" + ) + print("\nRESULT: RED (gate cannot proceed without bench.py)") + return 1 + ok(f"bench.py exists at {BENCH}") + + src = open(BENCH, encoding="utf-8", errors="replace").read() + + # ---- 2. PEP 723 header, empty dependencies (stdlib only) -------------- + if re.search(r"^#\s*///\s*script\s*$", src, re.MULTILINE): + ok("bench.py has a PEP 723 `# /// script` metadata block") + else: + bad( + f"expected a PEP 723 block starting with `# /// script`, found " + f"none, at {BENCH} โ€” add the inline metadata block " + f"(`# /// script`, `# requires-python = ...`, " + f"`# dependencies = []`, `# ///`) at the top of bench.py" + ) + dep_lines = [l for l in src.splitlines() if re.search(r"^\#\s*dependencies\s*=", l)] + if dep_lines and not re.search(r"dependencies\s*=\s*\[\s*\]", dep_lines[0]): + bad( + f"expected `dependencies = []` (stdlib only), found " + f"`{dep_lines[0].strip()}`, at {BENCH} โ€” remove all third-party " + f"dependencies from the PEP 723 block; the plan requires Python " + f"standard library only" + ) + else: + ok("PEP 723 dependencies are empty (stdlib only)") + + # ---- 3. all imports are stdlib ----------------------------------------- + try: + tree = ast.parse(src) + roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for a in node.names: + roots.add(a.name.split(".")[0]) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + roots.add(node.module.split(".")[0]) + non_std = sorted(r for r in roots if r not in sys.stdlib_module_names) + if non_std: + bad( + f"expected only stdlib imports, found non-stdlib {non_std}, " + f"at {BENCH} โ€” rewrite bench.py using only the Python " + f"standard library" + ) + else: + ok("bench.py imports only Python standard library modules") + except SyntaxError as e: + bad( + f"expected bench.py to be valid Python, found SyntaxError " + f"`{one_line(str(e))}`, at {BENCH} โ€” fix the syntax error" + ) + + # ---- 4. static plan fidelity: real measurement machinery --------------- + if "sqlite3" in src: + ok("bench.py uses sqlite3 (real inserts, not simulated)") + else: + bad( + f"expected bench.py to import/use sqlite3, found no mention, at " + f"{BENCH} โ€” the benchmark must perform real SQLite inserts per " + f"BENCH_PLAN.md" + ) + if re.search(r"subprocess|multiprocessing", src): + ok("bench.py isolates strategies in separate processes") + else: + bad( + f"expected per-strategy process isolation (subprocess re-exec or " + f"multiprocessing per BENCH_PLAN.md), found neither, at {BENCH} โ€” " + f"run each strategy in its own fresh process so peak-RAM readings " + f"cannot pollute each other" + ) + if re.search(r"ru_maxrss|getrusage", src): + ok("bench.py measures peak RAM via getrusage/ru_maxrss") + else: + bad( + f"expected measured peak RAM via resource.getrusage(...).ru_maxrss " + f"per BENCH_PLAN.md, found no getrusage/ru_maxrss, at {BENCH} โ€” " + f"measure whole-process peak RSS inside each isolated worker; " + f"do not assume or hardcode memory figures" + ) + if re.search(r"1_000_000|1000000", src): + ok("bench.py targets N=1,000,000 rows") + else: + bad( + f"expected N=1,000,000 rows (literal 1_000_000 or 1000000), found " + f"neither, at {BENCH} โ€” the plan requires 1M rows for every " + f"full-run strategy (baseline may be sampled and scaled)" + ) + + # ---- 5. `uv run bench.py` runs to completion --------------------------- + pre_marker = time.time() + proc = None + ran_ok = False + try: + proc = subprocess.run( + ["uv", "run", "bench.py"], + cwd=ROOT, + capture_output=True, + text=True, + timeout=RUN_TIMEOUT, + ) + if proc.returncode == 0: + ok("`uv run bench.py` ran to completion (exit 0)") + ran_ok = True + else: + bad( + f"expected `uv run bench.py` to exit 0, found exit " + f"{proc.returncode}, at {BENCH} โ€” fix the crash; stderr tail: " + f"{one_line(proc.stderr or proc.stdout or '(empty)')}" + ) + except FileNotFoundError: + bad( + f"expected `uv` on PATH so `uv run bench.py` works, found uv " + f"missing, at {ROOT} โ€” ensure the script is runnable with exactly " + f"`uv run bench.py`; do not substitute another runner" + ) + # best-effort fallback so content checks below still give feedback + try: + proc = subprocess.run( + [sys.executable, "bench.py"], + cwd=ROOT, + capture_output=True, + text=True, + timeout=RUN_TIMEOUT, + ) + except Exception: + proc = None + except subprocess.TimeoutExpired: + bad( + f"expected `uv run bench.py` to finish within {RUN_TIMEOUT}s " + f"(plan budget ~3 minutes), found it still running, at {BENCH} โ€” " + f"sample the naive autocommit baseline (plan: 20,000 rows scaled " + f"x50) instead of running it for the full 1M rows" + ) + + # ---- 6. RESULTS.md written by the run ---------------------------------- + if not os.path.isfile(RESULTS): + bad( + f"expected the run to write RESULTS.md, found no file, at " + f"{RESULTS} โ€” make bench.py write RESULTS.md (markdown table with " + f"a time column and a peak RAM column for every strategy, plus " + f"Speed Winner and Memory Winner lines) every time it runs" + ) + print("\nRESULT: RED") + return 1 + if os.path.getmtime(RESULTS) >= pre_marker - 2: + ok("RESULTS.md was (re)written by this `uv run bench.py` invocation") + else: + bad( + f"expected RESULTS.md to be rewritten by the run just executed, " + f"found a stale file (mtime predates the run), at {RESULTS} โ€” " + f"bench.py itself must write RESULTS.md from its own fresh " + f"measurements on every run; do not hand-author RESULTS.md" + ) + + text = open(RESULTS, encoding="utf-8", errors="replace").read() + + # ---- 7. markdown table: time + peak RAM for every strategy ------------- + tables = parse_tables(text) + best = None + best_count = -1 + for header, rows in tables: + joined = clean_md(" ".join(" ".join(r) for r in rows)) + count = sum(1 for s in STRATEGIES if s in joined) + if count > best_count: + best_count = count + best = (header, rows) + + times: dict[str, float] = {} + rams: dict[str, float] = {} + if best is None or best_count == 0: + bad( + f"expected a markdown table whose rows name the 7 plan strategies " + f"{STRATEGIES}, found no such table, at {RESULTS} โ€” write one " + f"markdown table with one row per strategy using the plan's " + f"strategy names" + ) + else: + header, rows = best + time_col = next( + (i for i, h in enumerate(header) if re.search(r"time|sec|duration", h, re.I)), + None, + ) + ram_candidates = [ + i for i, h in enumerate(header) if re.search(r"ram|rss|mem", h, re.I) + ] + ram_col = next( + (i for i in ram_candidates if re.search(r"peak", header[i], re.I)), + ram_candidates[0] if ram_candidates else None, + ) + if time_col is None: + bad( + f"expected a time column (header matching time/sec/duration), " + f"found headers {header}, at {RESULTS} โ€” add a measured time " + f"column to the results table" + ) + else: + ok(f"results table has a time column (`{header[time_col]}`)") + if ram_col is None: + bad( + f"expected a peak RAM column (header matching RAM/RSS/mem), " + f"found headers {header}, at {RESULTS} โ€” add a measured peak " + f"RAM column to the results table" + ) + else: + ok(f"results table has a peak RAM column (`{header[ram_col]}`)") + + if time_col is not None and ram_col is not None: + for strat in STRATEGIES: + row = next( + (r for r in rows if strat in clean_md(" ".join(r))), None + ) + if row is None: + bad( + f"expected a table row for strategy `{strat}`, found " + f"none, at {RESULTS} โ€” every strategy in BENCH_PLAN.md " + f"must have its own measured row" + ) + continue + t = first_float(row[time_col]) if time_col < len(row) else None + m = first_float(row[ram_col]) if ram_col < len(row) else None + if t is None or t <= 0: + bad( + f"expected a positive measured time for `{strat}`, " + f"found `{row[time_col] if time_col < len(row) else '(missing cell)'}`, " + f"at {RESULTS} โ€” record the real measured wall time" + ) + else: + times[strat] = t + if m is None or m <= 0: + bad( + f"expected a positive measured peak RAM for `{strat}`, " + f"found `{row[ram_col] if ram_col < len(row) else '(missing cell)'}`, " + f"at {RESULTS} โ€” record the real measured peak RSS" + ) + else: + rams[strat] = m + if len(times) == len(STRATEGIES): + ok("every plan strategy has a positive measured time in the table") + if len(rams) == len(STRATEGIES): + ok("every plan strategy has a positive measured peak RAM in the table") + + # ---- 8. declared winners ------------------------------------------------ + def declared(kind: str): + m = re.search(kind + r"\s*winner[^\n]*", text, re.I) + if not m: + return None, None + line = clean_md(m.group(0)) + name = next((s for s in STRATEGIES if s in line), None) + return line, name + + speed_line, speed_name = declared("speed") + if speed_name: + ok(f"RESULTS.md declares a Speed Winner: {speed_name}") + else: + bad( + f"expected a `Speed Winner:` line naming one plan strategy, found " + f"`{speed_line or 'no such line'}`, at {RESULTS} โ€” declare the " + f"Speed Winner using the plan's strategy name" + ) + mem_line, mem_name = declared("memory") + if mem_name: + ok(f"RESULTS.md declares a Memory Winner: {mem_name}") + else: + bad( + f"expected a `Memory Winner:` line naming one plan strategy, found " + f"`{mem_line or 'no such line'}`, at {RESULTS} โ€” declare the " + f"Memory Winner using the plan's strategy name" + ) + + # ---- 9. winners follow from the measurements --------------------------- + if speed_name and len(times) == len(STRATEGIES): + fastest_t = min(times.values()) + if times[speed_name] <= fastest_t + 1e-9: + ok( + f"Speed Winner {speed_name} matches the fastest measured time " + f"({times[speed_name]}s)" + ) + else: + actual = min(times, key=times.get) + bad( + f"expected the Speed Winner to be the fastest strategy in the " + f"table (`{actual}` at {times[actual]}s), found `{speed_name}` " + f"at {times[speed_name]}s, at {RESULTS} โ€” derive the winner " + f"from the measurements, not assumptions" + ) + if mem_name and len(rams) == len(STRATEGIES): + if mem_name == BASELINE: + bad( + f"expected the Memory Winner to exclude the sampled baseline, " + f"found `{BASELINE}`, at {RESULTS} โ€” per BENCH_PLAN.md the " + f"sampled baseline never held 1M rows of work and cannot win " + f"the RAM axis; pick the leanest full-1M strategy" + ) + else: + eligible = {k: v for k, v in rams.items() if k != BASELINE} + lean = min(eligible.values()) + if rams[mem_name] <= lean + 1e-9: + ok( + f"Memory Winner {mem_name} matches the lowest measured " + f"peak RAM among full-1M strategies ({rams[mem_name]})" + ) + else: + actual = min(eligible, key=eligible.get) + bad( + f"expected the Memory Winner to have the lowest peak RAM " + f"among full-1M strategies (`{actual}` at {eligible[actual]}), " + f"found `{mem_name}` at {rams[mem_name]}, at {RESULTS} โ€” " + f"derive the winner from the measurements" + ) + + # ---- 10. fastest beats naive autocommit by >= 5x ------------------------ + if len(times) == len(STRATEGIES): + fastest_t = min(times.values()) + speedup = times[BASELINE] / fastest_t if fastest_t > 0 else 0.0 + if speedup >= 5.0: + ok( + f"fastest strategy beats naive autocommit by {speedup:.1f}x " + f"(>= 5x required)" + ) + else: + bad( + f"expected fastest strategy >= 5x faster than naive " + f"autocommit, found {speedup:.1f}x ({times[BASELINE]}s vs " + f"{fastest_t}s), at {RESULTS} โ€” use one big transaction / " + f"executemany / set-based insert per BENCH_PLAN.md so the " + f"speedup is real" + ) + else: + bad( + f"expected complete measured times for all 7 strategies to verify " + f"the >=5x speedup, found only {sorted(times)} parseable, at " + f"{RESULTS} โ€” fix the results table first" + ) + + if not ran_ok: + pass # the uv-run FAIL above already carries the instruction + + failed = _results.count(False) + print( + f"\nRESULT: {'GREEN' if failed == 0 else 'RED'} " + f"({_results.count(True)} passed, {failed} failed)" + ) + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/live_final_generation/harness-artifacts/prompt.md b/live_final_generation/harness-artifacts/prompt.md new file mode 100644 index 0000000..262958b --- /dev/null +++ b/live_final_generation/harness-artifacts/prompt.md @@ -0,0 +1 @@ +Read BENCH_PLAN.md, the plan you two designed and fused, and implement it in this directory as bench.py, an astral uv single file script. Python standard library only, runnable with uv run bench.py. Definition of Done is defined by a gate YOU design. At minimum it proves uv run bench.py runs to completion, writes RESULTS.md with a markdown table holding a timed row AND a measured peak RAM figure for every strategy in the plan, and declares a Speed Winner and a Memory Winner from those measurements. Measured, not assumed. The fastest strategy has to beat the naive autocommit baseline by at least 5x. Baseline the gate RED before you build anything. Never shim, wrap, or modify global system tools. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/triage-round-3.md b/live_final_generation/harness-artifacts/triage-round-3.md new file mode 100644 index 0000000..dd654a4 --- /dev/null +++ b/live_final_generation/harness-artifacts/triage-round-3.md @@ -0,0 +1,11 @@ +**Diagnosis** โ€” `GATE DEFECT:` The builder's work is correct and complete; the gate can never pass it. `gate.py:61` defines `clean_md(s) = re.sub(r"[`*_]", "", s)` โ€” intended to strip markdown emphasis, it also deletes every underscore from the text being searched. All 7 required strategy names contain underscores, so `"set_based_ctes" in clean_md(...)` is mathematically unsatisfiable for any possible RESULTS.md (the gate's own round-2/3 output proves it: it echoes `setbasedctes` / `waltunedgen` โ€” mangled by the gate, not written that way). This single line causes all four FAILs (table match count 0 โ†’ "no such table", both winner-name checks, empty parsed times โ†’ 5x check). I verified the real state: `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` contains a well-formed table with all 7 exact plan names, positive measured times and peak-RAM figures, a Speed Winner (`set_based_ctes`, 0.1768s = table minimum), a Memory Winner (`wal_tuned_gen`, 25.657 MB = minimum among full-1M rows, baseline excluded), and a 2030x speedup vs naive โ€” every substantive requirement is already met. The builder's round-3 report is accurate; rounds 2โ€“3 are identical because there is nothing left for the builder to fix. + +**Do exactly this** +1. **HUMAN/HARNESS (not builder):** patch `/tmp/fusion-harness-K48sbW/gate.py` line 61 from `return re.sub(r"[`*_]", "", s)` to `return re.sub(r"[`*]", "", s)`. Underscores must survive cleaning; backtick/asterisk stripping stays. No other gate change is needed. +2. **BUILDER:** change nothing. Leave `/private/tmp/ddc0f4d0/fusion-harness/bench.py` and `RESULTS.md` exactly as they are โ€” they satisfy every check once matching works (RESULTS.md is regenerated fresh each run, so mtime/consistency checks will pass). +3. Re-run the gate once the patch lands; expect GREEN. + +**Do NOT** +- Do NOT rename strategies to underscore-free variants (e.g., `setbasedctes`) or strip underscores/backticks in RESULTS.md to appease the broken matcher โ€” that diverges from BENCH_PLAN.md's canonical names and is gate-gaming. +- Do NOT let the builder edit `/tmp/fusion-harness-K48sbW/gate.py` itself; the fix is the human's call, flagged here. +- Do NOT burn rounds 4โ€“5 re-running the unmodified gate against unmodified files โ€” the failure is deterministic until line 61 is patched. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/triage-round-4.md b/live_final_generation/harness-artifacts/triage-round-4.md new file mode 100644 index 0000000..a6aa2f1 --- /dev/null +++ b/live_final_generation/harness-artifacts/triage-round-4.md @@ -0,0 +1,11 @@ +**Diagnosis** โ€” `GATE DEFECT:` (unchanged, second escalation for the same root cause). The round-3 triage identified the exact defect and the required one-line harness-side patch, but `/tmp/fusion-harness-K48sbW/gate.py:61` still reads `return re.sub(r"[`*_]", "", s)` โ€” the patch was never applied. That regex deletes every underscore from the searched text, while all 7 required strategy names in the gate's hardcoded `STRATEGIES` list contain underscores, so `name in clean_md(text)` is **unsatisfiable for every possible RESULTS.md** โ€” no builder edit of any project file can ever pass checks 7โ€“10. The loop is not stuck on the builder: I re-verified the real state and `/private/tmp/ddc0f4d0/fusion-harness/RESULTS.md` is correct and freshly measured (round 4: `set_based_ctes` 0.1667s speed winner = table minimum, `executemany_gen` 25.608 MB memory winner, 1733x โ‰ฅ 5x, all 7 exact snake_case names present โ€” the winner even changed between rounds 3 and 4, proving values are measured, not hardcoded). The gate's own FAIL lines echo `setbasedctes`/`executemanygen` โ€” names mangled by the gate itself, not by the builder. The builder's round-4 report is accurate; it correctly made no changes. + +**Do exactly this** +1. **HUMAN/HARNESS OPERATOR โ€” this is the only action that can produce GREEN:** edit `/tmp/fusion-harness-K48sbW/gate.py` line 61 from `return re.sub(r"[`*_]", "", s)` to `return re.sub(r"[`*]", "", s)`. One character class change; nothing else in the gate needs touching. Apply it **before** round 5 โ€” it is the final round under the cap, and running it unpatched guarantees a false RED. +2. **BUILDER:** make no changes. `/private/tmp/ddc0f4d0/fusion-harness/bench.py` and `RESULTS.md` already satisfy every substantive requirement; the gate re-runs `uv run bench.py` itself and regenerates RESULTS.md, so all mtime/consistency checks will pass. +3. Re-run the gate once patched; expect GREEN (15/15 style output, exit 0). + +**Do NOT** +- Do NOT spend round 5 re-running the unpatched gate โ€” the failure is deterministic and proven unsatisfiable; that would exhaust the cap on a known-defective check. +- Builder: do NOT attempt workarounds (renaming strategies without underscores, zero-width characters, duplicate underscore-free rows) โ€” the gate's needle list itself contains underscores, so no RESULTS.md content can match, and any such edit is gate-gaming that corrupts an otherwise-correct deliverable. +- Builder: do NOT edit `/tmp/fusion-harness-K48sbW/gate.py` yourself โ€” the fix is flagged for the human, per the escalation contract. \ No newline at end of file diff --git a/live_final_generation/harness-artifacts/validator.md b/live_final_generation/harness-artifacts/validator.md new file mode 100644 index 0000000..7a4a0c7 --- /dev/null +++ b/live_final_generation/harness-artifacts/validator.md @@ -0,0 +1,3 @@ +Gate written to /tmp/fusion-harness-K48sbW/gate.py. + +It verifies, in order: BENCH_PLAN.md intact; `bench.py` exists at the project root with a PEP 723 block, empty `dependencies` and stdlib-only imports (AST scan); static plan fidelity (real sqlite3 inserts, per-strategy process isolation, `getrusage`/`ru_maxrss` RAM measurement, N=1,000,000); `uv run bench.py` exits 0 within the ~3-minute budget; RESULTS.md is freshly (re)written by that very run; its markdown table has a time column and a peak-RAM column with positive measured values for all 7 plan strategies; Speed Winner and Memory Winner are declared and are consistent with the table (memory winner excludes the sampled baseline per plan); and the fastest strategy beats naive autocommit by โ‰ฅ5x. Currently RED โ€” bench.py does not exist yet. \ No newline at end of file diff --git a/live_final_generation/project-root/BENCH_PLAN.md b/live_final_generation/project-root/BENCH_PLAN.md new file mode 100644 index 0000000..5aa1497 --- /dev/null +++ b/live_final_generation/project-root/BENCH_PLAN.md @@ -0,0 +1,340 @@ +# BENCH_PLAN โ€” SQLite Bulk Insert Benchmark (fused design) + +Fusion of two independent designs: **[ARCHITECT] anthropic/claude-fable-5** and **[BUILDER] openai/gpt-5.6-sol**. Where the designs disagreed, the simpler one was chosen (reasons inline and in the closing section). The fused script was **built and executed** โ€” every number below is measured, not projected from the source answers. + +- Fused runnable script: `/tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py` (also embedded in full at the end of this plan) +- Run with: `uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py` +- One astral-uv single-file script (PEP 723 header, `dependencies = []`), Python stdlib only +- Measured total wall time: **6.8 s** โ€” well inside the 3-minute budget + +## Method + +**Self-isolating architecture** [ARCHITECT]. The script is both orchestrator and worker: it re-execs itself (`sys.executable __file__ --worker NAME`) once per strategy, so every strategy runs in a **fresh process against a fresh temp database**. Peak-RSS readings physically cannot pollute each other โ€” this satisfies the isolated-process RAM requirement with zero third-party dependencies. [BUILDER] independently arrived at the same fresh-subprocess-per-trial isolation, so this is consensus architecture. + +**Peak RAM** = whole-process `resource.getrusage(...).ru_maxrss`, normalized (bytes on macOS, KBร—1024 on Linux) [both]. A **no-op calibration worker** measures bare-interpreter RSS (~20.3 MB here) so the table reports both absolute peak and delta-over-baseline [ARCHITECT]. + +**Timing boundary** [consensus]: the clock covers row generation + bind + insert + commit; connection setup, schema creation, and validation sit outside the timer. Generation is identical work for every strategy, so including it is neutral โ€” excluding it would unfairly flatter the list-materializing strategy [ARCHITECT]. WAL runs additionally pay for `wal_checkpoint(TRUNCATE)` *inside* the timed region, so WAL is not credited with "finishing" while a million rows still sit in the `-wal` sidecar [BUILDER]. + +### Strategies (7) + +| # | Strategy | What it does | Source | +|---|----------|--------------|--------| +| 1 | `naive_autocommit` | per-row implicit transaction โ€” the baseline (sampled, see below) | both | +| 2 | `one_big_txn_loop` | `BEGIN` + `execute()` loop + one `COMMIT` | both | +| 3 | `executemany_list` | one txn, all 1M tuples materialized first โ€” the classic mistake, kept to expose its RAM cost | [ARCHITECT] | +| 4 | `executemany_gen` | one txn, `executemany` fed a generator | both | +| 5 | `wal_tuned_gen` | `journal_mode=WAL` + `synchronous=NORMAL` + strategy 4 (WAL tuning combined with the predicted Python-bound winner, per the request) | both | +| 6 | `max_tuned_gen` | `journal_mode=OFF` + `synchronous=OFF` + strategy 4 โ€” fastest way to push Python-resident rows | [ARCHITECT] | +| 7 | `set_based_ctes` | **the "beats all" candidate**: `INSERT โ€ฆ SELECT` over a recursive CTE that generates the *identical* rows inside SQLite โ€” zero Pythonโ†’SQLite binding crossings; run at OFF/OFF so it competes at the same durability tier as #6 | [BUILDER] | + +Chosen over discarded alternatives: +- [BUILDER]'s set-based candidate replaces [ARCHITECT]'s `sqlite3` CLI `.import`: **simpler** (no external binary, no CSV temp file, no graceful-skip logic โ€” pure stdlib in-process SQL) *and* it measured ~2ร— faster than everything else anyway. +- [BUILDER]'s chunked multi-row `VALUES` strategy was dropped: the variable-limit math and chunk assembly add complexity, and its niche (fewer Pythonโ†”SQLite crossings) is bracketed by `executemany_gen` below it and `set_based_ctes` above it. +- [ARCHITECT]'s 64 MB `cache_size` pragma was dropped from the tuned strategies in favor of [BUILDER]'s bounded-default-cache stance: **simpler**, and [ARCHITECT]'s own measurements showed the big cache costs ~50 MB of resident memory for essentially zero speed gain. (Fused result: `wal_tuned_gen` peaks at 24 MB instead of ARCHITECT's 71 MB.) + +## Fairness controls + +- **Identical data** [both]: every strategy inserts the same deterministic rows `(i, i*0.5, "payload-{i:012d}")` โ€” one Python definition, no RNG. `i*0.5` is exact in floating point, so SQL-generated and Python-generated values are bit-identical. +- **Identical schema** [both]: `CREATE TABLE t (a INTEGER, b REAL, c TEXT)` (INTEGER/REAL/TEXT storage classes covered), fresh temp DB per worker, deleted afterward including `-wal`/`-shm`. [ARCHITECT]'s 3-column schema chosen over [BUILDER]'s 6-column `events` table as the simpler design that still exercises the same code paths. +- **Integrity gate, hard-fail** [fused]: every worker verifies `count(*)`, the closed-form `sum(a)` checksum, and `sum(length(c))` [ARCHITECT's gate], plus spot-row equality at indices `{0, 1, N/2, Nโˆ’1}` against the Python row function [distilled from BUILDER's digest] โ€” required so the set-based strategy *proves* it wrote identical data rather than just the right number of rows. No strategy can win by writing less or writing different bytes. This is simpler than [BUILDER]'s 11-field digest + `quick_check` while closing the same loophole. +- **Honest sampling, declared up front** [both]: `naive_autocommit` is fsync-bound (one journal sync per row); a full run would take ~3 minutes alone. It runs **20,000 real rows scaled ร—50**, marked `*` in the table. Everything else runs the full 1,000,000. [ARCHITECT]'s 20kร—50 chosen over [BUILDER]'s 10kร—100 โ€” equal simplicity, smaller scaling factor is the more honest extrapolation (both project to ~190 s regardless). +- **Baseline excluded from the memory verdict** [both, independently]: the sampled baseline never held 1M rows of work, so it cannot win the RAM axis. +- **Single run per strategy, fixed order** [ARCHITECT]: chosen over [BUILDER]'s 2ร—-with-median and seeded order shuffling as the simpler design. Justification: two full executions of the fused benchmark showed per-strategy spread <0.05 s while the winner gaps are โ‰ฅ0.19 s, and process isolation already removes state carryover. +- **Durability labeling** [BUILDER]: OFF/OFF strategies (#6, #7) are flagged in the output as *rebuildable staging loads only* โ€” a crash mid-load can corrupt the DB. They race in the same heats but carry the warning label. + +## Exact output format + +``` +SQLite bulk insert benchmark | N=1,000,000 rows | py sqlite + + [done] : s ( rows[ sampled]) โ† one line per isolated process + +Strategy Time 1M rows Speedup Peak RSS RSS -base +---------------------------------------------------------------------- + s[*] x MB MB โ† sorted fastest-first +---------------------------------------------------------------------- +* sampled at 20,000 rows, honestly scaled x50 | base RSS MB (no-op interpreter) +note: max_tuned_gen and set_based_ctes run journal_mode=OFF/synchronous=OFF -> rebuildable staging loads only. + +Speed Winner : (s, x vs naive) +Memory Winner: (MB peak, +MB over baseline) +``` + +Table layout is [ARCHITECT]'s (fewer columns, speedup included, sorted fastest-first โ€” simpler than [BUILDER]'s 7-column trial-level table); the `Speed Winner:`/`Memory Winner:` terminal lines are [BUILDER]'s exact phrasing. + +## Measured results (fused run: py 3.12.13, sqlite 3.50.4, macOS / Apple Silicon) + +| Strategy | Time 1M rows | Speedup | Peak RSS | RSS โˆ’base | +|---|---:|---:|---:|---:| +| **set_based_ctes** | **0.17s** | **1126.2x** | 24.3 MB | 4.1 MB | +| max_tuned_gen | 0.37s | 518.4x | 24.1 MB | 3.9 MB | +| executemany_gen | 0.38s | 508.8x | 24.1 MB | 3.9 MB | +| **wal_tuned_gen** | 0.40s | 475.6x | **23.9 MB** | **3.7 MB** | +| executemany_list | 0.49s | 393.5x | 226.8 MB | 206.6 MB | +| one_big_txn_loop | 0.51s | 374.9x | 24.0 MB | 3.8 MB | +| naive_autocommit | 191.15s* | 1.0x | 22.7 MB | 2.5 MB | + +\* sampled at 20,000 rows, scaled ร—50. Base RSS 20.2 MB (no-op interpreter). Total wall time 6.8 s. + +Key findings the fused design makes visible: +1. **The single transaction is ~99% of the win** โ€” 375ร— from `BEGIN`/`COMMIT` alone [ARCHITECT's finding, confirmed]. +2. **Eliminating the million Pythonโ†’SQLite binding crossings is the last 2ร—** โ€” set-based insertion at 0.17 s [BUILDER's finding, confirmed on the identical schema]. +3. **`executemany_list` is strictly dominated**: slower than the generator *and* ~200 MB heavier [ARCHITECT's finding, confirmed]. +4. **Memory is a four-way statistical tie** among all streaming strategies (23.9โ€“24.3 MB, within ยฑ0.4 MB across repeat runs): once you stream rows instead of materializing them, SQLite's bounded page cache makes peak RSS nearly strategy-invariant. + +## Declared Winners + +- **Speed Winner: `set_based_ctes`** โ€” `INSERT โ€ฆ SELECT` over a recursive CTE, OFF/OFF pragmas: **0.17 s for 1M rows, ~1126ร— over naive autocommit.** Caveats carried from [BUILDER]: rebuildable-staging durability only, and it applies only when rows are derivable in SQL. For arbitrary Python-resident rows, the practical speed winner is `max_tuned_gen` (0.37 s), with `wal_tuned_gen` (0.40 s) the fastest *durable-enough* option โ€” the same practical hierarchy both source models converged on. +- **Memory Winner: `wal_tuned_gen`** โ€” **23.9 MB peak, +3.7 MB over a bare interpreter**, as selected by the declared rule (minimum peak RSS among full-1M runs, speed as tiebreak) in both verification runs. Honest note: this is a statistical tie with `one_big_txn_loop`, `executemany_gen`, and `max_tuned_gen` (all within 0.4 MB); the real memory lesson is *stream, never materialize* โ€” the list variant costs 206 MB extra for nothing. + +--- + +## Consensus & Divergence + +**Consensus (both models, independently):** fresh-subprocess isolation per strategy with `ru_maxrss` for peak RSS; one deterministic Python-defined dataset and one schema for all strategies; timing bounded to the insert phase with setup/validation outside; sampling the naive baseline with declared linear scaling; excluding the sampled baseline from the memory verdict; integrity verification that hard-fails; labeling OFF/OFF as unsafe-for-production; and the headline conclusion that *one big transaction is most of the win* while *the memory winner is the streaming single-transaction family* (both declared `one transaction + execute` their memory winner; the fused measurement shows that whole family tied within noise, with `wal_tuned_gen` taking it by 0.1โ€“0.2 MB under the fused rule). + +**Divergences and rulings** (simpler option preferred per the fusion brief): +- *Beats-all candidate*: [BUILDER gpt-5.6-sol]'s set-based `INSERTโ€ฆSELECT` **kept**; [ARCHITECT claude-fable-5]'s CLI `.import` **discarded** โ€” set-based is simpler (no external binary, CSV staging, or skip logic) and empirically faster; the CLI route also strains the "stdlib only" spirit. +- *Speed winner dispute*: [ARCHITECT] declared `max_tuned_gen`, [BUILDER] declared OFF/OFF `INSERT-SELECT`. Settled by running both on the identical fused schema: **[BUILDER]'s pick wins** (0.17 s vs 0.37 s). +- *Schema & data*: [ARCHITECT]'s 3-column table **kept** over [BUILDER]'s 6-column `events` table โ€” simpler, same storage classes exercised. +- *Repeats & trial ordering*: [ARCHITECT]'s single run in fixed order **kept** over [BUILDER]'s median-of-2 with seeded shuffle โ€” simpler, and measured run-to-run spread (<0.05 s) is far below decision margins. +- *Verification*: [ARCHITECT]'s count+checksum gate **kept as the base**, extended with a slim length-digest + spot-row check distilled from [BUILDER]'s 11-field digest โ€” the minimum needed to prove the SQL-generated rows are byte-identical; [BUILDER]'s full digest and `PRAGMA quick_check` **discarded** as redundant for this schema. +- *Cache tuning*: [ARCHITECT]'s 64 MB `cache_size` **discarded** in favor of [BUILDER]'s bounded-cache stance โ€” simpler, and ARCHITECT's own data showed +50 MB RSS for no speed benefit. +- *WAL checkpoint in timed region*: [BUILDER]'s control **adopted** โ€” not a simplification but a fairness correction; without it WAL under-reports its true cost. +- *Baseline sample size*: [ARCHITECT]'s 20kร—50 **kept** over [BUILDER]'s 10kร—100 โ€” equal complexity, smaller extrapolation factor. +- *Also discarded*: [BUILDER]'s multi-row `VALUES` strategy (complexity without a podium finish โ€” dominated on both ends), [BUILDER]'s Windows `PeakWorkingSetSize` branch (POSIX `ru_maxrss` is simpler; noted as an easy portability extension), and [ARCHITECT]'s `executemany_list`โ€ฆ no โ€” that one was **kept**: three extra lines that buy the single most vivid RAM lesson in the table. + +--- + +## Appendix โ€” the fused script (verbatim, as executed) + +```python +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +""" +FUSED SQLite bulk-insert benchmark (claude-fable-5 + gpt-5.6-sol). + +1,000,000 rows. Speed + peak RAM. Stdlib only. Run: + uv run /tmp/fused-sqlite-bench-claude-fable-5-gpt-5.6-sol.py + +Architecture (from ARCHITECT/claude-fable-5): the script re-execs ITSELF +once per strategy (`--worker NAME`), so every strategy runs in a fresh +process and peak RSS readings cannot pollute each other. A no-op +calibration worker measures bare-interpreter RSS for a delta column. + +Beyond-the-list candidate (from BUILDER/gpt-5.6-sol): set-based +INSERT ... SELECT over a recursive CTE that generates the IDENTICAL rows +inside SQLite, verified by digest + spot-check against the Python rows. +WAL runs pay for wal_checkpoint(TRUNCATE) inside the timed region. +""" + +import json +import os +import resource +import sqlite3 +import subprocess +import sys +import tempfile +import time + +N = 1_000_000 +NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/sample) in report +SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)" +INSERT = "INSERT INTO t VALUES (?,?,?)" + +SET_BASED_SQL = """ +WITH RECURSIVE seq(i) AS ( + SELECT 0 + UNION ALL + SELECT i + 1 FROM seq WHERE i + 1 < ? +) +INSERT INTO t SELECT i, i * 0.5, printf('payload-%012d', i) FROM seq +""" + + +# ---------------------------------------------------------------- data ---- +def make_row(i): + return (i, i * 0.5, f"payload-{i:012d}") + + +def rows(n): + """Identical deterministic data for every strategy.""" + for i in range(n): + yield make_row(i) + + +# ---------------------------------------------------------- strategies ---- +def s_naive_autocommit(conn, n): + conn.isolation_level = None # every INSERT commits: journal sync per row + for r in rows(n): + conn.execute(INSERT, r) + + +def s_one_big_txn_loop(conn, n): + conn.execute("BEGIN") + for r in rows(n): + conn.execute(INSERT, r) + conn.commit() + + +def s_executemany_list(conn, n): + data = list(rows(n)) # deliberately materialized: shows RAM cost + conn.execute("BEGIN") + conn.executemany(INSERT, data) + conn.commit() + + +def s_executemany_gen(conn, n): + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_wal_tuned_gen(conn, n): + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + # Fairness (BUILDER): WAL must not finish with 1M rows still in the -wal + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + + +def s_max_tuned_gen(conn, n): + """Python-bound speed candidate: one txn + executemany(generator) + + journaling/sync disabled for the load. Rebuildable staging only.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_set_based(conn, n): + """Beyond-the-list candidate (BUILDER): SQLite generates the identical + rows itself; zero Python->SQLite binding crossings. OFF/OFF pragmas so + it is compared at the same durability tier as max_tuned_gen.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.execute(SET_BASED_SQL, (n,)) + conn.commit() + + +STRATEGIES = { + "naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N), + "one_big_txn_loop": (s_one_big_txn_loop, N), + "executemany_list": (s_executemany_list, N), + "executemany_gen": (s_executemany_gen, N), + "wal_tuned_gen": (s_wal_tuned_gen, N), + "max_tuned_gen": (s_max_tuned_gen, N), + "set_based_ctes": (s_set_based, N), +} + + +# --------------------------------------------------------------- worker ---- +def rss_bytes(): + v = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return v if sys.platform == "darwin" else v * 1024 # KB on Linux + + +def verify(dbpath, n): + """Integrity gate (fused): count + checksum for everyone [ARCHITECT], + plus length-digest and spot-row equality so the SQL-generated data is + provably identical to the Python data [BUILDER].""" + conn = sqlite3.connect(dbpath) + cnt, chk, clen = conn.execute( + "SELECT count(*), sum(a), sum(length(c)) FROM t").fetchone() + ok = (cnt == n and chk == n * (n - 1) // 2 and clen == 20 * n) + for i in (0, 1, n // 2, n - 1): + got = conn.execute("SELECT a, b, c FROM t WHERE a=?", (i,)).fetchone() + ok = ok and got is not None and tuple(got) == make_row(i) + conn.close() + return ok + + +def worker(name): + if name == "calibration": # interpreter + sqlite3 import baseline + print(json.dumps({"name": name, "rows": 0, "seconds": 0.0, + "peak_rss": rss_bytes(), "ok": True})) + return + fn, n = STRATEGIES[name] + dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir()) + try: + conn = sqlite3.connect(dbpath) + conn.execute(SCHEMA) + t0 = time.perf_counter() # timer: generation + bind + insert + commit + fn(conn, n) + dt = time.perf_counter() - t0 + conn.close() + ok = verify(dbpath, n) # validation outside the timer + print(json.dumps({"name": name, "rows": n, "seconds": dt, + "peak_rss": rss_bytes(), "ok": ok})) + finally: + for p in (dbpath, dbpath + "-wal", dbpath + "-shm"): + if os.path.exists(p): + os.remove(p) + + +# ----------------------------------------------------------- orchestrator -- +def spawn(name): + out = subprocess.run([sys.executable, os.path.abspath(__file__), + "--worker", name], + capture_output=True, text=True, check=True) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def main(): + t_total = time.perf_counter() + print(f"SQLite bulk insert benchmark | N={N:,} rows | " + f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} " + f"{sys.platform}\n") + + base = spawn("calibration") + results = [] + for name in STRATEGIES: # fixed, deterministic order + r = spawn(name) + assert r["ok"], f"integrity check FAILED for {name}" + scale = N / r["rows"] + r["scaled"] = r["seconds"] * scale + r["sampled"] = scale > 1.0 + results.append(r) + print(f" [done] {name}: {r['seconds']:.3f}s " + f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})") + + naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit") + print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}" + f"{'Peak RSS':>12}{'RSS -base':>12}") + print("-" * 70) + for r in sorted(results, key=lambda r: r["scaled"]): + mark = "*" if r["sampled"] else " " + print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}" + f"{naive / r['scaled']:>9.1f}x" + f"{r['peak_rss'] / 1e6:>10.1f}MB" + f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB") + print("-" * 70) + print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled " + f"x{N // NAIVE_SAMPLE_N} | base RSS " + f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)") + print("note: max_tuned_gen and set_based_ctes run journal_mode=OFF/" + "synchronous=OFF -> rebuildable staging loads only.") + + fastest = min(results, key=lambda r: r["scaled"]) + full = [r for r in results if not r["sampled"]] # baseline excluded + leanest = min(full, key=lambda r: (r["peak_rss"], r["scaled"])) + print(f"\nSpeed Winner : {fastest['name']} " + f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)") + print(f"Memory Winner: {leanest['name']} " + f"({leanest['peak_rss'] / 1e6:.1f}MB peak, " + f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)") + print(f"\ntotal benchmark wall time: {time.perf_counter() - t_total:.1f}s") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--worker": + worker(sys.argv[2]) + else: + main() +``` diff --git a/live_final_generation/project-root/RESULTS.md b/live_final_generation/project-root/RESULTS.md new file mode 100644 index 0000000..7125cad --- /dev/null +++ b/live_final_generation/project-root/RESULTS.md @@ -0,0 +1,25 @@ +# SQLite Bulk Insert Benchmark Results + +Fresh measurements: **1,000,000 target rows**, Python 3.12.13, SQLite 3.50.4, macOS-26.5.2-arm64-arm-64bit. + +The timer includes row generation, binding, insertion, and commit. WAL timing also includes `wal_checkpoint(TRUNCATE)`. Each strategy ran in a fresh process against a fresh database. + +| Strategy | Measured rows | Time 1M rows (s) | Speedup vs naive | Peak RAM (MB) | RSS over base (MB) | +|---|---:|---:|---:|---:|---:| +| `set_based_ctes` | 1,000,000 | 0.1700 | 1147.98x | 26.051 | 4.014 | +| `executemany_gen` | 1,000,000 | 0.3480 | 560.77x | 25.625 | 3.588 | +| `max_tuned_gen` | 1,000,000 | 0.3484 | 560.15x | 25.641 | 3.604 | +| `executemany_list` | 1,000,000 | 0.3761 | 518.88x | 228.737 | 206.701 | +| `wal_tuned_gen` | 1,000,000 | 0.3918 | 498.07x | 25.903 | 3.867 | +| `one_big_txn_loop` | 1,000,000 | 0.6071 | 321.41x | 25.854 | 3.817 | +| `naive_autocommit` | 20,000 (sampled) | 195.1362* | 1.00x | 24.183 | 2.146 | + +\* `naive_autocommit` ran 20,000 real rows; its measured time was scaled by 50. Its RAM reading is measured, not scaled, and it is excluded from the memory-winner decision. + +`max_tuned_gen` and `set_based_ctes` use `journal_mode=OFF`/`synchronous=OFF`; they are for rebuildable staging loads only. + +Bare-worker peak RSS calibration: **22.036 MB**. +Total benchmark wall time: **6.857 s**. + +**Speed Winner: `set_based_ctes`** โ€” 0.1700 s, 1147.98x faster than naive. +**Memory Winner: `executemany_gen`** โ€” 25.625 MB peak RSS, +3.588 MB over the calibration worker. diff --git a/live_final_generation/project-root/bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py b/live_final_generation/project-root/bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py new file mode 100644 index 0000000..fd50e8d --- /dev/null +++ b/live_final_generation/project-root/bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py @@ -0,0 +1,233 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +""" +SQLite bulk-insert benchmark: 1,000,000 rows, speed + peak RAM, per-strategy +process isolation. Stdlib only. Run: uv run bench-sqlite-bulk-ARCHITECT-anthropic-claude-fable-5.py + +Design (ARCHITECT / anthropic-claude-fable-5): + * Orchestrator re-execs THIS file as a subprocess per strategy + (`--worker NAME`), so peak RSS readings never pollute each other. + * Every strategy inserts IDENTICAL deterministic data into an IDENTICAL + fresh schema, verified by count + checksum. + * Timing covers the full insert phase (row generation + bind + commit), + identical work for all strategies. + * The naive autocommit baseline is fsync-bound (~50-200 rows/ms is + impossible; it does one journal sync per row). It is SAMPLED at 20,000 + rows and honestly scaled x50; the table flags it with '*'. + * Peak RAM = ru_maxrss of the worker process (normalized to bytes across + macOS/Linux), plus a no-op calibration worker so a delta over + interpreter baseline is reported. +""" + +import json +import os +import resource +import sqlite3 +import subprocess +import sys +import tempfile +import time + +N = 1_000_000 +NAIVE_SAMPLE_N = 20_000 # honest sample; scaled x(N/NAIVE_SAMPLE_N) in report +SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)" +INSERT = "INSERT INTO t VALUES (?,?,?)" +EXPECTED_SUM = lambda n: n * (n - 1) // 2 # sum of column a for n rows + + +# ---------------------------------------------------------------- data ---- +def rows(n): + """Identical deterministic data for every strategy.""" + for i in range(n): + yield (i, i * 0.5, f"payload-{i:012d}") + + +# ---------------------------------------------------------- strategies ---- +def s_naive_autocommit(conn, n): + conn.isolation_level = None # every INSERT commits (journal sync per row) + for r in rows(n): + conn.execute(INSERT, r) + + +def s_one_big_txn_loop(conn, n): + conn.execute("BEGIN") + for r in rows(n): + conn.execute(INSERT, r) + conn.commit() + + +def s_executemany_list(conn, n): + data = list(rows(n)) # deliberately materialized: shows the RAM cost + conn.execute("BEGIN") + conn.executemany(INSERT, data) + conn.commit() + + +def s_executemany_gen(conn, n): + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_wal_tuned_gen(conn, n): + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA cache_size=-64000") # 64 MB page cache + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_max_tuned_gen(conn, n): + """Predicted overall speed winner within stdlib: one txn + + executemany(generator) + journaling/sync disabled for the load.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("PRAGMA cache_size=-64000") + conn.execute("PRAGMA temp_store=MEMORY") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +STRATEGIES = { + "naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N), + "one_big_txn_loop": (s_one_big_txn_loop, N), + "executemany_list": (s_executemany_list, N), + "executemany_gen": (s_executemany_gen, N), + "wal_tuned_gen": (s_wal_tuned_gen, N), + "max_tuned_gen": (s_max_tuned_gen, N), +} + +# Optional bonus (external `sqlite3` CLI, skipped gracefully if absent). +# Same identical data, same isolation, timing includes writing the CSV. +CLI_STRATEGY = "cli_dot_import" + + +def run_cli_import(dbpath, n): + import csv + import shutil + + exe = shutil.which("sqlite3") + if not exe: + return False + csvpath = dbpath + ".csv" + with open(csvpath, "w", newline="") as f: + w = csv.writer(f) + for r in rows(n): + w.writerow(r) + subprocess.run( + [exe, dbpath, SCHEMA + ";", ".mode csv", f".import {csvpath} t"], + check=True, capture_output=True, + ) + os.remove(csvpath) + return True + + +# --------------------------------------------------------------- worker ---- +def rss_bytes(who=resource.RUSAGE_SELF): + v = resource.getrusage(who).ru_maxrss + return v if sys.platform == "darwin" else v * 1024 # KB on Linux + + +def worker(name): + if name == "calibration": # no-op: interpreter + sqlite3 import baseline + print(json.dumps({"name": name, "rows": 0, "seconds": 0.0, + "peak_rss": rss_bytes(), "ok": True})) + return + + dbpath = tempfile.mktemp(suffix=".db", dir=tempfile.gettempdir()) + try: + if name == CLI_STRATEGY: + t0 = time.perf_counter() + ok = run_cli_import(dbpath, N) + dt = time.perf_counter() - t0 + if not ok: + print(json.dumps({"name": name, "skipped": True})) + return + n = N + else: + fn, n = STRATEGIES[name] + conn = sqlite3.connect(dbpath) + conn.execute(SCHEMA) + t0 = time.perf_counter() + fn(conn, n) + dt = time.perf_counter() - t0 + conn.close() + + vconn = sqlite3.connect(dbpath) + cnt, chk = vconn.execute("SELECT count(*), sum(a) FROM t").fetchone() + vconn.close() + peak = max(rss_bytes(), rss_bytes(resource.RUSAGE_CHILDREN)) + print(json.dumps({ + "name": name, "rows": cnt, "seconds": dt, "peak_rss": peak, + "ok": cnt == n and chk == EXPECTED_SUM(n), + })) + finally: + for p in (dbpath, dbpath + "-wal", dbpath + "-shm", dbpath + ".csv"): + if os.path.exists(p): + os.remove(p) + + +# ----------------------------------------------------------- orchestrator -- +def spawn(name): + out = subprocess.run([sys.executable, os.path.abspath(__file__), + "--worker", name], + capture_output=True, text=True, check=True) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def main(): + print(f"SQLite bulk insert benchmark | N={N:,} rows | " + f"py {sys.version.split()[0]} sqlite {sqlite3.sqlite_version} " + f"{sys.platform}\n") + + base = spawn("calibration") + results = [] + order = list(STRATEGIES) + [CLI_STRATEGY] + for name in order: + r = spawn(name) + if r.get("skipped"): + print(f" [skip] {name} (sqlite3 CLI not on PATH)") + continue + assert r["ok"], f"integrity check FAILED for {name}" + scale = N / r["rows"] + r["scaled"] = r["seconds"] * scale + r["sampled"] = scale > 1.0 + results.append(r) + print(f" [done] {name}: {r['seconds']:.3f}s " + f"({r['rows']:,} rows{' sampled' if r['sampled'] else ''})") + + naive = next(r["scaled"] for r in results if r["name"] == "naive_autocommit") + print(f"\n{'Strategy':<22}{'Time 1M rows':>14}{'Speedup':>10}" + f"{'Peak RSS':>12}{'RSS -base':>12}") + print("-" * 70) + for r in sorted(results, key=lambda r: r["scaled"]): + mark = "*" if r["sampled"] else " " + print(f"{r['name']:<22}{r['scaled']:>12.2f}s{mark}" + f"{naive / r['scaled']:>9.1f}x" + f"{r['peak_rss'] / 1e6:>10.1f}MB" + f"{(r['peak_rss'] - base['peak_rss']) / 1e6:>10.1f}MB") + print("-" * 70) + print(f"* sampled at {NAIVE_SAMPLE_N:,} rows, honestly scaled " + f"x{N // NAIVE_SAMPLE_N} | base RSS " + f"{base['peak_rss'] / 1e6:.1f}MB (no-op interpreter)") + + fastest = min(results, key=lambda r: r["scaled"]) + # memory winner among full-speed strategies (exclude the naive baseline) + fast = [r for r in results if not r["sampled"]] + leanest = min(fast, key=lambda r: (r["peak_rss"], r["scaled"])) + print(f"\nSPEED WINNER : {fastest['name']} " + f"({fastest['scaled']:.2f}s, {naive / fastest['scaled']:.0f}x vs naive)") + print(f"MEMORY WINNER: {leanest['name']} " + f"({leanest['peak_rss'] / 1e6:.1f}MB peak, " + f"+{(leanest['peak_rss'] - base['peak_rss']) / 1e6:.1f}MB over baseline)") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--worker": + worker(sys.argv[2]) + else: + main() diff --git a/live_final_generation/project-root/bench.py b/live_final_generation/project-root/bench.py new file mode 100644 index 0000000..f2ce755 --- /dev/null +++ b/live_final_generation/project-root/bench.py @@ -0,0 +1,393 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Benchmark seven SQLite bulk-insert strategies on speed and peak RAM. + +Run with: + uv run bench.py + +The orchestrator re-executes this file once per strategy. Each worker uses a +fresh process and a fresh SQLite database, so process peak-RSS measurements +cannot leak from one strategy into another. The run writes RESULTS.md next to +this script from fresh measurements every time. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import platform +import resource +import sqlite3 +import subprocess +import sys +import tempfile +import time +from typing import Any, Callable, Iterator + +N = 1_000_000 +NAIVE_SAMPLE_N = 20_000 +SCHEMA = "CREATE TABLE t (a INTEGER, b REAL, c TEXT)" +INSERT = "INSERT INTO t VALUES (?, ?, ?)" +RESULTS_PATH = Path(__file__).resolve().with_name("RESULTS.md") + +SET_BASED_SQL = """ +WITH RECURSIVE seq(i) AS ( + SELECT 0 + UNION ALL + SELECT i + 1 FROM seq WHERE i + 1 < ? +) +INSERT INTO t +SELECT i, i * 0.5, printf('payload-%012d', i) +FROM seq +""" + +Row = tuple[int, float, str] +StrategyFunction = Callable[[sqlite3.Connection, int], None] + + +def make_row(i: int) -> Row: + """Return the sole canonical definition of one benchmark row.""" + return (i, i * 0.5, f"payload-{i:012d}") + + +def rows(n: int) -> Iterator[Row]: + """Generate identical deterministic data for every Python-bound strategy.""" + for i in range(n): + yield make_row(i) + + +# ---------------------------------------------------------------- strategies + +def s_naive_autocommit(conn: sqlite3.Connection, n: int) -> None: + """Baseline: every INSERT is its own durable transaction.""" + conn.isolation_level = None + for row in rows(n): + conn.execute(INSERT, row) + + +def s_one_big_txn_loop(conn: sqlite3.Connection, n: int) -> None: + conn.execute("BEGIN") + for row in rows(n): + conn.execute(INSERT, row) + conn.commit() + + +def s_executemany_list(conn: sqlite3.Connection, n: int) -> None: + # Deliberately materialized so the benchmark measures the classic RAM cost. + data = list(rows(n)) + conn.execute("BEGIN") + conn.executemany(INSERT, data) + conn.commit() + + +def s_executemany_gen(conn: sqlite3.Connection, n: int) -> None: + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_wal_tuned_gen(conn: sqlite3.Connection, n: int) -> None: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + # Include write-back cost instead of ending with the rows in a WAL sidecar. + busy, _wal_pages, _checkpointed = conn.execute( + "PRAGMA wal_checkpoint(TRUNCATE)" + ).fetchone() + if busy: + raise RuntimeError("WAL checkpoint was unexpectedly busy") + + +def s_max_tuned_gen(conn: sqlite3.Connection, n: int) -> None: + """Maximum Python-bound speed; safe only for a rebuildable staging DB.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.executemany(INSERT, rows(n)) + conn.commit() + + +def s_set_based_ctes(conn: sqlite3.Connection, n: int) -> None: + """Generate identical rows inside SQLite, avoiding per-row Python binds.""" + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + conn.execute("BEGIN") + conn.execute(SET_BASED_SQL, (n,)) + conn.commit() + + +STRATEGIES: dict[str, tuple[StrategyFunction, int]] = { + "naive_autocommit": (s_naive_autocommit, NAIVE_SAMPLE_N), + "one_big_txn_loop": (s_one_big_txn_loop, N), + "executemany_list": (s_executemany_list, N), + "executemany_gen": (s_executemany_gen, N), + "wal_tuned_gen": (s_wal_tuned_gen, N), + "max_tuned_gen": (s_max_tuned_gen, N), + "set_based_ctes": (s_set_based_ctes, N), +} + + +# ------------------------------------------------------------------- workers + +def rss_bytes() -> int: + """Return this process's measured high-water resident set size in bytes.""" + ru_maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return int(ru_maxrss if sys.platform == "darwin" else ru_maxrss * 1024) + + +def verify(db_path: Path, n: int) -> bool: + """Prove row count, checksums, lengths, and representative rows match.""" + conn = sqlite3.connect(db_path) + try: + count, checksum, text_length = conn.execute( + "SELECT count(*), sum(a), sum(length(c)) FROM t" + ).fetchone() + valid = ( + count == n + and checksum == n * (n - 1) // 2 + and text_length == 20 * n + ) + for i in (0, 1, n // 2, n - 1): + actual = conn.execute( + "SELECT a, b, c FROM t WHERE a = ?", (i,) + ).fetchone() + valid = valid and actual is not None and tuple(actual) == make_row(i) + return bool(valid) + finally: + conn.close() + + +def worker(name: str) -> None: + if name == "calibration": + print( + json.dumps( + { + "name": name, + "rows": 0, + "seconds": 0.0, + "peak_rss": rss_bytes(), + "ok": True, + } + ) + ) + return + + function, n = STRATEGIES[name] + with tempfile.TemporaryDirectory(prefix=f"sqlite-bulk-{name}-") as temp_dir: + db_path = Path(temp_dir) / "benchmark.sqlite3" + conn = sqlite3.connect(db_path) + try: + conn.execute(SCHEMA) + started = time.perf_counter() + function(conn, n) + seconds = time.perf_counter() - started + finally: + conn.close() + + valid = verify(db_path, n) + result = { + "name": name, + "rows": n, + "seconds": seconds, + "peak_rss": rss_bytes(), + "ok": valid, + } + print(json.dumps(result, separators=(",", ":"))) + + +def spawn(name: str) -> dict[str, Any]: + completed = subprocess.run( + [sys.executable, os.path.abspath(__file__), "--worker", name], + check=True, + capture_output=True, + text=True, + timeout=220, + ) + output_lines = completed.stdout.strip().splitlines() + if not output_lines: + raise RuntimeError(f"worker {name} produced no result") + return json.loads(output_lines[-1]) + + +# ---------------------------------------------------------------- reporting + +def measured_rows(result: dict[str, Any]) -> str: + suffix = " (sampled)" if result["sampled"] else "" + return f"{result['rows']:,}{suffix}" + + +def build_results_markdown( + results: list[dict[str, Any]], + base_rss: int, + total_wall_seconds: float, +) -> tuple[str, dict[str, Any], dict[str, Any]]: + baseline = next(r for r in results if r["name"] == "naive_autocommit") + baseline_time = float(baseline["scaled_seconds"]) + ordered = sorted(results, key=lambda result: result["scaled_seconds"]) + fastest = ordered[0] + full_runs = [result for result in results if not result["sampled"]] + leanest = min( + full_runs, + key=lambda result: (result["peak_rss"], result["scaled_seconds"]), + ) + + lines = [ + "# SQLite Bulk Insert Benchmark Results", + "", + ( + f"Fresh measurements: **{N:,} target rows**, Python " + f"{platform.python_version()}, SQLite {sqlite3.sqlite_version}, " + f"{platform.platform()}." + ), + "", + ( + "The timer includes row generation, binding, insertion, and commit. " + "WAL timing also includes `wal_checkpoint(TRUNCATE)`. Each strategy " + "ran in a fresh process against a fresh database." + ), + "", + "| Strategy | Measured rows | Time 1M rows (s) | Speedup vs naive | Peak RAM (MB) | RSS over base (MB) |", + "|---|---:|---:|---:|---:|---:|", + ] + + for result in ordered: + scaled = float(result["scaled_seconds"]) + peak_mb = int(result["peak_rss"]) / 1_000_000 + delta_mb = (int(result["peak_rss"]) - base_rss) / 1_000_000 + time_mark = "*" if result["sampled"] else "" + lines.append( + f"| `{result['name']}` | {measured_rows(result)} | " + f"{scaled:.4f}{time_mark} | {baseline_time / scaled:.2f}x | " + f"{peak_mb:.3f} | {delta_mb:.3f} |" + ) + + fastest_time = float(fastest["scaled_seconds"]) + leanest_peak_mb = int(leanest["peak_rss"]) / 1_000_000 + leanest_delta_mb = (int(leanest["peak_rss"]) - base_rss) / 1_000_000 + lines.extend( + [ + "", + ( + f"\\* `naive_autocommit` ran {NAIVE_SAMPLE_N:,} real rows; its " + f"measured time was scaled by {N // NAIVE_SAMPLE_N}. Its RAM " + "reading is measured, not scaled, and it is excluded from the " + "memory-winner decision." + ), + "", + ( + "`max_tuned_gen` and `set_based_ctes` use " + "`journal_mode=OFF`/`synchronous=OFF`; they are for rebuildable " + "staging loads only." + ), + "", + f"Bare-worker peak RSS calibration: **{base_rss / 1_000_000:.3f} MB**.", + f"Total benchmark wall time: **{total_wall_seconds:.3f} s**.", + "", + ( + f"**Speed Winner: `{fastest['name']}`** โ€” {fastest_time:.4f} s, " + f"{baseline_time / fastest_time:.2f}x faster than naive." + ), + ( + f"**Memory Winner: `{leanest['name']}`** โ€” " + f"{leanest_peak_mb:.3f} MB peak RSS, " + f"{leanest_delta_mb:+.3f} MB over the calibration worker." + ), + "", + ] + ) + return "\n".join(lines), fastest, leanest + + +def definition_of_done_gate( + results: list[dict[str, Any]], + markdown: str, + fastest: dict[str, Any], + leanest: dict[str, Any], +) -> None: + """Hard-fail unless fresh measurements satisfy the benchmark contract.""" + expected = set(STRATEGIES) + actual = {str(result["name"]) for result in results} + if actual != expected: + raise AssertionError(f"strategy mismatch: expected {expected}, got {actual}") + if not all(result["ok"] for result in results): + failed = [result["name"] for result in results if not result["ok"]] + raise AssertionError(f"integrity verification failed: {failed}") + if not all( + float(result["seconds"]) > 0 and int(result["peak_rss"]) > 0 + for result in results + ): + raise AssertionError("every strategy must have measured positive time and RSS") + for result in results: + required_rows = NAIVE_SAMPLE_N if result["name"] == "naive_autocommit" else N + if result["rows"] != required_rows: + raise AssertionError(f"wrong row count for {result['name']}") + if result["name"] not in markdown: + raise AssertionError(f"RESULTS.md omitted {result['name']}") + baseline = next(r for r in results if r["name"] == "naive_autocommit") + speedup = float(baseline["scaled_seconds"]) / float(fastest["scaled_seconds"]) + if speedup < 5.0: + raise AssertionError(f"fastest strategy achieved only {speedup:.2f}x") + if fastest["name"] not in markdown or leanest["name"] not in markdown: + raise AssertionError("winner declaration missing from RESULTS.md") + if "Speed Winner:" not in markdown or "Memory Winner:" not in markdown: + raise AssertionError("RESULTS.md lacks winner lines") + if leanest["name"] == "naive_autocommit": + raise AssertionError("sampled baseline cannot be Memory Winner") + + +def main() -> None: + total_started = time.perf_counter() + print( + f"SQLite bulk insert benchmark | N={N:,} rows | " + f"Python {platform.python_version()} | SQLite {sqlite3.sqlite_version}" + ) + print("Each strategy runs in a fresh isolated worker process.\n") + + calibration = spawn("calibration") + results: list[dict[str, Any]] = [] + for name in STRATEGIES: + result = spawn(name) + if not result["ok"]: + raise RuntimeError(f"integrity check failed for {name}") + scale = N / int(result["rows"]) + result["scaled_seconds"] = float(result["seconds"]) * scale + result["sampled"] = scale != 1.0 + results.append(result) + sample_note = " sampled" if result["sampled"] else "" + print( + f"[done] {name}: {result['seconds']:.4f}s, " + f"{result['rows']:,} rows{sample_note}, " + f"peak {result['peak_rss'] / 1_000_000:.3f} MB" + ) + + total_wall_seconds = time.perf_counter() - total_started + markdown, fastest, leanest = build_results_markdown( + results, int(calibration["peak_rss"]), total_wall_seconds + ) + definition_of_done_gate(results, markdown, fastest, leanest) + RESULTS_PATH.write_text(markdown, encoding="utf-8") + + print(f"\nWrote {RESULTS_PATH}") + print( + f"Speed Winner: {fastest['name']} " + f"({fastest['scaled_seconds']:.4f}s measured/scaled to 1M)" + ) + print( + f"Memory Winner: {leanest['name']} " + f"({leanest['peak_rss'] / 1_000_000:.3f} MB measured peak RSS)" + ) + print("Definition-of-Done gate: PASS") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--worker": + worker(sys.argv[2]) + elif len(sys.argv) == 1: + main() + else: + raise SystemExit("usage: uv run bench.py [--worker STRATEGY]") diff --git a/live_final_generation/project-root/sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py b/live_final_generation/project-root/sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py new file mode 100644 index 0000000..b1a351a --- /dev/null +++ b/live_final_generation/project-root/sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py @@ -0,0 +1,631 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.11" +# /// +"""SQLite bulk-insert benchmark: elapsed time and process peak RSS. + +Run: + uv run sqlite-bulk-benchmark-BUILDER-openai-gpt-5.6-sol.py + +Every timed trial executes in a fresh child process against a fresh database. +The deliberately slow autocommit case uses a prefix sample and reports a +clearly marked linear projection to one million rows. +""" + +from __future__ import annotations + +import argparse +import gc +import itertools +import json +import os +import platform +import random +import sqlite3 +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Iterator, Sequence + +IDENTITY = "BUILDER-openai-gpt-5.6-sol" +DEFAULT_ROWS = 1_000_000 +DEFAULT_BASELINE_ROWS = 10_000 +DEFAULT_REPEATS = 2 +EXECUTEMANY_BATCH_ROWS = 10_000 +MULTI_VALUES_TARGET_ROWS = 500 +COLUMN_COUNT = 6 + +INSERT_ONE = """ +INSERT INTO events + (id, account_id, created_at, amount_cents, status, payload) +VALUES (?, ?, ?, ?, ?, ?) +""" + +SCHEMA = """ +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + account_id INTEGER NOT NULL, + created_at INTEGER NOT NULL, + amount_cents INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ('new', 'paid', 'sent', 'void')), + payload TEXT NOT NULL +) +""" + +SET_BASED_INSERT = """ +WITH RECURSIVE +seq(i) AS ( + VALUES(0) + UNION ALL + SELECT i + 1 FROM seq WHERE i + 1 < ? +), +generated(i, mix) AS ( + SELECT i, (i * 1103515245 + 12345) % 2147483648 + FROM seq +) +INSERT INTO events + (id, account_id, created_at, amount_cents, status, payload) +SELECT + i, + mix % 100003, + 1700000000 + i, + (mix % 2000001) - 1000000, + CASE i % 4 + WHEN 0 THEN 'new' + WHEN 1 THEN 'paid' + WHEN 2 THEN 'sent' + ELSE 'void' + END, + printf( + 'event-%010d-%010x-abcdefghijklmnopqrstuvwxyz0123456789', + i, + mix + ) +FROM generated +""" + +DIGEST_SQL = """ +SELECT + count(*), + sum(id), + sum(account_id), + sum(created_at), + sum(amount_cents), + sum(length(status)), + sum(length(payload)), + min(id), + max(id), + min(payload), + max(payload) +FROM events +""" + + +@dataclass(frozen=True) +class Strategy: + key: str + label: str + journal_mode: str + synchronous: str + implementation: str + durability: str + sampled: bool = False + + +STRATEGIES: tuple[Strategy, ...] = ( + Strategy( + "naive_autocommit", + "naive autocommit", + "DELETE", + "FULL", + "naive", + "durable", + sampled=True, + ), + Strategy( + "one_big_transaction", + "one transaction + execute", + "DELETE", + "FULL", + "one_transaction", + "durable", + ), + Strategy( + "executemany_streamed", + "streamed executemany batches", + "DELETE", + "FULL", + "executemany", + "durable", + ), + Strategy( + "wal_multi_values", + "WAL/NORMAL + multi-VALUES", + "WAL", + "NORMAL", + "multi_values", + "consistent; latest commit power-loss risk", + ), + Strategy( + "off_multi_values", + "OFF/OFF + multi-VALUES", + "OFF", + "OFF", + "multi_values", + "rebuildable staging only", + ), + Strategy( + "wal_set_based", + "WAL/NORMAL + INSERT-SELECT", + "WAL", + "NORMAL", + "set_based", + "consistent; latest commit power-loss risk", + ), + Strategy( + "off_set_based", + "OFF/OFF + INSERT-SELECT", + "OFF", + "OFF", + "set_based", + "rebuildable staging only", + ), +) +STRATEGY_BY_KEY = {strategy.key: strategy for strategy in STRATEGIES} + + +Row = tuple[int, int, int, int, str, str] +Digest = tuple[int | str, ...] + + +def make_row(i: int) -> Row: + """The sole Python definition of benchmark data.""" + mix = (i * 1103515245 + 12345) % 2147483648 + status = ("new", "paid", "sent", "void")[i % 4] + return ( + i, + mix % 100003, + 1700000000 + i, + (mix % 2000001) - 1000000, + status, + f"event-{i:010d}-{mix:010x}-abcdefghijklmnopqrstuvwxyz0123456789", + ) + + +def generated_rows(start: int, stop: int) -> Iterator[Row]: + for i in range(start, stop): + yield make_row(i) + + +def expected_digest(row_count: int) -> Digest: + count = 0 + id_sum = account_sum = created_sum = amount_sum = 0 + status_length_sum = payload_length_sum = 0 + min_payload: str | None = None + max_payload: str | None = None + for row in generated_rows(0, row_count): + row_id, account_id, created_at, amount, status, payload = row + count += 1 + id_sum += row_id + account_sum += account_id + created_sum += created_at + amount_sum += amount + status_length_sum += len(status) + payload_length_sum += len(payload) + if min_payload is None or payload < min_payload: + min_payload = payload + if max_payload is None or payload > max_payload: + max_payload = payload + return ( + count, + id_sum, + account_sum, + created_sum, + amount_sum, + status_length_sum, + payload_length_sum, + 0 if row_count else None, + row_count - 1 if row_count else None, + min_payload, + max_payload, + ) + + +def peak_rss_bytes() -> int: + if os.name == "nt": + # ctypes is standard library. PeakWorkingSetSize is the Windows + # equivalent of process peak RSS. + import ctypes + from ctypes import wintypes + + class ProcessMemoryCounters(ctypes.Structure): + _fields_ = ( + ("cb", wintypes.DWORD), + ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ) + + counters = ProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + get_current_process = ctypes.windll.kernel32.GetCurrentProcess + get_current_process.restype = wintypes.HANDLE + ok = ctypes.windll.psapi.GetProcessMemoryInfo( + get_current_process(), ctypes.byref(counters), counters.cb + ) + if not ok: + raise ctypes.WinError() + return int(counters.PeakWorkingSetSize) + + import resource + + raw = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # macOS reports bytes; Linux and the BSDs report KiB. + if sys.platform == "darwin": + return int(raw) + return int(raw) * 1024 + + +def setup_connection(db_path: Path, strategy: Strategy) -> sqlite3.Connection: + connection = sqlite3.connect(db_path, isolation_level=None, timeout=60.0) + connection.execute("PRAGMA page_size=4096") + actual_journal = connection.execute( + f"PRAGMA journal_mode={strategy.journal_mode}" + ).fetchone()[0] + if str(actual_journal).upper() != strategy.journal_mode: + raise RuntimeError( + f"requested journal_mode={strategy.journal_mode}, got {actual_journal}" + ) + connection.execute(f"PRAGMA synchronous={strategy.synchronous}") + + # Common controls: bounded cache, no mmap RSS ambiguity, disk-backed temp + # structures, and spill enabled. These are identical for every strategy. + connection.execute("PRAGMA cache_size=-2048") + connection.execute("PRAGMA cache_spill=ON") + connection.execute("PRAGMA temp_store=FILE") + connection.execute("PRAGMA mmap_size=0") + connection.execute("PRAGMA foreign_keys=ON") + connection.execute(SCHEMA) + return connection + + +def finish_commit(connection: sqlite3.Connection, strategy: Strategy) -> None: + connection.commit() + # Include WAL's eventual write-back cost, rather than declaring victory + # while the complete database still resides in a large sidecar file. + if strategy.journal_mode == "WAL": + result = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + if result[0] != 0: + raise RuntimeError(f"WAL checkpoint was busy: {result!r}") + + +def insert_naive(connection: sqlite3.Connection, row_count: int, strategy: Strategy) -> None: + # isolation_level=None makes every statement its own transaction. + for row in generated_rows(0, row_count): + connection.execute(INSERT_ONE, row) + + +def insert_one_transaction( + connection: sqlite3.Connection, row_count: int, strategy: Strategy +) -> None: + connection.execute("BEGIN IMMEDIATE") + for row in generated_rows(0, row_count): + connection.execute(INSERT_ONE, row) + finish_commit(connection, strategy) + + +def insert_executemany( + connection: sqlite3.Connection, row_count: int, strategy: Strategy +) -> None: + connection.execute("BEGIN IMMEDIATE") + for start in range(0, row_count, EXECUTEMANY_BATCH_ROWS): + stop = min(start + EXECUTEMANY_BATCH_ROWS, row_count) + # An iterator, not a list: batching does not materialize input rows. + connection.executemany(INSERT_ONE, generated_rows(start, stop)) + finish_commit(connection, strategy) + + +def insert_multi_values( + connection: sqlite3.Connection, row_count: int, strategy: Strategy +) -> None: + try: + variable_limit = connection.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) + except AttributeError: + variable_limit = 999 + batch_rows = max( + 1, min(MULTI_VALUES_TARGET_ROWS, variable_limit // COLUMN_COUNT) + ) + placeholder = "(" + ",".join("?" for _ in range(COLUMN_COUNT)) + ")" + + connection.execute("BEGIN IMMEDIATE") + for start in range(0, row_count, batch_rows): + stop = min(start + batch_rows, row_count) + batch = list(generated_rows(start, stop)) + statement = ( + "INSERT INTO events " + "(id, account_id, created_at, amount_cents, status, payload) VALUES " + + ",".join(itertools.repeat(placeholder, len(batch))) + ) + bindings = [value for row in batch for value in row] + connection.execute(statement, bindings) + finish_commit(connection, strategy) + + +def insert_set_based( + connection: sqlite3.Connection, row_count: int, strategy: Strategy +) -> None: + connection.execute("BEGIN IMMEDIATE") + connection.execute(SET_BASED_INSERT, (row_count,)) + finish_commit(connection, strategy) + + +IMPLEMENTATIONS = { + "naive": insert_naive, + "one_transaction": insert_one_transaction, + "executemany": insert_executemany, + "multi_values": insert_multi_values, + "set_based": insert_set_based, +} + + +def worker(strategy_key: str, db_path: Path, row_count: int) -> int: + strategy = STRATEGY_BY_KEY[strategy_key] + if db_path.exists(): + db_path.unlink() + connection = setup_connection(db_path, strategy) + implementation = IMPLEMENTATIONS[strategy.implementation] + + gc_was_enabled = gc.isenabled() + gc.disable() + started_ns = time.perf_counter_ns() + try: + implementation(connection, row_count, strategy) + except BaseException: + if connection.in_transaction: + connection.rollback() + raise + finally: + if gc_was_enabled: + gc.enable() + elapsed_ns = time.perf_counter_ns() - started_ns + + # Snapshot peak before the untimed validation query. + rss_bytes = peak_rss_bytes() + digest = tuple(connection.execute(DIGEST_SQL).fetchone()) + quick_check = connection.execute("PRAGMA quick_check").fetchone()[0] + connection.close() + + print( + json.dumps( + { + "strategy": strategy.key, + "rows": row_count, + "elapsed_seconds": elapsed_ns / 1_000_000_000, + "peak_rss_bytes": rss_bytes, + "digest": digest, + "quick_check": quick_check, + }, + separators=(",", ":"), + ) + ) + return 0 + + +def format_seconds_list(results: Sequence[dict[str, object]]) -> str: + return ",".join(f"{float(result['elapsed_seconds']):.3f}" for result in results) + + +def format_rss_list(results: Sequence[dict[str, object]]) -> str: + mib = 1024 * 1024 + return ",".join(f"{int(result['peak_rss_bytes']) / mib:.1f}" for result in results) + + +def median(values: Sequence[float]) -> float: + ordered = sorted(values) + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + return (ordered[middle - 1] + ordered[middle]) / 2 + + +def print_table( + grouped: dict[str, list[dict[str, object]]], + total_rows: int, + expected_by_count: dict[int, Digest], +) -> tuple[Strategy, Strategy]: + headers = ( + "Strategy", + "Mode", + "Rows/run", + "Trial time(s)", + "Time@1M(s)", + "Peak RAM/run MiB", + "Data", + ) + rows: list[tuple[str, ...]] = [] + normalized_times: dict[str, float] = {} + peak_by_strategy: dict[str, int] = {} + + for strategy in STRATEGIES: + results = grouped[strategy.key] + elapsed = [float(result["elapsed_seconds"]) for result in results] + row_count = int(results[0]["rows"]) + projected = median(elapsed) * total_rows / row_count + normalized_times[strategy.key] = projected + peak_by_strategy[strategy.key] = max( + int(result["peak_rss_bytes"]) for result in results + ) + verified = all( + tuple(result["digest"]) == expected_by_count[int(result["rows"])] + and result["quick_check"] == "ok" + for result in results + ) + projection_marker = "*" if strategy.sampled else "" + rows.append( + ( + strategy.label, + f"{strategy.journal_mode}/{strategy.synchronous}", + f"{row_count:,}", + format_seconds_list(results), + f"{projected:.3f}{projection_marker}", + format_rss_list(results), + "OK" if verified else "FAIL", + ) + ) + + widths = [len(header) for header in headers] + for row in rows: + for index, value in enumerate(row): + widths[index] = max(widths[index], len(value)) + + def render(row: Sequence[str]) -> str: + return " | ".join(value.ljust(widths[i]) for i, value in enumerate(row)) + + print(render(headers)) + print("-+-".join("-" * width for width in widths)) + for row in rows: + print(render(row)) + + speed_key = min(normalized_times, key=normalized_times.get) + # The sampled baseline is not eligible for the memory title: its peak RSS + # was measured honestly but it did not execute the full row count. + full_keys = [strategy.key for strategy in STRATEGIES if not strategy.sampled] + memory_key = min(full_keys, key=peak_by_strategy.get) + return STRATEGY_BY_KEY[speed_key], STRATEGY_BY_KEY[memory_key] + + +def orchestrator(args: argparse.Namespace) -> int: + if args.rows <= 0 or args.baseline_rows <= 0 or args.repeats <= 0: + raise SystemExit("--rows, --baseline-rows, and --repeats must be positive") + baseline_rows = min(args.baseline_rows, args.rows) + + print("SQLite bulk insert benchmark") + print(f"Identity: {IDENTITY}") + print(f"Python: {platform.python_version()} ({sys.executable})") + print(f"SQLite: {sqlite3.sqlite_version}") + print(f"Platform: {platform.platform()}") + print(f"Rows: {args.rows:,}") + print( + f"Autocommit sample: {baseline_rows:,} rows; its Time@1M is a linear projection" + ) + print(f"Full-strategy trials: {args.repeats}") + print( + "Timed interval: first BEGIN/INSERT through COMMIT; WAL trials also include " + "TRUNCATE checkpoint" + ) + print("Peak RAM: per-worker process ru_maxrss; every trial uses a fresh process") + print() + + print("Computing independent expected data digests...") + expected_by_count = { + baseline_rows: expected_digest(baseline_rows), + args.rows: expected_digest(args.rows), + } + + jobs: list[tuple[Strategy, int, int]] = [] + for strategy in STRATEGIES: + trial_count = 1 if strategy.sampled else args.repeats + row_count = baseline_rows if strategy.sampled else args.rows + for trial in range(1, trial_count + 1): + jobs.append((strategy, trial, row_count)) + random.Random(0x5A17E).shuffle(jobs) + + grouped: dict[str, list[dict[str, object]]] = { + strategy.key: [] for strategy in STRATEGIES + } + script_path = Path(__file__).resolve() + child_environment = os.environ.copy() + child_environment["PYTHONHASHSEED"] = "0" + + with tempfile.TemporaryDirectory(prefix=f"sqlite-bench-{IDENTITY}-") as temp_name: + temp_dir = Path(temp_name) + for job_number, (strategy, trial, row_count) in enumerate(jobs, start=1): + db_path = temp_dir / f"{IDENTITY}-{strategy.key}-trial-{trial}.sqlite3" + print( + f"[{job_number:02d}/{len(jobs):02d}] {strategy.label}, " + f"trial {trial}, {row_count:,} rows...", + flush=True, + ) + completed = subprocess.run( + [ + sys.executable, + str(script_path), + "--worker", + strategy.key, + "--db", + str(db_path), + "--worker-rows", + str(row_count), + ], + check=False, + capture_output=True, + text=True, + env=child_environment, + timeout=args.worker_timeout, + ) + if completed.returncode != 0: + raise RuntimeError( + f"worker failed for {strategy.key}:\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + result = json.loads(completed.stdout.strip().splitlines()[-1]) + grouped[strategy.key].append(result) + + for results in grouped.values(): + results.sort(key=lambda item: float(item["elapsed_seconds"])) + + print("\nResults") + speed_winner, memory_winner = print_table( + grouped, args.rows, expected_by_count + ) + print( + "\n* naive autocommit Time@1M is measured-time ร— " + f"{args.rows:,}/{baseline_rows:,}; its RAM is measured, not projected." + ) + print( + "OFF/OFF rows are semantically identical but the database is rebuildable-only; " + "a crash can corrupt it." + ) + print( + "INSERT-SELECT is applicable when rows can be generated in SQL or selected " + "from an attached SQLite source." + ) + print(f"\nSpeed Winner: {speed_winner.label}") + print(f"Memory Winner: {memory_winner.label}") + return 0 + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=DEFAULT_ROWS) + parser.add_argument("--baseline-rows", type=int, default=DEFAULT_BASELINE_ROWS) + parser.add_argument("--repeats", type=int, default=DEFAULT_REPEATS) + parser.add_argument("--worker-timeout", type=float, default=180.0) + parser.add_argument("--worker", choices=tuple(STRATEGY_BY_KEY)) + parser.add_argument("--db", type=Path) + parser.add_argument("--worker-rows", type=int) + args = parser.parse_args(argv) + if args.worker and (args.db is None or args.worker_rows is None): + parser.error("--worker requires --db and --worker-rows") + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.worker: + return worker(args.worker, args.db, args.worker_rows) + return orchestrator(args) + + +if __name__ == "__main__": + raise SystemExit(main())