refactor(sleep): decouple engine to top-level skillopt_sleep/ (zero research dep)

Open-source-tool / research-code separation:
  - git mv skillopt/sleep/ -> skillopt_sleep/ (top-level, sibling to the research
    skillopt/ package). History preserved as renames.
  - All imports skillopt.sleep.* -> skillopt_sleep.*.
  - Vendor the validation gate into skillopt_sleep/gate.py (a self-contained copy
    of skillopt.evaluation.gate). The engine now has ZERO dependency on the
    research package — verified: grep finds no `from skillopt.` in skillopt_sleep/,
    and consolidate's gate resolves to skillopt_sleep.gate.
  - Plugin scripts/commands/skill call `-m skillopt_sleep`.

29 tests pass; `python -m skillopt_sleep` runs standalone.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>

skillopt_sleep/experiments/__init__.py

@@ -0,0 +1 @@
+"""SkillOpt-Sleep experiments."""

skillopt_sleep/experiments/gbrain_bench.py

@@ -0,0 +1,119 @@
+"""SkillOpt-Sleep — gbrain-evals benchmark adapter.
+
+Loads gbrain-evals' `skillopt-v1` benchmark (deficient skills + train/held-out
+task sets with rule-based judges) into our TaskRecord format, so we can run the
+SkillOpt-Sleep cycle against the SAME suite gbrain publishes a scorecard for:
+
+  docs/benchmarks/2026-06-03-skillopt.md  — "4/4 skills 0 -> 1.00"
+
+Each gbrain seed dir has:
+  SKILL.md          — the deliberately deficient starting skill
+  benchmark.jsonl   — training tasks  {task_id, task, judge:{kind:"rule",checks}}
+  held-out.jsonl    — held-out tasks (same judge shape, unseen items)
+
+We map:
+  benchmark.jsonl -> TaskRecords with split="replay"
+  held-out.jsonl  -> TaskRecords with split="holdout"
+  judge           -> TaskRecord.judge (+ reference_kind="rule")
+
+This lets us reproduce gbrain's headline result with our engine and either the
+claude or codex backend, scoring locally via skillopt_sleep.judges (no judge API).
+"""
+from __future__ import annotations
+
+import json
+import os
+from typing import Dict, List, Optional, Tuple
+
+from skillopt_sleep.types import TaskRecord
+
+
+SEED_DIRS = {
+    "brief-writer": "seed-missing-structure",
+    "thorough-analyst": "seed-verbose",
+    "advisor": "seed-no-verdict",
+    "quick-answerer": "seed-no-brain-first",
+}
+
+
+def _load_jsonl(path: str) -> List[dict]:
+    out: List[dict] = []
+    if not os.path.exists(path):
+        return out
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                try:
+                    out.append(json.loads(line))
+                except Exception:
+                    pass
+    return out
+
+
+def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
+    return TaskRecord(
+        id=f"{seed}:{rec.get('task_id', '')}",
+        project=f"gbrain/{seed}",
+        intent=str(rec.get("task", "")),
+        reference_kind="rule",
+        judge=rec.get("judge", {}) or {},
+        tags=[f"seed:{seed}"],
+        split=split,
+    )
+
+
+def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34,
+              split_seed: int = 42) -> Tuple[str, List[TaskRecord]]:
+    """Return (deficient_skill_md, tasks) for one gbrain seed.
+
+    Faithful split mapping:
+      * gbrain held-out.jsonl  -> our ``test`` (the true final measure)
+      * gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val``
+        (val gates updates; train drives reflect)
+    All tasks are origin='real' (gbrain provides no synthetic tasks).
+    """
+    import hashlib
+    sub = SEED_DIRS.get(seed, seed)
+    seed_dir = os.path.join(data_root, sub)
+    skill_path = os.path.join(seed_dir, "SKILL.md")
+    skill = ""
+    if os.path.exists(skill_path):
+        with open(skill_path, encoding="utf-8") as f:
+            skill = f.read()
+    tasks: List[TaskRecord] = []
+    # benchmark pool -> train/val
+    val_cut = int(round(val_fraction * 100))
+    for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
+        t = _to_task(rec, seed=seed, split="train")
+        bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100
+        t.split = "val" if bucket < val_cut else "train"
+        tasks.append(t)
+    # held-out -> test
+    for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
+        tasks.append(_to_task(rec, seed=seed, split="test"))
+    # guarantee a non-empty val
+    if not any(t.split == "val" for t in tasks):
+        train_only = [t for t in tasks if t.split == "train"]
+        if train_only:
+            train_only[0].split = "val"
+    return skill, tasks
+
+
+def available_seeds(data_root: str) -> List[str]:
+    return [s for s, sub in SEED_DIRS.items()
+            if os.path.isdir(os.path.join(data_root, sub))]
+
+
+def find_data_root(explicit: str = "") -> Optional[str]:
+    """Locate eval/data/skillopt-v1 from common clone locations."""
+    cands = [explicit] if explicit else []
+    cands += [
+        os.path.expanduser("~/git/gbrain-evals/eval/data/skillopt-v1"),
+        "/tmp/gbrain-evals/eval/data/skillopt-v1",
+        os.path.expanduser("~/gbrain-evals/eval/data/skillopt-v1"),
+    ]
+    for c in cands:
+        if c and os.path.isdir(c):
+            return c
+    return None

skillopt_sleep/experiments/personas.py

@@ -0,0 +1,86 @@
+"""SkillOpt-Sleep — persona task fixtures for the validation experiment.
+
+Each persona is a list of TaskRecords with EXACT checkable references and a
+`rule:<key>` tag naming the single skill rule that makes the task solvable
+(consumed by MockBackend). This lets the experiment prove — deterministically,
+with no API — that nightly consolidation lifts a held-out score and that the
+gate blocks regressions.
+
+Personas mirror the user's framing: programmer / researcher / analyst.
+"""
+from __future__ import annotations
+
+from typing import List
+
+from skillopt_sleep.types import TaskRecord
+
+
+def _t(i, intent, ref, rule, project="/personas/demo", outcome="fail") -> TaskRecord:
+    return TaskRecord(
+        id=f"persona_{rule}_{i}",
+        project=project,
+        intent=intent,
+        context_excerpt="",
+        attempted_solution="",
+        outcome=outcome,
+        reference_kind="exact",
+        reference=ref,
+        tags=[f"rule:{rule}"],
+        source_sessions=[f"sess_{i}"],
+    )
+
+
+def researcher_persona() -> List[TaskRecord]:
+    """Researcher who always wants arXiv ids wrapped in <answer> tags."""
+    items = [
+        ("Give me the arXiv id for the SkillOpt paper", "arXiv:2605.23904"),
+        ("What's the arXiv id of the Attention paper?", "arXiv:1706.03762"),
+        ("arXiv id for the GAN paper?", "arXiv:1406.2661"),
+        ("arXiv id for BERT?", "arXiv:1810.04805"),
+        ("arXiv id for the ResNet paper?", "arXiv:1512.03385"),
+        ("arXiv id for the Adam optimizer paper?", "arXiv:1412.6980"),
+        ("arXiv id for Dropout?", "arXiv:1207.0580"),
+        ("arXiv id for the Transformer-XL paper?", "arXiv:1901.02860"),
+        ("arXiv id for word2vec?", "arXiv:1301.3781"),
+        ("arXiv id for the VAE paper?", "arXiv:1312.6114"),
+        ("arXiv id for batch norm?", "arXiv:1502.03167"),
+        ("arXiv id for GPT-3?", "arXiv:2005.14165"),
+    ]
+    # Both rules required: format the id (arxiv-id) AND wrap in answer tags.
+    out: List[TaskRecord] = []
+    for i, (q, a) in enumerate(items):
+        t = _t(i, q, a, "wrap-answer")
+        t.tags = ["rule:wrap-answer", "rule:arxiv-id"]
+        out.append(t)
+    return out
+
+
+def programmer_persona() -> List[TaskRecord]:
+    """Programmer who wants imperative-mood commit subjects."""
+    items = [
+        ("commit message for adding a login form", "Add login form"),
+        ("commit message for fixing the null pointer bug", "Fix null pointer in parser"),
+        ("commit message for updating the README", "Update README"),
+        ("commit message for removing dead code", "Remove dead code"),
+        ("commit message for bumping the version", "Bump version to 1.2.0"),
+        ("commit message for refactoring the auth module", "Refactor auth module"),
+        ("commit message for adding tests", "Add unit tests for scheduler"),
+        ("commit message for fixing the CI pipeline", "Fix CI pipeline"),
+    ]
+    return [_t(i, q, a, "commit-imperative") for i, (q, a) in enumerate(items)]
+
+
+def harmful_edit_task() -> TaskRecord:
+    """A task whose 'fix' is a known-bad rule; used to prove the gate rejects
+    regressions. The MockBackend proposes the harmful rule on this failure,
+    but applying it does NOT raise the held-out score, so the gate must reject.
+    """
+    t = _t(99, "answer this freely", "THIS_WILL_NOT_MATCH", "__harmful__")
+    t.reference = "an-answer-that-the-harmful-rule-cannot-produce"
+    return t
+
+
+PERSONAS = {
+    "researcher": researcher_persona,
+    "programmer": programmer_persona,
+}

skillopt_sleep/experiments/report.py

@@ -0,0 +1,132 @@
+"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard.
+
+Usage:
+  python -m skillopt_sleep.experiments.report --in docs/sleep/sweep.jsonl \
+      --out docs/sleep/benchmark_report.md
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any, Dict, List
+
+
+def _load(path: str) -> List[Dict[str, Any]]:
+    rows = []
+    if os.path.exists(path):
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        rows.append(json.loads(line))
+                    except Exception:
+                        pass
+    return rows
+
+
+def _fmt_model(backend: str, model: str) -> str:
+    m = model or "default"
+    return f"{backend}:{m}"
+
+
+def render(rows: List[Dict[str, Any]]) -> str:
+    direct = [r for r in rows if r.get("cfg", {}).get("kind") in ("direct", "dual") and "error" not in r]
+    transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r]
+    errors = [r for r in rows if "error" in r]
+
+    out: List[str] = []
+    out.append("# SkillOpt-Sleep — benchmark report")
+    out.append("")
+    out.append("Auto-generated from `sweep.jsonl`. Benchmark: "
+               "[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` "
+               "(deficient skills, train/held-out split, local rule judge — no judge-API).")
+    out.append("Held-out scores are computed by the harness, not the optimizer.")
+    out.append("")
+
+    # ── direct improvement table ──────────────────────────────────────────
+    out.append("## Direct improvement (optimize, then deploy)")
+    out.append("")
+    out.append("| Optimizer → Target | Seed | Held-out before | Held-out after | Nights | Tokens |")
+    out.append("|---|---|---|---|---|---|")
+    for r in direct:
+        c = r["cfg"]
+        if c.get("kind") == "dual":
+            label = (f"{_fmt_model(c['optimizer_backend'], c.get('optimizer_model',''))}"
+                     f" → {_fmt_model(c['target_backend'], c.get('target_model',''))}")
+        else:
+            m = _fmt_model(c["backend"], c.get("model", ""))
+            label = f"{m} → {m}"
+        out.append(f"| {label} | {c['seed']} | "
+                   f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | "
+                   f"{r.get('tokens','?')} |")
+    if direct:
+        n_imp = sum(1 for r in direct if r.get("improved"))
+        out.append("")
+        out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**")
+    out.append("")
+
+    # ── transfer table ────────────────────────────────────────────────────
+    if transfer:
+        out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)")
+        out.append("")
+        out.append("The price-difference story: spend cheap tokens optimizing overnight, "
+                   "then deploy the frozen skill on any model with no further optimization.")
+        out.append("")
+        out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |")
+        out.append("|---|---|---|---|---|---|")
+        for r in transfer:
+            c = r["cfg"]
+            s = _fmt_model(c["source_backend"], c.get("source_model", ""))
+            t = _fmt_model(c["target_backend"], c.get("target_model", ""))
+            out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | "
+                       f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |")
+        n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0)
+        out.append("")
+        out.append(f"**{n_pos}/{len(transfer)} transfers were positive** "
+                   "(frozen skill helped a different model than it was optimized on).")
+        out.append("")
+
+    # ── errors (honest reporting) ─────────────────────────────────────────
+    if errors:
+        out.append("## Configs that errored (reported, not hidden)")
+        out.append("")
+        for r in errors:
+            out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}")
+        out.append("")
+
+    out.append("## How to reproduce")
+    out.append("")
+    out.append("```bash")
+    out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals")
+    out.append("python -m skillopt_sleep.experiments.sweep --plan full \\")
+    out.append("    --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl")
+    out.append("python -m skillopt_sleep.experiments.report \\")
+    out.append("    --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md")
+    out.append("```")
+    out.append("")
+    return "\n".join(out)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report")
+    ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl")
+    ap.add_argument("--out", default="docs/sleep/benchmark_report.md")
+    args = ap.parse_args(argv)
+
+    rows = _load(args.inp)
+    if not rows:
+        print(f"no rows in {args.inp}", file=sys.stderr)
+        return 1
+    md = render(rows)
+    os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
+    with open(args.out, "w") as f:
+        f.write(md)
+    print(f"wrote {args.out} ({len(rows)} rows)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillopt_sleep/experiments/run_experiment.py

@@ -0,0 +1,178 @@
+"""SkillOpt-Sleep — validation experiment.
+
+Answers the question the user posed: *does nightly offline self-evolution
+actually improve the agent?*  Runs deterministically with the MockBackend
+(no API key, reproducible) and is the acceptance test for the whole idea.
+
+What it proves:
+  1. MONOTONIC LIFT — over N sleep nights, the held-out score rises from a
+     baseline (empty skill/memory) toward 1.0 as the gate accepts the
+     general rules the persona's tasks require.
+  2. GATE SAFETY — an injected harmful edit is REJECTED (held-out score does
+     not improve), so a bad nightly proposal can never be adopted.
+  3. PLUMBING — harvest->mine->replay->consolidate->stage->adopt all run and
+     the adopted artifact, re-scored, retains the lift.
+
+Run:
+    python -m skillopt_sleep.experiments.run_experiment
+    python -m skillopt_sleep.experiments.run_experiment --persona programmer --nights 3
+    python -m skillopt_sleep.experiments.run_experiment --backend anthropic   # real lift
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+from typing import List
+
+from skillopt_sleep.backend import get_backend
+from skillopt_sleep.consolidate import consolidate
+from skillopt_sleep.experiments.personas import (
+    PERSONAS,
+    harmful_edit_task,
+    researcher_persona,
+)
+from skillopt_sleep.memory import ensure_skill_scaffold
+from skillopt_sleep.replay import aggregate_scores, replay_batch
+from skillopt_sleep.types import TaskRecord
+
+
+def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
+                   metric: str = "mixed", w: float = 0.5) -> float:
+    from skillopt_sleep.consolidate import select_gate_score
+    # the persona experiment uses a 2-way split (train/val, no test); score on val
+    holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks
+    pairs = replay_batch(backend, holdout, skill, memory)
+    h, s = aggregate_scores(pairs)
+    return select_gate_score(h, s, metric, w)
+
+
+def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock",
+        edit_budget: int = 4, seed: int = 42, model: str = "", codex_path: str = "",
+        limit_tasks: int = 0) -> dict:
+    from skillopt_sleep.mine import assign_splits
+
+    make = PERSONAS.get(persona, researcher_persona)
+    items = make()
+    if limit_tasks and limit_tasks < len(items):
+        items = items[:limit_tasks]
+    tasks = assign_splits(items, holdout_fraction=0.34, seed=seed)
+    backend = get_backend(backend_name, model=model, codex_path=codex_path)
+    is_mock = (backend.name == "mock")
+
+    # start from an empty managed skill + empty memory
+    skill = ensure_skill_scaffold("", name="skillopt-sleep-learned",
+                                  description="Learned preferences.")
+    memory = ""
+
+    baseline = _score_holdout(backend, tasks, skill, memory)
+    trace = [{"night": 0, "holdout_score": round(baseline, 4), "action": "baseline",
+              "n_edits": 0}]
+
+    for night in range(1, nights + 1):
+        res = consolidate(
+            backend, tasks, skill, memory,
+            edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
+            evolve_skill=True, evolve_memory=True, night=night,
+        )
+        if res.accepted:
+            skill, memory = res.new_skill, res.new_memory
+        trace.append({
+            "night": night,
+            "holdout_score": round(res.candidate_score, 4),
+            "action": res.gate_action,
+            "accepted": res.accepted,
+            "n_edits": len(res.applied_edits),
+            "edits": [e.content for e in res.applied_edits],
+            "n_rejected": len(res.rejected_edits),
+        })
+        # converged: stop early if perfect
+        if res.candidate_score >= 0.999:
+            break
+
+    after = _score_holdout(backend, tasks, skill, memory)
+
+    # ── gate-safety probe (mock only; it relies on the mock's known bad rule) ──
+    harmful_rejected = None
+    if is_mock:
+        harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
+                                      holdout_fraction=0.5, seed=seed)
+        _ = _score_holdout(backend, harmful_tasks, skill, memory)
+        res_h = consolidate(backend, harmful_tasks, skill, memory,
+                            edit_budget=edit_budget, gate_metric="mixed",
+                            evolve_skill=True, evolve_memory=False, night=nights + 1)
+        harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"]  # type: ignore[attr-defined]
+        harmful_rejected = (harmful_rule_text not in res_h.new_skill)
+
+    result = {
+        "persona": persona,
+        "backend": backend.name,
+        "model": model or "(default)",
+        "n_tasks": len(tasks),
+        "nights_run": len(trace) - 1,
+        "baseline_holdout": round(baseline, 4),
+        "after_holdout": round(after, 4),
+        "lift": round(after - baseline, 4),
+        "improved": after > baseline,
+        "gate_blocks_harmful": harmful_rejected,  # None for real backends
+        "tokens_used": backend.tokens_used(),
+        "final_skill_excerpt": skill[-500:],
+        "trace": trace,
+    }
+    return result
+
+
+def _assert(cond: bool, msg: str) -> None:
+    if not cond:
+        print(f"FAIL: {msg}")
+        raise SystemExit(1)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment")
+    ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys()))
+    ap.add_argument("--nights", type=int, default=4)
+    ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
+    ap.add_argument("--model", default="", help="backend model override")
+    ap.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
+    ap.add_argument("--edit-budget", type=int, default=4)
+    ap.add_argument("--limit-tasks", type=int, default=0, help="cap #tasks (control API cost)")
+    ap.add_argument("--json", action="store_true")
+    ap.add_argument("--assert-improves", action="store_true",
+                    help="exit nonzero unless lift>0 (and, for mock, gate blocks harmful edit)")
+    args = ap.parse_args(argv)
+
+    res = run(args.persona, nights=args.nights, backend_name=args.backend,
+              edit_budget=args.edit_budget, model=args.model,
+              codex_path=args.codex_path, limit_tasks=args.limit_tasks)
+
+    if args.json:
+        print(json.dumps(res, ensure_ascii=False, indent=2))
+    else:
+        print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} "
+              f"backend={res['backend']} model={res['model']} ===")
+        print(f"tasks: {res['n_tasks']}   tokens(approx): {res['tokens_used']}")
+        print(f"baseline held-out : {res['baseline_holdout']}")
+        print(f"after  held-out   : {res['after_holdout']}   (lift {res['lift']:+.4f})")
+        if res["gate_blocks_harmful"] is not None:
+            print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
+        print("trace:")
+        for row in res["trace"]:
+            edits = "; ".join(row.get("edits", []))[:80]
+            print(f"  night {row['night']}: holdout={row['holdout_score']} "
+                  f"{row['action']} (+{row['n_edits']} edits) {edits}")
+
+    if args.assert_improves:
+        _assert(res["improved"], "held-out score did not improve")
+        if res["gate_blocks_harmful"] is not None:
+            _assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
+            print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
+        else:
+            print("\nPASS: nightly consolidation improves held-out score (real backend).")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillopt_sleep/experiments/run_gbrain.py

@@ -0,0 +1,209 @@
+"""SkillOpt-Sleep — run the gbrain-evals skillopt-v1 benchmark with our engine.
+
+Reproduces gbrain's "Result 1 — skills measurably improve" scorecard
+(docs/benchmarks/2026-06-03-skillopt.md) using SkillOpt-Sleep's
+consolidate() loop and either the claude or codex backend.
+
+For each deficient seed skill:
+  1. score the held-out tasks with the ORIGINAL skill            -> before
+  2. run N consolidation nights on the training tasks (gated)     -> evolve skill
+  3. score the held-out tasks with the EVOLVED skill             -> after
+
+Held-out scoring is done locally by the rule judge (no judge API). Only the
+agent's `attempt` (and the optimizer's `reflect`) spend tokens.
+
+Usage:
+    python -m skillopt_sleep.experiments.run_gbrain --backend mock
+    python -m skillopt_sleep.experiments.run_gbrain --backend claude --seeds brief-writer --nights 2
+    python -m skillopt_sleep.experiments.run_gbrain --backend codex  --data-root /tmp/gbrain-evals/eval/data/skillopt-v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Dict, List, Optional
+
+from skillopt_sleep.backend import build_backend, get_backend
+from skillopt_sleep.consolidate import consolidate, select_gate_score
+from skillopt_sleep.experiments.gbrain_bench import (
+    available_seeds,
+    find_data_root,
+    load_seed,
+)
+from skillopt_sleep.replay import aggregate_scores, replay_batch
+
+
+def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5):
+    sub = [t for t in tasks if t.split == split]
+    if not sub:  # fall back to val, then everything, so we never score on nothing
+        sub = [t for t in tasks if t.split == "val"] or tasks
+    pairs = replay_batch(backend, sub, skill, memory)
+    h, s = aggregate_scores(pairs)
+    return h, s, select_gate_score(h, s, metric, w)
+
+
+def run_seed(backend, seed: str, skill: str, tasks: List, *,
+             nights: int = 3, edit_budget: int = 4, gate_mode: str = "on",
+             slow_update: bool = True, rollouts_k: int = 1,
+             limit_replay: int = 0, limit_holdout: int = 0) -> dict:
+    memory = ""
+    # optionally cap each split to control API cost / latency.
+    # limit_replay caps train; limit_holdout caps BOTH val and test.
+    if limit_replay or limit_holdout:
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
+        if limit_replay:
+            train = train[:limit_replay]
+        if limit_holdout:
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
+    # final measure is TEST (the gbrain held-out set); val gates internally
+    bh, bs, bscore = _score(backend, tasks, skill, memory, split="test")
+    trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}]
+    cur = skill
+    first_night_skill = skill
+    for night in range(1, nights + 1):
+        res = consolidate(
+            backend, tasks, cur, memory,
+            edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
+            gate_mode=gate_mode, rollouts_k=rollouts_k,
+            evolve_skill=True, evolve_memory=False, night=night,
+        )
+        if res.accepted:
+            cur = res.new_skill
+        if night == 1:
+            first_night_skill = cur
+        # report the TEST score each night (independent of the val gate)
+        th, _ts, _ = _score(backend, tasks, cur, memory, split="test")
+        trace.append({
+            "night": night,
+            "val_hard": round(res.holdout_candidate, 3),
+            "test_hard": round(th, 3),
+            "action": res.gate_action,
+            "accepted": res.accepted,
+            "edits": [e.content for e in res.applied_edits],
+        })
+        if th >= 0.999:
+            break
+
+    # ── SLOW UPDATE: consolidate cross-night experience into the protected
+    # long-term field. Runs regardless of gate mode (it is what preserves
+    # long-term memory even when the gate is OFF).
+    slow_text = None
+    if nights >= 2 and slow_update:
+        try:
+            from skillopt_sleep.slow_update import run_slow_update, replace_slow_field
+            val_tasks = [t for t in tasks if t.split == "val"] or tasks
+            prev_pairs = replay_batch(backend, val_tasks, first_night_skill, memory)
+            curr_pairs = replay_batch(backend, val_tasks, cur, memory)
+            slow_text = run_slow_update(
+                backend, prev_skill=first_night_skill, curr_skill=cur,
+                prev_pairs=[(t, r) for t, r in prev_pairs],
+                curr_pairs=[(t, r) for t, r in curr_pairs],
+            )
+            if slow_text:
+                cur = replace_slow_field(cur, slow_text)
+        except Exception:
+            slow_text = None
+
+    ah, as_, ascore = _score(backend, tasks, cur, memory, split="test")
+    return {
+        "seed": seed,
+        "held_out_before": round(bh, 3),
+        "held_out_after": round(ah, 3),
+        "improved": ah > bh,
+        "nights": len(trace) - 1,
+        "trace": trace,
+        "slow_update": slow_text,
+        "final_skill_tail": cur[-400:],
+    }
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Run gbrain-evals skillopt-v1 with SkillOpt-Sleep")
+    ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
+    ap.add_argument("--model", default="")
+    ap.add_argument("--optimizer-backend", default="", help="route reflect/judge here (dual)")
+    ap.add_argument("--optimizer-model", default="")
+    ap.add_argument("--target-backend", default="", help="route attempt here (dual)")
+    ap.add_argument("--target-model", default="")
+    ap.add_argument("--codex-path", default="")
+    ap.add_argument("--data-root", default="", help="path to eval/data/skillopt-v1")
+    ap.add_argument("--seeds", default="", help="comma list; default = all available")
+    ap.add_argument("--nights", type=int, default=3)
+    ap.add_argument("--edit-budget", type=int, default=4)
+    ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"],
+                    help="on/hard/soft = validation-gated; off = greedy (no hard filter)")
+    ap.add_argument("--rollouts-k", type=int, default=1,
+                    help=">1 = multi-rollout contrastive reflection per task")
+    ap.add_argument("--budget-tokens", type=int, default=0,
+                    help="approx token budget; auto-plans nights x rollouts when set")
+    ap.add_argument("--budget-minutes", type=float, default=0.0)
+    ap.add_argument("--preferences", default="", help="free-text user preferences (prior for reflect)")
+    ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)")
+    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)")
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args(argv)
+
+    data_root = find_data_root(args.data_root)
+    if not data_root:
+        print("ERROR: could not find eval/data/skillopt-v1. Clone gbrain-evals and pass --data-root.",
+              file=sys.stderr)
+        return 2
+
+    seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
+    backend = build_backend(
+        backend=args.backend, model=args.model,
+        optimizer_backend=args.optimizer_backend, optimizer_model=args.optimizer_model,
+        target_backend=args.target_backend, target_model=args.target_model,
+        codex_path=args.codex_path, preferences=args.preferences,
+    )
+
+    results = []
+    for seed in seeds:
+        skill, tasks = load_seed(data_root, seed)
+        if not tasks:
+            continue
+        # budget auto-planning: derive nights x rollouts_k from a token budget
+        nights, rollouts_k = args.nights, args.rollouts_k
+        if args.budget_tokens:
+            from skillopt_sleep.budget import Budget, plan_depth
+            n_train = len([t for t in tasks if t.split == "train"]) or len(tasks)
+            nights, rollouts_k = plan_depth(
+                Budget(max_tokens=args.budget_tokens), n_tasks=n_train,
+                default_nights=args.nights, default_k=args.rollouts_k,
+            )
+            if not args.json:
+                print(f"  [budget] {args.budget_tokens} tok -> nights={nights} rollouts_k={rollouts_k}")
+        r = run_seed(backend, seed, skill, tasks, nights=nights,
+                     edit_budget=args.edit_budget, rollouts_k=rollouts_k,
+                     gate_mode=("off" if args.gate == "off" else "on"),
+                     limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
+        results.append(r)
+        if not args.json:
+            print(f"  {seed:<18} held-out {r['held_out_before']:.2f} -> {r['held_out_after']:.2f}"
+                  f"  ({'IMPROVED' if r['improved'] else 'no change'}, {r['nights']} nights)")
+
+    n_improved = sum(1 for r in results if r["improved"])
+    summary = {
+        "benchmark": "gbrain-evals/skillopt-v1",
+        "backend": backend.name,
+        "model": args.model or "(default)",
+        "n_seeds": len(results),
+        "n_improved": n_improved,
+        "tokens_used": backend.tokens_used(),
+        "results": results,
+    }
+    if args.json:
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+    else:
+        print(f"\n=== {n_improved}/{len(results)} seeds improved on held-out "
+              f"(backend={backend.name}, ~{backend.tokens_used()} tokens) ===")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillopt_sleep/experiments/run_transfer.py

@@ -0,0 +1,155 @@
+"""SkillOpt-Sleep — skill-transfer experiment (sleep scenario).
+
+Answers: "if I optimize a skill while the agent sleeps using a CHEAP model,
+does the learned skill still help an EXPENSIVE model at deploy time?" — and the
+reverse. This is the SkillOpt paper's cross-model transfer result, reproduced
+in the sleep setting, and it is the core price-difference value proposition:
+spend cheap tokens overnight, deploy the frozen skill anywhere.
+
+Protocol, per gbrain seed:
+  1. baseline_target = held-out score of the DEFICIENT skill, run on TARGET model
+  2. optimize the skill for N nights using the SOURCE model (attempt+reflect)
+  3. transferred = held-out score of the LEARNED skill, run on TARGET model,
+     with NO further optimization
+  4. (reference) direct = held-out score of a skill optimized AND run on TARGET
+
+Report baseline / direct / transferred, mirroring SkillOpt Table "transfer".
+
+Usage:
+  python -m skillopt_sleep.experiments.run_transfer \
+     --source-backend claude --source-model haiku \
+     --target-backend claude --target-model sonnet \
+     --seeds brief-writer --nights 2
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from skillopt_sleep.backend import get_backend
+from skillopt_sleep.consolidate import consolidate, select_gate_score
+from skillopt_sleep.experiments.gbrain_bench import (
+    available_seeds, find_data_root, load_seed,
+)
+from skillopt_sleep.replay import aggregate_scores, replay_batch
+
+
+def _holdout_hard(backend, tasks, skill, memory="") -> float:
+    # transfer is measured on the true held-out TEST split
+    ho = [t for t in tasks if t.split == "test"]
+    if not ho:
+        ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks
+    pairs = replay_batch(backend, ho, skill, memory)
+    h, _s = aggregate_scores(pairs)
+    return h
+
+
+def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str:
+    cur = skill
+    for night in range(1, nights + 1):
+        res = consolidate(backend, tasks, cur, "",
+                          edit_budget=edit_budget, gate_metric="mixed",
+                          evolve_skill=True, evolve_memory=False, night=night)
+        if res.accepted:
+            cur = res.new_skill
+        if res.holdout_candidate >= 0.999:
+            break
+    return cur
+
+
+def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget,
+             limit_replay, limit_holdout, do_direct=True) -> dict:
+    if limit_replay or limit_holdout:
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
+        if limit_replay:
+            train = train[:limit_replay]
+        if limit_holdout:
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
+
+    baseline_target = _holdout_hard(target, tasks, skill)
+
+    # optimize on SOURCE, evaluate frozen skill on TARGET
+    learned_on_source = _optimize(source, skill, tasks, nights=nights, edit_budget=edit_budget)
+    transferred = _holdout_hard(target, tasks, learned_on_source)
+
+    direct = None
+    if do_direct:
+        learned_on_target = _optimize(target, skill, tasks, nights=nights, edit_budget=edit_budget)
+        direct = _holdout_hard(target, tasks, learned_on_target)
+
+    return {
+        "seed": seed,
+        "baseline_target": round(baseline_target, 3),
+        "direct_target": (round(direct, 3) if direct is not None else None),
+        "transferred": round(transferred, 3),
+        "transfer_gain": round(transferred - baseline_target, 3),
+        "learned_skill_tail": learned_on_source[-300:],
+    }
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="SkillOpt-Sleep cross-model transfer")
+    ap.add_argument("--source-backend", default="claude")
+    ap.add_argument("--source-model", default="haiku")
+    ap.add_argument("--target-backend", default="claude")
+    ap.add_argument("--target-model", default="sonnet")
+    ap.add_argument("--codex-path", default="")
+    ap.add_argument("--data-root", default="")
+    ap.add_argument("--seeds", default="brief-writer")
+    ap.add_argument("--nights", type=int, default=2)
+    ap.add_argument("--edit-budget", type=int, default=4)
+    ap.add_argument("--limit-replay", type=int, default=3)
+    ap.add_argument("--limit-holdout", type=int, default=3)
+    ap.add_argument("--no-direct", action="store_true", help="skip the direct reference (saves cost)")
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args(argv)
+
+    data_root = find_data_root(args.data_root)
+    if not data_root:
+        print("ERROR: gbrain-evals skillopt-v1 data not found; pass --data-root", file=sys.stderr)
+        return 2
+
+    source = get_backend(args.source_backend, model=args.source_model, codex_path=args.codex_path)
+    target = get_backend(args.target_backend, model=args.target_model, codex_path=args.codex_path)
+
+    seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
+    results = []
+    for seed in seeds:
+        skill, tasks = load_seed(data_root, seed)
+        if not tasks:
+            continue
+        r = run_seed(seed, skill, tasks, source=source, target=target,
+                     nights=args.nights, edit_budget=args.edit_budget,
+                     limit_replay=args.limit_replay, limit_holdout=args.limit_holdout,
+                     do_direct=not args.no_direct)
+        results.append(r)
+        if not args.json:
+            d = f" direct={r['direct_target']}" if r['direct_target'] is not None else ""
+            print(f"  {seed:<16} baseline={r['baseline_target']:.2f}"
+                  f" transferred={r['transferred']:.2f}{d}"
+                  f"  (gain {r['transfer_gain']:+.2f})")
+
+    summary = {
+        "experiment": "skillopt-sleep/transfer",
+        "source": f"{args.source_backend}:{args.source_model}",
+        "target": f"{args.target_backend}:{args.target_model}",
+        "tokens_source": source.tokens_used(),
+        "tokens_target": target.tokens_used(),
+        "results": results,
+    }
+    if args.json:
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+    else:
+        print(f"\n=== transfer {summary['source']} -> {summary['target']}: "
+              f"{sum(1 for r in results if r['transfer_gain'] > 0)}/{len(results)} positive ===")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillopt_sleep/experiments/sweep.py

@@ -0,0 +1,164 @@
+"""SkillOpt-Sleep — benchmark sweep driver.
+
+Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in
+one process, appending each result to a JSONL file as it finishes. Designed to
+run unattended in the background; safe to interrupt (already-written rows
+survive) and resume (skip configs whose row already exists).
+
+Then `report.py` turns the JSONL into a presented Markdown scorecard.
+
+Usage:
+  python -m skillopt_sleep.experiments.sweep --plan quick   --out docs/sleep/sweep.jsonl
+  python -m skillopt_sleep.experiments.sweep --plan full    --out docs/sleep/sweep.jsonl
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, List
+
+from skillopt_sleep.backend import build_backend, get_backend
+from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed
+from skillopt_sleep.experiments.run_gbrain import run_seed as bench_seed
+from skillopt_sleep.experiments.run_transfer import run_seed as transfer_seed
+
+
+# Plans: lists of config dicts. Kept small per-run to bound cost/latency.
+def _direct_cfg(backend, model, seed, nights=2):
+    return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights}
+
+
+def _dual_cfg(opt_backend, opt_model, tgt_backend, tgt_model, seed, nights=2):
+    # a 'direct' run on a DualBackend: strong optimizer proposes, weak target runs
+    return {"kind": "dual", "optimizer_backend": opt_backend, "optimizer_model": opt_model,
+            "target_backend": tgt_backend, "target_model": tgt_model, "seed": seed, "nights": nights}
+
+
+def _transfer_cfg(sb, sm, tb, tm, seed, nights=2):
+    return {"kind": "transfer", "source_backend": sb, "source_model": sm,
+            "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights}
+
+
+PLANS: Dict[str, List[Dict[str, Any]]] = {
+    # one cheap seed each, both backends — fast sanity
+    "quick": [
+        _direct_cfg("claude", "haiku", "brief-writer", 1),
+        _direct_cfg("codex", "", "brief-writer", 2),
+    ],
+    # SkillOpt-faithful: STRONG optimizer (sonnet) proposes, WEAK target (haiku)
+    # runs — the reliable config. Plus Codex self-optimized. All 4 gbrain seeds,
+    # including quick-answerer (real tool loop).
+    "direct": [
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "advisor"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "thorough-analyst"),
+        _dual_cfg("claude", "sonnet", "claude", "haiku", "quick-answerer"),
+        _direct_cfg("codex", "", "brief-writer"),
+        _direct_cfg("codex", "", "advisor"),
+        _direct_cfg("codex", "", "quick-answerer"),
+    ],
+    # the price-difference story: optimize cheap, deploy expensive (and reverse)
+    "transfer": [
+        _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"),
+        _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"),
+    ],
+}
+PLANS["full"] = PLANS["direct"] + PLANS["transfer"]
+
+
+def _cfg_key(c: Dict[str, Any]) -> str:
+    return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False)
+
+
+def _load_done(out_path: str) -> set:
+    done = set()
+    if os.path.exists(out_path):
+        with open(out_path) as f:
+            for line in f:
+                try:
+                    row = json.loads(line)
+                    if "cfg_key" in row:
+                        done.add(row["cfg_key"])
+                except Exception:
+                    pass
+    return done
+
+
+def _append(out_path: str, row: Dict[str, Any]) -> None:
+    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
+    with open(out_path, "a") as f:
+        f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str,
+            limit_replay: int, limit_holdout: int) -> Dict[str, Any]:
+    seed = cfg["seed"]
+    skill, tasks = load_seed(data_root, seed)
+    t0 = time.time()
+    if cfg["kind"] in ("direct", "dual"):
+        if cfg["kind"] == "dual":
+            be = build_backend(
+                optimizer_backend=cfg["optimizer_backend"], optimizer_model=cfg.get("optimizer_model", ""),
+                target_backend=cfg["target_backend"], target_model=cfg.get("target_model", ""),
+                codex_path=codex_path,
+            )
+        else:
+            be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path)
+        r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"],
+                       limit_replay=limit_replay, limit_holdout=limit_holdout)
+        out = {"baseline": r["held_out_before"], "after": r["held_out_after"],
+               "improved": r["improved"], "tokens": be.tokens_used()}
+    else:
+        src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path)
+        tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path)
+        r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"],
+                          edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout,
+                          do_direct=False)
+        out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"],
+               "transfer_gain": r["transfer_gain"],
+               "tokens": src.tokens_used() + tgt.tokens_used()}
+    out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)})
+    return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep")
+    ap.add_argument("--plan", default="quick", choices=list(PLANS.keys()))
+    ap.add_argument("--out", default="docs/sleep/sweep.jsonl")
+    ap.add_argument("--data-root", default="")
+    ap.add_argument("--codex-path", default="")
+    ap.add_argument("--limit-replay", type=int, default=3)
+    ap.add_argument("--limit-holdout", type=int, default=3)
+    args = ap.parse_args(argv)
+
+    data_root = find_data_root(args.data_root)
+    if not data_root:
+        print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr)
+        return 2
+
+    plan = PLANS[args.plan]
+    done = _load_done(args.out)
+    print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}")
+    for i, cfg in enumerate(plan, 1):
+        key = _cfg_key(cfg)
+        if key in done:
+            print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}")
+            continue
+        print(f"[sweep] ({i}/{len(plan)}) running: {cfg}", flush=True)
+        try:
+            row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout)
+        except Exception as e:  # never let one config kill the sweep
+            row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"}
+        _append(args.out, row)
+        print(f"[sweep]   -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}", flush=True)
+    print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())