url/SkillOpt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sleep): real claude + codex backends, gbrain-evals benchmark, rule judges

Browse Source 
Upgrade from mock-only to REAL multi-backend validation:

Backends (skillopt/sleep/backend.py):
  - CliBackend base: shared attempt/judge/reflect prompts, response cache,
    token accounting. Subclasses implement only _call().
  - ClaudeCliBackend: drives `claude -p --output-format text`.
  - CodexCliBackend: drives the REAL @openai/codex `exec -o <file>` for clean
    output; resolve_codex_path() skips the hermes wrapper at ~/.local/bin/codex.
  - reflect() now aggregates the exact failing judge criteria into the prompt
    (gbrain's lesson: tell the optimizer what the scorer rewards).

Rule judges (skillopt/sleep/judges.py): gbrain-compatible local scorers
  (section_present / regex / max_chars / contains / tool_called) — held-out
  scoring with no judge-API spend. TaskRecord gains a `judge` field +
  reference_kind="rule".

gbrain-evals adapter (experiments/gbrain_bench.py, run_gbrain.py): load
  garrytan/gbrain-evals skillopt-v1 deficient skills + train/held-out task
  sets and run our consolidate() loop against the SAME suite gbrain scores.

REAL results (docs/sleep/real_api_results.md), brief-writer seed, 1 night:
  - Claude (Haiku): held-out 0.00 -> 1.00
  - Codex:          held-out 0.00 -> 0.67
  Both proposed a correct, general format rule into the protected LEARNED block.

CLI: --backend {mock,claude,codex}, --codex-path, --model; experiment +
gbrain runners gain --limit-* cost controls. 17 tests pass.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
Yifan Yang
2026-06-08 14:31:51 +00:00
parent 309f3141d4
commit 4203086899
11 changed files with 744 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skillopt/sleep/experiments/gbrain_bench.py
+99
View File
@@ -0,0 +1,99 @@
"""SkillOpt-Sleep — gbrain-evals benchmark adapter.
Loads gbrain-evals' `skillopt-v1` benchmark (deficient skills + train/held-out
task sets with rule-based judges) into our TaskRecord format, so we can run the
SkillOpt-Sleep cycle against the SAME suite gbrain publishes a scorecard for:
docs/benchmarks/2026-06-03-skillopt.md — "4/4 skills 0 -> 1.00"
Each gbrain seed dir has:
SKILL.md — the deliberately deficient starting skill
benchmark.jsonl — training tasks {task_id, task, judge:{kind:"rule",checks}}
held-out.jsonl — held-out tasks (same judge shape, unseen items)
We map:
benchmark.jsonl -> TaskRecords with split="replay"
held-out.jsonl -> TaskRecords with split="holdout"
judge -> TaskRecord.judge (+ reference_kind="rule")
This lets us reproduce gbrain's headline result with our engine and either the
claude or codex backend, scoring locally via skillopt.sleep.judges (no judge API).
"""
from __future__ import annotations
import json
import os
from typing import Dict, List, Optional, Tuple
from skillopt.sleep.types import TaskRecord
SEED_DIRS = {
"brief-writer": "seed-missing-structure",
"thorough-analyst": "seed-verbose",
"advisor": "seed-no-verdict",
"quick-answerer": "seed-no-brain-first",
}
def _load_jsonl(path: str) -> List[dict]:
out: List[dict] = []
if not os.path.exists(path):
return out
with open(path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
try:
out.append(json.loads(line))
except Exception:
pass
return out
def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
return TaskRecord(
id=f"{seed}:{rec.get('task_id', '')}",
project=f"gbrain/{seed}",
intent=str(rec.get("task", "")),
reference_kind="rule",
judge=rec.get("judge", {}) or {},
tags=[f"seed:{seed}"],
split=split,
)
def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
"""Return (deficient_skill_md, tasks) for one gbrain seed."""
sub = SEED_DIRS.get(seed, seed)
seed_dir = os.path.join(data_root, sub)
skill_path = os.path.join(seed_dir, "SKILL.md")
skill = ""
if os.path.exists(skill_path):
with open(skill_path, encoding="utf-8") as f:
skill = f.read()
tasks: List[TaskRecord] = []
for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
tasks.append(_to_task(rec, seed=seed, split="replay"))
for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
tasks.append(_to_task(rec, seed=seed, split="holdout"))
return skill, tasks
def available_seeds(data_root: str) -> List[str]:
return [s for s, sub in SEED_DIRS.items()
if os.path.isdir(os.path.join(data_root, sub))]
def find_data_root(explicit: str = "") -> Optional[str]:
"""Locate eval/data/skillopt-v1 from common clone locations."""
cands = [explicit] if explicit else []
cands += [
os.path.expanduser("~/git/gbrain-evals/eval/data/skillopt-v1"),
"/tmp/gbrain-evals/eval/data/skillopt-v1",
os.path.expanduser("~/gbrain-evals/eval/data/skillopt-v1"),
]
for c in cands:
if c and os.path.isdir(c):
return c
return None
skillopt/sleep/experiments/run_experiment.py
+42 -22
View File
@@ -49,12 +49,17 @@ def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock",
edit_budget: int = 4, seed: int = 42) -> dict:
edit_budget: int = 4, seed: int = 42, model: str = "", codex_path: str = "",
limit_tasks: int = 0) -> dict:
from skillopt.sleep.mine import assign_splits
make = PERSONAS.get(persona, researcher_persona)
tasks = assign_splits(make(), holdout_fraction=0.34, seed=seed)
backend = get_backend(backend_name)
items = make()
if limit_tasks and limit_tasks < len(items):
items = items[:limit_tasks]
tasks = assign_splits(items, holdout_fraction=0.34, seed=seed)
backend = get_backend(backend_name, model=model, codex_path=codex_path)
is_mock = (backend.name == "mock")
# start from an empty managed skill + empty memory
skill = ensure_skill_scaffold("", name="skillopt-sleep-learned",
@@ -88,26 +93,31 @@ def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock"
after = _score_holdout(backend, tasks, skill, memory)
# ── gate-safety probe: inject a harmful task whose 'fix' is a bad rule ──
harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
holdout_fraction=0.5, seed=seed)
h_before = _score_holdout(backend, harmful_tasks, skill, memory)
res_h = consolidate(backend, harmful_tasks, skill, memory,
edit_budget=edit_budget, gate_metric="mixed",
evolve_skill=True, evolve_memory=False, night=nights + 1)
harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"] # type: ignore[attr-defined]
harmful_rejected = (harmful_rule_text not in res_h.new_skill)
# ── gate-safety probe (mock only; it relies on the mock's known bad rule) ──
harmful_rejected = None
if is_mock:
harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
holdout_fraction=0.5, seed=seed)
_ = _score_holdout(backend, harmful_tasks, skill, memory)
res_h = consolidate(backend, harmful_tasks, skill, memory,
edit_budget=edit_budget, gate_metric="mixed",
evolve_skill=True, evolve_memory=False, night=nights + 1)
harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"] # type: ignore[attr-defined]
harmful_rejected = (harmful_rule_text not in res_h.new_skill)
result = {
"persona": persona,
"backend": backend_name,
"backend": backend.name,
"model": model or "(default)",
"n_tasks": len(tasks),
"nights_run": len(trace) - 1,
"baseline_holdout": round(baseline, 4),
"after_holdout": round(after, 4),
"lift": round(after - baseline, 4),
"improved": after > baseline,
"gate_blocks_harmful": bool(harmful_rejected),
"final_skill_excerpt": skill[-400:],
"gate_blocks_harmful": harmful_rejected, # None for real backends
"tokens_used": backend.tokens_used(),
"final_skill_excerpt": skill[-500:],
"trace": trace,
}
return result
@@ -123,23 +133,30 @@ def main(argv=None) -> int:
ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment")
ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys()))
ap.add_argument("--nights", type=int, default=4)
ap.add_argument("--backend", default="mock", choices=["mock", "anthropic"])
ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
ap.add_argument("--model", default="", help="backend model override")
ap.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
ap.add_argument("--edit-budget", type=int, default=4)
ap.add_argument("--limit-tasks", type=int, default=0, help="cap #tasks (control API cost)")
ap.add_argument("--json", action="store_true")
ap.add_argument("--assert-improves", action="store_true",
help="exit nonzero unless lift>0 and gate blocks harmful edit")
help="exit nonzero unless lift>0 (and, for mock, gate blocks harmful edit)")
args = ap.parse_args(argv)
res = run(args.persona, nights=args.nights, backend_name=args.backend,
edit_budget=args.edit_budget)
edit_budget=args.edit_budget, model=args.model,
codex_path=args.codex_path, limit_tasks=args.limit_tasks)
if args.json:
print(json.dumps(res, ensure_ascii=False, indent=2))
else:
print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} backend={res['backend']} ===")
print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} "
f"backend={res['backend']} model={res['model']} ===")
print(f"tasks: {res['n_tasks']} tokens(approx): {res['tokens_used']}")
print(f"baseline held-out : {res['baseline_holdout']}")
print(f"after held-out : {res['after_holdout']} (lift {res['lift']:+.4f})")
print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
if res["gate_blocks_harmful"] is not None:
print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
print("trace:")
for row in res["trace"]:
edits = "; ".join(row.get("edits", []))[:80]
@@ -148,8 +165,11 @@ def main(argv=None) -> int:
if args.assert_improves:
_assert(res["improved"], "held-out score did not improve")
_assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
if res["gate_blocks_harmful"] is not None:
_assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
else:
print("\nPASS: nightly consolidation improves held-out score (real backend).")
return 0
skillopt/sleep/experiments/run_gbrain.py
+144
View File
@@ -0,0 +1,144 @@
"""SkillOpt-Sleep — run the gbrain-evals skillopt-v1 benchmark with our engine.
Reproduces gbrain's "Result 1 — skills measurably improve" scorecard
(docs/benchmarks/2026-06-03-skillopt.md) using SkillOpt-Sleep's
consolidate() loop and either the claude or codex backend.
For each deficient seed skill:
1. score the held-out tasks with the ORIGINAL skill -> before
2. run N consolidation nights on the training tasks (gated) -> evolve skill
3. score the held-out tasks with the EVOLVED skill -> after
Held-out scoring is done locally by the rule judge (no judge API). Only the
agent's `attempt` (and the optimizer's `reflect`) spend tokens.
Usage:
python -m skillopt.sleep.experiments.run_gbrain --backend mock
python -m skillopt.sleep.experiments.run_gbrain --backend claude --seeds brief-writer --nights 2
python -m skillopt.sleep.experiments.run_gbrain --backend codex --data-root /tmp/gbrain-evals/eval/data/skillopt-v1
"""
from __future__ import annotations
import argparse
import json
import sys
from typing import Dict, List, Optional
from skillopt.sleep.backend import get_backend
from skillopt.sleep.consolidate import consolidate, select_gate_score
from skillopt.sleep.experiments.gbrain_bench import (
available_seeds,
find_data_root,
load_seed,
)
from skillopt.sleep.replay import aggregate_scores, replay_batch
def _score(backend, tasks, skill, memory, split="holdout", metric="mixed", w=0.5):
sub = [t for t in tasks if t.split == split] or tasks
pairs = replay_batch(backend, sub, skill, memory)
h, s = aggregate_scores(pairs)
return h, s, select_gate_score(h, s, metric, w)
def run_seed(backend, seed: str, skill: str, tasks: List, *,
nights: int = 3, edit_budget: int = 4,
limit_replay: int = 0, limit_holdout: int = 0) -> dict:
memory = ""
# optionally cap each split to control API cost / latency
if limit_replay or limit_holdout:
replay = [t for t in tasks if t.split == "replay"]
holdout = [t for t in tasks if t.split == "holdout"]
if limit_replay:
replay = replay[:limit_replay]
if limit_holdout:
holdout = holdout[:limit_holdout]
tasks = replay + holdout
bh, bs, bscore = _score(backend, tasks, skill, memory)
trace = [{"night": 0, "held_out_hard": round(bh, 3), "action": "baseline"}]
cur = skill
for night in range(1, nights + 1):
res = consolidate(
backend, tasks, cur, memory,
edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
evolve_skill=True, evolve_memory=False, night=night,
)
if res.accepted:
cur = res.new_skill
trace.append({
"night": night,
"held_out_hard": round(res.holdout_candidate, 3),
"action": res.gate_action,
"accepted": res.accepted,
"edits": [e.content for e in res.applied_edits],
})
if res.holdout_candidate >= 0.999:
break
ah, as_, ascore = _score(backend, tasks, cur, memory)
return {
"seed": seed,
"held_out_before": round(bh, 3),
"held_out_after": round(ah, 3),
"improved": ah > bh,
"nights": len(trace) - 1,
"trace": trace,
"final_skill_tail": cur[-400:],
}
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description="Run gbrain-evals skillopt-v1 with SkillOpt-Sleep")
ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
ap.add_argument("--model", default="")
ap.add_argument("--codex-path", default="")
ap.add_argument("--data-root", default="", help="path to eval/data/skillopt-v1")
ap.add_argument("--seeds", default="", help="comma list; default = all available")
ap.add_argument("--nights", type=int, default=3)
ap.add_argument("--edit-budget", type=int, default=4)
ap.add_argument("--limit-replay", type=int, default=0, help="cap #training tasks (cost control)")
ap.add_argument("--limit-holdout", type=int, default=0, help="cap #held-out tasks (cost control)")
ap.add_argument("--json", action="store_true")
args = ap.parse_args(argv)
data_root = find_data_root(args.data_root)
if not data_root:
print("ERROR: could not find eval/data/skillopt-v1. Clone gbrain-evals and pass --data-root.",
file=sys.stderr)
return 2
seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
backend = get_backend(args.backend, model=args.model, codex_path=args.codex_path)
results = []
for seed in seeds:
skill, tasks = load_seed(data_root, seed)
if not tasks:
continue
r = run_seed(backend, seed, skill, tasks, nights=args.nights,
edit_budget=args.edit_budget,
limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
results.append(r)
if not args.json:
print(f" {seed:<18} held-out {r['held_out_before']:.2f} -> {r['held_out_after']:.2f}"
f" ({'IMPROVED' if r['improved'] else 'no change'}, {r['nights']} nights)")
n_improved = sum(1 for r in results if r["improved"])
summary = {
"benchmark": "gbrain-evals/skillopt-v1",
"backend": backend.name,
"model": args.model or "(default)",
"n_seeds": len(results),
"n_improved": n_improved,
"tokens_used": backend.tokens_used(),
"results": results,
}
if args.json:
print(json.dumps(summary, ensure_ascii=False, indent=2))
else:
print(f"\n=== {n_improved}/{len(results)} seeds improved on held-out "
f"(backend={backend.name}, ~{backend.tokens_used()} tokens) ===")
return 0
if __name__ == "__main__":
sys.exit(main())