Yifan Yang
4203086899
Upgrade from mock-only to REAL multi-backend validation:
Backends (skillopt/sleep/backend.py):
- CliBackend base: shared attempt/judge/reflect prompts, response cache,
token accounting. Subclasses implement only _call().
- ClaudeCliBackend: drives `claude -p --output-format text`.
- CodexCliBackend: drives the REAL @openai/codex `exec -o <file>` for clean
output; resolve_codex_path() skips the hermes wrapper at ~/.local/bin/codex.
- reflect() now aggregates the exact failing judge criteria into the prompt
(gbrain's lesson: tell the optimizer what the scorer rewards).
Rule judges (skillopt/sleep/judges.py): gbrain-compatible local scorers
(section_present / regex / max_chars / contains / tool_called) — held-out
scoring with no judge-API spend. TaskRecord gains a `judge` field +
reference_kind="rule".
gbrain-evals adapter (experiments/gbrain_bench.py, run_gbrain.py): load
garrytan/gbrain-evals skillopt-v1 deficient skills + train/held-out task
sets and run our consolidate() loop against the SAME suite gbrain scores.
REAL results (docs/sleep/real_api_results.md), brief-writer seed, 1 night:
- Claude (Haiku): held-out 0.00 -> 1.00
- Codex: held-out 0.00 -> 0.67
Both proposed a correct, general format rule into the protected LEARNED block.
CLI: --backend {mock,claude,codex}, --codex-path, --model; experiment +
gbrain runners gain --limit-* cost controls. 17 tests pass.
Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
100 lines
3.3 KiB
Python
100 lines
3.3 KiB
Python
"""SkillOpt-Sleep — gbrain-evals benchmark adapter.
|
|
|
|
Loads gbrain-evals' `skillopt-v1` benchmark (deficient skills + train/held-out
|
|
task sets with rule-based judges) into our TaskRecord format, so we can run the
|
|
SkillOpt-Sleep cycle against the SAME suite gbrain publishes a scorecard for:
|
|
|
|
docs/benchmarks/2026-06-03-skillopt.md — "4/4 skills 0 -> 1.00"
|
|
|
|
Each gbrain seed dir has:
|
|
SKILL.md — the deliberately deficient starting skill
|
|
benchmark.jsonl — training tasks {task_id, task, judge:{kind:"rule",checks}}
|
|
held-out.jsonl — held-out tasks (same judge shape, unseen items)
|
|
|
|
We map:
|
|
benchmark.jsonl -> TaskRecords with split="replay"
|
|
held-out.jsonl -> TaskRecords with split="holdout"
|
|
judge -> TaskRecord.judge (+ reference_kind="rule")
|
|
|
|
This lets us reproduce gbrain's headline result with our engine and either the
|
|
claude or codex backend, scoring locally via skillopt.sleep.judges (no judge API).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
from skillopt.sleep.types import TaskRecord
|
|
|
|
|
|
SEED_DIRS = {
|
|
"brief-writer": "seed-missing-structure",
|
|
"thorough-analyst": "seed-verbose",
|
|
"advisor": "seed-no-verdict",
|
|
"quick-answerer": "seed-no-brain-first",
|
|
}
|
|
|
|
|
|
def _load_jsonl(path: str) -> List[dict]:
|
|
out: List[dict] = []
|
|
if not os.path.exists(path):
|
|
return out
|
|
with open(path, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
out.append(json.loads(line))
|
|
except Exception:
|
|
pass
|
|
return out
|
|
|
|
|
|
def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
|
|
return TaskRecord(
|
|
id=f"{seed}:{rec.get('task_id', '')}",
|
|
project=f"gbrain/{seed}",
|
|
intent=str(rec.get("task", "")),
|
|
reference_kind="rule",
|
|
judge=rec.get("judge", {}) or {},
|
|
tags=[f"seed:{seed}"],
|
|
split=split,
|
|
)
|
|
|
|
|
|
def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
|
|
"""Return (deficient_skill_md, tasks) for one gbrain seed."""
|
|
sub = SEED_DIRS.get(seed, seed)
|
|
seed_dir = os.path.join(data_root, sub)
|
|
skill_path = os.path.join(seed_dir, "SKILL.md")
|
|
skill = ""
|
|
if os.path.exists(skill_path):
|
|
with open(skill_path, encoding="utf-8") as f:
|
|
skill = f.read()
|
|
tasks: List[TaskRecord] = []
|
|
for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
|
|
tasks.append(_to_task(rec, seed=seed, split="replay"))
|
|
for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
|
|
tasks.append(_to_task(rec, seed=seed, split="holdout"))
|
|
return skill, tasks
|
|
|
|
|
|
def available_seeds(data_root: str) -> List[str]:
|
|
return [s for s, sub in SEED_DIRS.items()
|
|
if os.path.isdir(os.path.join(data_root, sub))]
|
|
|
|
|
|
def find_data_root(explicit: str = "") -> Optional[str]:
|
|
"""Locate eval/data/skillopt-v1 from common clone locations."""
|
|
cands = [explicit] if explicit else []
|
|
cands += [
|
|
os.path.expanduser("~/git/gbrain-evals/eval/data/skillopt-v1"),
|
|
"/tmp/gbrain-evals/eval/data/skillopt-v1",
|
|
os.path.expanduser("~/gbrain-evals/eval/data/skillopt-v1"),
|
|
]
|
|
for c in cands:
|
|
if c and os.path.isdir(c):
|
|
return c
|
|
return None
|