feat(sleep): 3-way train/val/test split + gate_mode on|off

Data-split refactor (the anti-overfitting foundation the user asked for):
  - TaskRecord gains split∈{train,val,test} and origin∈{real,dream}.
  - assign_splits: real tasks deterministically split into val/test (disjoint);
    DREAM-augmented tasks (origin='dream') NEVER enter val/test — they only go to
    train. val gates updates; test is the final held-out measure.
  - gbrain loader maps its held-out.jsonl -> test, benchmark.jsonl -> train/val,
    so the gbrain held-out stays the true final score.
  - consolidate(): train drives reflect, val gates; adds gate_mode='off' (greedy,
    no hard filter) reporting val movement (greedy_improved/regressed/flat).
  - run_gbrain/transfer/experiment score on test (val fallback); run_gbrain gains
    --gate on|off. Legacy replay/holdout names normalized.

New test proves dream tasks never land in val/test. 21 tests pass; mock
experiment + gate=off both green.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>

skillopt/sleep/experiments/gbrain_bench.py

@@ -63,8 +63,17 @@ def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
     )
 
 
-def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
-    """Return (deficient_skill_md, tasks) for one gbrain seed."""
+def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34,
+              split_seed: int = 42) -> Tuple[str, List[TaskRecord]]:
+    """Return (deficient_skill_md, tasks) for one gbrain seed.
+
+    Faithful split mapping:
+      * gbrain held-out.jsonl  -> our ``test`` (the true final measure)
+      * gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val``
+        (val gates updates; train drives reflect)
+    All tasks are origin='real' (gbrain provides no synthetic tasks).
+    """
+    import hashlib
     sub = SEED_DIRS.get(seed, seed)
     seed_dir = os.path.join(data_root, sub)
     skill_path = os.path.join(seed_dir, "SKILL.md")
@@ -73,10 +82,21 @@ def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
         with open(skill_path, encoding="utf-8") as f:
             skill = f.read()
     tasks: List[TaskRecord] = []
+    # benchmark pool -> train/val
+    val_cut = int(round(val_fraction * 100))
     for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
-        tasks.append(_to_task(rec, seed=seed, split="replay"))
+        t = _to_task(rec, seed=seed, split="train")
+        bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100
+        t.split = "val" if bucket < val_cut else "train"
+        tasks.append(t)
+    # held-out -> test
     for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
-        tasks.append(_to_task(rec, seed=seed, split="holdout"))
+        tasks.append(_to_task(rec, seed=seed, split="test"))
+    # guarantee a non-empty val
+    if not any(t.split == "val" for t in tasks):
+        train_only = [t for t in tasks if t.split == "train"]
+        if train_only:
+            train_only[0].split = "val"
     return skill, tasks
 
 

skillopt/sleep/experiments/run_experiment.py

@@ -42,7 +42,8 @@ from skillopt.sleep.types import TaskRecord
 def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
                    metric: str = "mixed", w: float = 0.5) -> float:
     from skillopt.sleep.consolidate import select_gate_score
-    holdout = [t for t in tasks if t.split == "holdout"] or tasks
+    # the persona experiment uses a 2-way split (train/val, no test); score on val
+    holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks
     pairs = replay_batch(backend, holdout, skill, memory)
     h, s = aggregate_scores(pairs)
     return select_gate_score(h, s, metric, w)

skillopt/sleep/experiments/run_gbrain.py

@@ -34,47 +34,56 @@ from skillopt.sleep.experiments.gbrain_bench import (
 from skillopt.sleep.replay import aggregate_scores, replay_batch
 
 
-def _score(backend, tasks, skill, memory, split="holdout", metric="mixed", w=0.5):
-    sub = [t for t in tasks if t.split == split] or tasks
+def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5):
+    sub = [t for t in tasks if t.split == split]
+    if not sub:  # fall back to val, then everything, so we never score on nothing
+        sub = [t for t in tasks if t.split == "val"] or tasks
     pairs = replay_batch(backend, sub, skill, memory)
     h, s = aggregate_scores(pairs)
     return h, s, select_gate_score(h, s, metric, w)
 
 
 def run_seed(backend, seed: str, skill: str, tasks: List, *,
-             nights: int = 3, edit_budget: int = 4,
+             nights: int = 3, edit_budget: int = 4, gate_mode: str = "on",
              limit_replay: int = 0, limit_holdout: int = 0) -> dict:
     memory = ""
-    # optionally cap each split to control API cost / latency
+    # optionally cap each split to control API cost / latency.
+    # limit_replay caps train; limit_holdout caps BOTH val and test.
     if limit_replay or limit_holdout:
-        replay = [t for t in tasks if t.split == "replay"]
-        holdout = [t for t in tasks if t.split == "holdout"]
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
         if limit_replay:
-            replay = replay[:limit_replay]
+            train = train[:limit_replay]
         if limit_holdout:
-            holdout = holdout[:limit_holdout]
-        tasks = replay + holdout
-    bh, bs, bscore = _score(backend, tasks, skill, memory)
-    trace = [{"night": 0, "held_out_hard": round(bh, 3), "action": "baseline"}]
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
+    # final measure is TEST (the gbrain held-out set); val gates internally
+    bh, bs, bscore = _score(backend, tasks, skill, memory, split="test")
+    trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}]
     cur = skill
     for night in range(1, nights + 1):
         res = consolidate(
             backend, tasks, cur, memory,
             edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
-            evolve_skill=True, evolve_memory=False, night=night,
+            gate_mode=gate_mode, evolve_skill=True, evolve_memory=False, night=night,
         )
         if res.accepted:
             cur = res.new_skill
+        # report the TEST score each night (independent of the val gate)
+        th, _ts, _ = _score(backend, tasks, cur, memory, split="test")
         trace.append({
             "night": night,
-            "held_out_hard": round(res.holdout_candidate, 3),
+            "val_hard": round(res.holdout_candidate, 3),
+            "test_hard": round(th, 3),
             "action": res.gate_action,
             "accepted": res.accepted,
             "edits": [e.content for e in res.applied_edits],
         })
-        if res.holdout_candidate >= 0.999:
+        if th >= 0.999:
             break
-    ah, as_, ascore = _score(backend, tasks, cur, memory)
+    ah, as_, ascore = _score(backend, tasks, cur, memory, split="test")
     return {
         "seed": seed,
         "held_out_before": round(bh, 3),
@@ -99,8 +108,10 @@ def main(argv=None) -> int:
     ap.add_argument("--seeds", default="", help="comma list; default = all available")
     ap.add_argument("--nights", type=int, default=3)
     ap.add_argument("--edit-budget", type=int, default=4)
-    ap.add_argument("--limit-replay", type=int, default=0, help="cap #training tasks (cost control)")
-    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #held-out tasks (cost control)")
+    ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"],
+                    help="on/hard/soft = validation-gated; off = greedy (no hard filter)")
+    ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)")
+    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)")
     ap.add_argument("--json", action="store_true")
     args = ap.parse_args(argv)
 
@@ -125,6 +136,7 @@ def main(argv=None) -> int:
             continue
         r = run_seed(backend, seed, skill, tasks, nights=args.nights,
                      edit_budget=args.edit_budget,
+                     gate_mode=("off" if args.gate == "off" else "on"),
                      limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
         results.append(r)
         if not args.json:

skillopt/sleep/experiments/run_transfer.py

@@ -37,7 +37,10 @@ from skillopt.sleep.replay import aggregate_scores, replay_batch
 
 
 def _holdout_hard(backend, tasks, skill, memory="") -> float:
-    ho = [t for t in tasks if t.split == "holdout"] or tasks
+    # transfer is measured on the true held-out TEST split
+    ho = [t for t in tasks if t.split == "test"]
+    if not ho:
+        ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks
     pairs = replay_batch(backend, ho, skill, memory)
     h, _s = aggregate_scores(pairs)
     return h
@@ -59,13 +62,15 @@ def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str:
 def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget,
              limit_replay, limit_holdout, do_direct=True) -> dict:
     if limit_replay or limit_holdout:
-        replay = [t for t in tasks if t.split == "replay"]
-        holdout = [t for t in tasks if t.split == "holdout"]
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
         if limit_replay:
-            replay = replay[:limit_replay]
+            train = train[:limit_replay]
         if limit_holdout:
-            holdout = holdout[:limit_holdout]
-        tasks = replay + holdout
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
 
     baseline_target = _holdout_hard(target, tasks, skill)