feat(sleep): 3-way train/val/test split + gate_mode on|off

Data-split refactor (the anti-overfitting foundation the user asked for):
  - TaskRecord gains split∈{train,val,test} and origin∈{real,dream}.
  - assign_splits: real tasks deterministically split into val/test (disjoint);
    DREAM-augmented tasks (origin='dream') NEVER enter val/test — they only go to
    train. val gates updates; test is the final held-out measure.
  - gbrain loader maps its held-out.jsonl -> test, benchmark.jsonl -> train/val,
    so the gbrain held-out stays the true final score.
  - consolidate(): train drives reflect, val gates; adds gate_mode='off' (greedy,
    no hard filter) reporting val movement (greedy_improved/regressed/flat).
  - run_gbrain/transfer/experiment score on test (val fallback); run_gbrain gains
    --gate on|off. Legacy replay/holdout names normalized.

New test proves dream tasks never land in val/test. 21 tests pass; mock
experiment + gate=off both green.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
Yifan Yang
2026-06-08 14:31:51 +00:00
parent 99ec2caf6b
commit 6f1351edb9
10 changed files with 220 additions and 82 deletions
+26 -7
View File
@@ -105,14 +105,31 @@ class TestMine(unittest.TestCase):
self.assertEqual(ok[0].outcome, "success")
def test_split_stable_and_nonempty(self):
tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
tasks = assign_splits(researcher_persona(), val_fraction=0.34, seed=42)
splits = {t.split for t in tasks}
self.assertIn("replay", splits)
self.assertIn("holdout", splits)
self.assertIn("train", splits)
self.assertIn("val", splits)
# stable across calls
again = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
again = assign_splits(researcher_persona(), val_fraction=0.34, seed=42)
self.assertEqual([t.split for t in tasks], [t.split for t in again])
def test_dream_never_in_val_or_test(self):
# the anti-overfitting guarantee: origin='dream' tasks only ever land in train
from skillopt.sleep.types import TaskRecord
real = researcher_persona()
dream = [TaskRecord(id=f"d{i}", project="/p", intent=f"dream {i}",
origin="dream", derived_from="r0") for i in range(5)]
tasks = assign_splits(real + dream, val_fraction=0.3, test_fraction=0.3, seed=7)
for t in tasks:
if t.origin == "dream":
self.assertEqual(t.split, "train")
# val and test contain ONLY real tasks
for t in tasks:
if t.split in ("val", "test"):
self.assertEqual(t.origin, "real")
# and val/test are disjoint (a task is in exactly one split)
self.assertTrue(any(t.split == "val" for t in tasks))
class TestConsolidateGate(unittest.TestCase):
def test_accepts_helpful_rejects_harmful(self):
@@ -169,11 +186,13 @@ class TestGbrainLoader(unittest.TestCase):
self.skipTest("gbrain-evals data not present")
skill, tasks = load_seed(root, "brief-writer")
self.assertTrue(skill)
self.assertTrue(any(t.split == "holdout" for t in tasks))
# gbrain held-out maps to our 'test'; benchmark pool to train/val
self.assertTrue(any(t.split == "test" for t in tasks))
self.assertTrue(any(t.split == "val" for t in tasks))
self.assertTrue(all(t.reference_kind == "rule" for t in tasks))
# the deficient skill must FAIL its own held-out checks (baseline 0)
# the deficient skill must FAIL its own held-out (test) checks (baseline 0)
from skillopt.sleep.judges import score_rule_judge
ho = [t for t in tasks if t.split == "holdout"][0]
ho = [t for t in tasks if t.split == "test"][0]
self.assertEqual(score_rule_judge(ho.judge, skill)[0], 0.0)