url/SkillOpt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sleep): 3-way train/val/test split + gate_mode on|off

Browse Source 
Data-split refactor (the anti-overfitting foundation the user asked for):
  - TaskRecord gains split∈{train,val,test} and origin∈{real,dream}.
  - assign_splits: real tasks deterministically split into val/test (disjoint);
    DREAM-augmented tasks (origin='dream') NEVER enter val/test — they only go to
    train. val gates updates; test is the final held-out measure.
  - gbrain loader maps its held-out.jsonl -> test, benchmark.jsonl -> train/val,
    so the gbrain held-out stays the true final score.
  - consolidate(): train drives reflect, val gates; adds gate_mode='off' (greedy,
    no hard filter) reporting val movement (greedy_improved/regressed/flat).
  - run_gbrain/transfer/experiment score on test (val fallback); run_gbrain gains
    --gate on|off. Legacy replay/holdout names normalized.

New test proves dream tasks never land in val/test. 21 tests pass; mock
experiment + gate=off both green.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
Yifan Yang
2026-06-08 14:31:51 +00:00
parent 99ec2caf6b
commit 6f1351edb9
10 changed files with 220 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skillopt/sleep/consolidate.py
+59 -33
View File
@@ -52,14 +52,26 @@ class ConsolidationResult:
def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]:
replay = [t for t in tasks if t.split == "replay"]
holdout = [t for t in tasks if t.split == "holdout"]
# be robust if a split is empty
if not replay:
replay = tasks
if not holdout:
holdout = tasks
return replay, holdout
"""Return (train_tasks, val_tasks).
train drives reflect; val gates updates. test is held out entirely from
consolidation and is scored by the caller. Accepts legacy split names
(replay->train, holdout->val) for robustness.
"""
def _norm(s: str) -> str:
return {"replay": "train", "holdout": "val"}.get(s, s)
train = [t for t in tasks if _norm(t.split) == "train"]
val = [t for t in tasks if _norm(t.split) == "val"]
# be robust if a split is empty: fall back so a night still does something,
# but never silently use test as val.
test = [t for t in tasks if _norm(t.split) == "test"]
if not val:
# prefer train as the gate reference over nothing; last resort all-but-test
val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks
if not train:
train = val
return train, val
def consolidate(
@@ -71,25 +83,30 @@ def consolidate(
edit_budget: int = 4,
gate_metric: str = "mixed",
gate_mixed_weight: float = 0.5,
gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy)
evolve_skill: bool = True,
evolve_memory: bool = True,
night: int = 1,
) -> ConsolidationResult:
"""Run one consolidation epoch: reflect -> bounded edit -> gate.
Skill and memory are evolved in sequence (skill first if both enabled),
each behind the same held-out gate, so each document only changes when it
demonstrably helps on the user's held-out tasks.
"""
replay_tasks, holdout_tasks = _split(tasks)
train tasks drive reflect; val tasks gate the update (test is held out by the
caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve
requirement) — the user opts out of hard filtering — but val scores are still
recorded so the report shows whether quality moved.
# ── baseline on held-out slice (the gate reference) ──────────────────
base_pairs = replay_batch(backend, holdout_tasks, skill, memory)
Skill and memory are evolved in sequence (skill first if both enabled).
"""
train_tasks, val_tasks = _split(tasks)
gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
# ── baseline on the VAL slice (the gate reference) ────────────────────
base_pairs = replay_batch(backend, val_tasks, skill, memory)
base_hard, base_soft = aggregate_scores(base_pairs)
base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
# ── reflect over replay-split failures/successes ─────────────────────
train_pairs = replay_batch(backend, replay_tasks, skill, memory)
# ── reflect over TRAIN-split failures/successes ───────────────────────
train_pairs = replay_batch(backend, train_tasks, skill, memory)
failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0]
successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0]
@@ -104,14 +121,15 @@ def consolidate(
new_doc, applied = apply_edits(doc, edits)
if not applied:
return doc
# evaluate candidate on the held-out slice
# score the candidate on the VAL slice
trial_skill = new_doc if which == "skill" else cand_skill
trial_memory = new_doc if which == "memory" else cand_memory
pairs = replay_batch(backend, holdout_tasks, trial_skill, trial_memory)
pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
h, s = aggregate_scores(pairs)
cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
if cand_score > base_score:
base_score = cand_score
# gate OFF: accept greedily (no regression check); gate ON: strict improve
if gate_off or cand_score > base_score:
base_score = max(base_score, cand_score)
all_applied.extend(applied)
return new_doc
all_rejected.extend(applied)
@@ -126,7 +144,7 @@ def consolidate(
if evolve_memory:
# re-evaluate failures under the (possibly improved) skill
train_pairs2 = replay_batch(backend, replay_tasks, cand_skill, cand_memory)
train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory)
failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0]
successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0]
edits_m = backend.reflect(
@@ -135,19 +153,29 @@ def consolidate(
)
cand_memory = _gate_apply(cand_memory, edits_m, "memory")
# ── final gate decision (use the repo gate for the canonical action) ──
final_pairs = replay_batch(backend, holdout_tasks, cand_skill, cand_memory)
# ── final decision, scored on the VAL slice ───────────────────────────
final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
final_hard, final_soft = aggregate_scores(final_pairs)
final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
if _HAVE_REPO_GATE:
if gate_off:
# greedy mode: keep whatever edits we applied; report quality movement
accepted = bool(all_applied)
if final_score > base_gate_score:
action = "greedy_improved"
elif final_score < base_gate_score:
action = "greedy_regressed"
else:
action = "greedy_flat" if all_applied else "greedy_noop"
elif _HAVE_REPO_GATE:
gate = evaluate_gate(
candidate_skill=cand_skill,
cand_hard=final_hard,
current_skill=skill,
current_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
current_score=base_gate_score,
best_skill=skill,
best_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
best_score=base_gate_score,
best_step=night - 1,
global_step=night,
cand_soft=final_soft,
@@ -155,17 +183,15 @@ def consolidate(
mixed_weight=gate_mixed_weight,
)
action = gate.action
accepted = bool(all_applied) and final_score > base_gate_score
else:
action = "accept" if final_score > base_soft else "reject"
accepted = bool(all_applied) and final_score > select_gate_score(
base_hard, base_soft, gate_metric, gate_mixed_weight
)
action = "accept" if final_score > base_gate_score else "reject"
accepted = bool(all_applied) and final_score > base_gate_score
return ConsolidationResult(
accepted=accepted,
gate_action=action,
baseline_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
baseline_score=base_gate_score,
candidate_score=final_score,
new_skill=cand_skill if accepted else skill,
new_memory=cand_memory if accepted else memory,