feat(trainer): final-skill val + best promotion; keep best unpolluted by slow_update
- slow_update force-inject now writes current_skill ONLY (best_skill stays a faithful val-best snapshot, never receives un-validated slow_update content) - after training, run one val on the final skill; if its gate score beats the incumbent best, promote final to best (updates best_skill/best_step/best_origin) - trainer now evaluates final skill on test itself (reuses best test result when final==best); records final_selection_* and final_test_* in summary.json - spreadsheetbench: head+tail truncate the post-execution verification report at source to fix multi-MB conversation bloat Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Cuzyoung
+171
-15
@@ -1543,13 +1543,13 @@ class ReflACTTrainer:
|
||||
elif action in {
|
||||
"accept", "accept_new_best", "force_accept",
|
||||
}:
|
||||
# Force-accept mode: re-apply to both current & best.
|
||||
# Force-accept mode: re-apply guidance to
|
||||
# current_skill only. best_skill must remain a
|
||||
# faithful snapshot of the val-best step and must
|
||||
# NOT receive force-injected slow-update content.
|
||||
current_skill = replace_slow_update_field(
|
||||
current_skill, slow_saved["slow_update_content"],
|
||||
)
|
||||
best_skill = replace_slow_update_field(
|
||||
best_skill, slow_saved["slow_update_content"],
|
||||
)
|
||||
elif epoch == 1:
|
||||
# Epoch 1: inject empty placeholder
|
||||
os.makedirs(slow_dir, exist_ok=True)
|
||||
@@ -1557,7 +1557,7 @@ class ReflACTTrainer:
|
||||
current_origin = f"slow_update_placeholder_epoch_{epoch:02d}"
|
||||
_save_skill(out_root, global_step, current_skill)
|
||||
with open(os.path.join(out_root, "best_skill.md"), "w") as f:
|
||||
f.write(best_skill if best_score > current_score else current_skill)
|
||||
f.write(best_skill)
|
||||
with open(slow_done_path, "w") as f:
|
||||
json.dump({"action": "inject_placeholder", "epoch": epoch}, f, indent=2)
|
||||
_persist_runtime_state(global_step)
|
||||
@@ -1778,16 +1778,15 @@ class ReflACTTrainer:
|
||||
else:
|
||||
# ── Force-accept mode (default) ──────────────────
|
||||
# The epoch-level longitudinal guidance is injected
|
||||
# into both current_skill and best_skill
|
||||
# unconditionally — it must not be gated by
|
||||
# step-level selection scores.
|
||||
# into current_skill ONLY, so training continues
|
||||
# with the accumulated slow memory. best_skill is
|
||||
# left untouched: it must remain a faithful snapshot
|
||||
# of the val-best step (which may be a pre-slow step
|
||||
# such as S_0 carrying no slow_update field at all).
|
||||
slow_content = slow_result["slow_update_content"]
|
||||
current_skill = replace_slow_update_field(
|
||||
current_skill, slow_content,
|
||||
)
|
||||
best_skill = replace_slow_update_field(
|
||||
best_skill, slow_content,
|
||||
)
|
||||
# Update caches so downstream steps use the
|
||||
# slow-update-injected skill for hashing.
|
||||
slow_candidate_hash = skill_hash(current_skill)
|
||||
@@ -1798,7 +1797,7 @@ class ReflACTTrainer:
|
||||
|
||||
print(
|
||||
f" [slow update] force-injected into "
|
||||
f"current & best "
|
||||
f"current only "
|
||||
f"({len(slow_content)} chars), "
|
||||
f"{slow_time}s"
|
||||
)
|
||||
@@ -1951,10 +1950,70 @@ class ReflACTTrainer:
|
||||
baseline_test_soft = None
|
||||
test_hard = None
|
||||
test_soft = None
|
||||
final_test_hard = None
|
||||
final_test_soft = None
|
||||
final_selection_hard = None
|
||||
final_selection_soft = None
|
||||
|
||||
if cfg["eval_test"]:
|
||||
task_types = adapter.get_task_types()
|
||||
|
||||
# ── Final skill validation (valid_seen) + best promotion ─────
|
||||
# The final (last) skill may carry an epoch-end slow_update that
|
||||
# was force-injected WITHOUT a val pass (use_gate=false or
|
||||
# slow_update_gate_with_selection=false), so it never competed for
|
||||
# best. Run one real val on the final skill; if its gate score
|
||||
# beats the incumbent best, PROMOTE it to best so that best is the
|
||||
# true val-argmax over all skills (including the final slow_update).
|
||||
# When final == best, reuse the existing val score (no rollout).
|
||||
try:
|
||||
if skill_hash(current_skill) == skill_hash(best_skill):
|
||||
final_selection_hard, final_selection_soft = best_score, None
|
||||
print(
|
||||
"\n [final skill == best skill] "
|
||||
f"final_selection_hard={best_score:.4f} (reused)"
|
||||
)
|
||||
else:
|
||||
fval_env, fval_n = _build_eval_env(
|
||||
split="valid_seen",
|
||||
env_num=cfg["sel_env_num"],
|
||||
seed=seed,
|
||||
)
|
||||
fval_dir = os.path.join(out_root, "final_selection_eval")
|
||||
fval_results = adapter.rollout(fval_env, current_skill, fval_dir)
|
||||
final_selection_hard, final_selection_soft = compute_score(fval_results)
|
||||
final_gate_score = select_gate_score(
|
||||
final_selection_hard, final_selection_soft,
|
||||
gate_metric, gate_mixed_weight,
|
||||
)
|
||||
print(
|
||||
f"\n [final skill val] items={fval_n} "
|
||||
f"final_selection_hard={final_selection_hard:.4f} "
|
||||
f"gate={final_gate_score:.4f} "
|
||||
f"(best={best_score:.4f})"
|
||||
)
|
||||
if final_gate_score > best_score:
|
||||
# Promote: the final (slow-updated) skill is val-better
|
||||
# than the incumbent best. Make it the new best so the
|
||||
# subsequent BEST-skill test rollout evaluates it and
|
||||
# best/final test scores coincide.
|
||||
print(
|
||||
f" [promote] final {final_gate_score:.4f} > "
|
||||
f"best {best_score:.4f} → final becomes new best "
|
||||
f"(step {global_step}, origin {current_origin})"
|
||||
)
|
||||
best_skill = current_skill
|
||||
best_score = final_gate_score
|
||||
best_step = global_step
|
||||
best_origin = current_origin
|
||||
with open(os.path.join(out_root, "best_skill.md"), "w") as f:
|
||||
f.write(best_skill)
|
||||
_persist_runtime_state(global_step)
|
||||
except Exception as _e: # noqa: BLE001
|
||||
final_selection_hard = None
|
||||
final_selection_soft = None
|
||||
print(f"\n [final skill val FAILED: {_e!r}]")
|
||||
|
||||
# Baseline: S_0 on test set (valid_unseen)
|
||||
print(f"\n{'='*60}")
|
||||
print(" BASELINE TEST — evaluate initial skill on Test set (valid_unseen)")
|
||||
@@ -2023,13 +2082,87 @@ class ReflACTTrainer:
|
||||
f, indent=2, ensure_ascii=False,
|
||||
)
|
||||
|
||||
# Final skill (last skill in trajectory) on test set.
|
||||
# Distinct from best_skill: with use_gate=False every candidate is
|
||||
# force-accepted so the final skill is whatever the last step
|
||||
# produced; with use_gate=True it is the last accepted skill, which
|
||||
# may differ from the best-on-val skill. We always evaluate it so
|
||||
# every run reports baseline / best-on-val / final on test.
|
||||
# Guarded so a failure here never prevents summary.json from being
|
||||
# written (the orchestrator's post-hoc safety net fills it in).
|
||||
try:
|
||||
if skill_hash(current_skill) == skill_hash(best_skill):
|
||||
# Final == best: reuse results, skip a redundant rollout.
|
||||
final_test_hard, final_test_soft = test_hard, test_soft
|
||||
final_test_dir = os.path.join(out_root, "test_eval_final")
|
||||
os.makedirs(final_test_dir, exist_ok=True)
|
||||
with open(os.path.join(final_test_dir, "summary.json"), "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
k: {
|
||||
"total": b["total"],
|
||||
"hard_acc": b["hard"] / max(b["total"], 1),
|
||||
}
|
||||
for k, b in best_buckets.items()
|
||||
},
|
||||
f, indent=2, ensure_ascii=False,
|
||||
)
|
||||
print(
|
||||
"\n [final skill == best skill] "
|
||||
f"final_test_hard={final_test_hard:.4f} (reused)"
|
||||
)
|
||||
else:
|
||||
print(f"\n{'='*60}")
|
||||
print(" FINAL SKILL TEST — evaluate last skill on Test set (valid_unseen)")
|
||||
print(f"{'='*60}")
|
||||
test_env3, test_n3 = _build_eval_env(
|
||||
split="valid_unseen",
|
||||
env_num=cfg["test_env_num"],
|
||||
seed=seed,
|
||||
)
|
||||
print(f" Test items: {test_n3}")
|
||||
final_test_dir = os.path.join(out_root, "test_eval_final")
|
||||
final_test_results = adapter.rollout(test_env3, current_skill, final_test_dir)
|
||||
final_test_hard, final_test_soft = compute_score(final_test_results)
|
||||
final_buckets = _compute_task_type_buckets(final_test_results, task_types)
|
||||
print("\n === Final Skill Test Results ===")
|
||||
for task_type in task_types + ["overall"]:
|
||||
b = final_buckets.get(task_type, {"total": 0, "hard": 0})
|
||||
t = max(b["total"], 1)
|
||||
print(
|
||||
f" {task_type:<40s}: "
|
||||
f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}"
|
||||
)
|
||||
with open(os.path.join(final_test_dir, "summary.json"), "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
k: {
|
||||
"total": b["total"],
|
||||
"hard_acc": b["hard"] / max(b["total"], 1),
|
||||
}
|
||||
for k, b in final_buckets.items()
|
||||
},
|
||||
f, indent=2, ensure_ascii=False,
|
||||
)
|
||||
except Exception as _e: # noqa: BLE001
|
||||
final_test_hard = None
|
||||
final_test_soft = None
|
||||
print(f"\n [final skill test FAILED: {_e!r}] "
|
||||
"— will be filled by post-hoc eval")
|
||||
|
||||
# Comparison
|
||||
delta_hard = (test_hard or 0) - (baseline_test_hard or 0)
|
||||
print(f"\n === Improvement (best vs baseline) ===")
|
||||
print(f"\n === Improvement vs baseline (init S_0) ===")
|
||||
print(
|
||||
f" hard: {baseline_test_hard:.4f} -> {test_hard:.4f} "
|
||||
f" [2] best-on-val hard: {baseline_test_hard:.4f} -> {test_hard:.4f} "
|
||||
f"(delta={delta_hard:+.4f})"
|
||||
)
|
||||
if final_test_hard is not None:
|
||||
final_delta_hard = (final_test_hard or 0) - (baseline_test_hard or 0)
|
||||
print(
|
||||
f" [3] final/last hard: {baseline_test_hard:.4f} -> {final_test_hard:.4f} "
|
||||
f"(delta={final_delta_hard:+.4f})"
|
||||
)
|
||||
|
||||
# ── Global summary ───────────────────────────────────────────────
|
||||
total_wall = time.time() - t_loop_start
|
||||
@@ -2061,6 +2194,8 @@ class ReflACTTrainer:
|
||||
skill_hash(skill_init), (None, None),
|
||||
)[0],
|
||||
"best_selection_hard": best_score,
|
||||
"final_selection_hard": final_selection_hard,
|
||||
"final_selection_soft": final_selection_soft,
|
||||
"best_step": best_step,
|
||||
"current_origin": current_origin,
|
||||
"best_origin": best_origin,
|
||||
@@ -2073,11 +2208,18 @@ class ReflACTTrainer:
|
||||
"baseline_test_soft": baseline_test_soft,
|
||||
"test_hard": test_hard,
|
||||
"test_soft": test_soft,
|
||||
"final_test_hard": final_test_hard,
|
||||
"final_test_soft": final_test_soft,
|
||||
"test_delta_hard": (
|
||||
(test_hard or 0) - (baseline_test_hard or 0)
|
||||
if test_hard is not None
|
||||
else None
|
||||
),
|
||||
"final_test_delta_hard": (
|
||||
(final_test_hard or 0) - (baseline_test_hard or 0)
|
||||
if final_test_hard is not None
|
||||
else None
|
||||
),
|
||||
"total_wall_time_s": round(total_wall, 1),
|
||||
"token_summary": token_summary,
|
||||
}
|
||||
@@ -2098,8 +2240,22 @@ class ReflACTTrainer:
|
||||
f" epoch {es['epoch']}: accept={es['accepts']} reject={es['rejects']} "
|
||||
f"best={es['best_score_at_epoch_end']:.4f}"
|
||||
)
|
||||
if baseline_test_hard is not None:
|
||||
print("\n === TEST scores (3 skills, split=valid_unseen) ===")
|
||||
print(
|
||||
f" [1] init/baseline (S_0) : "
|
||||
f"test_hard={baseline_test_hard:.4f}"
|
||||
)
|
||||
if test_hard is not None:
|
||||
print(f" test_hard={test_hard:.4f} test_soft={test_soft:.4f}")
|
||||
print(
|
||||
f" [2] best-on-val (step {best_step})".ljust(37)
|
||||
+ f": test_hard={test_hard:.4f} test_soft={test_soft:.4f}"
|
||||
)
|
||||
if final_test_hard is not None:
|
||||
print(
|
||||
f" [3] final/last skill : "
|
||||
f"test_hard={final_test_hard:.4f} test_soft={final_test_soft:.4f}"
|
||||
)
|
||||
if token_summary.get("_total"):
|
||||
t = token_summary["_total"]
|
||||
print(
|
||||
|
||||
Reference in New Issue
Block a user