url/SkillOpt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(spreadsheetbench)+optimizer: fix verify-feedback bloat, drop optimizer-side truncation, soft-disable gate

Browse Source 
A. SpreadsheetBench verification-feedback bloat
   - rollout.py _auto_verify_output: use official _compare_cell_value (was
     repr() equality, which falsely flagged 5 vs 5.0 / None vs ""); collapse
     correct-and-empty cells into a count so large sparse answer ranges no
     longer flood feedback with MBs of None=None noise.
   - codegen_agent.py _build_eval_feedback: only list WRONG cells, collapse
     correct ones into a count.
   Scoring is unaffected (evaluate() is independent); this only fixes the
   target model's multi-turn solving feedback.

B. Remove optimizer-side truncation (bloat source now fixed)
   - reflect.py: drop _MAX_TRAJ_CHARS cap and all per-field clips.
   - update_modes.py / clip.py / lr_autonomous.py: describe_item /
     short_item_summary no longer truncate; raise ranking/lr token budget.
   - trainer.py _format_step_buffer: full task_ids / target.
   - slow_update.py: full comparison samples.

C. Soft-disable gate
   - config.py / trainer.py: use_gate=false no longer raises; validation still
     runs but candidates are force-accepted (new force_accept branch + log).

Misc: aggregate.py merge token budget 4096 -> 16384.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Cuzyoung
2026-06-01 11:23:08 +00:00
parent f64a41397c
commit 372fd56c1e
10 changed files with 138 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skillopt/config.py
-7
View File
@@ -189,13 +189,6 @@ def flatten_config(cfg: dict) -> dict:
flat: dict[str, Any] = {}
evaluation_section = cfg.get("evaluation", {})
if isinstance(evaluation_section, dict) and evaluation_section.get("use_gate") is False:
raise ValueError(
"Gate validation is mandatory in this branch. Remove "
"`evaluation.use_gate: false` from the config."
)
# Apply the explicit mapping
for dotted, flat_key in _FLATTEN_MAP.items():
section, key = dotted.split(".", 1)
skillopt/engine/trainer.py
+40 -11
View File
@@ -24,7 +24,7 @@ from collections import defaultdict
from skillopt.datasets.base import BatchSpec
from skillopt.envs.base import EnvAdapter
from skillopt.evaluation.gate import evaluate_gate, select_gate_score
from skillopt.evaluation.gate import GateResult, evaluate_gate, select_gate_score
from skillopt.gradient.aggregate import merge_patches
from skillopt.optimizer.meta_skill import run_meta_skill
from skillopt.optimizer.clip import rank_and_select
@@ -467,7 +467,7 @@ def _format_step_buffer(buffer: list[dict]) -> str:
# Failure patterns
for p in entry.get("failure_patterns", []):
ids = ", ".join(p["task_ids"][:3])
ids = ", ".join(p["task_ids"])
parts.append(f' - "{p["pattern"]}" (×{p["count"]}, tasks: {ids})')
# Rejected edits (only present on reject)
@@ -484,7 +484,7 @@ def _format_step_buffer(buffer: list[dict]) -> str:
content = e.get("content", "")
target = e.get("target", "")
if target:
parts.append(f' {i}. [{op}] target="{target[:80]}" → "{content}"')
parts.append(f' {i}. [{op}] target="{target}" → "{content}"')
else:
parts.append(f' {i}. [{op}] "{content}"')
else:
@@ -863,11 +863,10 @@ class ReflACTTrainer:
sel_cache[sh] = (rec["selection_hard"], rec["selection_soft"])
# ── Baseline evaluation on selection set ─────────────────────────
if cfg.get("use_gate") is False:
raise ValueError(
"Gate validation is mandatory in this branch. Remove "
"`evaluation.use_gate=false` from the config."
)
# `use_gate=False` keeps validation running (selection rollout +
# scoring are unconditional below) but force-accepts every candidate
# instead of gating it; final skill is chosen manually afterwards.
use_gate = cfg.get("use_gate", True) is not False
gate_metric = str(cfg.get("gate_metric", "hard")).strip().lower()
if gate_metric not in {"hard", "soft", "mixed"}:
raise ValueError(
@@ -887,6 +886,8 @@ class ReflACTTrainer:
if gate_metric == "mixed"
else ""
)
+ ("" if use_gate
else " (DISABLED → validation runs, candidates force-accepted)")
)
slow_gate_with_selection = bool(
cfg.get("slow_update_gate_with_selection", False)
@@ -1346,10 +1347,31 @@ class ReflACTTrainer:
cand_soft=cand_soft,
metric=gate_metric,
mixed_weight=gate_mixed_weight,
)
) if use_gate else None
cand_gate_score = select_gate_score(
cand_hard, cand_soft, gate_metric, gate_mixed_weight,
)
if not use_gate:
# Validation ran (scores recorded above) but the gate is
# disabled: force-accept the candidate as the new current
# skill. Best-so-far is still tracked for convenience; the
# final skill is selected manually from the trajectory.
if cand_gate_score > best_score:
fa_best_skill = candidate_skill
fa_best_score = cand_gate_score
fa_best_step = global_step
else:
fa_best_skill = best_skill
fa_best_score = best_score
fa_best_step = best_step
gate = GateResult(
action="force_accept",
current_skill=candidate_skill,
current_score=cand_gate_score,
best_skill=fa_best_skill,
best_score=fa_best_score,
best_step=fa_best_step,
)
step_rec["gate_metric"] = gate_metric
step_rec["candidate_gate_score"] = cand_gate_score
step_rec["action"] = gate.action
@@ -1360,9 +1382,11 @@ class ReflACTTrainer:
best_skill = gate.best_skill
best_score = gate.best_score
best_step = gate.best_step
if gate.action in {"accept", "accept_new_best"}:
if gate.action in {"accept", "accept_new_best", "force_accept"}:
current_origin = f"step_{global_step:04d}"
if gate.action == "accept_new_best":
if gate.action == "accept_new_best" or (
gate.action == "force_accept" and best_step == global_step
):
best_origin = current_origin
if gate_metric == "hard":
@@ -1384,6 +1408,11 @@ class ReflACTTrainer:
f" [6/6 EVALUATE] ACCEPT "
f"{score_label} > current={prev_current:.4f}"
)
elif gate.action == "force_accept":
print(
f" [6/6 EVALUATE] FORCE-ACCEPT (gate disabled) "
f"{score_label}"
)
else:
print(
f" [6/6 EVALUATE] REJECT "
skillopt/envs/spreadsheetbench/codegen_agent.py
+9 -4
View File
@@ -54,8 +54,8 @@ def _build_eval_feedback(verify_report: str) -> str:
output and whether each cell is correct or wrong.
"""
import re
lines = ["Your code executed successfully but produced incorrect results.",
"The following cells have wrong values:"]
wrong_lines = []
n_correct = 0
for raw_line in verify_report.splitlines():
raw_line = raw_line.strip()
if not raw_line:
@@ -68,9 +68,14 @@ def _build_eval_feedback(verify_report: str) -> str:
if m:
cell, got_val, mark = m.groups()
if mark == "✗":
lines.append(f" {cell}: your output = {got_val} (WRONG)")
wrong_lines.append(f" {cell}: your output = {got_val} (WRONG)")
else:
lines.append(f" {cell}: correct ✓")
n_correct += 1
lines = ["Your code executed successfully but produced incorrect results.",
"The following cells have wrong values:"]
lines.extend(wrong_lines)
if n_correct:
lines.append(f" ({n_correct} other cells are correct.)")
lines.append(
"\nPlease analyze the spreadsheet data more carefully and fix the code. "
"Return a complete corrected Python script inside a ```python``` block."
skillopt/envs/spreadsheetbench/rollout.py
+23 -2
View File
@@ -26,7 +26,9 @@ from concurrent.futures import (
import openpyxl
from skillopt.envs.spreadsheetbench.react_agent import run_react
from skillopt.envs.spreadsheetbench.evaluator import evaluate, _generate_cell_names
from skillopt.envs.spreadsheetbench.evaluator import (
evaluate, _generate_cell_names, _compare_cell_value,
)
from skillopt.envs.spreadsheetbench.executor import run_generated_code
@@ -129,11 +131,30 @@ def _auto_verify_output(
lines.append(f" Sheet '{sheet_name}' NOT FOUND in output.")
continue
n_correct_skipped = 0
for cn in cell_names:
gv = ws_gold[cn].value if ws_gold else "N/A"
pv = ws_pred[cn].value
match = "✓" if repr(gv) == repr(pv) else "✗"
# Use the official cell comparator so this report's ✓/✗ agrees
# with the real scorer (evaluate). repr() equality would wrongly
# flag e.g. 5 vs 5.0 or None vs "" as mismatches and mislead the
# model into "fixing" cells that already pass scoring.
ok_cell = ws_gold is not None and _compare_cell_value(gv, pv)
match = "✓" if ok_cell else "✗"
# Skip cells that are correct AND empty on both sides: for large
# answer ranges (e.g. C2:C5000) the vast majority are empty
# (got=None, expected=None ✓) and would otherwise flood the
# report with hundreds of thousands of noise chars, burying the
# few real ✗ lines. We only emit wrong cells and non-empty
# correct cells; empty-correct cells are collapsed into a count.
if ok_cell and gv in (None, "") and pv in (None, ""):
n_correct_skipped += 1
continue
lines.append(f" {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}")
if n_correct_skipped:
lines.append(
f" (+{n_correct_skipped} empty cells correct, omitted)"
)
# Also check if any cells in the output contain formula strings
formula_cells = []
skillopt/gradient/aggregate.py
+2 -2
View File
@@ -46,7 +46,7 @@ def _merge_batch(
response, _ = chat_optimizer(
system=system_prompt,
user=user,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 4096,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 16384,
retries=3,
stage="merge",
)
@@ -231,7 +231,7 @@ def merge_patches(
response, _ = chat_optimizer(
system=merge_final_prompt,
user=user,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 4096,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 16384,
retries=3,
stage="merge",
)
skillopt/gradient/reflect.py
+22 -25
View File
@@ -43,19 +43,21 @@ from skillopt.utils import extract_json
# ── Trajectory formatting ────────────────────────────────────────────────────
_MAX_TRAJ_CHARS = 12_000
def _clip_text(value, limit: int | None = None) -> str:
"""Render optional trajectory fields. Truncation is disabled: the optimizer
is given the full content so it can see exactly what the agent saw/did.
def _clip_text(value, limit: int) -> str:
"""Render optional trajectory fields safely before truncation."""
``limit`` is accepted for backward compatibility but ignored.
"""
if value is None:
return ""
return str(value)[:limit]
return str(value)
def fmt_trajectory(
conversation: list[dict],
max_chars: int = _MAX_TRAJ_CHARS,
max_chars: int | None = None,
) -> str:
"""Format a conversation list into analyst-readable text.
@@ -69,37 +71,32 @@ def fmt_trajectory(
lines: list[str] = []
for item in conversation:
if not isinstance(item, dict):
lines.append(f"[agent] {_clip_text(item, 500)}")
lines.append(f"[agent] {_clip_text(item)}")
continue
if item.get("type") == "tool_call":
cmd = _clip_text(item.get("cmd"), 500)
obs = _clip_text(item.get("obs"), 800)
cmd = _clip_text(item.get("cmd"))
obs = _clip_text(item.get("obs"))
lines.append(f"[action] {cmd}")
lines.append(f"[obs] {obs}")
elif "action" in item and "env_feedback" in item:
step = item.get("step", "?")
reasoning = _clip_text(item.get("reasoning"), 300)
action = _clip_text(item.get("action"), 200)
feedback = _clip_text(item.get("env_feedback"), 500)
reasoning = _clip_text(item.get("reasoning"))
action = _clip_text(item.get("action"))
feedback = _clip_text(item.get("env_feedback"))
if reasoning:
lines.append(f"[step {step} think] {reasoning}")
lines.append(f"[step {step} action] {action}")
lines.append(f"[step {step} obs] {feedback}")
elif item.get("role") == "system":
# Post-execution verification / enrichment info
msg = _clip_text(item.get("content"), 2000)
msg = _clip_text(item.get("content"))
lines.append(f"[verification] {msg}")
else:
msg = _clip_text(item.get("content"), 500)
msg = _clip_text(item.get("content"))
role = item.get("role", "agent")
lines.append(f"[{role}] {msg}")
text = "\n".join(lines)
if len(text) > max_chars:
head = text[: max_chars // 2]
tail = text[-max_chars // 2 :]
text = head + "\n...[middle truncated]...\n" + tail
return text
return "\n".join(lines)
# ── Minibatch trajectory formatting ──────────────────────────────────────────
@@ -157,7 +154,7 @@ def fmt_minibatch_trajectories(
if reference_text:
header += (
f"\n#### Hidden Reference\n"
f"{reference_text[:4000]}\n"
f"{reference_text}\n"
)
# ── Append target context (what the agent saw) ──────────────
@@ -170,7 +167,7 @@ def fmt_minibatch_trajectories(
if target_prompt:
header += (
f"\n#### Target System Prompt\n"
f"{target_prompt[:3000]}\n"
f"{target_prompt}\n"
)
user_prompt = item.get("target_user_prompt", "")
@@ -182,7 +179,7 @@ def fmt_minibatch_trajectories(
if user_prompt:
header += (
f"\n#### Target User Prompt\n"
f"{user_prompt[:3000]}\n"
f"{user_prompt}\n"
)
if os.environ.get("REFLACT_CODEX_TRACE_TO_OPTIMIZER", "0") == "1":
@@ -214,7 +211,7 @@ def fmt_minibatch_trajectories(
if preview:
header += (
f"\n#### Spreadsheet Preview\n"
f"{preview[:3000]}\n"
f"{preview}\n"
)
parts.append(header + "\n" + traj_text)
@@ -323,7 +320,7 @@ def run_error_analyst_minibatch(
try:
response, _ = chat_optimizer(
system=actual_system, user=user,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 4096,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 16384,
retries=3,
stage="analyst",
)
@@ -398,7 +395,7 @@ def run_success_analyst_minibatch(
try:
response, _ = chat_optimizer(
system=actual_system, user=user,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 4096,
max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 16384,
retries=3,
stage="analyst",
)
skillopt/optimizer/clip.py
+2 -2
View File
@@ -57,7 +57,7 @@ def rank_and_select(
# Build the edit pool description for the optimizer
edits_desc = []
for i, edit in enumerate(edits):
edits_desc.append(f"[{i}] {describe_item(edit, update_mode, max_chars=500)}")
edits_desc.append(f"[{i}] {describe_item(edit, update_mode)}")
user = (
f"## Current Skill\n{skill_content}\n\n"
@@ -74,7 +74,7 @@ def rank_and_select(
try:
response, _ = chat_optimizer(
system=load_prompt(prompt_name), user=user,
max_completion_tokens=2048, retries=3, stage="ranking",
max_completion_tokens=16384, retries=3, stage="ranking",
)
result = extract_json(response)
if result and "selected_indices" in result:
skillopt/optimizer/lr_autonomous.py
+2 -2
View File
@@ -48,7 +48,7 @@ def decide_autonomous_learning_rate(
items = get_payload_items(merged_patch, update_mode)
available = len(items)
item_lines = [
f"[{idx}] {describe_item(item, update_mode, max_chars=700)}"
f"[{idx}] {describe_item(item, update_mode)}"
for idx, item in enumerate(items)
]
user = (
@@ -76,7 +76,7 @@ def decide_autonomous_learning_rate(
response, _ = chat_optimizer(
system=load_prompt("lr_autonomous"),
user=user,
max_completion_tokens=2048,
max_completion_tokens=16384,
retries=3,
stage="lr_autonomous",
)
skillopt/optimizer/slow_update.py
+28 -28
View File
@@ -91,18 +91,21 @@ def replace_slow_update_field(skill: str, new_content: str) -> str:
# ── Comparison text builder ─────────────────────────────────────────────────
# NOTE: The character limits below (whole-trajectory cap + the per-field caps in
# _read_trajectory and the comparison metadata) only trim the comparison samples
# fed to the slow-update optimizer. They exist to cut token usage and speed up the
# call; they do NOT affect what gets written into the skill. If you need richer
# context for the longitudinal comparison, feel free to raise them.
_MAX_TRAJ_CHARS = 3000
# NOTE: Character-length limits on the comparison samples fed to the slow-update /
# meta-skill optimizer have been REMOVED. Previously a whole-trajectory cap plus
# per-field caps (cmd/obs/reasoning/etc.) and comparison-metadata caps
# (task/answer/fail_reason) trimmed this context to save optimizer tokens and
# speed up the call. They never affected what gets written into the skill — only
# how much longitudinal context the optimizer sees. We now pass everything through
# at full length: the comparison input is as long as the source data is.
def _clip_text(value, limit: int) -> str:
def _clip_text(value, limit: int | None = None) -> str:
# Truncation disabled: return the full text. The `limit` argument is kept only
# for call-site compatibility and is intentionally ignored (see NOTE above).
if value is None:
return ""
return str(value)[:limit]
return str(value)
def _read_trajectory(rollout_dir: str, task_id: str) -> str:
@@ -122,35 +125,32 @@ def _read_trajectory(rollout_dir: str, task_id: str) -> str:
for entry in conversation:
if not isinstance(entry, dict):
continue
# Per-field caps (cmd/obs/reasoning/etc.) keep each trajectory compact to
# save tokens / time; raise them if you want fuller step detail.
# Per-field truncation removed: feed each step's full cmd/obs/reasoning/
# action/feedback/content (see NOTE above).
if entry.get("type") == "tool_call":
cmd = _clip_text(entry.get("cmd"), 500)
obs = _clip_text(entry.get("obs"), 800)
cmd = _clip_text(entry.get("cmd"))
obs = _clip_text(entry.get("obs"))
lines.append(f"[action] {cmd}")
lines.append(f"[obs] {obs}")
elif "action" in entry and "env_feedback" in entry:
step = entry.get("step", "?")
reasoning = _clip_text(entry.get("reasoning"), 300)
action = _clip_text(entry.get("action"), 200)
feedback = _clip_text(entry.get("env_feedback"), 500)
reasoning = _clip_text(entry.get("reasoning"))
action = _clip_text(entry.get("action"))
feedback = _clip_text(entry.get("env_feedback"))
if reasoning:
lines.append(f"[step {step} think] {reasoning}")
lines.append(f"[step {step} action] {action}")
lines.append(f"[step {step} obs] {feedback}")
elif entry.get("role") == "system":
msg = _clip_text(entry.get("content"), 1000)
msg = _clip_text(entry.get("content"))
lines.append(f"[verification] {msg}")
else:
msg = _clip_text(entry.get("content"), 500)
msg = _clip_text(entry.get("content"))
role = entry.get("role", "agent")
lines.append(f"[{role}] {msg}")
text = "\n".join(lines)
if len(text) > _MAX_TRAJ_CHARS:
half = _MAX_TRAJ_CHARS // 2
text = text[:half] + "\n...[truncated]...\n" + text[-half:]
return text
# Whole-trajectory truncation removed: return the full formatted trajectory.
return "\n".join(lines)
# ── Structured comparison pairs ─────────────────────────────────────────────
@@ -228,7 +228,7 @@ def save_comparison_pairs(pairs: list[dict], out_path: str) -> None:
for p in pairs:
slim.append({
"id": p["id"],
"task": p["task"][:300],
"task": p["task"],
"category": p["category"],
"prev": p["prev"],
"curr": p["curr"],
@@ -276,16 +276,16 @@ def format_comparison_text(pairs: list[dict]) -> str:
prev = e["prev"]
curr = e["curr"]
lines.append(
f"\n#### Task {e['id']}: {e['task'][:300]}\n"
f"\n#### Task {e['id']}: {e['task']}\n"
f"- Prev epoch: {'PASS' if prev['hard'] else 'FAIL'} "
f"(soft={prev['soft']:.2f}) — answer: {str(prev['predicted_answer'])[:200]}\n"
f"(soft={prev['soft']:.2f}) — answer: {str(prev['predicted_answer'])}\n"
f"- Curr epoch: {'PASS' if curr['hard'] else 'FAIL'} "
f"(soft={curr['soft']:.2f}) — answer: {str(curr['predicted_answer'])[:200]}"
f"(soft={curr['soft']:.2f}) — answer: {str(curr['predicted_answer'])}"
)
if curr.get("fail_reason"):
lines.append(f"- Curr fail reason: {curr['fail_reason'][:300]}")
lines.append(f"- Curr fail reason: {curr['fail_reason']}")
if prev.get("fail_reason") and not prev["hard"]:
lines.append(f"- Prev fail reason: {prev['fail_reason'][:300]}")
lines.append(f"- Prev fail reason: {prev['fail_reason']}")
if show_traj:
if e.get("prev_trajectory"):
skillopt/optimizer/update_modes.py
+10 -11
View File
@@ -70,7 +70,7 @@ def truncate_payload(container: dict, max_items: int, mode: str | None) -> dict:
return container
def describe_item(item: dict, mode: str | None, *, max_chars: int = 240) -> str:
def describe_item(item: dict, mode: str | None, *, max_chars: int | None = None) -> str:
if not isinstance(item, dict):
return ""
if is_full_rewrite_minibatch_mode(mode):
@@ -84,7 +84,7 @@ def describe_item(item: dict, mode: str | None, *, max_chars: int = 240) -> str:
parts.append(f"support={item.get('support_count')}")
new_skill = str(item.get("new_skill", "")).strip()
if new_skill:
parts.append(f"new_skill_preview={new_skill[:120]!r}")
parts.append(f"new_skill_preview={new_skill!r}")
text = " ".join(parts)
elif is_rewrite_mode(mode):
parts = [
@@ -109,28 +109,27 @@ def describe_item(item: dict, mode: str | None, *, max_chars: int = 240) -> str:
if item.get("support_count") is not None:
parts.append(f"support={item.get('support_count')}")
text = " ".join(parts)
if len(text) <= max_chars:
return text
return text[: max_chars - 3].rstrip() + "..."
# Truncation disabled: the optimizer is given the full item description.
return text
def short_item_summary(item: dict, mode: str | None, *, max_chars: int = 200) -> dict[str, Any]:
def short_item_summary(item: dict, mode: str | None, *, max_chars: int | None = None) -> dict[str, Any]:
if is_full_rewrite_minibatch_mode(mode):
return {
"title": str(item.get("title", ""))[:max_chars],
"title": str(item.get("title", "")),
"change_summary": [
str(x)[:max_chars] for x in item.get("change_summary", [])[:3]
str(x) for x in item.get("change_summary", [])
] if isinstance(item.get("change_summary"), list) else [],
"source_type": item.get("source_type", ""),
}
if is_rewrite_mode(mode):
return {
"type": item.get("type", "?"),
"title": str(item.get("title", ""))[:max_chars],
"instruction": str(item.get("instruction", ""))[:max_chars],
"title": str(item.get("title", "")),
"instruction": str(item.get("instruction", "")),
}
return {
"op": item.get("op", "?"),
"content": str(item.get("content", ""))[:max_chars],
"content": str(item.get("content", "")),
"target": item.get("target", ""),
}