Initial commit

skillopt/envs/spreadsheetbench/__init__.py

@@ -0,0 +1,5 @@
+"""SpreadsheetBench environment adapter for ReflACT."""
+
+from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter
+
+__all__ = ["SpreadsheetBenchAdapter"]

skillopt/envs/spreadsheetbench/adapter.py

@@ -0,0 +1,309 @@
+"""SpreadsheetBench environment adapter for ReflACT.
+
+Connects the ReflACT training loop to SpreadsheetBench by implementing
+:class:`~skillopt.envs.base.EnvAdapter`.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+from skillopt.gradient.deep_probe import generate_deep_probe_instruction
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs.base import EnvAdapter
+from skillopt.envs.spreadsheetbench.dataloader import SpreadsheetBenchDataLoader
+from skillopt.envs.spreadsheetbench.rollout import (
+    process_one,
+    run_spreadsheet_batch,
+    run_spreadsheet_batch_codegen,
+)
+from skillopt.gradient.reflect import run_minibatch_reflect
+from skillopt.model import get_student_backend, is_student_exec_backend
+
+
+# Task types used for per-category breakdowns
+TASK_TYPES = ["cell_level", "sheet_level"]
+
+
+class SpreadsheetBenchAdapter(EnvAdapter):
+    """SpreadsheetBench environment adapter."""
+
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        data_root: str = "",
+        mode: str = "single",
+        max_turns: int = 30,
+        exec_timeout: int = 600,
+        workers: int = 64,
+        analyst_workers: int = 16,
+        failure_only: bool = False,
+        minibatch_size: int = 8,
+        edit_budget: int = 4,
+        seed: int = 42,
+        use_deep_reflect: bool = False,
+        deep_reflect_failures: int = 4,
+        deep_reflect_successes: int = 2,
+    ) -> None:
+        self.data_root = data_root
+        self.mode = mode  # "single", "multi", or "react"
+        self.max_turns = max_turns
+        self.exec_timeout = exec_timeout
+        self.workers = workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.use_deep_reflect = use_deep_reflect
+        self.deep_reflect_failures = deep_reflect_failures
+        self.deep_reflect_successes = deep_reflect_successes
+        self.dataloader = SpreadsheetBenchDataLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            data_root=data_root,
+            seed=seed,
+        )
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        if is_student_exec_backend() and self.mode != "single":
+            raise NotImplementedError(
+                "Exec student backends are currently supported only for SpreadsheetBench mode=single."
+            )
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def rollout(
+        self,
+        env_manager,
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict]:
+        """Run agent on all items and return results.
+
+        Dispatches based on ``self.mode``:
+          - ``"single"`` / ``"multi"``: codegen agent (no tool-call)
+          - ``"react"``: ReAct agent with tool-call (legacy)
+        """
+        items = env_manager  # For static datasets, env_manager is a list of items
+        results_path = os.path.join(out_dir, "results.jsonl")
+        os.makedirs(out_dir, exist_ok=True)
+
+        # Resume support
+        if os.path.exists(results_path):
+            existing: list[dict] = []
+            with open(results_path) as f:
+                for line in f:
+                    try:
+                        existing.append(json.loads(line))
+                    except Exception:
+                        pass
+            if existing:
+                return existing
+
+        if self.mode in ("single", "multi"):
+            results = run_spreadsheet_batch_codegen(
+                items=items,
+                data_root=self.data_root,
+                out_root=out_dir,
+                skill_content=skill_content,
+                mode=self.mode,
+                max_turns=self.max_turns,
+                max_api_workers=self.workers,
+                task_timeout=self.exec_timeout,
+                use_eval_feedback=kwargs.get("use_eval_feedback", False),
+                diagnostic_mode=kwargs.get("diagnostic_mode", False),
+                diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
+                diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"),
+            )
+        else:
+            results = run_spreadsheet_batch(
+                items=items,
+                data_root=self.data_root,
+                out_root=out_dir,
+                skill_content=skill_content,
+                max_turns=self.max_turns,
+                max_api_workers=self.workers,
+                task_timeout=max(600, int(self.exec_timeout) + 60),
+                diagnostic_mode=kwargs.get("diagnostic_mode", False),
+                diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
+                diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"),
+            )
+
+        with open(results_path, "w") as f:
+            for r in results:
+                f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+        return results
+
+    def reflect(
+        self,
+        results: list[dict],
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict | None]:
+        """Analyze rollout results and produce patches (minibatch mode)."""
+        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
+        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
+        random_seed = kwargs.get("random_seed")
+        step_buffer_context = kwargs.get("step_buffer_context", "")
+        meta_skill_context = kwargs.get("meta_skill_context", "")
+
+        return run_minibatch_reflect(
+            results=results,
+            skill_content=skill_content,
+            prediction_dir=prediction_dir,
+            patches_dir=patches_dir,
+            workers=self.analyst_workers,
+            failure_only=self.failure_only,
+            minibatch_size=self.minibatch_size,
+            edit_budget=self.edit_budget,
+            random_seed=random_seed,
+            error_system=self.get_error_minibatch_prompt(),
+            success_system=self.get_success_minibatch_prompt(),
+            step_buffer_context=step_buffer_context,
+            meta_skill_context=meta_skill_context,
+            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
+        )
+
+    def deep_reflect(
+        self,
+        results: list[dict],
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict | None]:
+        if not self.use_deep_reflect:
+            return []
+
+        env_manager = kwargs.get("env_manager")
+        if not isinstance(env_manager, list):
+            return []
+
+        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
+        random_seed = kwargs.get("random_seed")
+        step_buffer_context = kwargs.get("step_buffer_context", "")
+        meta_skill_context = kwargs.get("meta_skill_context", "")
+        codex_backend = get_student_backend() == "codex_exec"
+        selected_items = self.select_representative_items(
+            results,
+            env_manager,
+            n_failures=self.deep_reflect_failures,
+            n_successes=self.deep_reflect_successes,
+            seed=random_seed,
+        )
+        if not selected_items:
+            return []
+
+        selected_ids = {str(item["id"]) for item in selected_items}
+        selected_results = [row for row in results if str(row.get("id")) in selected_ids]
+        selected_examples = (
+            self.attach_codex_probe_context(selected_results, prediction_dir)
+            if codex_backend
+            else selected_results
+        )
+        selected_metadata = [
+            {
+                "id": str(item["id"]),
+                "instruction_type": str(item.get("instruction_type") or ""),
+                "answer_position": str(item.get("answer_position") or ""),
+            }
+            for item in selected_items
+        ]
+
+        deep_dir = os.path.join(out_dir, "deep_reflect")
+        rollout_dir = os.path.join(deep_dir, "rollout")
+        patches_dir = os.path.join(deep_dir, "patches")
+        os.makedirs(deep_dir, exist_ok=True)
+        print(
+            f"    [2b/6 DEEP REFLECT setup] selected={len(selected_items)} "
+            f"mode={self.mode}"
+        )
+        probe = generate_deep_probe_instruction(
+            skill_content=skill_content,
+            items=selected_examples,
+            prediction_dir=prediction_dir,
+            system_prompt=self.get_codex_deep_probe_prompt() if codex_backend else self.get_deep_probe_prompt(),
+            step_buffer_context=step_buffer_context,
+            meta_skill_context=meta_skill_context,
+            output_requirements=[
+                "- The instruction must ask for a short structured diagnostic readout before the student writes code or starts tool use.",
+                "- The readout should focus on task family, source/target region, and decisive transformation rule.",
+                "- The student must still complete the original spreadsheet task.",
+                "- Keep the readout concise and avoid exhaustive cell enumeration.",
+                "- The instruction text should be ready to append directly to the student's prompt.",
+            ],
+        )
+        if not probe:
+            return []
+        diagnostic_trace_context_by_id = None
+        if codex_backend:
+            selected_items, diagnostic_trace_context_by_id, probe = self.resolve_codex_probe_target(
+                selected_items=selected_items,
+                selected_examples=selected_examples,
+                prediction_dir=prediction_dir,
+                probe=probe,
+            )
+
+        with open(os.path.join(deep_dir, "probe.json"), "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    **probe,
+                    "selected_examples": selected_metadata,
+                },
+                f,
+                ensure_ascii=False,
+                indent=2,
+            )
+
+        deep_results = self.rollout(
+            selected_items,
+            skill_content,
+            rollout_dir,
+            diagnostic_mode=True,
+            diagnostic_instruction=probe["probe_instruction"],
+            diagnostic_trace_context_by_id=diagnostic_trace_context_by_id,
+        )
+        return run_minibatch_reflect(
+            results=deep_results,
+            skill_content=skill_content,
+            prediction_dir=os.path.join(rollout_dir, "predictions"),
+            patches_dir=patches_dir,
+            workers=self.analyst_workers,
+            failure_only=self.failure_only,
+            minibatch_size=self.minibatch_size,
+            edit_budget=self.edit_budget,
+            random_seed=random_seed,
+            error_system=self.get_error_minibatch_prompt(),
+            success_system=self.get_success_minibatch_prompt(),
+            step_buffer_context=step_buffer_context,
+            meta_skill_context=meta_skill_context,
+            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
+        )
+
+    def get_task_types(self) -> list[str]:
+        return list(TASK_TYPES)

skillopt/envs/spreadsheetbench/codegen_agent.py

@@ -0,0 +1,704 @@
+"""Codegen agent for SpreadsheetBench — no tool-call, pure code generation.
+
+Two modes:
+  - **single**: One LLM call → extract ```python``` block → done.
+  - **multi**: Up to max_turns LLM calls; after each, execute code and
+    feed errors back for correction.
+
+This matches the official SpreadsheetBench evaluation setting (LLM generates
+a Python code block, no function-calling / tool-use).
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import signal
+import time
+
+import openpyxl
+
+
+# ── Timeout helper ──────────────────────────────────────────────────────────
+
+class TaskTimeout(Exception):
+    """Raised when a task exceeds its time budget."""
+
+
+def _timeout_handler(signum, frame):
+    raise TaskTimeout("Task timed out")
+
+from skillopt.model.azure_openai import (
+    get_reasoning_effort,
+    get_student_client,
+    _needs_responses_api,
+    tracker,
+)
+from skillopt.model import get_codex_exec_config, get_student_backend, is_student_exec_backend
+from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
+from skillopt.prompts import load_prompt
+from skillopt.envs.spreadsheetbench.executor import run_generated_code
+from skillopt.envs.spreadsheetbench.evaluator import evaluate
+
+
+# ── Eval feedback helper (no golden value leakage) ─────────────────────────
+
+def _build_eval_feedback(verify_report: str) -> str:
+    """Build Student feedback from a verify report, hiding expected values.
+
+    The verify report contains lines like:
+        Sheet1!D2: got=None, expected=0 ✗
+        Sheet1!D10: got=None, expected=None ✓
+
+    We strip the ``expected=...`` part so the Student sees only its own
+    output and whether each cell is correct or wrong.
+    """
+    import re
+    lines = ["Your code executed successfully but produced incorrect results.",
+             "The following cells have wrong values:"]
+    for raw_line in verify_report.splitlines():
+        raw_line = raw_line.strip()
+        if not raw_line:
+            continue
+        # Match enrichment lines like "  Sheet1!D2: got=None, expected=0 ✗"
+        m = re.match(
+            r"(\S+!?\w+):\s*got=(.+?),\s*expected=.+?\s*(✓|✗)$",
+            raw_line,
+        )
+        if m:
+            cell, got_val, mark = m.groups()
+            if mark == "✗":
+                lines.append(f"  {cell}: your output = {got_val} (WRONG)")
+            else:
+                lines.append(f"  {cell}: correct ✓")
+    lines.append(
+        "\nPlease analyze the spreadsheet data more carefully and fix the code. "
+        "Return a complete corrected Python script inside a ```python``` block."
+    )
+    return "\n".join(lines)
+
+
+# ── Workbook preview (same as official prompt.py) ────────────────────────────
+
+def _preview_workbook(path: str, max_rows: int = 5, max_cols: int = 20) -> str:
+    """Generate a text preview of the first few rows of each sheet."""
+    wb = openpyxl.load_workbook(path, data_only=False)
+    chunks: list[str] = []
+    for sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        chunks.append(
+            f"## Sheet: {sheet_name}  "
+            f"(dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})"
+        )
+        for row in ws.iter_rows(
+            min_row=1,
+            max_row=min(ws.max_row, max_rows),
+            max_col=min(ws.max_column, max_cols),
+            values_only=False,
+        ):
+            cells = []
+            for cell in row:
+                v = cell.value
+                if v is None:
+                    cells.append(f"{cell.coordinate}=")
+                else:
+                    s = str(v)
+                    if len(s) > 40:
+                        s = s[:37] + "..."
+                    cells.append(f"{cell.coordinate}={s}")
+            chunks.append(" | ".join(cells))
+        if ws.max_row > max_rows:
+            chunks.append(f"... ({ws.max_row - max_rows} more rows)")
+        chunks.append("")
+    wb.close()
+    return "\n".join(chunks)
+
+
+# ── Code extraction (same as official prompt.py) ────────────────────────────
+
+def extract_code(text: str) -> str:
+    """Extract the first ```python``` fenced code block from LLM output."""
+    if "```" not in text:
+        return text.strip()
+    start = text.find("```")
+    nl = text.find("\n", start)
+    end = text.find("```", nl + 1)
+    if nl == -1 or end == -1:
+        return text.strip()
+    return text[nl + 1 : end].strip()
+
+
+# ── Prompt construction (official SpreadsheetBench prompts) ─────────────────
+
+
+def _build_system(skill_content: str) -> str:
+    base = load_prompt("codegen_system", env="spreadsheetbench")
+    if skill_content.strip():
+        base += f"\n\n## Skill\n{skill_content.strip()}"
+    return base
+
+
+def _build_user(
+    instruction: str,
+    input_xlsx: str,
+    instruction_type: str = "",
+    answer_position: str = "",
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> str:
+    try:
+        preview = _preview_workbook(input_xlsx)
+    except Exception as e:  # noqa: BLE001
+        preview = f"(failed to preview workbook: {e})"
+    extra = ""
+    if instruction_type:
+        extra += f"\nInstruction type: {instruction_type}"
+    if answer_position:
+        extra += f"\nExpected answer position: {answer_position}"
+    task_suffix = "Return only a ```python``` code block."
+    diagnostic = ""
+    if diagnostic_mode and diagnostic_instruction.strip():
+        task_suffix = (
+            "First provide a short diagnostic readout that follows the training "
+            "instruction below, then return a single complete ```python``` code block."
+        )
+        diagnostic = f"\n\n# Training readout\n{diagnostic_instruction.strip()}"
+    prefix = ""
+    if diagnostic_trace_context.strip():
+        prefix = (
+            "# Previous Codex Trace Snapshot\n"
+            "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
+            f"{diagnostic_trace_context.strip()}\n\n"
+        )
+    return (
+        f"{prefix}"
+        f"# Instruction\n{instruction}\n{extra}\n\n"
+        f"# Input spreadsheet preview\n{preview}\n\n"
+        "# Task\n"
+        "Write a Python script that reads the workbook from the variable `INPUT_PATH`, "
+        "applies the instruction, and writes the modified workbook to `OUTPUT_PATH`. "
+        "Preserve all other cells unchanged. "
+        "The preview may be truncated — do not hardcode row counts or assume the data ends at the last previewed row; "
+        "iterate over all actual rows in the workbook instead. "
+        f"{task_suffix}"
+        f"{diagnostic}"
+    )
+
+
+# ── LLM call with retry ────────────────────────────────────────────────────
+
+def _llm_call_with_retry(call_fn, *, retries: int = 5, timeout: int = 120):
+    """Wrap an LLM API call with retry and per-call timeout."""
+    last_err = None
+    for attempt in range(retries):
+        try:
+            return call_fn(timeout=timeout)
+        except Exception as e:  # noqa: BLE001
+            last_err = e
+            sleep = min(2 ** attempt + random.random(), 60)
+            time.sleep(sleep)
+    raise RuntimeError(f"LLM call failed after {retries} retries: {last_err}")
+
+
+def _get_deployment() -> str:
+    from skillopt.model import azure_openai as _llm
+    return _llm.STUDENT_DEPLOYMENT
+
+
+def _build_codex_skill(skill_content: str) -> str:
+    return render_skill_md(
+        skill_content,
+        description="Dynamic ReflACT skill for solving the current SpreadsheetBench task.",
+        preamble=(
+            "Use this skill when solving the current SpreadsheetBench task in this workspace.\n"
+            "Write a single self-contained Python solution to `solution.py`.\n"
+            "The solution must operate on the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n"
+            "You may inspect `input.xlsx` and run `python run_solution.py` to validate locally,\n"
+            "but do not hardcode values from the preview or from one specific workbook."
+        ),
+    )
+
+
+def _build_codex_task(
+    instruction: str,
+    input_xlsx: str,
+    instruction_type: str,
+    answer_position: str,
+    *,
+    diagnostic_mode: bool,
+    diagnostic_instruction: str,
+    diagnostic_trace_context: str,
+) -> str:
+    prompt = _build_user(
+        instruction,
+        input_xlsx,
+        instruction_type,
+        answer_position,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+    return (
+        f"{prompt}\n\n"
+        "## Codex Harness Task\n"
+        "- Read `.agents/skills/skillopt-student/SKILL.md` before writing code; do not call a Skill tool.\n"
+        "- Read and optionally inspect `input.xlsx` in this workspace.\n"
+        "- Write the final Python solution to `solution.py`.\n"
+        "- The script should use the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n"
+        "- If you want to validate locally, run `python run_solution.py`.\n"
+        "- Do not return a code fence as the primary artifact; the source of truth is `solution.py`.\n"
+    )
+
+
+def _build_codex_driver() -> str:
+    return (
+        "import pathlib\n"
+        "import re\n"
+        "import sys\n"
+        "import traceback\n\n"
+        'INPUT_PATH = "input.xlsx"\n'
+        'OUTPUT_PATH = "output.xlsx"\n'
+        "code = pathlib.Path('solution.py').read_text(encoding='utf-8')\n"
+        "code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n"
+        "globals_dict = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n"
+        "try:\n"
+        "    exec(compile(code, 'solution.py', 'exec'), globals_dict, globals_dict)\n"
+        "except Exception:\n"
+        "    traceback.print_exc()\n"
+        "    sys.exit(2)\n"
+    )
+
+
+def _prepare_codex_workspace(
+    *,
+    instruction: str,
+    input_xlsx: str,
+    output_path: str,
+    instruction_type: str,
+    answer_position: str,
+    skill_content: str,
+    diagnostic_mode: bool,
+    diagnostic_instruction: str,
+    diagnostic_trace_context: str,
+    workspace_name: str = "codex_single",
+) -> tuple[str, str, str, str]:
+    task_out_dir = os.path.dirname(output_path)
+    work_dir = os.path.join(task_out_dir, workspace_name)
+    skill_md = _build_codex_skill(skill_content)
+    task_md = _build_codex_task(
+        instruction,
+        input_xlsx,
+        instruction_type,
+        answer_position,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+    prompt = (
+        "Read `.agents/skills/skillopt-student/SKILL.md` directly; do not call a Skill tool.\n"
+        "Read `task.md`, inspect `input.xlsx` if useful, and write the final solution to `solution.py`.\n"
+        "You may run `python run_solution.py` to validate the script locally.\n"
+        "In your final response, briefly confirm whether `solution.py` was written and summarize the approach."
+    )
+    prepare_workspace(
+        work_dir=work_dir,
+        skill_md=skill_md,
+        task_text=task_md,
+        extra_files={"run_solution.py": _build_codex_driver()},
+        copy_files=[(input_xlsx, "input.xlsx")],
+    )
+
+    return work_dir, skill_md, task_md, prompt
+
+
+def _run_exec_backend(
+    *,
+    work_dir: str,
+    prompt: str,
+    model: str,
+    timeout: int,
+) -> tuple[str, str]:
+    return run_student_exec(
+        work_dir=work_dir,
+        prompt=prompt,
+        model=model,
+        timeout=timeout,
+        allow_file_edits=True,
+    )
+
+
+# ── Chat (no tools) ────────────────────────────────────────────────────────
+
+def _chat_call(
+    client,
+    deployment: str,
+    messages: list[dict],
+    max_output_tokens: int,
+    llm_timeout: int = 120,
+) -> str:
+    """Single LLM call, no tools. Returns raw text."""
+    reasoning_effort = get_reasoning_effort()
+    if _needs_responses_api(deployment):
+        # Responses API
+        system = ""
+        api_input = []
+        for m in messages:
+            if m["role"] == "system":
+                system = m["content"]
+            else:
+                api_input.append({"role": m["role"], "content": m["content"]})
+        resp = _llm_call_with_retry(lambda timeout: client.responses.create(
+            model=deployment,
+                instructions=system,
+                input=api_input,
+                max_output_tokens=max_output_tokens,
+                **({"reasoning": {"effort": reasoning_effort}} if reasoning_effort else {}),
+                timeout=timeout,
+            ), timeout=llm_timeout)
+        if hasattr(resp, "usage") and resp.usage:
+            tracker.record(
+                "rollout",
+                getattr(resp.usage, "input_tokens", 0) or 0,
+                getattr(resp.usage, "output_tokens", 0) or 0,
+            )
+        text = getattr(resp, "output_text", None) or ""
+        if text:
+            return text
+        for item in getattr(resp, "output", None) or []:
+            for part in getattr(item, "content", []):
+                if getattr(part, "type", "") == "output_text":
+                    return part.text or ""
+        return ""
+    else:
+        # Chat Completions API — no tools
+        kwargs = {
+            "model": deployment,
+            "messages": messages,
+            "max_completion_tokens": max_output_tokens,
+        }
+        if reasoning_effort is not None:
+            kwargs["reasoning_effort"] = reasoning_effort
+        resp = _llm_call_with_retry(lambda timeout: client.chat.completions.create(
+            **kwargs,
+            timeout=timeout,
+        ), timeout=llm_timeout)
+        if resp.usage:
+            tracker.record(
+                "rollout",
+                resp.usage.prompt_tokens or 0,
+                resp.usage.completion_tokens or 0,
+            )
+        return resp.choices[0].message.content or ""
+
+
+# ── Public API ──────────────────────────────────────────────────────────────
+
+def run_single(
+    instruction: str,
+    input_xlsx: str,
+    output_path: str,
+    instruction_type: str = "",
+    answer_position: str = "",
+    skill_content: str = "",
+    max_output_tokens: int = 16384,
+    llm_timeout: int = 120,
+    task_timeout: int = 300,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Single-round code generation. One LLM call, no tools.
+
+    Args:
+        llm_timeout: Per-LLM-call timeout in seconds (default 120).
+        task_timeout: Total task timeout in seconds (default 300).
+
+    Returns ``{"code": str, "raw": str, "n_turns": 1}``.
+    """
+    if is_student_exec_backend():
+        deadline = time.time() + task_timeout
+        deployment = _get_deployment()
+        work_dir, skill_md, task_md, prompt = _prepare_codex_workspace(
+            instruction=instruction,
+            input_xlsx=input_xlsx,
+            output_path=output_path,
+            instruction_type=instruction_type,
+            answer_position=answer_position,
+            skill_content=skill_content,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=diagnostic_trace_context,
+        )
+        remaining = max(10, int(deadline - time.time()))
+        effective_timeout = min(task_timeout, remaining)
+        final_message, raw = _run_exec_backend(
+            work_dir=work_dir,
+            prompt=prompt,
+            model=deployment,
+            timeout=effective_timeout,
+        )
+        solution_path = os.path.join(work_dir, "solution.py")
+        if os.path.exists(solution_path):
+            with open(solution_path, encoding="utf-8") as f:
+                code = f.read()
+        else:
+            code = extract_code(final_message or raw)
+        return {
+            "code": code,
+            "raw": raw or final_message,
+            "n_turns": 1,
+            "conversation": [{"role": "assistant", "content": final_message or raw}],
+            "student_system_prompt": skill_md,
+            "student_user_prompt": f"{prompt}\n\n## Task File\n\n{task_md}",
+        }
+
+    deadline = time.time() + task_timeout
+    client = get_student_client()
+    deployment = _get_deployment()
+    system = _build_system(skill_content)
+    user = _build_user(
+        instruction,
+        input_xlsx,
+        instruction_type,
+        answer_position,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+
+    remaining = max(10, int(deadline - time.time()))
+    effective_timeout = min(llm_timeout, remaining)
+    raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout)
+    time.sleep(3)  # Rate-limit cooldown after successful LLM call
+    code = extract_code(raw)
+
+    return {
+        "code": code,
+        "raw": raw,
+        "n_turns": 1,
+        "conversation": [{"role": "assistant", "content": raw}],
+        "student_system_prompt": system,
+        "student_user_prompt": user,
+    }
+
+
+def run_multi(
+    instruction: str,
+    input_xlsx: str,
+    output_path: str,
+    instruction_type: str = "",
+    answer_position: str = "",
+    skill_content: str = "",
+    max_turns: int = 5,
+    max_output_tokens: int = 16384,
+    llm_timeout: int = 120,
+    task_timeout: int = 600,
+    gold_path: str = "",
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Multi-round code generation with execution feedback. No tools.
+
+    Each round: LLM generates code → execute → if error, feed back and retry.
+
+    Args:
+        llm_timeout: Per-LLM-call timeout in seconds (default 120).
+        task_timeout: Total task timeout in seconds (default 600).
+        gold_path: Path to golden answer xlsx for eval feedback during
+            training.  When non-empty, a successful execution is followed
+            by an eval check; if the output is wrong the agent receives
+            cell-level feedback (without revealing expected values) and
+            gets another turn.  Leave empty for eval/test to avoid
+            data leakage.
+
+    Returns ``{"code": str, "raw": str, "n_turns": int, "conversation": [...]}``.
+    """
+    if is_student_exec_backend():
+        deadline = time.time() + task_timeout
+        deployment = _get_deployment()
+        work_dir, skill_md, task_md, initial_prompt = _prepare_codex_workspace(
+            instruction=instruction,
+            input_xlsx=input_xlsx,
+            output_path=output_path,
+            instruction_type=instruction_type,
+            answer_position=answer_position,
+            skill_content=skill_content,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=diagnostic_trace_context,
+            workspace_name="codex_multi",
+        )
+        prompt = (
+            f"{initial_prompt}\n\n"
+            "## Multi-Turn Repair Mode\n"
+            "- This is turn 1. Write or overwrite `solution.py`.\n"
+            "- After each turn, the harness will execute your `solution.py`; if it fails, you will receive feedback and may revise it.\n"
+            "- Keep the script general: use `INPUT_PATH` and `OUTPUT_PATH`, and do not hardcode one workbook's values."
+        )
+        conversation: list[dict] = []
+        code = ""
+        raw = ""
+        final_message = ""
+        solution_path = os.path.join(work_dir, "solution.py")
+
+        for turn in range(max_turns):
+            remaining = deadline - time.time()
+            if remaining <= 10:
+                break
+
+            effective_timeout = max(10, int(remaining))
+            final_message, raw = _run_exec_backend(
+                work_dir=work_dir,
+                prompt=prompt,
+                model=deployment,
+                timeout=effective_timeout,
+            )
+            conversation.append({"role": "assistant", "content": final_message or raw})
+
+            if os.path.exists(solution_path):
+                with open(solution_path, encoding="utf-8") as f:
+                    code = f.read()
+            else:
+                code = extract_code(final_message or raw)
+                if code.strip():
+                    with open(solution_path, "w", encoding="utf-8") as f:
+                        f.write(code)
+
+            if not code.strip():
+                feedback = (
+                    "No usable `solution.py` or Python code block was produced. "
+                    "Write a complete `solution.py` that reads `INPUT_PATH` and saves `OUTPUT_PATH`."
+                )
+            else:
+                ok, err = run_generated_code(code, input_xlsx, output_path)
+                if ok:
+                    if gold_path and answer_position:
+                        from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output
+                        eval_result = evaluate(
+                            output_path, gold_path, instruction_type, answer_position,
+                        )
+                        if eval_result["ok"]:
+                            break
+                        verify = _auto_verify_output(output_path, gold_path, answer_position)
+                        feedback = _build_eval_feedback(verify)
+                    else:
+                        break
+                else:
+                    feedback = (
+                        "The current `solution.py` raised an error during harness execution:\n\n"
+                        f"```\n{err[:3000]}\n```\n\n"
+                        "Revise `solution.py` to fix the error. Keep using `INPUT_PATH` and `OUTPUT_PATH`."
+                    )
+
+            feedback_path = os.path.join(work_dir, f"feedback_turn_{turn + 1:02d}.md")
+            with open(feedback_path, "w", encoding="utf-8") as f:
+                f.write(feedback)
+            conversation.append({"role": "user", "content": feedback})
+            prompt = (
+                f"The previous `solution.py` was evaluated and needs another revision.\n"
+                f"Read `{os.path.basename(feedback_path)}` and update `solution.py` accordingly.\n"
+                "You may run `python run_solution.py` for a local syntax/runtime check, but the harness will run the final code separately.\n"
+                "Do not hardcode workbook-specific answers; preserve unrelated cells."
+            )
+
+        return {
+            "code": code,
+            "raw": raw or final_message,
+            "n_turns": len([m for m in conversation if m["role"] == "assistant"]),
+            "conversation": conversation,
+            "student_system_prompt": skill_md,
+            "student_user_prompt": f"{initial_prompt}\n\n## Task File\n\n{task_md}",
+        }
+
+    deadline = time.time() + task_timeout
+    client = get_student_client()
+    deployment = _get_deployment()
+    system = _build_system(skill_content)
+    user = _build_user(
+        instruction,
+        input_xlsx,
+        instruction_type,
+        answer_position,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+
+    messages: list[dict] = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+    conversation: list[dict] = []
+    code = ""
+    raw = ""
+
+    for turn in range(max_turns):
+        remaining = deadline - time.time()
+        if remaining <= 10:
+            # Not enough time for another round
+            break
+
+        effective_timeout = min(llm_timeout, int(remaining))
+        raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout)
+        time.sleep(3)  # Rate-limit cooldown after successful LLM call
+        code = extract_code(raw)
+        conversation.append({"role": "assistant", "content": raw})
+        messages.append({"role": "assistant", "content": raw})
+
+        if not code.strip():
+            # No code extracted — ask again
+            feedback = (
+                "No Python code block was found in your response. "
+                "Please return a complete Python script inside a ```python``` block."
+            )
+            messages.append({"role": "user", "content": feedback})
+            conversation.append({"role": "user", "content": feedback})
+            continue
+
+        # Execute the code
+        ok, err = run_generated_code(code, input_xlsx, output_path)
+        if ok:
+            # Execution succeeded — check correctness if gold_path available
+            if gold_path and answer_position:
+                from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output
+                eval_result = evaluate(
+                    output_path, gold_path, instruction_type, answer_position,
+                )
+                if eval_result["ok"]:
+                    break  # Genuinely correct — stop
+
+                # Output is wrong — build feedback without leaking golden values
+                verify = _auto_verify_output(output_path, gold_path, answer_position)
+                feedback = _build_eval_feedback(verify)
+                messages.append({"role": "user", "content": feedback})
+                conversation.append({"role": "user", "content": feedback})
+                continue
+            else:
+                # No gold path (eval/test) — accept execution success
+                break
+
+        # Execution failed — feed error back
+        feedback = (
+            f"The code raised an error during execution:\n\n"
+            f"```\n{err[:3000]}\n```\n\n"
+            f"Please fix the code and return a complete corrected Python script "
+            f"inside a ```python``` block."
+        )
+        messages.append({"role": "user", "content": feedback})
+        conversation.append({"role": "user", "content": feedback})
+
+    return {
+        "code": code,
+        "raw": raw,
+        "n_turns": turn + 1,
+        "conversation": conversation,
+        "student_system_prompt": system,
+        "student_user_prompt": user,
+    }

skillopt/envs/spreadsheetbench/dataloader.py

@@ -0,0 +1,37 @@
+"""SpreadsheetBench task dataloader."""
+from __future__ import annotations
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+class SpreadsheetBenchDataLoader(SplitDataLoader):
+    """SpreadsheetBench dataloader.
+
+    Each split directory contains a .json file (JSON array of task items).
+    Spreadsheet files referenced by items live under a separate ``data_root``.
+    """
+
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        data_root: str = "",
+        seed: int = 42,
+        limit: int = 0,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+        self.data_root = data_root

skillopt/envs/spreadsheetbench/evaluator.py

@@ -0,0 +1,158 @@
+"""Cell-value evaluator faithful to the official SpreadsheetBench
+`evaluation/evaluation.py` (https://github.com/RUCKBReasoning/SpreadsheetBench).
+
+Key rules (copied from the official `transform_value` / `compare_cell_value`):
+  * numeric values (int/float and numeric strings) are compared after
+    ``round(float(v), 2)`` — a fixed 2-decimal quantization (NOT a tolerance);
+  * ``datetime.time`` is stringified and the trailing microseconds stripped;
+  * ``datetime.datetime`` is converted to an Excel serial day and rounded
+    to an integer day;
+  * an empty string ``""`` and ``None`` are considered equal, but otherwise
+    ``type(v1) != type(v2)`` fails the comparison.
+
+Format/style comparison is deliberately NOT performed — the official
+reference evaluator also skips it (the relevant lines are commented out
+in `cell_level_compare`). soft vs hard is defined at the run_bench level
+across a task's multiple test cases, not here.
+"""
+from __future__ import annotations
+
+import datetime
+import os
+import re
+
+import openpyxl
+
+
+# ---------- value transform / compare (official port) ----------
+
+def _datetime_to_float(dt: datetime.datetime) -> float:
+    excel_start_date = datetime.datetime(1899, 12, 30)
+    delta = dt - excel_start_date
+    return delta.days + delta.seconds / 86400.0
+
+
+def _transform_value(v):
+    if isinstance(v, bool):
+        # openpyxl can return Python bool; official code doesn't special-case
+        # bools, but round(float(True), 2) == 1.0 which breaks 1 vs True. Keep
+        # parity with the official transform by promoting bool -> float.
+        return round(float(v), 2)
+    if isinstance(v, (int, float)):
+        return round(float(v), 2)
+    if isinstance(v, datetime.time):
+        return str(v)[:-3]
+    if isinstance(v, datetime.datetime):
+        return round(_datetime_to_float(v), 0)
+    if isinstance(v, str):
+        try:
+            return round(float(v), 2)
+        except ValueError:
+            return v
+    return v
+
+
+def _compare_cell_value(v1, v2) -> bool:
+    v1 = _transform_value(v1)
+    v2 = _transform_value(v2)
+    if (v1 == "" and v2 is None) or (v1 is None and v2 == ""):
+        return True
+    if (v1 == "" and v2 == "") or (v1 is None and v2 is None):
+        return True
+    if type(v1) is not type(v2):
+        return False
+    return v1 == v2
+
+
+# ---------- range parsing (official port) ----------
+
+def _col_num2name(n: int) -> str:
+    name = ""
+    while n > 0:
+        n, r = divmod(n - 1, 26)
+        name = chr(65 + r) + name
+    return name
+
+
+def _col_name2num(name: str) -> int:
+    num = 0
+    for c in name:
+        num = num * 26 + (ord(c) - ord("A") + 1)
+    return num
+
+
+def _parse_range(range_str: str):
+    start_cell, end_cell = range_str.split(":")
+    sc = "".join(ch for ch in start_cell if ch.isalpha())
+    sr = "".join(ch for ch in start_cell if ch.isdigit())
+    ec = "".join(ch for ch in end_cell if ch.isalpha())
+    er = "".join(ch for ch in end_cell if ch.isdigit())
+    return (_col_name2num(sc), int(sr)), (_col_name2num(ec), int(er))
+
+
+def _generate_cell_names(range_str: str):
+    if ":" not in range_str:
+        return [range_str]
+    (sc, sr), (ec, er) = _parse_range(range_str)
+    cols = [_col_num2name(i) for i in range(sc, ec + 1)]
+    return [f"{c}{r}" for c in cols for r in range(sr, er + 1)]
+
+
+def _cell_level_compare(wb_gt, wb_proc, sheet_name: str, cell_range: str):
+    if sheet_name not in wb_proc.sheetnames:
+        return False, f"worksheet not found: {sheet_name}"
+    ws_gt = wb_gt[sheet_name]
+    ws_proc = wb_proc[sheet_name]
+    for cn in _generate_cell_names(cell_range):
+        cg = ws_gt[cn]
+        cp = ws_proc[cn]
+        if not _compare_cell_value(cg.value, cp.value):
+            return False, f"value@{sheet_name}!{cn}: gt={cg.value!r} pred={cp.value!r}"
+    return True, ""
+
+
+# ---------- public API ----------
+
+def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tuple[bool, str]:
+    """Return (ok, msg). Single test-case comparison, official semantics."""
+    if not os.path.exists(proc_file):
+        return False, "file not exist"
+    try:
+        wb_gt = openpyxl.load_workbook(filename=gt_file, data_only=True)
+        wb_proc = openpyxl.load_workbook(filename=proc_file, data_only=True)
+    except Exception as e:  # noqa: BLE001
+        return False, f"load error: {e}"
+    try:
+        ok_all = True
+        msg_first = ""
+        for scr in (answer_position or "").split(","):
+            scr = scr.strip()
+            if not scr:
+                continue
+            if "!" in scr:
+                sheet_name, cell_range = scr.split("!", 1)
+                sheet_name = sheet_name.strip().strip("'\"")
+            else:
+                sheet_name = wb_gt.sheetnames[0]
+                cell_range = scr
+            cell_range = cell_range.strip().strip("'\"")
+            ok, msg = _cell_level_compare(wb_gt, wb_proc, sheet_name, cell_range)
+            if not ok:
+                ok_all = False
+                if not msg_first:
+                    msg_first = msg
+        return ok_all, msg_first
+    finally:
+        wb_gt.close()
+        wb_proc.close()
+
+
+def evaluate(pred_path: str, gold_path: str,
+             instruction_type: str, answer_position: str) -> dict:
+    """Single test-case evaluate. soft/hard aggregation happens in run_bench."""
+    ok, msg = compare_workbooks(gold_path, pred_path, answer_position)
+    return {
+        "ok": ok,
+        "reason": msg,
+        "instruction_type": instruction_type,
+    }

skillopt/envs/spreadsheetbench/executor.py

@@ -0,0 +1,67 @@
+"""Execute LLM-generated Python code against an input xlsx to produce an output xlsx."""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import textwrap
+
+
+RUNNER_TEMPLATE = textwrap.dedent(
+    """
+    import os, sys, traceback
+    INPUT_PATH = {input_path!r}
+    OUTPUT_PATH = {output_path!r}
+    try:
+    {user_code_indented}
+    except Exception:
+        traceback.print_exc()
+        sys.exit(2)
+    """
+)
+
+# Regex to strip user-defined INPUT_PATH / OUTPUT_PATH assignments,
+# since the runner template injects the correct values.
+_PATH_ASSIGN_RE = re.compile(
+    r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE
+)
+
+
+def _strip_path_assignments(code: str) -> str:
+    """Remove INPUT_PATH/OUTPUT_PATH assignments from user code."""
+    return _PATH_ASSIGN_RE.sub("", code)
+
+
+def run_generated_code(code: str, input_path: str, output_path: str, timeout: int = 120) -> tuple[bool, str]:
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    cleaned = _strip_path_assignments(code)
+    indented = textwrap.indent(cleaned, "    ")
+    script = RUNNER_TEMPLATE.format(
+        input_path=input_path,
+        output_path=output_path,
+        user_code_indented=indented,
+    )
+    with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
+        f.write(script)
+        tmp = f.name
+    try:
+        proc = subprocess.run(
+            [sys.executable, tmp],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        if proc.returncode != 0:
+            return False, (proc.stdout + "\n" + proc.stderr).strip()
+        if not os.path.exists(output_path):
+            return False, "output file was not created"
+        return True, ""
+    except subprocess.TimeoutExpired:
+        return False, f"timeout after {timeout}s"
+    finally:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass

skillopt/envs/spreadsheetbench/prompts/analyst_error.md

@@ -0,0 +1,46 @@
+You are an expert failure-analysis agent for spreadsheet manipulation tasks.
+
+You will be given MULTIPLE failed agent trajectories from a single minibatch
+and the current skill document.
+Your job is to identify the most important COMMON failure patterns across
+the batch and propose a concise set of skill edits.
+
+## Failure Type Categories
+- **rule_missing**: the skill lacks a relevant rule for this type of task
+- **rule_wrong**: an existing skill rule is misleading or incorrect
+- **rule_ignored**: the skill has the right rule but the agent did not follow it
+- **data_exploration**: the agent did not read enough data from the spreadsheet
+- **code_error**: the agent's code has a bug unrelated to the skill
+- **other**: none of the above
+
+## Analysis Process
+1. Read ALL failed trajectories in the minibatch.
+2. Identify the most prevalent, systematic failure patterns across them.
+3. For each pattern, classify its failure type.
+4. Propose skill edits that address the COMMON patterns — not individual edge cases.
+5. Edits must be generalizable; do not hardcode task-specific values
+   (file paths, cell addresses, expected values).
+6. Only patch gaps in the skill — do not duplicate existing content.
+7. If the failure is because the agent did not read enough spreadsheet rows/columns
+   to understand the data, propose a patch encouraging broader data exploration.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the highest-impact patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object (no markdown fences, no extra text):
+{
+  "batch_size": <number of trajectories analysed>,
+  "failure_summary": [
+    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
+  ],
+  "patch": {
+    "reasoning": "<why these edits address the batch's common failures>",
+    "edits": [
+      {"op": "append",       "content": "<markdown to add at end of skill>"},
+      {"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<exact text to replace>",              "content": "<replacement>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+Only include edits that are needed. "edits" can be an empty list if no patch is warranted.

skillopt/envs/spreadsheetbench/prompts/analyst_success.md

@@ -0,0 +1,32 @@
+You are an expert success-pattern analyst for AI spreadsheet agents.
+
+You will be given MULTIPLE successful agent trajectories from a single minibatch
+and the current skill document. Your job is to identify generalizable behavior
+patterns that are COMMON across the batch and worth encoding in the skill.
+
+## Rules
+- Only propose patches for patterns NOT already covered in the skill.
+- Focus on patterns that appear across MULTIPLE trajectories in the batch.
+- Be concise. Patterns must generalize beyond specific tasks.
+- Prefer reinforcing existing sections over adding new top-level sections.
+- If the agents' success involved reading enough data rows or using a smart
+  exploration strategy, consider reinforcing that in the patch.
+
+You will be told the maximum number of edits (the budget L). Produce AT MOST L edits,
+focusing on the most broadly applicable patterns. You may produce fewer if warranted.
+
+Respond ONLY with a valid JSON object:
+{
+  "batch_size": <number of trajectories analysed>,
+  "success_patterns": ["<pattern 1>", "<pattern 2>"],
+  "patch": {
+    "reasoning": "<why these patterns are worth encoding>",
+    "edits": [
+      {"op": "append",       "content": "<markdown>"},
+      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+"edits" may be empty if the skill already covers all observed patterns.

skillopt/envs/spreadsheetbench/prompts/codegen_system.md

@@ -0,0 +1 @@
+You are an expert Python programmer specializing in spreadsheet manipulation. You will be given a user instruction together with a preview of an input .xlsx file. Your job is to write a single self-contained Python script that reads the input file at the path stored in the variable INPUT_PATH, performs the requested manipulation, and saves the result to OUTPUT_PATH. Use only the standard library, openpyxl, and pandas. Do not print anything. Do not use input(). Do not hardcode file paths. Return ONLY the Python code inside a single ```python ... ``` fenced block.

skillopt/envs/spreadsheetbench/prompts/critical_rules.md

@@ -0,0 +1,9 @@
+## Critical Rules (MUST follow)
+1. NEVER write Excel formulas to cells that will be graded on their displayed value.
+   openpyxl does NOT compute formulas -- the evaluator will see None.
+   Instead, compute results in Python and write literal values (numbers/strings).
+2. After saving the workbook, ALWAYS reopen and verify the written values:
+   `wb2 = openpyxl.load_workbook(OUTPUT_PATH); print(wb2[sheet][cell].value)`
+3. Use the `write_file` tool to create solution.py -- it avoids shell escaping issues.
+   Do NOT use `echo "..." > solution.py` for multi-line scripts.
+

skillopt/envs/spreadsheetbench/prompts/deep_probe.md

@@ -0,0 +1,35 @@
+You are an expert diagnostic-probe designer for spreadsheet manipulation tasks.
+
+You will design one short diagnostic instruction to append to the student's
+existing SpreadsheetBench prompt for a handful of representative trajectories.
+
+The goal is to expose whether the student already knows the right task
+decomposition, source range, target range, and transformation rule without
+substantially changing the current scaffold.
+
+## Hard Constraints
+1. Do NOT substantially change the student's current scaffold.
+2. Do NOT prescribe a brand-new full algorithm.
+3. Do NOT ask for exhaustive cell-by-cell enumeration.
+4. Keep the diagnostic readout brief and structured.
+5. The student must still complete the original spreadsheet task.
+6. Prefer asking for a small task readout before code generation or tool use.
+7. Never ask for hidden reference content or golden values.
+
+## Good Probe Targets
+- task family: filter / sort / dedup / lookup / aggregate / reshape
+- source sheet/range and target sheet/range
+- decisive grouping / matching / sorting key
+- one or two representative cells or rows and how they should be derived
+- whether the solution must be dynamic rather than hardcoded
+
+## Bad Probe Targets
+- full derivation of every output cell
+- dumping all rows or all formulas
+- imposing a long new checklist that was not already implicit
+
+Respond ONLY with a valid JSON object:
+{
+  "reasoning": "<why this probe reveals the latent skill gap>",
+  "probe_instruction": "<the exact instruction text to append to the student prompt>"
+}

skillopt/envs/spreadsheetbench/prompts/react_system.md

@@ -0,0 +1,21 @@
+You are an expert spreadsheet manipulation agent.
+
+{critical_rules}{skill_section}## Tools
+You have two tools:
+- `bash` -- execute any shell command and receive its output.
+- `write_file` -- write content to a file (path, content). Use this for solution.py.
+
+## Protocol
+1. Explore the input spreadsheet to understand its structure (sheets, headers, row count).
+2. Use the `write_file` tool to create `solution.py` in the current directory.
+   solution.py MUST start with:
+       INPUT_PATH  = "<exact input path given in the task>"
+       OUTPUT_PATH = "<exact output path given in the task>"
+   Then perform the manipulation and save the result to OUTPUT_PATH.
+   Use only: standard library, openpyxl, pandas.
+3. Run `python solution.py` via `bash` and verify the output was created.
+4. Fix any errors and re-run until the output is correct.
+5. Once OUTPUT_PATH exists and is correct, stop calling tools.
+
+Do NOT use any libraries other than standard library, openpyxl, and pandas.
+Do NOT hardcode cell values from the preview -- iterate over actual rows.

skillopt/envs/spreadsheetbench/react_agent.py

@@ -0,0 +1,395 @@
+"""ReAct agent with bash tool for SpreadsheetBench evaluation.
+
+Adapted from workspace-yqh/refleAct/spreadsheetbench/src/react_agent.py.
+
+Uses the unified ``skillopt.model`` router so SpreadsheetBench follows the same
+backend selection as the rest of the framework.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+
+from skillopt.model import chat_student_messages
+from skillopt.prompts import load_prompt
+
+# ── Tool schemas ─────────────────────────────────────────────────────────────
+
+BASH_TOOL_CHAT = {
+    "type": "function",
+    "function": {
+        "name": "bash",
+        "description": (
+            "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). "
+            "Use Python to read / write Excel files."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "cmd": {"type": "string", "description": "Bash command to execute."}
+            },
+            "required": ["cmd"],
+        },
+    },
+}
+
+BASH_TOOL_RESPONSES = {
+    "type": "function",
+    "name": "bash",
+    "description": (
+        "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). "
+        "Use Python to read / write Excel files."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "cmd": {"type": "string", "description": "Bash command to execute."}
+        },
+        "required": ["cmd"],
+    },
+}
+
+WRITE_FILE_TOOL_CHAT = {
+    "type": "function",
+    "function": {
+        "name": "write_file",
+        "description": (
+            "Write content to a file. Use this instead of echo/cat for multi-line "
+            "Python scripts to avoid shell escaping issues."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "description": "File path to write (relative to working directory).",
+                },
+                "content": {
+                    "type": "string",
+                    "description": "File content to write.",
+                },
+            },
+            "required": ["path", "content"],
+        },
+    },
+}
+
+WRITE_FILE_TOOL_RESPONSES = {
+    "type": "function",
+    "name": "write_file",
+    "description": (
+        "Write content to a file. Use this instead of echo/cat for multi-line "
+        "Python scripts to avoid shell escaping issues."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "path": {
+                "type": "string",
+                "description": "File path to write (relative to working directory).",
+            },
+            "content": {
+                "type": "string",
+                "description": "File content to write.",
+            },
+        },
+        "required": ["path", "content"],
+    },
+}
+
+# ── System prompt ─────────────────────────────────────────────────────────────
+
+
+def _build_system(skill_content: str) -> str:
+    if skill_content.strip():
+        skill_section = f"## Skill\n{skill_content.strip()}\n\n"
+    else:
+        skill_section = ""
+    return load_prompt("react_system", env="spreadsheetbench").format(
+        critical_rules=load_prompt("critical_rules", env="spreadsheetbench"),
+        skill_section=skill_section,
+    )
+
+
+def _build_user(
+    instruction: str,
+    input_path: str,
+    output_path: str,
+    instruction_type: str,
+    answer_position: str,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> str:
+    parts = []
+    if diagnostic_trace_context.strip():
+        parts.append(
+            "# Previous Codex Trace Snapshot\n"
+            "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
+            f"{diagnostic_trace_context.strip()}"
+        )
+    parts.extend([
+        f"# Instruction\n{instruction}",
+        f"# Input file\n{input_path}",
+        f"# Output file\n{output_path}",
+    ])
+    if instruction_type:
+        parts.append(f"# Instruction type\n{instruction_type}")
+    if answer_position:
+        parts.append(f"# Answer position\n{answer_position}")
+    if diagnostic_mode and diagnostic_instruction.strip():
+        parts.append(f"# Training readout\n{diagnostic_instruction.strip()}")
+    parts.append(
+        "Manipulate the input spreadsheet according to the instruction "
+        "and save the result to the output file."
+    )
+    return "\n\n".join(parts)
+
+
+# ── File write (bypass shell escaping) ────────────────────────────────────────
+
+def _write_file(path: str, content: str, work_dir: str) -> str:
+    """Write content to a file, bypassing shell escaping issues."""
+    try:
+        full_path = os.path.join(work_dir, path) if not os.path.isabs(path) else path
+        parent = os.path.dirname(full_path)
+        if parent:
+            os.makedirs(parent, exist_ok=True)
+        with open(full_path, "w") as f:
+            f.write(content)
+        return f"File written: {full_path} ({len(content)} chars)"
+    except Exception as e:  # noqa: BLE001
+        return f"[write_file error: {e}]"
+
+
+# ── Auto-verification ─────────────────────────────────────────────────────────
+
+def _auto_verify(work_dir: str) -> str:
+    """Auto-verify output xlsx after solution.py runs."""
+    import glob as _glob
+
+    sol_path = os.path.join(work_dir, "solution.py")
+    output_path = None
+    if os.path.exists(sol_path):
+        with open(sol_path) as f:
+            for line in f:
+                stripped = line.strip()
+                if stripped.startswith("OUTPUT_PATH"):
+                    try:
+                        val = stripped.split("=", 1)[1].strip()
+                        output_path = val.strip("'\"").strip()
+                    except Exception:  # noqa: BLE001
+                        pass
+                    break
+
+    if not output_path or not os.path.exists(output_path):
+        xlsx_files = [
+            f for f in _glob.glob(os.path.join(work_dir, "*.xlsx"))
+            if "_pred" in os.path.basename(f)
+        ]
+        if xlsx_files:
+            output_path = xlsx_files[0]
+
+    if not output_path or not os.path.exists(output_path):
+        return (
+            "\n\n[AUTO-VERIFY] WARNING: Output file not found! "
+            "Make sure OUTPUT_PATH is correct and wb.save(OUTPUT_PATH) is called."
+        )
+
+    try:
+        import openpyxl
+
+        wb_formula = openpyxl.load_workbook(output_path, data_only=False)
+        wb_value = openpyxl.load_workbook(output_path, data_only=True)
+        lines = [f"\n\n[AUTO-VERIFY] Output file exists: {output_path}"]
+
+        sn = wb_formula.sheetnames[0]
+        ws_f = wb_formula[sn]
+        ws_v = wb_value[sn]
+        lines.append(f"  Sheet '{sn}': {ws_f.dimensions}")
+
+        for row in ws_v.iter_rows(
+            min_row=1, max_row=min(5, ws_v.max_row), values_only=True,
+        ):
+            lines.append(f"    {list(row)}")
+
+        none_cells: list[str] = []
+        for row_f, row_v in zip(
+            ws_f.iter_rows(min_row=1, max_row=min(30, ws_f.max_row)),
+            ws_v.iter_rows(min_row=1, max_row=min(30, ws_v.max_row)),
+        ):
+            for cf, cv in zip(row_f, row_v):
+                formula_val = cf.value
+                cached_val = cv.value
+                if (
+                    isinstance(formula_val, str)
+                    and formula_val.startswith("=")
+                    and cached_val is None
+                ):
+                    none_cells.append(cf.coordinate)
+
+        if none_cells:
+            lines.append(
+                f"  WARNING: {len(none_cells)} cells have formulas but NO cached "
+                f"value -- evaluator will see None: {none_cells[:10]}"
+            )
+            lines.append(
+                "  FIX: Compute values in Python and write literal "
+                "numbers/strings instead of formulas."
+            )
+        else:
+            lines.append("  All cells have concrete values. Looks good.")
+
+        wb_formula.close()
+        wb_value.close()
+        return "\n".join(lines)
+    except Exception as e:  # noqa: BLE001
+        return f"\n\n[AUTO-VERIFY] Could not inspect output: {e}"
+
+
+# ── Bash execution ────────────────────────────────────────────────────────────
+
+def _run_bash(cmd: str, work_dir: str, timeout: int = 60) -> str:
+    try:
+        proc = subprocess.run(
+            cmd,
+            shell=True,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            cwd=work_dir,
+        )
+        out = (proc.stdout + proc.stderr).strip()
+    except subprocess.TimeoutExpired:
+        return f"[timeout after {timeout}s]"
+    except Exception as e:  # noqa: BLE001
+        return f"[error: {e}]"
+    if len(out) > 4000:
+        out = out[:3800] + f"\n...[truncated, {len(out)} total chars]"
+    result = out or "(no output)"
+
+    if "solution.py" in cmd and "python" in cmd.lower():
+        result += _auto_verify(work_dir)
+
+    return result
+
+
+def _assistant_tool_calls(message) -> list[dict]:
+    tool_calls = getattr(message, "tool_calls", None) or []
+    return [
+        tool_call.model_dump() if hasattr(tool_call, "model_dump") else dict(tool_call)
+        for tool_call in tool_calls
+    ]
+
+
+def _react_loop(
+    system: str,
+    user: str,
+    work_dir: str,
+    max_turns: int,
+    max_output_tokens: int,
+) -> dict:
+    messages: list[dict] = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+    conversation: list[dict] = []
+    n_turns = 0
+
+    for _ in range(max_turns):
+        message, _ = chat_student_messages(
+            messages=messages,
+            tools=[BASH_TOOL_CHAT, WRITE_FILE_TOOL_CHAT],
+            tool_choice="auto",
+            max_completion_tokens=max_output_tokens,
+            retries=5,
+            stage="rollout",
+            return_message=True,
+        )
+
+        assistant_text = str(getattr(message, "content", "") or "")
+        tool_calls = _assistant_tool_calls(message)
+        assistant_payload: dict = {"role": "assistant", "content": assistant_text}
+        if tool_calls:
+            assistant_payload["tool_calls"] = tool_calls
+        messages.append(assistant_payload)
+
+        if not tool_calls:
+            conversation.append({"type": "message", "content": assistant_text})
+            break
+
+        for tool_call in tool_calls:
+            n_turns += 1
+            function = tool_call.get("function", {}) or {}
+            try:
+                args = json.loads(str(function.get("arguments", "{}") or "{}"))
+            except json.JSONDecodeError:
+                args = {}
+
+            if function.get("name") == "write_file":
+                obs = _write_file(
+                    args.get("path", ""),
+                    args.get("content", ""),
+                    work_dir,
+                )
+                conversation.append({
+                    "type": "tool_call",
+                    "cmd": f"[write_file] {args.get('path', '')}",
+                    "obs": obs,
+                })
+            else:
+                cmd = args.get("cmd", "")
+                obs = _run_bash(cmd, work_dir)
+                conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs})
+
+            messages.append(
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call.get("id", ""),
+                    "content": obs,
+                }
+            )
+
+    return {"conversation": conversation, "n_turns": n_turns}
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+def run_react(
+    instruction: str,
+    input_path: str,
+    output_path: str,
+    work_dir: str,
+    instruction_type: str = "",
+    answer_position: str = "",
+    skill_content: str = "",
+    max_turns: int = 30,
+    max_output_tokens: int = 4096,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Run the ReAct agent for one task.
+
+    Returns:
+        {
+          "conversation": [...],  # list of {type, cmd/content, obs?}
+          "n_turns": int,         # number of bash tool calls made
+        }
+    """
+    system = _build_system(skill_content)
+    user = _build_user(
+        instruction,
+        input_path,
+        output_path,
+        instruction_type,
+        answer_position,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        diagnostic_trace_context=diagnostic_trace_context,
+    )
+    result = _react_loop(system, user, work_dir, max_turns, max_output_tokens)
+    result["student_system_prompt"] = system
+    result["student_user_prompt"] = user
+    return result

skillopt/envs/spreadsheetbench/reflect.py

@@ -0,0 +1,4 @@
+"""SpreadsheetBench Reflect stage.
+
+Prompts are now loaded from .md files by the base adapter.
+"""

skillopt/envs/spreadsheetbench/rollout.py

@@ -0,0 +1,921 @@
+"""SpreadsheetBench rollout — codegen & ReAct batch execution.
+
+Provides:
+  - process_one_codegen(): single/multi-round code generation (no tool-call)
+  - run_spreadsheet_batch_codegen(): batch wrapper for codegen
+  - process_one(): ReAct agent with tool-call (legacy)
+  - run_spreadsheet_batch(): batch wrapper for ReAct (legacy)
+  - load_items(): load benchmark .json/.jsonl files
+"""
+from __future__ import annotations
+
+import glob as _glob
+import json
+import os
+import shutil
+import tempfile
+import time
+import traceback
+from concurrent.futures import (
+    FIRST_COMPLETED,
+    ThreadPoolExecutor,
+    wait,
+    TimeoutError as FuturesTimeoutError,
+)
+
+import openpyxl
+
+from skillopt.envs.spreadsheetbench.react_agent import run_react
+from skillopt.envs.spreadsheetbench.evaluator import evaluate, _generate_cell_names
+from skillopt.envs.spreadsheetbench.executor import run_generated_code
+
+
+# ── Data loading ─────────────────────────────────────────────────────────────
+
+
+def load_items(path: str) -> list[dict]:
+    """Load a benchmark file. Supports both .jsonl and .json (list of dicts)."""
+    if path.endswith(".json"):
+        with open(path) as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            data = data.get("data") or list(data.values())
+        return list(data)
+    items = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                items.append(json.loads(line))
+    return items
+
+
+# ── Test case discovery ──────────────────────────────────────────────────────
+
+
+def _find_test_cases(task_dir: str) -> list[tuple[str, str, str]]:
+    """Return [(case_no, input_path, answer_path), ...] sorted by case_no.
+
+    Supports naming conventions used by SpreadsheetBench releases:
+      * ``{no}_{id}_input.xlsx``  + ``{no}_{id}_answer.xlsx``  (original)
+      * ``{no}_{id}_init.xlsx``   + ``{no}_{id}_golden.xlsx``  (verified_400)
+      * ``initial.xlsx``          + ``golden.xlsx``             (verified_400, no prefix)
+    """
+    cases: list[tuple[str, str, str]] = []
+    inputs = sorted(_glob.glob(os.path.join(task_dir, "*_input.xlsx")))
+    for ip in inputs:
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_input.xlsx", "_answer.xlsx")
+        if os.path.exists(ap):
+            cases.append((no, ip, ap))
+    inits = sorted(_glob.glob(os.path.join(task_dir, "*_init.xlsx")))
+    for ip in inits:
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_init.xlsx", "_golden.xlsx")
+        if os.path.exists(ap):
+            cases.append((no, ip, ap))
+
+    # Fallback: bare initial.xlsx + golden.xlsx (no numbered prefix)
+    if not cases:
+        bare_init = os.path.join(task_dir, "initial.xlsx")
+        bare_gold = os.path.join(task_dir, "golden.xlsx")
+        if os.path.exists(bare_init) and os.path.exists(bare_gold):
+            cases.append(("1", bare_init, bare_gold))
+
+    return cases
+
+
+# ── Auto-verify helper ──────────────────────────────────────────────────────
+
+
+def _auto_verify_output(
+    pred_path: str,
+    gold_path: str,
+    answer_position: str,
+) -> str:
+    """Reopen the predicted xlsx and compare cells at answer_position with gold.
+
+    Returns a human-readable verification report that can be appended to the
+    trajectory so the error analyst can see exactly what went wrong (e.g.
+    ``cell A1: got=None, expected=420``).
+    """
+    if not os.path.exists(pred_path):
+        return "Verification: output file does not exist."
+    try:
+        wb_pred = openpyxl.load_workbook(pred_path, data_only=True)
+        wb_gold = openpyxl.load_workbook(gold_path, data_only=True)
+    except Exception as e:
+        return f"Verification: could not open workbooks: {e}"
+
+    lines = ["## Output Verification"]
+    try:
+        for scr in (answer_position or "").split(","):
+            scr = scr.strip()
+            if not scr:
+                continue
+            if "!" in scr:
+                sheet_name, cell_range = scr.split("!", 1)
+                sheet_name = sheet_name.strip().strip("'\"")
+            else:
+                sheet_name = wb_gold.sheetnames[0]
+                cell_range = scr
+            cell_range = cell_range.strip().strip("'\"")
+
+            cell_names = _generate_cell_names(cell_range)
+            ws_pred = wb_pred[sheet_name] if sheet_name in wb_pred.sheetnames else None
+            ws_gold = wb_gold[sheet_name] if sheet_name in wb_gold.sheetnames else None
+
+            if ws_pred is None:
+                lines.append(f"  Sheet '{sheet_name}' NOT FOUND in output.")
+                continue
+
+            for cn in cell_names:
+                gv = ws_gold[cn].value if ws_gold else "N/A"
+                pv = ws_pred[cn].value
+                match = "✓" if repr(gv) == repr(pv) else "✗"
+                lines.append(f"  {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}")
+
+        # Also check if any cells in the output contain formula strings
+        formula_cells = []
+        for sn in wb_pred.sheetnames:
+            ws = wb_pred[sn]
+            for row in ws.iter_rows(max_row=min(ws.max_row, 200), values_only=False):
+                for cell in row:
+                    if isinstance(cell.value, str) and cell.value.startswith("="):
+                        formula_cells.append(f"{sn}!{cell.coordinate}={cell.value}")
+                        if len(formula_cells) >= 10:
+                            break
+                if len(formula_cells) >= 10:
+                    break
+            if len(formula_cells) >= 10:
+                break
+        if formula_cells:
+            lines.append(f"\n  WARNING: {len(formula_cells)} cells contain Excel formulas (openpyxl cannot evaluate them):")
+            for fc in formula_cells[:5]:
+                lines.append(f"    {fc}")
+            if len(formula_cells) > 5:
+                lines.append(f"    ... and {len(formula_cells) - 5} more")
+    finally:
+        wb_pred.close()
+        wb_gold.close()
+
+    return "\n".join(lines)
+
+
+# ── Per-task worker ──────────────────────────────────────────────────────────
+
+
+def process_one(
+    item: dict,
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    max_turns: int,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Run the ReAct agent on a single SpreadsheetBench task.
+
+    Returns a result dict compatible with ``compute_score()``.
+    """
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position_eval = f"{answer_sheet}!{answer_position}"
+    else:
+        answer_position_eval = answer_position
+
+    # Determine task_type from instruction_type
+    itype_lower = (instruction_type or "").lower()
+    if "cell" in itype_lower:
+        task_type = "cell_level"
+    elif "sheet" in itype_lower:
+        task_type = "sheet_level"
+    else:
+        task_type = "other"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {
+        "id": task_id,
+        "ok": False,
+        "instruction_type": instruction_type,
+        "task_type": task_type,
+        "task_description": instruction,
+        "phase": "setup",
+        "fail_reason": "",
+        "agent_ok": False,
+        "exec_ok": False,
+        "n_cases": 0,
+        "n_exec_pass": 0,
+        "n_pass": 0,
+        "soft": 0.0,
+        "hard": 0,
+        "n_turns": 0,
+        "cases": [],
+        "error": "",
+    }
+
+    try:
+        cases = _find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out_dir = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out_dir, exist_ok=True)
+
+        no1, ip1, _ = cases[0]
+        pred_path_1 = os.path.join(task_out_dir, f"{no1}_pred.xlsx")
+        student_prompt_parts = [
+            f"# Instruction\n{instruction}",
+            f"# Input file\n{ip1}",
+            f"# Output file\n{pred_path_1}",
+        ]
+        if instruction_type:
+            student_prompt_parts.append(f"# Instruction type\n{instruction_type}")
+        if answer_position_eval:
+            student_prompt_parts.append(f"# Answer position\n{answer_position_eval}")
+        if diagnostic_trace_context.strip():
+            student_prompt_parts.insert(
+                0,
+                "# Previous Codex Trace Snapshot\n"
+                "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
+                f"{diagnostic_trace_context.strip()}",
+            )
+        if diagnostic_mode and diagnostic_instruction.strip():
+            student_prompt_parts.append(f"# Training readout\n{diagnostic_instruction.strip()}")
+        student_user_prompt = "\n\n".join(student_prompt_parts)
+        try:
+            from skillopt.envs.spreadsheetbench.react_agent import _build_system
+            student_system_prompt = _build_system(skill_content)
+        except Exception:
+            student_system_prompt = ""
+        if student_system_prompt:
+            with open(os.path.join(task_out_dir, "student_system_prompt.txt"), "w") as f:
+                f.write(student_system_prompt)
+            result["student_system_prompt"] = student_system_prompt
+        with open(os.path.join(task_out_dir, "student_user_prompt.txt"), "w") as f:
+            f.write(student_user_prompt)
+        result["student_user_prompt"] = student_user_prompt
+
+        # ── Stage 1: run ReAct agent on test case 1 ─────────────────────
+        result["phase"] = "agent"
+
+        work_dir = tempfile.mkdtemp(prefix=f"react_{task_id}_")
+        try:
+            # Copy input so agent works in an isolated directory
+            work_input = os.path.join(work_dir, os.path.basename(ip1))
+            shutil.copy2(ip1, work_input)
+
+            agent_result = run_react(
+                instruction=instruction,
+                input_path=work_input,
+                output_path=pred_path_1,
+                work_dir=work_dir,
+                instruction_type=instruction_type,
+                answer_position=answer_position_eval,
+                skill_content=skill_content,
+                max_turns=max_turns,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+                diagnostic_trace_context=diagnostic_trace_context,
+            )
+            result["n_turns"] = agent_result.get("n_turns", 0)
+            if agent_result.get("student_system_prompt"):
+                with open(os.path.join(task_out_dir, "student_system_prompt.txt"), "w") as f:
+                    f.write(agent_result["student_system_prompt"])
+                result["student_system_prompt"] = agent_result["student_system_prompt"]
+            if agent_result.get("student_user_prompt"):
+                with open(os.path.join(task_out_dir, "student_user_prompt.txt"), "w") as f:
+                    f.write(agent_result["student_user_prompt"])
+                result["student_user_prompt"] = agent_result["student_user_prompt"]
+
+            # Save conversation log
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(
+                    agent_result.get("conversation", []),
+                    f, ensure_ascii=False, indent=2,
+                )
+
+            # Copy solution.py if the agent wrote one
+            solution_src = os.path.join(work_dir, "solution.py")
+            solution_dst = os.path.join(task_out_dir, "solution.py")
+            if os.path.exists(solution_src):
+                shutil.copy2(solution_src, solution_dst)
+
+        except Exception as e:
+            result["fail_reason"] = f"agent-error: {type(e).__name__}: {e}"
+            result["error"] = traceback.format_exc()
+            return result
+        finally:
+            shutil.rmtree(work_dir, ignore_errors=True)
+
+        result["agent_ok"] = True
+
+        # ── Stage 2: evaluate all test cases ─────────────────────────────
+        result["phase"] = "eval"
+        solution_path = os.path.join(task_out_dir, "solution.py")
+        all_exec = True
+
+        for i, (no, ip, ap) in enumerate(cases):
+            pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx")
+
+            if i > 0:
+                # Re-apply solution.py to subsequent test cases
+                if not os.path.exists(solution_path):
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": "no-solution-py"}
+                    )
+                    if not result["fail_reason"]:
+                        result["fail_reason"] = "no-solution-py-for-other-cases"
+                    continue
+
+                with open(solution_path) as f:
+                    code = f.read()
+
+                # Prepend new INPUT_PATH / OUTPUT_PATH
+                preamble = (
+                    f"INPUT_PATH  = {ip!r}\n"
+                    f"OUTPUT_PATH = {pred_path!r}\n"
+                )
+                full_code = preamble + code
+
+                ok_exec, err = run_generated_code(full_code, ip, pred_path)
+                if not ok_exec:
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": err[:500]}
+                    )
+                    if not result["fail_reason"]:
+                        tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown"
+                        result["fail_reason"] = f"exec-error: {tail}"
+                    continue
+
+            # ── Evaluate ─────────────────────────────────────────────────
+            if not os.path.exists(pred_path):
+                all_exec = False
+                result["cases"].append(
+                    {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"}
+                )
+                if not result["fail_reason"]:
+                    result["fail_reason"] = "output-not-found"
+                continue
+
+            result["n_exec_pass"] += 1
+            try:
+                ev = evaluate(pred_path, ap, instruction_type, answer_position_eval)
+            except Exception as e:  # noqa: BLE001
+                ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"}
+
+            if ev["ok"]:
+                result["n_pass"] += 1
+            else:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}"
+            result["cases"].append(
+                {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")}
+            )
+
+        result["exec_ok"] = all_exec
+        n_cases = result["n_cases"]
+        n_pass = result["n_pass"]
+        result["soft"] = (n_pass / n_cases) if n_cases else 0.0
+        result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]:
+            result["fail_reason"] = ""
+        return result
+
+    except Exception as e:  # noqa: BLE001
+        result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Batch runner ─────────────────────────────────────────────────────────────
+
+
+def run_spreadsheet_batch(
+    items: list[dict],
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    max_turns: int = 30,
+    max_api_workers: int = 64,
+    task_timeout: int = 600,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context_by_id: dict[str, str] | None = None,
+) -> list[dict]:
+    """Run the ReAct agent on all items with ThreadPoolExecutor.
+
+    Returns list of result dicts compatible with ``compute_score()``.
+    """
+    os.makedirs(out_root, exist_ok=True)
+
+    # Check for already-done items (resume support)
+    results_path = os.path.join(out_root, "results.jsonl")
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path) as f:
+            for line in f:
+                try:
+                    r = json.loads(line)
+                    done_ids.add(str(r["id"]))
+                    existing.append(r)
+                except Exception:
+                    pass
+
+    pending = [it for it in items if str(it["id"]) not in done_ids]
+    print(
+        f"  [spreadsheet rollout] total={len(items)} done={len(done_ids)} "
+        f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout}s"
+    )
+
+    if not pending:
+        return existing
+
+    t0 = time.time()
+    results = list(existing)
+    started_at: dict[str, float] = {}
+
+    def _timeout_result(item: dict) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "phase": "timeout",
+            "fail_reason": f"task-timeout-{task_timeout}s",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": "timeout",
+        }
+
+    def _error_result(item: dict, exc: Exception) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "phase": "error",
+            "fail_reason": f"unexpected: {type(exc).__name__}: {exc}",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": str(exc),
+        }
+
+    def _run_one(it: dict) -> dict:
+        started_at[str(it["id"])] = time.time()
+        return process_one(
+            it,
+            data_root,
+            out_root,
+            skill_content,
+            max_turns,
+            diagnostic_mode,
+            diagnostic_instruction,
+            (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
+        )
+
+    ex = ThreadPoolExecutor(max_workers=max_api_workers)
+    try:
+        futs = {ex.submit(_run_one, it): it for it in pending}
+        pending_futs = set(futs)
+        finished = 0
+        while pending_futs:
+            done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
+            now = time.time()
+            timed_out = [
+                fut for fut in pending_futs - done
+                if str(futs[fut]["id"]) in started_at
+                and now - started_at[str(futs[fut]["id"])] >= task_timeout
+            ]
+            for fut in done:
+                pending_futs.remove(fut)
+                item = futs[fut]
+                try:
+                    res = fut.result()
+                except FuturesTimeoutError:
+                    res = _timeout_result(item)
+                except Exception as e:  # noqa: BLE001
+                    res = _error_result(item, e)
+                results.append(res)
+                finished += 1
+                status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL")
+                dt = time.time() - t0
+                print(
+                    f"    {finished}/{len(pending)} id={res['id']:<10} {status}  "
+                    f"turns={res.get('n_turns', 0):<3} "
+                    f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+                    f"dt={dt:.0f}s"
+                )
+            for fut in timed_out:
+                pending_futs.remove(fut)
+                res = _timeout_result(futs[fut])
+                results.append(res)
+                finished += 1
+                status = "TIMEOUT"
+                dt = time.time() - t0
+                print(
+                    f"    {finished}/{len(pending)} id={res['id']:<10} {status}  "
+                    f"turns={res.get('n_turns', 0):<3} "
+                    f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+                    f"dt={dt:.0f}s"
+                )
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
+
+    return results
+
+
+# ── Codegen per-task worker (no tool-call) ──────────────────────────────────
+
+
+def process_one_codegen(
+    item: dict,
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    mode: str = "single",
+    max_turns: int = 5,
+    use_eval_feedback: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context: str = "",
+) -> dict:
+    """Run codegen agent (single or multi-round) on one SpreadsheetBench task.
+
+    This matches the official evaluation setting: LLM generates a Python code
+    block, no function-calling / tool-use.
+    """
+    from skillopt.envs.spreadsheetbench.codegen_agent import run_single, run_multi
+
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position_eval = f"{answer_sheet}!{answer_position}"
+    else:
+        answer_position_eval = answer_position
+
+    itype_lower = (instruction_type or "").lower()
+    if "cell" in itype_lower:
+        task_type = "cell_level"
+    elif "sheet" in itype_lower:
+        task_type = "sheet_level"
+    else:
+        task_type = "other"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {
+        "id": task_id,
+        "ok": False,
+        "instruction_type": instruction_type,
+        "task_type": task_type,
+        "task_description": instruction,
+        "phase": "setup",
+        "fail_reason": "",
+        "llm_ok": False,
+        "code_ok": False,
+        "exec_ok": False,
+        "n_cases": 0,
+        "n_exec_pass": 0,
+        "n_pass": 0,
+        "soft": 0.0,
+        "hard": 0,
+        "n_turns": 0,
+        "cases": [],
+        "error": "",
+    }
+
+    try:
+        cases = _find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out_dir = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out_dir, exist_ok=True)
+
+        # ── Save context for Teacher (Reflect stage) ──────────────────
+        from skillopt.envs.spreadsheetbench.codegen_agent import (
+            _preview_workbook, _build_system, _build_user,
+        )
+        first_input_for_preview = cases[0][1]
+        try:
+            preview_text = _preview_workbook(first_input_for_preview)
+        except Exception:
+            preview_text = "(preview failed)"
+        student_system = _build_system(skill_content)
+        student_user = _build_user(
+            instruction,
+            first_input_for_preview,
+            instruction_type,
+            answer_position_eval,
+            diagnostic_mode=diagnostic_mode,
+            diagnostic_instruction=diagnostic_instruction,
+            diagnostic_trace_context=diagnostic_trace_context,
+        )
+
+        with open(os.path.join(task_out_dir, "spreadsheet_preview.txt"), "w") as f:
+            f.write(preview_text)
+        with open(os.path.join(task_out_dir, "student_system_prompt.txt"), "w") as f:
+            f.write(student_system)
+        with open(os.path.join(task_out_dir, "student_user_prompt.txt"), "w") as f:
+            f.write(student_user)
+
+        result["spreadsheet_preview"] = preview_text
+        result["student_system_prompt"] = student_system
+        result["student_user_prompt"] = student_user
+
+        # ── LLM phase ──────────────────────────────────────────────────
+        result["phase"] = "llm"
+        first_input = cases[0][1]
+        first_gold = cases[0][2]
+        first_pred = os.path.join(task_out_dir, f"{cases[0][0]}_pred.xlsx")
+
+        try:
+            if mode == "multi":
+                agent_result = run_multi(
+                    instruction=instruction,
+                    input_xlsx=first_input,
+                    output_path=first_pred,
+                    instruction_type=instruction_type,
+                    answer_position=answer_position_eval,
+                    skill_content=skill_content,
+                    max_turns=max_turns,
+                    gold_path=first_gold if use_eval_feedback else "",
+                    diagnostic_mode=diagnostic_mode,
+                    diagnostic_instruction=diagnostic_instruction,
+                    diagnostic_trace_context=diagnostic_trace_context,
+                )
+            else:
+                agent_result = run_single(
+                    instruction=instruction,
+                    input_xlsx=first_input,
+                    output_path=first_pred,
+                    instruction_type=instruction_type,
+                    answer_position=answer_position_eval,
+                    skill_content=skill_content,
+                    diagnostic_mode=diagnostic_mode,
+                    diagnostic_instruction=diagnostic_instruction,
+                    diagnostic_trace_context=diagnostic_trace_context,
+                )
+        except Exception as e:  # noqa: BLE001
+            result["fail_reason"] = f"llm-call-failed: {type(e).__name__}: {e}"
+            result["error"] = traceback.format_exc()
+            return result
+
+        result["llm_ok"] = True
+        result["n_turns"] = agent_result.get("n_turns", 1)
+        code = agent_result.get("code", "")
+        raw = agent_result.get("raw", "")
+
+        # Save artifacts
+        with open(os.path.join(task_out_dir, "code.py"), "w") as f:
+            f.write(code)
+        with open(os.path.join(task_out_dir, "raw.txt"), "w") as f:
+            f.write(raw)
+        if agent_result.get("conversation"):
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(agent_result["conversation"], f, ensure_ascii=False, indent=2)
+
+        if not code.strip():
+            result["phase"] = "extract"
+            result["fail_reason"] = "empty-code-block"
+            return result
+        result["code_ok"] = True
+
+        # ── Exec + eval per test case ──────────────────────────────────
+        result["phase"] = "exec"
+        all_exec = True
+        # Collect enrichment info for the conversation/trajectory
+        enrichment_parts: list[str] = []
+
+        for no, ip, ap in cases:
+            pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx")
+
+            # For multi mode, the first case may already be produced
+            if not os.path.exists(pred_path):
+                ok_exec, err = run_generated_code(code, ip, pred_path)
+                if not ok_exec:
+                    all_exec = False
+                    result["cases"].append(
+                        {"no": no, "stage": "exec", "ok": False, "error": err[:500]}
+                    )
+                    if not result["fail_reason"]:
+                        tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown"
+                        result["fail_reason"] = f"exec-error: {tail}"
+                    enrichment_parts.append(
+                        f"## Execution (case {no})\nERROR: {err[:500]}"
+                    )
+                    continue
+
+            if not os.path.exists(pred_path):
+                all_exec = False
+                result["cases"].append(
+                    {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"}
+                )
+                if not result["fail_reason"]:
+                    result["fail_reason"] = "output-not-found"
+                continue
+
+            result["n_exec_pass"] += 1
+            try:
+                ev = evaluate(pred_path, ap, instruction_type, answer_position_eval)
+            except Exception as e:  # noqa: BLE001
+                ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"}
+
+            if ev["ok"]:
+                result["n_pass"] += 1
+            else:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}"
+            result["cases"].append(
+                {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")}
+            )
+
+            # Auto-verify: reopen output and compare cells at answer_position
+            if answer_position_eval:
+                verify_report = _auto_verify_output(pred_path, ap, answer_position_eval)
+                enrichment_parts.append(
+                    f"## Eval Result (case {no}): {'PASS' if ev['ok'] else 'FAIL'}\n"
+                    f"{ev.get('reason', '')}\n\n{verify_report}"
+                )
+
+        result["exec_ok"] = all_exec
+
+        # ── Enrich conversation with eval details ──────────────────────
+        if enrichment_parts:
+            enrichment_msg = "\n\n---\n\n".join(enrichment_parts)
+            conversation = agent_result.get("conversation", [])
+            conversation.append({
+                "role": "system",
+                "content": f"[POST-EXECUTION VERIFICATION]\n\n{enrichment_msg}",
+            })
+            # Re-save the enriched conversation
+            with open(os.path.join(task_out_dir, "conversation.json"), "w") as f:
+                json.dump(conversation, f, ensure_ascii=False, indent=2)
+        n_cases = result["n_cases"]
+        n_pass = result["n_pass"]
+        result["soft"] = (n_pass / n_cases) if n_cases else 0.0
+        result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]:
+            result["fail_reason"] = ""
+        return result
+
+    except Exception as e:  # noqa: BLE001
+        result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Codegen batch runner ────────────────────────────────────────────────────
+
+
+def run_spreadsheet_batch_codegen(
+    items: list[dict],
+    data_root: str,
+    out_root: str,
+    skill_content: str,
+    mode: str = "single",
+    max_turns: int = 5,
+    max_api_workers: int = 32,
+    task_timeout: int = 0,
+    use_eval_feedback: bool = False,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    diagnostic_trace_context_by_id: dict[str, str] | None = None,
+) -> list[dict]:
+    """Run codegen agent on all items (no tool-call).
+
+    Args:
+        mode: "single" or "multi".
+        task_timeout: Hard per-task timeout in seconds at the future level.
+            0 = auto (single: 300s, multi: 600s).
+    """
+    if task_timeout <= 0:
+        task_timeout = 300 if mode == "single" else 600
+
+    os.makedirs(out_root, exist_ok=True)
+
+    results_path = os.path.join(out_root, "results.jsonl")
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path) as f:
+            for line in f:
+                try:
+                    r = json.loads(line)
+                    done_ids.add(str(r["id"]))
+                    existing.append(r)
+                except Exception:
+                    pass
+
+    pending = [it for it in items if str(it["id"]) not in done_ids]
+    print(
+        f"  [spreadsheet codegen-{mode}] total={len(items)} done={len(done_ids)} "
+        f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout}s"
+    )
+
+    if not pending:
+        return existing
+
+    t0 = time.time()
+    results = list(existing)
+
+    started_at: dict[str, float] = {}
+
+    def _run_one(it: dict) -> dict:
+        started_at[str(it["id"])] = time.time()
+        return process_one_codegen(
+            it,
+            data_root,
+            out_root,
+            skill_content,
+            mode,
+            max_turns,
+            use_eval_feedback,
+            diagnostic_mode,
+            diagnostic_instruction,
+            (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
+        )
+
+    def _timeout_result(item: dict) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "instruction_type": item.get("instruction_type", ""),
+            "task_type": "other",
+            "phase": "timeout",
+            "fail_reason": f"task-timeout-{task_timeout}s",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": "timeout",
+        }
+
+    def _error_result(item: dict, e: Exception) -> dict:
+        return {
+            "id": str(item["id"]),
+            "ok": False,
+            "instruction_type": item.get("instruction_type", ""),
+            "task_type": "other",
+            "phase": "error",
+            "fail_reason": f"unexpected: {type(e).__name__}: {e}",
+            "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0,
+            "n_turns": 0, "cases": [], "error": str(e),
+        }
+
+    def _record(res: dict, i: int) -> None:
+        results.append(res)
+        status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL")
+        dt = time.time() - t0
+        print(
+            f"    {i}/{len(pending)} id={res['id']:<10} {status}  "
+            f"turns={res.get('n_turns', 0):<3} "
+            f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)}  "
+            f"dt={dt:.0f}s"
+        )
+
+    ex = ThreadPoolExecutor(max_workers=max_api_workers)
+    try:
+        futs = {ex.submit(_run_one, it): it for it in pending}
+        pending_futs = set(futs)
+        finished = 0
+        while pending_futs:
+            done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
+            now = time.time()
+            timed_out = [
+                fut for fut in pending_futs - done
+                if str(futs[fut]["id"]) in started_at
+                and now - started_at[str(futs[fut]["id"])] >= task_timeout
+            ]
+            for fut in done:
+                pending_futs.remove(fut)
+                item = futs[fut]
+                try:
+                    res = fut.result()
+                except FuturesTimeoutError:
+                    res = _timeout_result(item)
+                except Exception as e:  # noqa: BLE001
+                    res = _error_result(item, e)
+                finished += 1
+                _record(res, finished)
+            for fut in timed_out:
+                pending_futs.remove(fut)
+                fut.cancel()
+                finished += 1
+                _record(_timeout_result(futs[fut]), finished)
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
+
+    return results

skillopt/envs/spreadsheetbench/skills/initial.md

@@ -0,0 +1,56 @@
+# Spreadsheet Manipulation Skill (xlsx)
+
+## Overview
+This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python.
+
+**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation).
+Never use any other third-party libraries.
+
+---
+
+## Common Workflow
+
+1. **Explore** the input file: list sheets, inspect headers, check dimensions.
+2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top.
+3. **Execute** `python solution.py` and verify the output file was created.
+4. **Confirm** the target cells/range contain the expected values.
+
+---
+
+## Library Selection
+
+| Use case | Library |
+|----------|---------|
+| Preserve formulas, formatting, named ranges | `openpyxl` |
+| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` |
+| Simple cell read/write | `openpyxl` |
+
+**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges.
+When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`.
+
+---
+
+## solution.py Template
+
+```python
+import openpyxl
+import pandas as pd
+
+INPUT_PATH  = "..."   # set to the actual input path
+OUTPUT_PATH = "..."   # set to the actual output path
+
+wb = openpyxl.load_workbook(INPUT_PATH)
+ws = wb.active  # or wb["SheetName"]
+
+# --- perform manipulation ---
+
+wb.save(OUTPUT_PATH)
+```
+
+---
+
+## Output Requirements
+
+- Save the result to `OUTPUT_PATH`.
+- Do not hardcode row counts or column letters — iterate over actual rows in the workbook.
+- Preserve sheets and cells not mentioned in the instruction.

skillopt/envs/spreadsheetbench/skills/xlsx_initial.md

@@ -0,0 +1,56 @@
+# Spreadsheet Manipulation Skill (xlsx)
+
+## Overview
+This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python.
+
+**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation).
+Never use any other third-party libraries.
+
+---
+
+## Common Workflow
+
+1. **Explore** the input file: list sheets, inspect headers, check dimensions.
+2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top.
+3. **Execute** `python solution.py` and verify the output file was created.
+4. **Confirm** the target cells/range contain the expected values.
+
+---
+
+## Library Selection
+
+| Use case | Library |
+|----------|---------|
+| Preserve formulas, formatting, named ranges | `openpyxl` |
+| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` |
+| Simple cell read/write | `openpyxl` |
+
+**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges.
+When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`.
+
+---
+
+## solution.py Template
+
+```python
+import openpyxl
+import pandas as pd
+
+INPUT_PATH  = "..."   # set to the actual input path
+OUTPUT_PATH = "..."   # set to the actual output path
+
+wb = openpyxl.load_workbook(INPUT_PATH)
+ws = wb.active  # or wb["SheetName"]
+
+# --- perform manipulation ---
+
+wb.save(OUTPUT_PATH)
+```
+
+---
+
+## Output Requirements
+
+- Save the result to `OUTPUT_PATH`.
+- Do not hardcode row counts or column letters — iterate over actual rows in the workbook.
+- Preserve sheets and cells not mentioned in the instruction.

skillopt/envs/spreadsheetbench/skills/xlsx_skill0.md

@@ -0,0 +1,4 @@
+# No Skill
+
+This is a placeholder. No domain skill is loaded for this run.
+The agent relies solely on its parametric knowledge.