Merge pull request #150 from Yif-Yang/test/assert-fractional-degradation-score

test(sleep): assert fractional candidate score

.env.example

@@ -0,0 +1,60 @@
+# SkillOpt Environment Variables
+# Copy this file to .env and fill in your values.
+# Usage: set -a; source .env; set +a
+
+# ── Azure OpenAI (required for openai_chat backend) ──────────────────
+export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+export AZURE_OPENAI_API_VERSION=2024-12-01-preview
+# Authentication: choose one method
+#   Option 1: API Key
+export AZURE_OPENAI_API_KEY=
+export AZURE_OPENAI_AUTH_MODE=api_key
+#   Option 2: Azure CLI (no API key needed, recommended on Azure VMs)
+# export AZURE_OPENAI_AUTH_MODE=azure_cli
+#   Option 3: Managed Identity
+# export AZURE_OPENAI_AUTH_MODE=managed_identity
+# export AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID=your-client-id
+
+# ── OpenAI-compatible endpoints ──────────────────────────────────────
+# Path 1: generic research backend. Select openai_compatible explicitly as
+# model.optimizer_backend and/or model.target_backend.
+# export OPENAI_COMPATIBLE_BASE_URL=https://api.deepseek.com/v1
+# export OPENAI_COMPATIBLE_API_KEY=sk-...
+# export OPENAI_COMPATIBLE_MODEL=deepseek-chat
+# Per-role overrides use OPTIMIZER_OPENAI_COMPATIBLE_* and
+# TARGET_OPENAI_COMPATIBLE_* (BASE_URL, API_KEY, MODEL, TEMPERATURE,
+# MAX_TOKENS, TIMEOUT_SECONDS).
+# For scripts/train.py and scripts/eval_only.py, also set model.optimizer and
+# model.target in YAML (or via --cfg-options). Those role model values are
+# applied after backend initialization and override *_MODEL environment values.
+
+# Path 2: research openai_chat compatibility mode. This reuses the Azure-family
+# variables but creates a plain OpenAI client (no Azure auth or api-version).
+# export AZURE_OPENAI_ENDPOINT=https://api.openai.com/v1
+# export AZURE_OPENAI_API_KEY=sk-...
+# export AZURE_OPENAI_AUTH_MODE=openai_compatible
+
+# Path 3: SkillOpt-Sleep. `skillopt-sleep run --backend azure_openai` uses the
+# same three AZURE_* variables from path 2. Optional Sleep-only controls:
+# export SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=8192
+# export SKILLOPT_SLEEP_CHAT_EXTRA_BODY='{"provider_option": true}'
+
+# ── Claude Code CLI (for claude_chat backend) ─────────────────────────
+# Install and authenticate the `claude` CLI before use. For a non-default path:
+# export CLAUDE_CLI_BIN=/path/to/claude
+# ANTHROPIC_API_KEY is one authentication option understood by the CLI; SkillOpt
+# does not create a direct Anthropic API client for this backend.
+# export ANTHROPIC_API_KEY=sk-ant-...
+
+# ── Qwen Local Model (for qwen_chat backend) ────────────────────────
+# export QWEN_CHAT_BASE_URL=http://localhost:8000/v1
+# export QWEN_CHAT_MODEL=Qwen/Qwen3.5-4B
+# The train/eval entry points likewise override this model with
+# model.optimizer/model.target for the selected Qwen roles.
+
+# ── MiniMax (for minimax_chat backend) ──────────────────────────────
+# export MINIMAX_BASE_URL=https://api.minimax.io/v1
+# export MINIMAX_API_KEY=...
+# When MiniMax is the target, set model.minimax_model in YAML. The current
+# adapter shares one deployment across MiniMax roles; mixed-backend runs cannot
+# independently select a MiniMax optimizer model and a different target model.

.gitignore

@@ -1,15 +1,66 @@
 __pycache__/
 *.pyc
-data/
+*.egg-info/
+build/
+dist/
+site/
+
+data/*
+!data/README.md
+!data/searchqa_id_split/
+!data/searchqa_id_split/**
+!data/livemathematicianbench_id_split/
+!data/livemathematicianbench_id_split/**
+!data/docvqa_id_split/
+!data/docvqa_id_split/**
+!data/officeqa_id_split/
+!data/officeqa_id_split/**
+!data/spreadsheetbench_id_split/
+!data/spreadsheetbench_id_split/**
+!data/alfworld_path_split/
+!data/alfworld_path_split/**
 outputs/
 logs/
 external/
+# SkillOpt-Sleep runtime state (staging proposals, config, diagnostics, cron logs)
+.skillopt-sleep/
+# SkillOpt-Sleep handoff-backend round data (prompts/answers derived from transcripts)
+.skillopt-sleep-handoff/
+.skillopt-sleep-handoff.night*.done/
+
 /BabyVision/
 /MMRB/
 /SpreadsheetBench/
 /dl4ir-searchQA/
+
 configs/local/
 configs/**/*.local.yaml
 *.local.md
+*.secret.md
+*.bak
+
+.env
 .secrets/
 .codex_azure*/
+
+# Internal docs (not for open-source release)
+docs/ablation_plan.md
+docs/ablation_paper_tables.md
+docs/ablation_paper_tables.html
+docs/experiment_commands.md
+docs/slow_update_flowchart.md
+docs/session_memory.md
+docs/harness_fresh_machine_handoff.md
+docs/harness_monitoring_memory.md
+docs/harness_reproduction_secrets.secret.md
+docs/reflact_conda_env_export.yml
+docs/reflact_overview.html
+docs/render_ablation_paper_tables.py
+docs/让*
+.gradio/
+.venv
+
+# Local experiment launchers — contain machine-specific endpoints/identities, never commit
+tests/run_*.sh
+tests/launch_*.py
+*.launch.log

CHANGELOG.md

@@ -0,0 +1,171 @@
+# Changelog
+
+All notable changes to SkillOpt are documented here. This project adheres to
+[Semantic Versioning](https://semver.org/) and the format is based on
+[Keep a Changelog](https://keepachangelog.com/).
+
+## [Unreleased]
+
+### Added
+- **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep — runs the
+  sleep cycle with no model subprocess or API key: the engine writes each
+  pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the
+  user's own agent session answers into `answers/<id>.md`; re-running the
+  same command resumes statelessly from the answers (typically 3–6 rounds
+  per night). Mined tasks are pinned per night so answering sessions cannot
+  shift the task set. Ships a `/skillopt-sleep-handoff` Claude Code command
+  that automates the loop with fresh-context subagents to protect the
+  held-out gate (thanks @dimitarvdenev, #125).
+- **Generic OpenAI-compatible research backend** for optimizer and target
+  calls, with configurable base URL, API key, model, and timeout (thanks
+  @nankingjing, #115).
+- **OpenAI-compatible SkillOpt-Sleep endpoint support** for providers such as
+  DeepSeek and self-hosted vLLM servers (thanks @Alphaxalchemy, #129; hardened
+  in #138).
+- End-to-end wiring for the documented reflection `--preferences` option
+  (thanks @AKhozya, #131).
+
+### Changed
+- Claude Code's Sleep plugin can now use a `pip`/`uv`-installed
+  `skillopt-sleep` when no repository checkout is present (thanks
+  @ichoosetoaccept, #107).
+- Qwen reasoning-model requests now use `max_completion_tokens` and omit
+  unsupported temperature parameters (thanks @chirag127, #128).
+- Configuration files are read explicitly as UTF-8 (thanks @nankingjing,
+  #124).
+
+### Fixed
+- Preserve fractional rollout hard scores instead of coercing them to binary
+  values (thanks @zixuanguo786-ctrl, #104).
+- Reject duplicate and overlapping IDs while materializing SearchQA manifests
+  (thanks @zixuanguo786-ctrl, #105).
+- Make JSON-array extraction robust to unmatched braces and keep malformed
+  scans linear-time (thanks @zixuanguo786-ctrl, #103; follow-up #136).
+- Package Markdown prompt assets in wheels and tolerate Windows temporary-file
+  cleanup failures (thanks @nankingjing, #135; follow-up #137).
+- Exclude sub-agent transcripts and plugin-generated sessions from Sleep task
+  mining (thanks @codeL1985, #99).
+- Normalize validation-gate density against the proposed edits and handle
+  zero-edit candidates safely (thanks @SparshGarg999, #102).
+- Route optimizer-role MiniMax calls through the MiniMax backend (thanks
+  @jcforever1, #116).
+- Surface Claude CLI spawn failures instead of silently turning them into zero
+  scores (thanks @Phoenix0531-sudo, #126).
+- Improve Claude CLI behavior on Windows, including `.cmd` resolution and
+  long-prompt handling (thanks @codeL1985, #98).
+- Preserve the scheduler's established annealing contract while expanding its
+  endpoint and sequence coverage (thanks @nankingjing, #123; follow-up #133).
+
+### Security
+- Prevent managed-identity credentials from being sent to non-Azure or
+  non-HTTPS endpoints, and isolate compatible-provider request extensions
+  from native Azure mode in SkillOpt-Sleep (#138, following
+  @Alphaxalchemy's #129).
+
+### Tests
+- Strengthen SkillOpt-Sleep verifier-discipline assertions, including recorded
+  scores and gate actions (thanks @Tanmay9223, #96).
+- Add focused coverage for the validation-gate decision core and edit-budget
+  schedulers (thanks @nankingjing, #122, #123).
+
+### Acknowledgements 🙏
+Thank you to the contributors behind this unreleased work:
+@AKhozya, @Alphaxalchemy, @Phoenix0531-sudo, @SparshGarg999,
+@Tanmay9223, @chirag127, @codeL1985, @dimitarvdenev,
+@ichoosetoaccept, @jcforever1, @nankingjing, and
+@zixuanguo786-ctrl.
+
+## [0.2.0] — 2026-07-02
+
+The headline of this release is **SkillOpt-Sleep**: a nightly offline
+self-evolution engine that harvests a coding agent's real session
+transcripts, mines recurring tasks, replays them offline, and consolidates
+short-term experience into long-term memory and skills — all behind the same
+held-out validation gate that keeps SkillOpt training honest. It ships as a
+decoupled top-level package (`skillopt_sleep/`, zero dependency on the
+research code) and as the new `skillopt-sleep` CLI.
+
+### Added
+- **SkillOpt-Sleep engine** — nightly offline self-evolution cycle
+  (harvest → mine → replay → consolidate) behind a validation gate, exposed
+  as the `skillopt-sleep` console script and `python -m skillopt_sleep`.
+  - Multi-objective reward (accuracy / tokens / latency) with user preferences.
+  - Multi-rollout contrastive reflection under a token/time budget.
+  - Experience replay + controllable dream rollouts (opt-in).
+  - Slow-update long-term memory field (runs even with the gate off).
+  - 3-way train/val/test split with `gate_mode on|off`.
+  - Verifier-discipline validation gate, with a stress-test suite
+    (thanks @Tanmay9223, #87).
+- **Cross-tool backends & plugin shells** for Claude Code, Codex, Copilot,
+  Devin, and OpenClaw:
+  - Codex Desktop transcript harvesting, skill-first Codex integration, and a
+    reviewed task-file flow (thanks @Kirchberg, #48, #49, #60).
+  - GitHub Copilot backend (`CopilotCliBackend`) + research-engine MCP plugin
+    (thanks @Dongbumlee, #50).
+  - Devin plugin: MCP server + ATIF-v1.7 harvest (thanks @xerxes-y, #88).
+  - OpenClaw shell for SkillOpt-Sleep (thanks @Elzlxx, #59).
+- **SearchQA** split materialization helper and fail-fast on systemic rollout
+  failures, with a `searchqa` install extra (thanks @summerview1997,
+  #63, #64, #65).
+- WebUI environment loading and backend preflight (thanks @summerview1997, #63).
+
+### Changed
+- Decoupled the Sleep engine into a standalone top-level `skillopt_sleep/`
+  package with zero dependency on the research code.
+- Made `EnvAdapter.reflect` a shared default so reflect kwargs are no longer
+  dropped (thanks @imshunsuke, #44).
+- English-only pass across the engine, plugins, and docs.
+
+### Fixed
+- Windows robustness for the Claude/Codex backends, plus a hardened JSON
+  fallback path (thanks @Yif-Yang, #79).
+- Reject prose pseudo-JSON wrapped in single quotes/backticks (#82).
+- Surface Codex auth/model/version failures instead of silently scoring 0
+  (thanks @dmmdea, #92).
+- Redact secrets before persisting cycle diagnostics.
+- Configure the `qwen_chat`/`minimax` backends so local LLM endpoints work
+  (thanks @imrehg, #85).
+- Forward the Qwen target timeout and gate `enable_thinking` for vLLM targets
+  (thanks @mvanhorn, #40).
+- Make `--bare` conditional on `ANTHROPIC_API_KEY` (#68), add a
+  `SKILLOPT_SLEEP_PYTHON` override with a lookback-hours first-run fallback
+  (#74), and fix ALFWorld gamefile paths relative to `ALFWORLD_DATA`.
+
+### Packaging
+- Bump `skillopt`, `skillopt.__version__`, and `skillopt_sleep.__version__`
+  to `0.2.0`.
+- Restore `skillopt_webui` to the built wheel (it was dropped when the
+  `packages.find` include list was made explicit).
+- Add the `searchqa` extra and include `json_repair` in the `claude`, `qwen`,
+  and `all` extras.
+
+### Acknowledgements 🙏
+v0.2.0 landed thanks to our community contributors — thank you!
+
+- @Kirchberg — Codex Desktop harvesting, skill-first Codex integration,
+  reviewed task-file flow (#48, #49, #60)
+- @Dongbumlee — GitHub Copilot backend + research-engine MCP plugin (#50)
+- @summerview1997 — SearchQA materialization, rollout fail-fast, WebUI
+  preflight (#63, #64, #65)
+- @xerxes-y — Devin plugin: MCP server + ATIF-v1.7 harvest (#88)
+- @Elzlxx — OpenClaw shell for SkillOpt-Sleep (#59)
+- @imshunsuke — shared `EnvAdapter.reflect` default + docs fixes (#43, #44)
+- @mvanhorn — Qwen timeout forwarding + `enable_thinking` gating (#40)
+- @dmmdea — surface Codex auth/model/version failures (#92)
+- @Tanmay9223 — verifier-discipline stress test (#87)
+- @imrehg — `configure_qwen_chat` for local LLM endpoints (#85)
+- @samuelgoofus-boop — community contributions
+
+Special thanks to @Yif-Yang for driving the SkillOpt-Sleep engine.
+
+**Full changelog:** https://github.com/microsoft/SkillOpt/compare/v0.1.0...v0.2.0
+
+## [0.1.0] — 2026-06-02
+
+Initial public release: the full training loop (rollout → reflect →
+aggregate → select → update → evaluate), multi-backend support
+(OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and the
+WebUI dashboard.
+
+[0.2.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0
+[0.1.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0

CONTRIBUTING.md

@@ -0,0 +1,48 @@
+# Contributing to SkillOpt
+
+Thank you for your interest in contributing! SkillOpt welcomes contributions of all kinds.
+
+## Getting Started
+
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e ".[dev,docs]"
+```
+
+## How to Contribute
+
+### 🐛 Bug Reports
+Open a GitHub issue with reproduction steps, expected/actual behavior, and your config file (remove API keys).
+
+### 🔧 Add a Benchmark
+See the [guide](docs/guide/new-benchmark.md) and use the scaffold at
+`skillopt/envs/_template/`. Register the adapter lazily in both
+`scripts/train.py` and `scripts/eval_only.py`, and add focused tests.
+
+### 🤖 Add a Model Backend
+First check whether the built-in `openai_compatible` backend covers the
+provider. Otherwise follow the function-based backend contract in the
+[backend guide](docs/guide/new-backend.md), including routing, configuration,
+token accounting, and no-network tests.
+
+### 📝 Improve Documentation
+```bash
+python -m mkdocs serve   # Preview at http://localhost:8000
+```
+
+## Pull Request Process
+
+1. Fork the repo and create a feature branch
+2. Make changes and run focused tests plus `python -m pytest -q`
+3. Submit a PR with a clear description
+4. For documentation changes, run `python -m mkdocs build --strict`
+5. Ensure CI passes
+
+## Code Style
+- Follow existing patterns in the codebase
+- Use type hints for function signatures
+- Keep docstrings concise
+
+## License
+By contributing, you agree your contributions are licensed under the [MIT License](LICENSE).

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,557 +1,113 @@
-# ReflACT: Reflective Agent Tuning
+# SkillOpt: Executive Strategy for Self-Evolving Agent Skills
 
-ReflACT is a framework for optimizing an external skill document through iterative rollout, reflection, editing, and gated validation.
+*Train agent skills like you train neural networks — with epochs, (mini-)batchsize, learning rates, and validation gates — but without touching model weights.*
 
-It does **not** fine-tune model weights. Instead, it treats the skill document as the optimization target:
+[![Project Page](https://img.shields.io/badge/Project%20Page-SkillOpt-8dbb3c)](https://microsoft.github.io/SkillOpt/) [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b)](https://arxiv.org/abs/2605.23904) [![Project Video](https://img.shields.io/badge/Project%20Video-Watch%20Demo-ff0000)](https://youtu.be/JUBMDTCiM0M) [![PyPI](https://img.shields.io/badge/PyPI-skillopt-green.svg)](https://pypi.org/project/skillopt/) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 
-- the **student** model executes tasks with the current skill
-- the **teacher** model analyzes trajectories and proposes edits
-- the framework merges, ranks, applies, and validates those edits
-- only validated skill updates are kept
+<p align="center">
+  <a href="https://trendshift.io/repositories/38498?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-38498" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/38498/daily?language=Python" alt="microsoft%2FSkillOpt | Trendshift" width="250" height="55"/></a>
+  <a href="https://trendshift.io/repositories/38498?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-38498" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/38498/weekly?language=Python" alt="microsoft%2FSkillOpt | Trendshift" width="250" height="55"/></a>
+</p>
 
-This branch implements a full training loop with step-level skill optimization and optional epoch-level memory mechanisms (`slow_update`, `meta_skill`, `meta_reflect`).
+> 📖 **For installation, data preparation, training/eval commands, configuration, and framework internals, start with the versioned [SkillOpt documentation](https://github.com/microsoft/SkillOpt/blob/main/docs/index.md). A concise rendered overview is available in the [Documentation & Reproduction Guide](https://microsoft.github.io/SkillOpt/docs/guideline.html), and longer-form engineering analysis appears on the [Technical Blog](https://microsoft.github.io/SkillOpt/blog/). We also maintain a [Changelog](CHANGELOG.md) for released and unreleased changes.**
 
-## Method Overview
+---
 
-### Optimization Target
+## News 🔥🔥🔥
+- **[2026-07-02]** 🚀 **SkillOpt [v0.2.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) is out on [PyPI](https://pypi.org/project/skillopt/)!** Headline feature: **SkillOpt-Sleep**, a nightly offline self-evolution engine (harvest → mine → replay → consolidate behind a held-out validation gate), now shipped as the `skillopt-sleep` CLI. It also includes experimental multi-objective, replay, and dream-rollout controls; the main CLI keeps conservative defaults and does not expose every experiment-harness control as a flag. The release source adds integration shells for **Claude Code, Codex, Copilot, and Devin**, plus an **OpenClaw reference adaptation**; these plugin/MCP files live in the repository rather than the PyPI wheel. It also adds SearchQA split materialization, Windows robustness, and hardened JSON parsing. See the [release notes](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) for full release details and contributor acknowledgements.
+- **[2026-06-15]** 😴 **SkillOpt-Sleep (preview)** — a nightly offline self-evolution companion for local coding agents (Claude Code / Codex / Copilot): review past sessions, replay recurring tasks, and consolidate validated skills behind a held-out gate. See **[`docs/sleep/README.md`](docs/sleep/README.md)** for what it is, how to use it, and results.
+- **[2026-06-03]** 🎉 **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.**
+- **[2026-06-02]** 🎉 **SkillOpt [v0.1.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0) is now available on [PyPI](https://pypi.org/project/skillopt/)!** Install with `pip install skillopt`. This initial release includes the full training loop (rollout → reflect → aggregate → select → update → evaluate), multi-backend support (OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and WebUI dashboard.
 
-Each run maintains a mutable markdown skill document. The framework repeatedly improves that document instead of changing model parameters.
+---
 
-This gives a training-style loop for prompt / policy optimization:
+## Overview
 
-1. Roll out the current skill on a batch of tasks.
-2. Reflect on failures and successes.
-3. Merge patch proposals into a coherent candidate update.
-4. Rank and select a bounded number of edits.
-5. Apply those edits to produce a candidate skill.
-6. Validate the candidate skill on a held-out selection split.
-7. Keep the update only if the gate accepts it.
+Modern agent skills are usually hand-crafted, generated one-shot by a strong
+LLM, or evolved through loosely controlled self-revision — none of which
+behaves like a deep-learning optimizer for the skill itself, and none of
+which reliably improves over its starting point under feedback.
 
-### Per-Step Pipeline
+**SkillOpt treats the skill document as the trainable state of a frozen
+agent**, and trains it with the discipline that makes weight-space
+optimization reproducible. A separate optimizer model turns scored rollouts
+into bounded add / delete / replace edits on a single skill document; in the
+default paper-style path, a candidate edit is accepted only when it strictly
+improves a held-out validation score. A textual learning-rate budget, a rejected-edit buffer,
+and an epoch-wise slow / meta update make skill training stable while
+adding **zero inference-time model calls** at deployment.
 
-Every training step executes the following pipeline in `reflact/engine/trainer.py`:
+The deployed artifact is a compact `best_skill.md` (typically 300–2,000
+tokens) that runs against the unchanged target model. Across **six
+benchmarks, seven target models, and three execution harnesses** (direct
+chat, Codex CLI, Claude Code CLI), SkillOpt is best or tied-best on **all
+52 evaluated (model, benchmark, harness) cells** and on GPT-5.5 lifts the
+average no-skill accuracy by **+23.5 points in direct chat, +24.8 inside
+the Codex agentic loop, and +19.1 inside Claude Code**. Optimized skill
+artifacts transfer across model scales, between Codex and Claude Code
+harnesses, and to nearby benchmarks without further optimization.
 
-1. **Rollout**
-   The student model runs a batch of tasks using the current skill.
+For the full method, ablations, and per-cell results see the [paper](https://arxiv.org/abs/2605.23904); for a visual walkthrough of the loop see the [project page](https://microsoft.github.io/SkillOpt/); for deeper API / backend / benchmark docs see [`docs/`](docs/).
 
-2. **Reflect**
-   The teacher analyzes minibatches of trajectories and emits raw patches.
-   Failure-driven and success-driven patches are tracked separately.
+## 🎬 Demo Video
 
-3. **Aggregate**
-   Raw patches are merged hierarchically. Metadata such as `support_count` and `source_type` is carried into the merged patch so later ranking can use it.
+https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7
 
-4. **Select**
-   The teacher ranks the merged edit pool and keeps up to `edit_budget` edits.
+<p align="center">
+  <a href="https://youtu.be/JUBMDTCiM0M"><b>▶ Watch the full demo on YouTube</b></a>
+</p>
 
-5. **Update**
-   The selected edits are applied to the skill document. The framework records an `edit_apply_report.json` so you can see which edits actually landed, which were skipped, and why.
+---
 
-6. **Evaluate / Gate**
-   The candidate skill is evaluated on the selection split. Gate validation is mandatory in this branch. A candidate update is accepted only if it improves over the current selection score; a new global best is tracked separately.
+## Extensibility & WebUI
 
-### Within-Epoch Memory
+### Adding a new backend
 
-Inside an epoch, the trainer maintains a step buffer containing:
+A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`,
+`qwen_chat`, `minimax_chat`, `openai_compatible`, `codex_exec`,
+`claude_code_exec`). If a provider implements the OpenAI Chat Completions
+protocol, try the built-in `openai_compatible` backend before adding code. See
+[`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full
+contract; in short you add a `skillopt/model/<name>_backend.py` module,
+register it in `skillopt/model/common.py` + `backend_config.py`, and wire
+it through the router in `skillopt/model/__init__.py`. `qwen_backend.py`
+and `minimax_backend.py` are good templates.
 
-- compact failure-pattern summaries from previous steps
-- rejected edits and their score deltas
+### Adding a new benchmark
 
-That context is fed back into later reflection calls so the teacher can avoid repeating ineffective edits and can focus on unsolved error patterns.
+A benchmark = a `skillopt/envs/<name>/` package with an adapter, a data loader,
+a scored rollout helper, a YAML config, and optionally an initial seed skill.
+See
+[`docs/guide/new-benchmark.md`](docs/guide/new-benchmark.md) for the full
+contract; the simplest reference is `skillopt/envs/searchqa/`.
 
-### Epoch-Level Mechanisms
+### WebUI
 
-This branch supports three optional epoch-level mechanisms.
-
-#### Slow Update
-
-At the end of each epoch, `slow_update` compares the previous epoch’s terminal skill and current epoch’s terminal skill on a sampled train subset. It then writes longitudinal guidance into a protected slow-update region inside the skill document.
-
-Importantly, this guidance is **not** blindly written through. It is converted into a candidate skill and sent through the same selection gate as step-level updates.
-
-#### Meta Skill
-
-`meta_skill` is teacher-side cross-epoch memory. It does not directly edit the current skill. Instead, it writes a compact memory artifact describing longer-term patterns across adjacent epochs. That memory is loaded into later reflection / merge / ranking calls as extra context.
-
-#### Meta Reflect
-
-`meta_reflect` runs at epoch end over the step history of the current epoch. It looks at accepted and rejected directions from the whole epoch, proposes higher-level patch edits, applies them to a meta candidate, and then sends that candidate through the same selection gate.
-
-## What This Branch Guarantees
-
-The current implementation assumes the following as the mainline method contract:
-
-- gate validation is always on
-- the current skill, current score, best skill, and best score stay aligned
-- `slow_update` is gated before being committed
-- patch provenance (`source_type`, `support_count`) reaches selection
-- patch application is observable through per-edit reports
-- resume state is restored from `runtime_state.json` rather than inferred only from history
-- all benchmark model calls go through the unified backend router
-
-## Model Backends
-
-All model access now goes through the split teacher/student model layer in `reflact.model`.
-
-Supported teacher backends:
-
-- `openai_chat`
-- `claude_chat`
-
-Supported student backends:
-
-- `openai_chat`
-- `claude_chat`
-- `codex_exec`
-- `claude_code_exec`
-
-Recommended config shape:
-
-```yaml
-model:
-  teacher_backend: openai_chat
-  student_backend: codex_exec
-  teacher: gpt-5.4
-  student: gpt-5.4-codex
-  reasoning_effort: medium
-```
-
-Legacy `model.backend` and CLI flags like `--backend codex` still work. They are mapped onto the split backend model for backward compatibility.
-
-The same routing is used by:
-
-- training (`scripts/train.py`)
-- eval-only runs (`scripts/eval_only.py`)
-- SpreadsheetBench standalone prompt eval scripts
-- LiveMathematicianBench baseline eval script
-- benchmark rollout code inside the main framework
-
-### Azure OpenAI
-
-If you use `openai_chat`, configure either environment variables or config values:
+Launch the monitoring dashboard (optional):
 
 ```bash
-export AZURE_OPENAI_ENDPOINT="https://your-endpoint.openai.azure.com/"
-export AZURE_OPENAI_API_KEY="your-api-key"
-export AZURE_OPENAI_API_VERSION="2025-04-01-preview"
+pip install -e ".[webui]"
+python -m skillopt_webui.app
 ```
 
-The config supports both the old keys and the new explicit names:
-
-```yaml
-model:
-  azure_openai_endpoint: "..."
-  azure_openai_api_version: "..."
-  azure_openai_api_key: ""
-  azure_openai_auth_mode: api_key
-  azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  azure_openai_managed_identity_client_id: ""
-```
-
-`azure_openai_auth_mode` can be used for API-key auth or Azure AD / managed identity flows.
-
-### Exec Harness
-
-`codex_exec` and `claude_code_exec` run the student inside a workspace harness instead of a plain chat call. The harness writes task files, renders a dynamic `SKILL.md`, runs the student CLI, and saves raw execution artifacts such as:
-
-- `codex_raw.txt`
-- `codex_trace_summary.txt`
-- workspace-local task / skill files
-
-This branch keeps `meta_skill` and `apply_patch_with_report`, while upgrading the student path to the more realistic workspace-exec setup.
-
-### Trace-Aware Deep Reflect
-
-When `student_backend=codex_exec` and `gradient.use_deep_reflect=true`, deep reflection can probe a specific earlier Codex attempt:
-
-- the teacher sees a compact Codex trace summary
-- deep probe can target `probe_target_id`
-- the follow-up rollout can resume from `probe_after_step`
-
-This is wired for the dataset-backed environments in this branch.
-
-### Rewrite Mode
-
-Skill updates support two modes:
-
-- `optimizer.skill_update_mode=patch`
-- `optimizer.skill_update_mode=rewrite_from_suggestions`
-
-`patch` keeps the existing fine-grained edit application path and still records `edit_apply_report.json`.
-
-`rewrite_from_suggestions` asks the teacher to emit higher-level rewrite suggestions, then rewrites the whole skill in one pass. This is useful when patch edits become too fragmented.
-
-## Repository Layout
-
-```text
-reflact/
-  engine/
-    trainer.py                 main training loop
-  gradient/
-    reflect.py                 minibatch reflection
-    aggregate.py               hierarchical patch merge
-    deep_probe.py              diagnostic probing for deep reflect
-  optimizer/
-    clip.py                    edit ranking / selection
-    skill.py                   patch application + apply report
-    slow_update.py             epoch-level longitudinal guidance
-    meta_skill.py              teacher-side cross-epoch memory
-    meta_reflect.py            epoch-level macro editing
-  evaluation/
-    gate.py                    pure gate decision logic
-  model/
-    backend_config.py          teacher/student backend routing
-    azure_openai.py            Azure backend
-    codex_harness.py           workspace exec harness + Codex trace parsing
-    claude_backend.py          Claude backend
-  envs/
-    ...                        environment adapters and rollout logic
-scripts/
-  train.py                     unified training entry
-  eval_only.py                 evaluate one skill without training
-configs/
-  _base_/default.yaml          shared defaults
-  <env>/default.yaml           environment-specific configs
-```
-
-## Configuration
-
-Configs use structured YAML with `_base_` inheritance.
-
-The base config is `configs/_base_/default.yaml`. Key defaults in this branch are:
-
-- `model.teacher_backend = openai_chat`
-- `model.student_backend = openai_chat`
-- `model.reasoning_effort = medium`
-- `optimizer.use_slow_update = true`
-- `optimizer.use_meta_skill = true`
-- `optimizer.use_meta_reflect = false`
-- `gradient.use_deep_reflect = false`
-- `optimizer.skill_update_mode = patch`
-
-Default setting snapshot:
-
-```yaml
-model:
-  backend: azure_openai
-  teacher: gpt-5.4
-  student: gpt-5.4
-  teacher_backend: openai_chat
-  student_backend: openai_chat
-  reasoning_effort: medium
-  rewrite_reasoning_effort: ""
-  rewrite_max_completion_tokens: 64000
-  codex_exec_path: codex
-  codex_exec_sandbox: workspace-write
-  codex_exec_profile: ""
-  codex_exec_full_auto: false
-  codex_exec_reasoning_effort: none
-  claude_code_exec_path: claude
-  claude_code_exec_profile: ""
-  codex_trace_to_teacher: true
-
-train:
-  num_epochs: 4
-  train_size: 0
-  batch_size: 80
-  accumulation: 1
-  seed: 42
-
-gradient:
-  minibatch_size: 16
-  merge_batch_size: 16
-  analyst_workers: 16
-  max_analyst_rounds: 3
-  failure_only: false
-  use_deep_reflect: false
-  deep_reflect_failures: 4
-  deep_reflect_successes: 2
-
-optimizer:
-  learning_rate: 8
-  min_learning_rate: 2
-  lr_scheduler: cosine
-  skill_update_mode: patch
-  use_meta_reflect: false
-  meta_learning_rate: 8
-  use_slow_update: true
-  slow_update_samples: 20
-  use_meta_skill: true
-
-evaluation:
-  use_gate: true
-  sel_env_num: 0
-  test_env_num: 0
-  eval_test: true
-
-env:
-  split_mode: ratio
-  split_ratio: "2:1:7"
-  split_seed: 42
-```
-
-For the full source of truth, see [configs/_base_/default.yaml](/home/azureuser/workspace-yqh/skillopt_final/configs/_base_/default.yaml).
-
-Selected fields:
-
-| Section | Key | Meaning |
+| Flag | Default | Description |
 |---|---|---|
-| `model` | `teacher_backend` | teacher backend: `openai_chat` or `claude_chat` |
-| `model` | `student_backend` | student backend: chat backend or exec backend |
-| `model` | `teacher` | teacher model / deployment |
-| `model` | `student` | student model / deployment |
-| `model` | `reasoning_effort` | reasoning budget passed to the backend when supported |
-| `model` | `codex_trace_to_teacher` | include Codex trace summaries in teacher reflection context |
-| `train` | `num_epochs` | number of epochs |
-| `train` | `train_size` | expected train split size, or `0` to infer |
-| `train` | `batch_size` | tasks per rollout batch |
-| `train` | `accumulation` | number of rollout/reflect minibatches per step |
-| `gradient` | `minibatch_size` | trajectories per analyst minibatch |
-| `gradient` | `merge_batch_size` | patches per aggregate batch |
-| `gradient` | `use_deep_reflect` | enable diagnostic probe rollouts |
-| `gradient` | `max_analyst_rounds` | teacher reflection retries / refinement budget |
-| `optimizer` | `learning_rate` | max edits kept after selection |
-| `optimizer` | `lr_scheduler` | edit-budget scheduler |
-| `optimizer` | `use_slow_update` | epoch-level longitudinal guidance |
-| `optimizer` | `use_meta_skill` | teacher-side epoch memory |
-| `optimizer` | `use_meta_reflect` | epoch-level macro editing |
-| `optimizer` | `skill_update_mode` | `patch` or `rewrite_from_suggestions` |
-| `evaluation` | `sel_env_num` | selection set size (`0` means full split) |
-| `evaluation` | `test_env_num` | test set size (`0` means full split) |
+| `--port` | 7860 | Server port |
+| `--host` | `0.0.0.0` | Bind address |
+| `--share` | off | Create a public Gradio share link |
 
-### Important Branch Rule
+The default host listens on every network interface. Use
+`--host 127.0.0.1` for local-only access.
 
-`use_gate=false` is intentionally not supported in this branch. Gate validation is part of the method contract here.
+---
 
-If an old config still contains `evaluation.use_gate: false`, the loader / trainer will raise instead of silently continuing.
+## Citation
 
-## Supported Environments
-
-The main training entry and eval-only entry now register 11 environments:
-
-| Env | Default rollout shape | Current default split / data setting | Branch alignment |
-|---|---|---|---|
-| `alfworld` | environment-backed episodic rollout | native ALFWorld train/eval splits | in `reflact_new_zzw` |
-| `babyvision` | single-round multimodal QA | `split_mode=ratio` from raw metadata/images, or prepared `split_dir` | in `reflact_new_zzw` |
-| `docvqa` | single-round multimodal QA | `split_dir: data/docvqa_split` | in `reflact_new_zzw` |
-| `livemathematicianbench` | single-round QA | `split_mode=ratio` or prepared `split_dir` | in `reflact_new_zzw` |
-| `mathverse` | single-round multimodal math QA | `data_root: data/MathVerse`, split files loaded from `split_dir` when provided | in `reflact_new_zzw` |
-| `mmrb` | single-round multimodal reasoning QA | `split_mode=ratio` or prepared `split_dir` | in `reflact_new_zzw` |
-| `officeqa` | multi-turn tool loop | `split_dir: data/officeqa_split` plus `data_dirs: [data/officeqa_docs_official]` | in `reflact_new_zzw` |
-| `sealqa` | multi-turn tool loop | `split_dir: data/sealqa_split` | in `reflact_new_zzw` |
-| `searchqa` | single-round QA (`max_turns=1`) | `split_dir: data/searchqa_split` | in `reflact_new_zzw` |
-| `spreadsheetbench` | codegen loop, default `mode=multi`, `max_turns=30` | `split_dir: data/spreadsheetbench_split`, `data_root: data/spreadsheetbench_verified_400` | in `reflact_new_zzw`, default adjusted here to multi-round |
-| `swebench` | mini-swe-agent multi-step bug-fixing rollout | `split_mode=ratio`, `dataset_name=lite`, repo-stratified `2:1:7` split materialized under `out_root/_generated_splits/...` unless `split_dir` is provided | added here, aligned to `swe-bench-old` |
-
-## Data Expectations
-
-The standard two-mode dataset entry path is:
-
-- `split_mode: ratio`
-  - load raw data from `env.data_path`
-  - build a deterministic `train/`, `val/`, `test/` split under `env.split_output_dir` (or under `out_root/_generated_splits/` if unset)
-  - default ratio is explicitly `2:1:7`
-- `split_mode: split_dir`
-  - load an existing `env.split_dir` with `train/`, `val/`, `test/` subdirectories
-
-This currently applies to:
-
-- `searchqa`
-- `spreadsheetbench`
-- `babyvision`
-- `livemathematicianbench`
-- `mmrb`
-- `swebench`
-
-`ALFWorld` is the exception: it is environment-backed rather than JSON split-backed.
-
-The following environments currently expect prepared split directories or extra rooted assets rather than the generic ratio-split path:
-
-- `docvqa`
-- `mathverse`
-- `officeqa`
-- `sealqa`
-
-At a high level:
-
-- `SearchQA`: raw QA json / jsonl or pre-split QA json files
-- `SpreadsheetBench`: raw task manifest json plus spreadsheet task directory, or a pre-split task manifest
-- `ALFWorld`: installed game environment and configured eval/train splits
-- `BabyVision`: raw `meta_data.jsonl` plus images, or a pre-split directory
-- `DocVQA`: pre-split CSV / JSON data under `split_dir`
-- `LiveMathematicianBench`: raw monthly QA json files, or a pre-split directory
-- `MathVerse`: split files plus `data_root` image assets
-- `MMRB`: raw extracted dataset json files, or a pre-split directory
-- `OfficeQA`: pre-split metadata plus resolved office document directories
-- `SealQA`: pre-split metadata for tool-augmented QA tasks
-- `SWEBench`: HuggingFace SWE-bench dataset alias (`lite` / `verified` / `full`) or a prepared split directory
-
-### Split References Across Branches
-
-The split-related defaults are not identical across `skillopt-final`, `reflact_new_zzw`, `gepa`, and `swe-bench-old`. The practical reference points are:
-
-| Source branch | Explicit split settings / dirs |
-|---|---|
-| `skillopt-final` | `searchqa -> data/searchqa_split`; `spreadsheetbench -> data/spreadsheetbench_split`; `docvqa -> data/docvqa_split`; `officeqa -> data/officeqa_split`; `sealqa -> data/sealqa_split`; `swebench -> ratio split 2:1:7 over the default lite dataset, materialized under out_root/_generated_splits/...` |
-| `reflact_new_zzw` | Same 10-benchmark env set as above except no `swebench`; explicit split dirs are `data/searchqa_split`, `data/spreadsheetbench_split`, `data/docvqa_split`, `data/officeqa_split`, `data/sealqa_split`; `spreadsheetbench` there defaults to `mode=single`; `officeqa` uses `max_tool_turns=24`; `sealqa` uses `max_tool_turns=12` |
-| `gepa` | `configs/spreadsheetbench.yaml` uses `data.splits_dir = data/spreadsheetbench/splits`, `eval.mode = react`, `eval.max_turns = 20`; `configs/swebench.yaml` uses `dataset = SWE-bench/SWE-bench_Verified` with `train_size = 100`, `val_size = 50`, `test_size = 350` |
-| `swe-bench-old` | Repo-stratified `2:1:7` split over `SWE-Bench_Lite`, persisted as `outputs/.../split/train.json`, `selection.json`, `test.json`; the example split in that branch is `train=60`, `selection=33`, `test=207` |
-
-For the 10 benches shared with `reflact_new_zzw`, the current branch is now aligned on env coverage. The main intentional delta is `spreadsheetbench`: this branch defaults to multi-round codegen, while `reflact_new_zzw` kept `mode=single` by default.
-
-## Running Training
-
-Example:
-
-```bash
-python scripts/train.py --config configs/searchqa/default.yaml
+```bibtex
+@article{yang2026skillopt,
+  title={Skillopt: Executive strategy for self-evolving agent skills},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
+  journal={arXiv preprint arXiv:2605.23904},
+  year={2026}
+}
 ```
-
-Explicit 2:1:7 split from raw data:
-
-```bash
-python scripts/train.py \
-  --config configs/searchqa/default.yaml \
-  --split_mode ratio \
-  --data_path /path/to/searchqa_train_2000.json
-```
-
-Directly consume a prepared split directory:
-
-```bash
-python scripts/train.py \
-  --config configs/searchqa/default.yaml \
-  --split_mode split_dir \
-  --split_dir /path/to/searchqa_split
-```
-
-You can override structured config keys from the CLI:
-
-```bash
-python scripts/train.py \
-  --config configs/spreadsheetbench/default.yaml \
-  --cfg-options model.teacher_backend=openai_chat model.student_backend=codex_exec train.batch_size=40 optimizer.learning_rate=4
-```
-
-Legacy flat overrides still work for common keys:
-
-```bash
-python scripts/train.py \
-  --config configs/searchqa/default.yaml \
-  --backend azure_openai \
-  --teacher_model gpt-5.4 \
-  --student_model gpt-5.4 \
-  --reasoning_effort medium
-```
-
-Exec harness example:
-
-```bash
-python scripts/train.py \
-  --config configs/searchqa/default.yaml \
-  --teacher_backend openai_chat \
-  --student_backend codex_exec \
-  --teacher_model gpt-5.4 \
-  --student_model gpt-5.4-codex \
-  --use_deep_reflect true \
-  --skill_update_mode rewrite_from_suggestions
-```
-
-SWEBench example:
-
-```bash
-python scripts/train.py \
-  --config configs/swebench/default.yaml \
-  --cfg-options env.dataset_name=lite env.split_ratio=2:1:7
-```
-
-## Eval-Only and Standalone Evaluation
-
-Evaluate a specific skill without training:
-
-```bash
-python scripts/eval_only.py \
-  --config configs/searchqa/default.yaml \
-  --skill reflact/envs/searchqa/skills/initial.md
-```
-
-The same dataset entry modes apply in eval-only runs:
-
-- `--split_mode ratio --data_path ...`
-- `--split_mode split_dir --split_dir ...`
-
-Standalone scripts also exist for benchmark-specific comparisons, including:
-
-- `scripts/eval_prompt_custom.py`
-- `scripts/eval_prompt_official.py`
-- `scripts/eval_livemathematicianbench_baseline.py`
-
-These scripts now also support backend selection through the unified model layer.
-
-## Output Structure
-
-Each run writes a structured output directory under `out_root`.
-
-Important top-level artifacts:
-
-- `config.json` — flattened runtime config
-- `history.json` — per-step history records
-- `runtime_state.json` — resume state for current/best skill tracking
-- `best_skill.md` — current best validated skill
-- `skills/skill_vXXXX.md` — persisted skill snapshot per step
-
-Per-step artifacts live under `steps/step_XXXX/`, including:
-
-- `merged_patch.json`
-- `ranked_edits.json`
-- `candidate_skill.md`
-- `edit_apply_report.json`
-- `rewrite_result.json` when rewrite mode is enabled
-- `selection_eval/`
-- `trajectory_digest.json`
-- rollout and patch subdirectories
-
-Epoch-level artifacts live under:
-
-- `slow_update/epoch_XX/`
-- `meta_skill/epoch_XX/`
-- `meta_reflect/epoch_XX/`
-
-## Resume Behavior
-
-The trainer resumes from `runtime_state.json` when present. That state tracks:
-
-- last completed step
-- current skill path
-- current score
-- best skill path
-- best score
-- origin tags for current and best skill
-
-This is important because skill state can change at both step level and epoch level; resuming only from `history.json` is not sufficient for this branch’s method logic.
-
-## Notes
-
-- This repository focuses on skill optimization logic; datasets are not included.
-- Patch application is intentionally observable. Inspect `edit_apply_report.json` when candidate skills do not behave as expected.
-- `SpreadsheetBench` now defaults to `mode=multi`. If you run an exec student backend there, override back to `env.mode=single` because exec backends are still only wired for SpreadsheetBench single-mode rollout.
-- `SWEBench` follows the older mini-swe-agent + `swebench.harness.run_evaluation` path, so it requires the SWE-bench / Docker toolchain rather than the generic chat-only stack.
-- `slow_update` writes into a protected skill region and normal edits are prevented from overwriting that region directly.
-- `meta_skill` is context memory, not a direct skill edit.
-- `meta_reflect` is a gated skill edit stage, not just logging.
-
-## Minimal Setup
-
-```bash
-conda create -n reflact python=3.11
-conda activate reflact
-pip install openai pyyaml openpyxl
-```
-
-Depending on the environment, you may also need:
-
-```bash
-pip install datasets gymnasium numpy ray regex
-```
-
-For `SWEBench`, you also need a working Docker environment plus the SWE-bench / mini-swe-agent dependencies used in `swe-bench-old`.

blog/index.html

@@ -0,0 +1,149 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>SkillOpt Technical Blog</title>
+  <meta name="description" content="Technical notes from the SkillOpt team on skill optimization, evaluation, safety, and agent learning systems.">
+  <link rel="canonical" href="https://microsoft.github.io/SkillOpt/blog/">
+  <meta property="og:type" content="website">
+  <meta property="og:site_name" content="SkillOpt">
+  <meta property="og:title" content="SkillOpt Technical Blog">
+  <meta property="og:description" content="Technical notes on skill optimization, evaluation, safety, and agent learning systems.">
+  <meta property="og:url" content="https://microsoft.github.io/SkillOpt/blog/">
+  <meta property="og:image" content="https://microsoft.github.io/SkillOpt/skillopt-assets/teaser-1.png">
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:title" content="SkillOpt Technical Blog">
+  <meta name="twitter:description" content="Technical notes on skill optimization, evaluation, safety, and agent learning systems.">
+  <meta name="twitter:image" content="https://microsoft.github.io/SkillOpt/skillopt-assets/teaser-1.png">
+  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Crect width='100' height='100' rx='16' fill='%23245fc7'/%3E%3Ctext x='50' y='68' text-anchor='middle' font-size='58' font-family='Arial' font-weight='700' fill='white'%3ES%3C/text%3E%3C/svg%3E">
+  <style>
+    :root {
+      --ink: #172033;
+      --muted: #596579;
+      --line: #d9deea;
+      --paper: #fff;
+      --wash: #f7f9fc;
+      --blue: #245fc7;
+      --blue-soft: #eaf1ff;
+      --shadow: 0 14px 44px rgba(23, 32, 51, .08);
+    }
+    * { box-sizing: border-box; }
+    body {
+      margin: 0;
+      color: var(--ink);
+      background: var(--wash);
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+      line-height: 1.6;
+    }
+    a { color: var(--blue); }
+    .skip-link {
+      position: fixed;
+      left: 16px;
+      top: -80px;
+      z-index: 100;
+      padding: 10px 14px;
+      border-radius: 8px;
+      color: #fff;
+      background: var(--blue);
+    }
+    .skip-link:focus { top: 12px; }
+    header {
+      border-bottom: 1px solid var(--line);
+      background: rgba(255, 255, 255, .95);
+    }
+    .header-inner {
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      gap: 24px;
+      max-width: 1040px;
+      margin: 0 auto;
+      padding: 16px 24px;
+    }
+    .brand {
+      color: var(--ink);
+      font-weight: 750;
+      text-decoration: none;
+    }
+    nav { display: flex; flex-wrap: wrap; gap: 16px; font-size: 14px; }
+    nav a { color: var(--muted); text-decoration: none; }
+    nav a:hover, nav a:focus-visible { color: var(--blue); text-decoration: underline; }
+    main { max-width: 1040px; margin: 0 auto; padding: 72px 24px 96px; }
+    .eyebrow {
+      color: var(--blue);
+      font-size: 13px;
+      font-weight: 750;
+      letter-spacing: .08em;
+      text-transform: uppercase;
+    }
+    h1 { margin: 10px 0 16px; font-size: clamp(40px, 7vw, 68px); line-height: 1.02; }
+    .intro { max-width: 760px; color: var(--muted); font-size: 20px; }
+    .post-list { margin-top: 52px; }
+    .post-card {
+      display: grid;
+      grid-template-columns: minmax(0, 1fr) auto;
+      gap: 30px;
+      padding: 34px;
+      border: 1px solid var(--line);
+      border-radius: 16px;
+      background: var(--paper);
+      box-shadow: var(--shadow);
+    }
+    .post-card h2 { margin: 8px 0 12px; font-size: clamp(25px, 4vw, 36px); line-height: 1.15; }
+    .post-card h2 a { color: var(--ink); text-decoration: none; }
+    .post-card h2 a:hover, .post-card h2 a:focus-visible { color: var(--blue); text-decoration: underline; }
+    .meta { color: var(--muted); font-size: 14px; }
+    .summary { max-width: 720px; margin: 16px 0 0; color: #344054; }
+    .read-link { align-self: center; white-space: nowrap; font-weight: 700; }
+    footer {
+      max-width: 1040px;
+      margin: 0 auto;
+      padding: 26px 24px 42px;
+      border-top: 1px solid var(--line);
+      color: var(--muted);
+      font-size: 14px;
+    }
+    @media (max-width: 720px) {
+      .header-inner { align-items: flex-start; flex-direction: column; }
+      main { padding-top: 48px; }
+      .post-card { grid-template-columns: 1fr; padding: 24px; }
+      .read-link { justify-self: start; }
+    }
+  </style>
+</head>
+<body>
+  <a class="skip-link" href="#main-content">Skip to content</a>
+  <header>
+    <div class="header-inner">
+      <a class="brand" href="../">SkillOpt</a>
+      <nav aria-label="Site navigation">
+        <a href="../">Project</a>
+        <a href="../docs/guideline.html">Documentation</a>
+        <a href="https://arxiv.org/abs/2605.23904">Paper</a>
+        <a href="https://github.com/microsoft/SkillOpt">Code</a>
+      </nav>
+    </div>
+  </header>
+
+  <main id="main-content">
+    <span class="eyebrow">Microsoft Research · SkillOpt</span>
+    <h1>Technical Blog</h1>
+    <p class="intro">Scoped experiments and engineering notes on optimizing agent skills, evaluating text-space updates, and deploying self-improving systems responsibly.</p>
+
+    <section class="post-list" aria-labelledby="latest-posts">
+      <h2 id="latest-posts">Latest posts</h2>
+      <article class="post-card">
+        <div>
+          <div class="meta"><time datetime="2026-07-14">July 14, 2026</time> · Ziwei Zhou, Ziyang Gong, and Yifan Yang</div>
+          <h2><a href="gating-reflection-safe-updates/">Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep</a></h2>
+          <p class="summary">A three-part report with expanded SkillOpt ablations, a skill-aware reflection design with memory consolidation, and a controlled study of the SkillOpt-Sleep plugin.</p>
+        </div>
+        <a class="read-link" href="gating-reflection-safe-updates/" aria-label="Read Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep">Read article →</a>
+      </article>
+    </section>
+  </main>
+
+  <footer>SkillOpt Technical Blog · <a href="https://github.com/microsoft/SkillOpt">github.com/microsoft/SkillOpt</a></footer>
+</body>
+</html>

ckpt/README.md

@@ -0,0 +1,87 @@
+# Paper-aligned SkillOpt reference skills (GPT-5.5)
+
+This folder provides a subset of the paper's main Table 1 GPT-5.5 optimized
+skills as reference artifacts — one `gpt5.5_skill.md` per currently included
+benchmark. You can plug them into `scripts/eval_only.py` to evaluate the
+provided skills on a given split without re-running the training loop.
+
+> These are checkpoints associated with the paper, not a general-purpose
+> tool. They're here so you can verify the reported numbers and use the
+> skills as portable artifacts. If you want to *train* your own skill,
+> use `scripts/train.py` per the top-level README.
+>
+> This is the first optimized-skill artifact batch. We plan to continue
+> uploading remaining paper artifacts as they are cleaned and verified. All
+> six lightweight ID/path split manifests are already checked in under
+> `data/`; most still require materializing their upstream benchmark payload.
+
+## What's here
+
+| Benchmark | Skill artifact | Matching config |
+|---|---|---|
+| SearchQA | `ckpt/searchqa/gpt5.5_skill.md` | `configs/searchqa/default.yaml` |
+| ALFWorld | `ckpt/alfworld/gpt5.5_skill.md` | `configs/alfworld/default.yaml` |
+| DocVQA | `ckpt/docvqa/gpt5.5_skill.md` | `configs/docvqa/default.yaml` |
+| LiveMathematicianBench | `ckpt/livemath/gpt5.5_skill.md` | `configs/livemathematicianbench/default.yaml` |
+| OfficeQA | `ckpt/officeqa/gpt5.5_skill.md` | `configs/officeqa/default.yaml` |
+| SpreadsheetBench | `ckpt/spreadsheetbench/gpt5.5_skill.md` | `configs/spreadsheetbench/default.yaml` |
+
+Each file is a plain Markdown skill document (~2k–13k chars). It contains a
+protected `SLOW_UPDATE` section at the end that holds epoch-wise
+longitudinal guidance — that's expected, not a formatting issue.
+
+## How to evaluate a provided skill
+
+`scripts/eval_only.py` runs a single skill against a data split without
+invoking the optimizer. Example for SearchQA against the test split:
+
+```bash
+# The checked-in SearchQA split is ID-only; materialize full examples first.
+python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill ckpt/searchqa/gpt5.5_skill.md \
+  --split valid_unseen \
+  --split_dir data/searchqa_split \
+  --azure_openai_endpoint https://your-resource.openai.azure.com/ \
+  --azure_openai_auth_mode api_key \
+  --target_model gpt-5.5
+```
+
+Substitute the benchmark, config, skill path, and `--split_dir` to evaluate
+any of the other five. `--split valid_unseen` is the test split, `valid_seen`
+is the selection / validation split, `train` is the training split, and
+`all` runs all three.
+
+## On comparing to the paper numbers
+
+To compare against the paper-reported cells, use the same dataset split and
+scorer. SearchQA's ID manifest is checked in at `data/searchqa_id_split/` (400
+train / 200 selection / 1400 test); the materializer writes the runnable
+payload to `data/searchqa_split/`. All six lightweight split manifests are
+checked in under `data/`. ALFWorld's manifest records game-file paths; the
+other ID manifests still require you to materialize the corresponding
+upstream benchmark payload into the documented `split_dir`. See
+[`data/README.md`](../data/README.md) for the exact status of each benchmark.
+When using `split_mode: ratio` instead, the loader is deterministic from
+`split_seed` (default `42`) + `split_ratio` (default `2:1:7`), so a given
+`data_path` + seed reproduces across machines.
+
+## Why force-accept vs. gated slow-update matters
+
+These `ckpt/` skills were produced with the gated slow-update semantics
+described in paper Section 3.6:
+
+```yaml
+optimizer:
+  slow_update_gate_with_selection: true
+```
+
+Current `main` defaults to `false` (force-accept mode), a newer
+post-submission behavior where the slow-update guidance is written into
+`current_skill` and `best_skill` unconditionally at the epoch boundary. If
+you re-train with the current default, you may produce a *different*
+`best_skill.md` than the one checked in here. Both modes are supported; see
+the [configuration reference](../docs/reference/config.md).

ckpt/alfworld/gpt5.5_skill.md

@@ -0,0 +1,113 @@
+# ALFWorld Embodied Agent Skill
+
+## Overview
+This skill guides agents operating in the ALFWorld text-based embodied environment.
+The agent must complete household tasks by navigating rooms, interacting with objects,
+and using appliances. Actions must be chosen from the admissible action list provided
+at each step.
+
+**Output format**: Always output `<think>...</think>` for reasoning, then `<action>...</action>` for the chosen action.
+
+---
+
+## Task Types
+
+| Type | Goal | Key Steps |
+|------|------|-----------|
+| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y |
+| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place |
+
+### Pick Two Object Bookkeeping
+For `pick_two_obj_and_place`, choose one destination receptacle instance once it is opened/usable, and remember it as the target. Both object instances should be placed into that same remembered receptacle. After placing the first object, do not remove it again; if the second object was already seen, return directly to its remembered location rather than searching randomly. If the two objects are accidentally split across different receptacles, consolidate them into the chosen target receptacle.
+| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp |
+
+| Examine in Light detail | Final interaction | While holding X where a desklamp is visible, use the desklamp; do not try to place X on the lamp first. |
+| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X |
+| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X |
+| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X |
+
+---
+
+## General Principles
+
+1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next.
+2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty.
+
+- Prioritize semantically likely locations first, then broaden systematically: food in fridges/on countertops or dining tables; dishes/utensils/cookware on countertops, dining tables, stoveburners, cabinets, or drawers; office/bedroom items on desks, shelves, dressers, sidetables, or in drawers; newspapers on coffeetables, sidetables, sofas, or tvstands; toiletries/cleaning items near sinks, bathroom counters, shelves, carts, or cabinets.
+
+- For portable kitchen targets such as bread, mugs, cups, plates, bowls, and utensils, check broad exposed surfaces early: after one or two empty countertops, try dining tables or other open surfaces before opening many cabinets/drawers. For small office/bedroom targets, alternate drawers with exposed desks, shelves, sidetables, and dressers rather than exhausting drawers first.
+
+- Keep a persistent **searched set** of receptacle instances, e.g. `drawer 1`, `shelf 3`, `countertop 2`. Once an observation shows no needed target object there, mark it searched and do not call it “unexplored” later.
+   - If all locations in the current preferred class are searched, **broaden to any unvisited admissible `go to ...` location** instead of restarting the same sequence. Search broadly across surfaces, furniture, containers, and appliances when relevant.
+   - If a visible object is itself an openable/container-like object, such as a box, and opening/examining it is admissible, inspect it before leaving the area.
+3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere.
+
+- Pick up only the exact requested object type. Similar or related objects, such as a cup when the task asks for a mug, a spoon when it asks for a knife, or a pot when it asks for a pan, are distractors; leave them in place and mark that location searched for the target.
+4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination.
+
+- Do not repeatedly revisit the sink, microwave, fridge, or final destination before holding the target object. If you find the appliance early, remember its location, then resume searching unvisited object locations until the target object is acquired.
+
+- Use direct admissible appliance/tool commands immediately when available, such as `clean X with sinkbasin`, `heat X with microwave`, `cool X with fridge`, or `use desklamp`. Do not waste steps opening, closing, toggling, or examining the appliance unless the needed action is unavailable or opening is required for searching/placing.
+5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it.
+
+- Remember known destination receptacles and return directly to the same instance after pickup/transformation. If the destination is also a semantically likely source location, check/open it early rather than only after exhaustive search: food may already be in the fridge, utensils may be on the diningtable, newspapers may be on/near the sofa, and a target drawer can be opened early for pick-two tasks. If the object starts at the destination but needs cleaning/heating/cooling, take it out, transform it, then return to that same instance and place it back.
+6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero.
+7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location.
+8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions.
+
+---
+
+## Common Mistakes to Avoid
+
+- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them.
+- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately.
+- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required.
+- **Premature termination**: Do not stop the episode until all goal conditions are verified as met.
+- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead.
+
+### Hard Search-Loop Recovery
+
+- **Exact-instance lockout before pickup**: once a receptacle/surface instance has been observed and does not contain the target object, do not go back to that exact instance while still searching for the object. A phase change, such as holding the object or needing final delivery, is the only reason to return.
+- **Fast broadening threshold**: after 3-4 misses in the same receptacle class, switch to a different likely class or any unvisited admissible location instead of continuing or restarting that class, unless the target has already been seen there.
+- **No search reset by recency**: do not say a location is "unsearched" merely because it was not in the last few observations. The searched set is global for the whole episode.
+- **Finite-class exhaustion**: if all visible instances of a small class have been checked once, such as all stoveburners, diningtables, countertops, or shelves, mark that class exhausted for object search and do not start a second pass. Remember a usable destination instance, then search different receptacle classes.
+- **Unvisited beats likely-but-searched**: after several misses, prefer any admissible unvisited `go to`, `open`, or `examine` target over revisiting a semantically likely but already-searched location.
+- **Destination surfaces before pickup**: if the destination receptacle is also a likely object location, inspect each instance at most once before pickup. If it lacks the object, remember it as the final destination but stop using it as a search target until the object has been transformed and is ready to place.
+- **Kitchen item fallback**: for cookware and dishware, after checking obvious burners/tables/counters once, broaden to unsearched cabinets, drawers, shelves, sinkbasins, and other kitchen storage/surfaces rather than cycling among the obvious locations.
+
+### Strict Search Ledger Action Filter
+
+Before every empty-handed search action, apply this hard filter:
+
+1. If a required target object is visible, take it immediately.
+2. Otherwise choose an exact receptacle/surface/container instance whose contents have not yet been observed in the current object-search phase.
+3. Reject any `go to`, `examine`, or `open` action for an exact instance already observed to lack the target, even if it is semantically likely, nearby, recently mentioned, or the final destination type.
+4. If all likely instances are rejected by the ledger, broaden to any unvisited admissible location/class instead of restarting from instance 1 of a searched class.
+
+The searched ledger survives inventory checks, appliance visits, placing the first object in a pick-two task, and putting down an irrelevant inspected object/container. These events are not permission to rescan shelves, drawers, cabinets, tables, counters, or destination receptacles from the beginning.
+
+### Destination-as-Source Lockout
+
+When the final receptacle type is also a plausible source location, inspect each visible destination instance at most once before pickup. After it lacks the target, remember a usable destination instance and lock that exact instance out of object search until you are holding the required object ready for delivery. Do not alternate between destination instances and other searched source instances while still empty-handed.
+
+### Pick-Two Phase Memory
+
+After placing the first object in a pick-two task, do not begin a fresh room/class search. If another required instance was previously seen, return directly to that remembered source location for the second pickup. If no second instance is remembered, continue from the existing unsearched-location ledger rather than revisiting locations already checked before the first placement.
+
+<!-- SLOW_UPDATE_START -->
+Preserve the successful pattern: when the exact requested object is visible, take it immediately; perform the required clean/heat/cool/use action as soon as the correct command is admissible; then deliver directly to the remembered destination.
+
+Treat tool locations as tools, not repeated search targets. If a sinkbasin, fridge, microwave, desklamp, or destination receptacle has already been checked and does not contain the target while you are empty-handed, remember it for later but do not revisit it until you are holding the required object or ready to place/use it.
+
+Use a next-unsearched-instance pointer for every numbered class. If you leave cabinets, drawers, shelves, countertops, or stoveburners and later return to that class, resume at the lowest exact instance not yet observed; never restart at instance 1 and never revisit an instance already observed to lack the target.
+
+For pan-to-stoveburner tasks, search in a step-efficient order: make one quick pass over stoveburners only to find a pan or remember an empty destination, then leave stoveburners until delivery. Next check countertops/islands and sinkbasins. Then prioritize cabinets in numeric order, opening each closed cabinet and observing its contents, before low-yield drawers. Do not abandon cabinet search to revisit searched stoveburners, countertops, or drawers.
+
+For kettle/teapot clean-and-place tasks, after checking obvious countertops/islands, check stoveburners and sinkbasins once, then cabinets in numeric order. If several cabinets are empty, continue to the next unsearched cabinet or broaden to unvisited shelves/carts/dining tables; do not return to already searched countertops. Remember one open/empty cabinet as the final destination, but do not keep using searched cabinets as search targets.
+
+For dishsponge clean-and-place tasks, check sinkbasin and nearby countertops once, then search unvisited cabinets, drawers, shelves, carts, and other storage/surfaces. Because the sink is needed for cleaning, remember it after the first visit; do not go back to the sink while empty-handed just because the sponge is likely near it. Because shelf is the destination, remember a usable shelf after inspecting it once; after a shelf lacks the sponge, search only unvisited shelves or other unvisited locations until the sponge is found.
+
+When the step budget is running and you are still empty-handed, prefer any unvisited admissible location over any searched likely location. A location being semantically likely, useful later, or recently mentioned is never a reason to rescan it before acquisition.
+
+Do not let the broadening threshold cause class restarts. Broadening means move to a different unvisited class or continue at the next unsearched instance of a promising storage class; it never means cycling back through exact instances already observed.
+<!-- SLOW_UPDATE_END -->

ckpt/docvqa/gpt5.5_skill.md

@@ -0,0 +1,26 @@
+# DocVQA Skill
+
+## Visual Evidence Discipline
+- Read the document carefully before answering.
+- Prefer the smallest exact text span that answers the question.
+
+- For questions asking for a value, count, page number, date, or graph reading, return only the requested value span; omit nearby labels, category names, units, or explanatory words unless the question explicitly asks for them.
+- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question.
+
+## Exact Answer Discipline
+- Copy names, numbers, and dates exactly from the document whenever possible.
+
+- Preserve the document's exact spelling and punctuation for names and quoted phrases; do not substitute similar letters or change straight/curly quotes, spacing, or parentheses when the visible text provides them.
+- Prefer direct extraction over paraphrase.
+- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span.
+
+## Structured Layout Lookup
+- For tables, first find the row or entry named in the question, then read the value under the requested column, header, date, or category; answer with that cell only.
+- For forms, receipts, or labeled fields, locate the exact role, party, or field label mentioned in the question, then copy the filled-in value from the same line, box, block, or immediately adjacent field.
+- For table-of-contents, indexed, numbered, or bulleted lists, match the requested title, entry, or point number, then follow the same line or list item to the associated value; do not take a nearby value from another item.
+
+## Anchored Handwriting / Nearby Text
+- For handwritten or list/table questions with an anchor term, first locate the anchor, then inspect the immediately adjacent text in the same row, column, or nearby margin. If legible, provide the best-supported nearby span rather than leaving the answer blank.
+
+<!-- SLOW_UPDATE_START -->
+<!-- SLOW_UPDATE_END -->

ckpt/livemath/gpt5.5_skill.md

@@ -0,0 +1,35 @@
+# Live Mathematical MCQ Heuristics
+
+## Option Comparison
+
+### Meta-Options About Stronger Results
+- Treat options of the form “one of the remaining options is correct, but a stronger result can be proven” as serious candidates, especially when the question asks for the strongest statement.
+- If a concrete option is true but your theorem or derivation gives a strictly stronger conclusion not exactly listed, choose the meta-option rather than the weaker concrete statement.
+- When options are nested by strength, rank them explicitly before answering: e.g. finite-time blowup is stronger than merely “not globally bounded”; positive stable growth is stronger than ordinary unboundedness; sharper constants, rates, exceptional-set bounds, endpoint inclusion, or full equivalences are stronger than weaker asymptotic versions.
+- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case.
+- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when".
+
+## Theorem-Level Precision
+
+- Do not add converse, realization, or classification claims unless the theorem explicitly proves them. Phrases such as “conversely,” “every such parameter occurs,” “if and only if,” or “exactly all” add strength beyond a one-way implication.
+- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence.
+- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one.
+
+## Hypotheses
+
+### Exact Conditions and Thresholds
+- For biconditional/equivalence questions, reject conditions that are merely necessary or merely sufficient. A broader condition, such as congruence modulo a divisor instead of modulo the full modulus, is usually weaker and not equivalent unless the domain collapses the extra cases.
+- For threshold conditions, verify the exact sign and endpoint: distinguish \(\mu_0\) from \(-\mu_0\), \(<\) from \(\le\), and whether the equality case belongs to the positive, zero, or negative parameter regime.
+- When options differ by “for every” vs “for sufficiently large,” local vs global domains, strict vs non-strict inequalities, or dependence of constants, rank them by logical strength and match the sharpest justified version.
+- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions.
+- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily.
+
+## Final Answer
+- Output the final answer as the single option label only.
+
+## Exact Scope and Quantitative Wording
+- Distinguish global conclusions from localized or completed ones. Equivalence after localization, completion, or at each prime/scale is usually weaker than an unqualified equivalence.
+- In estimate-heavy options, compare every quantitative detail: exponent, derivative index range, constants and their parameter dependence, log factors, additive terms, one-sided vs two-sided notation, and pointwise vs uniform convergence.
+
+<!-- SLOW_UPDATE_START -->
+<!-- SLOW_UPDATE_END -->

ckpt/officeqa/gpt5.5_skill.md

@@ -0,0 +1,50 @@
+# OfficeQA Skill
+
+## Retrieval Discipline
+
+- When an external official time-series observation is needed, prefer the source's series/data-download/table page once identified. If exact-date or guessed-value searches return empty results, stop repeating them; broaden to the official series name/code plus `data` or `download` and use the table values.
+
+- Treat provided/oracle parsed pages as primary evidence: if they contain the relevant table and period, extract directly from them before searching elsewhere; search only for missing continuation pages, missing periods, or an official actual value not present.
+- Start by narrowing to the most likely candidate file before reading long passages.
+- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question.
+- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit.
+
+- If the requested date range extends beyond the provided/oracle page, first enumerate the required periods and verify that every period is present in evidence. Do not compute from a partial ledger or fill missing periods from memory; retrieve continuation pages, adjacent issues, or a later issue of the same table that contains the missing dates/revisions.
+
+## Evidence Discipline
+- Extract the exact value from the retrieved text before doing any arithmetic.
+- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in.
+
+- For Treasury financing narratives, label each amount by transaction role before calculating: offered amount, tenders/subscriptions received, tenders accepted, competitive/noncompetitive accepted, foreign or Government-account exchange tenders, refunding, and **new cash** are not interchangeable.
+- When converting currencies or scales, make a direction ledger first: source table unit, source currency, exchange-rate orientation (foreign currency per U.S. dollar means divide by the rate; U.S. dollars per foreign unit means multiply), and requested final unit.
+
+- For tables, align values by row label and exact column header, not proximity alone; watch for continued or unlabeled columns, footnotes, adjacent amount-versus-percent columns, fiscal-year versus calendar-year sections, and repeated month rows under different year blocks.
+- If the question asks for a transformed or derived quantity, compute only after confirming every operand.
+
+- For derived comparisons, preserve the direction and sign implied by the wording: “change from A to B” means B minus A; “former than latter” means former minus latter; “share accounted for by X” means X divided by the stated total; paired “gap” questions require computing each within-row difference before ranking.
+- For statistical, regression, correlation, and growth-rate questions, write a formula ledger before calculating: confirm the exact series/endpoints, ordered vector, elapsed intervals, and requested convention such as continuously compounded rate, CAGR, Pearson correlation, or OLS index/year choice.
+- For multi-stage questions where one table determines the period/entity used in another lookup, freeze that derived key with evidence first, then retrieve the second measure only for that exact month/year/reporting date/entity.
+
+- For inclusive time-series ranges, make a period-by-period ledger covering every requested month/year exactly once, preserving calendar versus fiscal basis, end-of-month or end-of-fiscal-month status, source units, and any specified adjustments.
+
+- For statistical transforms over time-series windows, confirm endpoint inclusion/exclusion exactly as worded, use consecutive time indices for trend regressions when appropriate, sort values before medians, and for logarithmic growth use ln(final/initial) before converting to the requested percentage format.
+
+## Final Answer Discipline
+
+- Before finalizing, enforce the requested unit and format: convert thousands/millions/billions or full nominal dollars as needed, then apply no-comma, fixed-decimal, whole-number, or nearest-tenth/thousandth formatting exactly as asked.
+- Return the final answer only after one last consistency check against the retrieved evidence.
+- Copy the final answer from a checked value, not from an unverified intermediate guess.
+
+## Statistical and Time-Series Calculation Checks
+
+- Before computing any statistic, write the intended formula and denominator convention. If the prompt explicitly says **population standard deviation**, divide by `n`; if it says **sample**, divide by `n-1`; for a z-score comparing one observation against a small set of comparison months/periods and no population convention is stated, estimate dispersion with the **sample** standard deviation of the comparison set. Do not round intermediate operands, weighted averages, logs, exchange-rate conversions, or standard deviations before the final requested rounding.
+- For long inclusive ranges, first enumerate the expected count of observations and the first/last period, then verify the ledger has exactly that count. Exclude totals, cumulative-to-date columns, comparable-period columns, estimates, and extra latest-month columns outside the requested calendar or fiscal range.
+- When a page contains multiple nearby sections with similar labels, use only the section whose title and row label match the requested measure exactly; do not compute from the first visible table if the requested measure/table title is absent or only partially shown.
+- For Treasury security quotations, obey the table's quote basis. If the table states that price decimals are 32nds, convert quotes such as `99.27` as `99 + 27/32`, not as decimal `99.27`. If a task asks for smoothing, averaging, or forecasting in a target currency using period-specific exchange rates, convert each period's observation to the target currency first unless the prompt explicitly says to compute in the source currency and convert only the final result.
+
+## Stricter Final Formatting
+
+- Match any requested output template exactly. Unless the prompt explicitly asks for unit words or explanatory text, return only the numeric value or requested list; do not append words such as `million`, `dollars`, `percent`, or `percentage points`. Include symbols/commas only when the prompt requests currency-formatted output or the answer format clearly requires them.
+
+<!-- SLOW_UPDATE_START -->
+<!-- SLOW_UPDATE_END -->

ckpt/searchqa/gpt5.5_skill.md

@@ -0,0 +1,71 @@
+# Question Answering Skill
+
+(No learned rules yet. Rules will be added through the reflection process.)
+
+## Concise Answer Normalization
+- Prefer the shortest unambiguous answer that directly satisfies the question. Do not include generic descriptors, legal suffixes, or expanded formal names unless the question specifically asks for the full official name or the descriptor is necessary to identify the entity.
+
+- If the answer appears inside a longer descriptive phrase, strip words that merely repeat the clue's requested type or modifiers already stated in the clue. For short-answer trivia, return the distinctive core entity or headword rather than role titles, product flavor adjectives, or place/facility designators, even when those words are part of a fuller official phrase, unless the full official name is explicitly requested.
+- For place/name-etymology questions asking for “the name” or “the word” that means something, answer the distinctive name/word itself rather than a larger phrase with a generic type label.
+
+- For natural geographic features, preserve conventional feature designators such as “Lake,” “River,” “Bay,” “Gorge,” “Mount,” or “Island” when they are part of the proper name or match the requested feature type. Do not shorten “Lake Okeechobee,” “Tampa Bay,” or “Olduvai Gorge” to an ambiguous base name merely to be concise.
+- For companies, brands, and organizations, answer the common distinctive name when sufficient; omit additions such as “Company,” “Corporation,” “Inc.,” etc. unless explicitly required.
+
+- Preserve the answer surface form supported by the strongest evidence when exact variants differ: spelling, capitalization, punctuation, and word order can matter. Do not substitute an equivalent official/common variant such as an alternate spelling or inverted institution name if a direct title/snippet/answer field gives the expected form.
+
+- When copying titles or quoted names, preserve ordinary ASCII punctuation from the evidence, especially straight apostrophes (`'`). Do not replace them with typographic curly quotes/apostrophes unless that exact stylized form is explicitly shown as the supported answer.
+
+- For nicknames, epithets, saints, and quoted titles, copy the supported surface form exactly, including spacing, capitalization, and conventional abbreviations such as “St.” Do not normalize a stylized or quoted form into a lowercase dictionary word or an expanded spelling when the clue/evidence points to the stylized answer.
+
+- For person answers in trivia or crossword-style clues, prefer the conventional supported name. Use just a surname, first name, or saint/regnal name only when the clue/source clearly expects that short form; otherwise use the canonical full personal name from the strongest evidence or answer field, especially when a lone given name would be ambiguous.
+- Return the grammatical base form expected by the clue. Do not add a plural `s` merely because a crossword source pluralizes a shared name or category; if the clue lists people sharing a first name, answer the singular given name.
+
+- For common-noun category answers, default to the singular dictionary headword in trivia/crossword-style clues, even if the clue uses plural words like “these,” “those,” “places,” or “items” for grammar. Use a plural only when the term is inherently plural or an answer field/source clearly gives a plural phrase.
+
+- For common-noun clues about things being replaced, used in place of, or substituted by another system/item, answer the broad headword for the thing replaced unless a narrowing modifier is required by the clue or answer field. Do not add adjectives such as “letter,” “regular,” or “standard” merely because they appear in explanatory context.
+- For fill-in-the-blank or definitional clues using words like “this” or “that,” provide a standalone noun phrase. Avoid context-dependent pronouns or possessives from the source text; use a natural article such as “the” when needed (e.g., answer “the highest point,” not “its highest point”).
+
+## Context-Grounded Evidence Matching
+- Start by identifying the most distinctive terms in the question: proper names, dates, titles, quoted phrases, unusual words, roles, relationships, and category descriptors.
+- Prioritize passages or document titles where several distinctive clue terms occur together, especially if the wording directly repeats or closely paraphrases the question.
+- Treat document titles as useful evidence: the answer is often named in a title while the snippet confirms the clue facts.
+
+- Do not assume the document title itself is the answer. If the requested type differs from the title entity, use the title as context and extract the matching typed entity from the snippet or clue relationship.
+
+- For “known as,” “called,” “defined as,” or category/type clues, choose the canonical term explicitly used in the strongest matching title/snippet or scraped answer field rather than inventing a related derivative or near-synonym from the clue wording. When multiple plausible candidates appear, prefer the candidate whose evidence directly states the requested relationship and repeats the most distinctive clue facts.
+- Ignore noisy results that only match generic words; prefer evidence that directly connects the clue facts to one specific entity.
+
+## Clue Interpretation and Answer Type
+- For Jeopardy-style wording such as “this man,” “this group,” “this film,” “this country,” “this system,” “he,” or “his wife,” infer the expected answer type before choosing the answer.
+- Use that expected type to validate candidates: answer with the concise person, place, title, organization, object, term, or phrase requested by the clue.
+
+- Treat modifiers attached to the requested type as hard filters, not background flavor: constraints like dates, “largest,” “2-letter-named,” “1978 remake,” “hot dog brand,” “dual throne,” or “on this company’s board” must all fit the candidate before you answer.
+- For clues centered on creative works such as books, films, plays, songs, poems, or other media, first determine whether the clue asks for the work itself, its creator, a performer or cast member, a character, a quotation source, or a setting. Verbs such as “wrote,” “directed,” “stars,” “played,” and “set in,” plus pronouns like “he” or “her,” usually determine the target.
+
+- For fill-in-style clues with placeholders such as “this,” “these,” or “one of these,” substitute each candidate back into the clue and choose the concise answer that makes the full phrase, title, or fact read correctly.
+- For terse clues that are just examples or names separated by commas, slashes, or “or,” infer the shared category, class, or synonym that links them, then answer with that concise common term.
+
+- For crossword-style clues, treat parenthetical numbers or stated letter counts as hard constraints on the answer length, and omit generic labels that would violate them. In dual-definition clues using wording like “X, or what Y does,” choose the single word that satisfies both senses and preserve the required inflected form.
+- If the clue references an unavailable image or link with wording like “seen here,” “pictured,” or parenthetical visual hints, rely on the textual clues and context to infer the answer; do not treat the missing image as necessary evidence.
+- If multiple snippets support the same entity, use that corroboration to choose the canonical/common form of the answer.
+
+## Trivia / Jeopardy Snippet Formats
+- Retrieved trivia snippets may contain the clue and answer in scraped formats such as `CATEGORY | clue | answer`, `clue. ANSWER`, or labels like `right:`.
+- When the question text matches the clue in such a snippet, extract the answer field or adjacent answer name, not the category or the whole clue sentence.
+
+## Common Clue Traps
+- Watch for inverse relationships: if the clue says “His third wife was Jiang Qing,” the requested answer is the husband, not Jiang Qing.
+
+- More generally, preserve relation direction in clues: “A is evidence of this B,” “A is related to this language,” or “home to these characters” asks for the target of the relationship, not the entity already named in the clue.
+
+- When a clue says examples, models, breeds, members, or items “include,” “like,” or “such as” named entities, treat those names as evidence for the requested parent class or entity. Answer the encompassing brand, animal, category, place, or term requested by “this,” not one of the examples already given.
+- If the question gives the start of a quotation or phrase, answer with the exact missing continuation from the context.
+
+- For song, poem, nursery-rhyme, or quotation clues, first decide whether the question asks for a missing word or phrase from the quote or for the associated creator, performer, or work; use pronouns and answer-type signals to choose the right target.
+- When a clue asks for a constrained form such as a first name, abbreviation, acronym, or lyric word, return that exact form rather than the fuller person, title, or explanation; preserve conventional punctuation or spelling when it is part of the requested form.
+- If the clue contains wordplay, quotation marks, or puns, treat them as hints, but answer with the real entity supported by the evidence.
+
+- If a clue includes a quoted title, quoted narration or lyric, named event, slogan, or other distinctive phrase but asks for an associated “this” entity, treat the quote or name as evidence to identify the requested person, work, place, group, category, source, or term; do not return the quoted anchor unless the clue explicitly asks for it.
+
+<!-- SLOW_UPDATE_START -->
+<!-- SLOW_UPDATE_END -->

ckpt/spreadsheetbench/gpt5.5_skill.md

@@ -0,0 +1,133 @@
+# Spreadsheet Manipulation Skill (xlsx)
+
+## Overview
+This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python.
+
+**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation).
+Never use any other third-party libraries.
+
+---
+
+## Common Workflow
+
+1. **Explore** the input file: list sheets, inspect headers, check dimensions.
+
+- Inspect actual workbook data beyond the preview, including nearby rows/columns, sample outputs, formulas, labels, headers, and any reference/example sheets such as `Output`, `Manual Result`, or `Desired...` tabs.
+
+- Treat existing filled cells in the requested output area or adjacent example tables as semantic examples for edge cases and expected formats, but still recompute and write the complete requested target range.
+   - Scan the used range for complete header groups, not just row 1. Tables may start in later rows/columns, have title rows above them, or have multiple source/result tables on the same sheet; use nearby labels and the requested output range to distinguish sources from destinations.
+   - Locate tables, fields, and target ranges by header text, nearby labels, and surrounding nonblank structure rather than fixed coordinates. Build header maps from actual cells when useful, e.g. `{str(cell.value).strip(): cell.column}`.
+2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top.
+3. **Execute** `python solution.py` and verify the output file was created.
+4. **Confirm** the target cells/range contain the expected values.
+
+---
+
+## Library Selection
+
+| Use case | Library |
+|----------|---------|
+| Preserve formulas, formatting, named ranges | `openpyxl` |
+| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` |
+| Simple cell read/write | `openpyxl` |
+
+**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges.
+When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`.
+
+**Formula evaluation caution**: `openpyxl` can write formulas but does **not** calculate them or update cached results. If the requested output will be checked as cell values, compute the result in Python and write literal values unless the user explicitly requires live formulas. When existing formulas are inputs to your logic, load a second workbook with `data_only=True` to read cached values while saving changes through the normal workbook:
+
+```python
+wb = openpyxl.load_workbook(INPUT_PATH)
+wb_values = openpyxl.load_workbook(INPUT_PATH, data_only=True)
+ws = wb["Sheet1"]
+ws_values = wb_values["Sheet1"]
+```
+
+Treat wording such as “write/fix a formula,” “SUMIFS/COUNTIFS,” “VBA,” or “macro” as a description of the spreadsheet logic unless the deliverable explicitly requires live formula text, an `.xlsm`, or a preserved VBA project. For normal `.xlsx` outputs, implement the equivalent logic in Python/openpyxl and write the computed final values to the requested cells so verification does not depend on Excel recalculation or macros.
+
+When the user provides an existing or broken formula, use it as a semantic specification: honor its referenced lookup ranges, criteria ranges, return ranges, aggregation intent, and error-handling behavior, then write the resulting values rather than guessing different source columns or leaving unevaluated formulas.
+
+---
+
+## solution.py Template
+
+```python
+import openpyxl
+import pandas as pd
+
+INPUT_PATH  = "..."   # set to the actual input path
+OUTPUT_PATH = "..."   # set to the actual output path
+
+wb = openpyxl.load_workbook(INPUT_PATH)
+ws = wb.active  # or wb["SheetName"]
+
+# --- perform manipulation ---
+
+wb.save(OUTPUT_PATH)
+```
+
+---
+
+## Output Requirements
+
+- Save the result to `OUTPUT_PATH`.
+- Do not hardcode row counts or column letters — iterate over actual rows in the workbook.
+- Preserve sheets and cells not mentioned in the instruction.
+
+## Matching and Target Range Hygiene
+
+- Choose the comparison operator from the instruction and examples: use `startswith` for “begins with”, substring search for “contains/search/occurrence”, and exact normalized equality only when a whole-cell match is implied.
+- Create small helper functions for comparisons and numeric parsing. Normalize text by trimming, collapsing repeated spaces/NBSPs, and casefolding; when names or labels have punctuation/spacing inconsistencies, consider punctuation-insensitive keys. Parse numeric text after removing commas/currency symbols while preserving signs and decimal points; skip `None`/blank and booleans for numeric tests, and handle placeholders such as `"-"`, `"$"`, `"$0"`, blanks, and numeric zero deliberately.
+- Normalize date keys deliberately: handle `datetime`/`date` objects, Excel serial numbers, and date-like strings, then compare at the granularity implied by the task, such as exact date, month, month/year, fiscal period, or year. For workday/date-window logic, compute the range in Python and exclude weekends/holidays as specified.
+
+- For monthly or period summary grids, canonicalize period labels from all sources: sheet names, title text, row/column headers, text months such as `March`, and actual date cells. Match summaries by normalized period plus the other stated criteria rather than by fixed month offsets or existing formulas.
+- For date ranges and rolling windows, infer endpoint inclusivity from wording and examples. Phrases like `X to Y`, `through`, and `up to`, or examples such as `2 to 5` meaning `4 days`, usually require inclusive boundary handling.
+- For time extraction or time-threshold logic, parse `datetime`, `time`, Excel serial/fractional times, and time-like strings into real Python `time`/`datetime` values. Write real time values with an Excel `number_format` such as `hh:mm:ss AM/PM`; do not write text substrings when the result should behave as a time.
+- For joins, deduplication, grouping, interval lookups, lookup grids, and ordered outputs, build explicit normalized keys, including composite keys when the task refers to multiple fields. Preserve original source order within each group unless sorting is explicitly requested.
+- For outputs that depend on other rows or lookup grids, make a first pass to build normalized dictionaries/groups/range structures, then a second pass to write results. Avoid nested full-sheet scans per row; split delimited tokens and ignore empty tokens, and treat error literals such as `#N/A` as meaningful sentinel values when the task refers to them.
+
+- For lookups, filters, joins, and label/header matching, normalize comparison keys consistently: trim whitespace, skip blanks explicitly, use case-insensitive text matching when appropriate, and treat numeric-looking IDs consistently (`330`, `330.0`, and `"330"`). Keep numeric outputs numeric; use `number_format` for display formatting instead of converting numbers to strings unless text is explicitly required.
+- When replacing a generated output area, clear only the instructed target range before writing new results so stale values/formulas do not remain. Preserve formatting, column widths, borders, formulas, and unrelated cells unless the instruction explicitly asks to change them.
+
+- If the instruction includes formatting changes, apply them exactly after writing values and only to the requested cells/range. Use `openpyxl` styles for fills, alignment, fonts, borders, and number formats; convert hex colors to ARGB when needed, for example `#FFC000` → `FFFFC000`. For “format as text,” set `number_format = '@'` and write string values when the expected cell values are text.
+
+- When the instruction names a destination range or columns, write derived results directly there. Do not insert rows/columns, relocate the source table, or sort/delete source records unless that structural change is explicitly requested.
+- For filtered lists, summaries, and aggregations, first collect all source records/results in memory, preserving the required order, then write from the first output row and clear leftover cells below the new results in the target columns. When adding rows, copy style/alignment/number format from an existing template row when appropriate; when deleting rows, delete from bottom to top to avoid row-index shifts.
+- Preserve intended blanks as empty cells (`None`) rather than placeholder text or `0` unless the task specifies otherwise.
+
+- For numeric aggregation, crosstab, SUMIFS-like, and INDEX/MATCH-style summary outputs, infer missing-match behavior from table semantics and examples: numeric summary grids usually require literal `0` for no matching records, while filtered lists or “show only once” outputs usually require blanks (`None`).
+- For blank-sensitive logic such as “if input is blank, output blank,” evaluate the driving input with `data_only=True` when it may itself be a formula, and write `None` for truly blank outputs rather than relying on a new formula returning `""`.
+
+## Robustness for Simple Fill Tasks
+
+- Prefer simple, auditable row/column loops over complex workbook XML parsing unless the task truly requires unsupported workbook internals. Before returning, run the script once to catch syntax/indentation errors and verify that representative target rows were actually written.
+
+<!-- SLOW_UPDATE_START -->
+When the user asks for a formula, macro, VBA code, or a fix to an Excel formula, still deliver the completed workbook state: compute the intended results in Python and write literal final values into the requested cells. Do not write formula strings unless the task explicitly says the output must contain live formulas.
+
+After writing, reload or inspect the saved workbook and verify that every requested/evaluated target cell contains a non-formula literal where a value is expected. If a target cell is still `None` unexpectedly, fix the script before finishing.
+
+Use existing formulas in the workbook as examples/specifications, not as output. If a cell contains a reference formula such as `=A25` or an INDEX/MATCH/SUMIFS pattern, parse what source cells/ranges/criteria it refers to, compute those results yourself, and overwrite the destination with the referenced or calculated value.
+
+For blank-sensitive formula tasks, compute the branch explicitly: if the driving source cell is truly blank, write `None`; otherwise write the actual result such as `0`, `1`, a category label, or a lookup value. Never rely on `IF(...,"",...)` formulas to be recalculated later.
+
+For lookup/category tasks, locate both the input rows and the lookup table by headers and nearby labels. Support exact keys, numeric-looking keys, and interval/range tables; then fill every destination row that has a driving input, not just the first visible example.
+
+For “every nth row” or OFFSET-style tasks, infer the source column, first source row, and step from the provided examples or formulas, then copy the actual source values into the requested output range as literals.
+
+For schedule/calendar fill tasks, build a cycle-day-to-periods mapping from the schedule/template area first, then fill the daily rows across all requested class columns based on each row’s cycle day. Preserve repeated/double periods exactly as shown by the template; do not leave formulas in the schedule cells.
+
+For INDEX/MATCH problems where the first row works but subsequent rows fail, treat row labels, column/year headers, region/type criteria, and expense/category labels as a multi-key lookup. Fill the whole result matrix with values from the source data table, using cached `data_only` values when source cells are formulas.
+
+For multi-step macro/VBA-style requests, implement every stated operation in the workbook, not just the first deletion/filtering step. Re-read the numbered requirements before saving and verify later computed columns, totals, and derived fields as well as the obvious filtered rows.
+
+When a target range includes special rows such as `Total`, `Grand Total`, `min`, `max`, constraints, headers, or blank separators, do not apply ordinary row logic blindly to those rows. Compute totals as aggregates when indicated, and leave constraint/header/blank cells untouched unless explicitly requested.
+
+For residual-balancing tasks, identify data rows separately from min/max constraint rows. Add positive residuals from unit 1 toward unit 5 without exceeding max values; subtract negative residuals from unit 5 toward unit 1 without going below min values; update only the unit cells in actual data rows.
+
+For time-threshold rows, decide per row whether it is a normal data row or a summary row. Normal rows use the before/after threshold rule; summary rows should aggregate the computed normal-row results if the workbook labels or examples indicate a total.
+
+Keep scripts simple enough to run cleanly. Avoid unnecessary dynamic code generation and fragile f-strings with regex expressions inside them. Always execute the final `solution.py`; fix any syntax, indentation, or runtime error, then verify representative target cells.
+
+If workbook cells contain arbitrary sample text that could be sensitive or trigger content filters, do not quote large raw cell contents in your response. Process them locally in Python with neutral variable names and output only the completed script/workbook changes.
+<!-- SLOW_UPDATE_END -->

configs/_base_/default.yaml

@@ -1,12 +1,12 @@
-# ReflACT default configuration — base for all environments.
+# SkillOpt default configuration — base for all environments.
 # Environment configs should inherit via: _base_: default.yaml
 
 model:
   backend: azure_openai
-  teacher: gpt-5.5
-  student: gpt-5.5
-  teacher_backend: openai_chat
-  student_backend: openai_chat
+  optimizer: gpt-5.5
+  target: gpt-5.5
+  optimizer_backend: openai_chat
+  target_backend: openai_chat
   reasoning_effort: medium
   rewrite_reasoning_effort: ""
   rewrite_max_completion_tokens: 64000
@@ -24,25 +24,37 @@ model:
   claude_code_exec_use_sdk: auto
   claude_code_exec_effort: medium
   claude_code_exec_max_thinking_tokens: 16384
-  codex_trace_to_teacher: true
-  azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
+  codex_trace_to_optimizer: true
+  azure_openai_endpoint: ""           # e.g. "https://your-resource.openai.azure.com/"
   azure_openai_api_version: "2024-12-01-preview"
   azure_openai_api_key: ""       # Fill locally if you do not export AZURE_OPENAI_API_KEY
-  azure_openai_auth_mode: azure_cli
+  azure_openai_auth_mode: ""           # empty → fall back to AZURE_OPENAI_AUTH_MODE env (default "azure_cli")
   azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
   azure_openai_managed_identity_client_id: ""
-  teacher_azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
-  teacher_azure_openai_api_version: "2024-12-01-preview"
-  teacher_azure_openai_api_key: ""
-  teacher_azure_openai_auth_mode: azure_cli
-  teacher_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  teacher_azure_openai_managed_identity_client_id: ""
-  student_azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
-  student_azure_openai_api_version: "2024-12-01-preview"
-  student_azure_openai_api_key: ""
-  student_azure_openai_auth_mode: azure_cli
-  student_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
-  student_azure_openai_managed_identity_client_id: ""
+  optimizer_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  optimizer_azure_openai_api_version: "2024-12-01-preview"
+  optimizer_azure_openai_api_key: ""
+  optimizer_azure_openai_auth_mode: ""           # empty → fall back to OPTIMIZER_AZURE_OPENAI_AUTH_MODE env, then shared
+  optimizer_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  optimizer_azure_openai_managed_identity_client_id: ""
+  target_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  target_azure_openai_api_version: "2024-12-01-preview"
+  target_azure_openai_api_key: ""
+  target_azure_openai_auth_mode: ""           # empty → fall back to TARGET_AZURE_OPENAI_AUTH_MODE env, then shared
+  target_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  target_azure_openai_managed_identity_client_id: ""
+
+  # MiniMax backend settings (minimax_chat target)
+  minimax_base_url: ""          # https://api.minimax.io/v1 if blank
+  minimax_api_key: ""
+  minimax_model: "MiniMax-M2.7"
+  minimax_temperature: "0.7"
+  minimax_max_tokens: "8000"
+  minimax_enable_thinking: "false"
+  optimizer_minimax_base_url: ""    # per-role override
+  target_minimax_base_url: ""       # per-role override
+  optimizer_minimax_api_key: ""
+  target_minimax_api_key: ""
 
 train:
   num_epochs: 4
@@ -57,9 +69,6 @@ gradient:
   analyst_workers: 16
   max_analyst_rounds: 3
   failure_only: false
-  use_deep_reflect: false
-  deep_reflect_failures: 4
-  deep_reflect_successes: 2
 
 optimizer:
   learning_rate: 4          # max edits per step (edit_budget)
@@ -67,12 +76,14 @@ optimizer:
   lr_scheduler: cosine      # constant / linear / cosine / autonomous
   lr_control_mode: fixed    # fixed / autonomous / none
   skill_update_mode: patch  # patch / rewrite_from_suggestions / full_rewrite_minibatch
-  use_meta_reflect: false
-  meta_learning_rate: 4     # max edits per epoch-level meta-reflect
   use_slow_update: true
   slow_update_samples: 20
+  slow_update_gate_with_selection: false
   longitudinal_pair_policy: mixed  # mixed / changed / unchanged
   use_meta_skill: true
+  use_skill_aware_reflection: false  # EmbodiSkill: split failures into SKILL_DEFECT (edit body) vs EXECUTION_LAPSE (protected appendix)
+  skill_aware_appendix_source: both  # both = success+failure emit appendix notes; failure_only = only EXECUTION_LAPSE (paper-faithful)
+  skill_aware_consolidate_threshold: 0  # 0 = off; >0 = LLM-consolidate the appendix when its note count exceeds N
 
 evaluation:
   use_gate: true
@@ -84,10 +95,9 @@ env:
   name: ""
   skill_init: ""
   split_mode: ratio       # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test
-  split_ratio: "2:1:7"    # explicit default for dataset-backed benchmarks: train:val:test
   split_seed: 42
   split_dir: ""
   data_path: ""
   split_output_dir: ""
-  exec_timeout: 120      # per student model/code-agent call timeout in seconds
+  exec_timeout: 120      # per target model/code-agent call timeout in seconds
   out_root: ""

configs/ablation_study/README.md

@@ -1,305 +0,0 @@
-# Ablation Study Configuration Manifest
-
-This folder records the final, reproducible settings for the ablation runs used
-in `docs/ablation_paper_tables.md`.
-
-It is intentionally separate from the benchmark default configs. The benchmark
-configs under `configs/<benchmark>/default.yaml` remain the source task configs;
-this folder records the exact matrix-level overrides, run roots, launch commands,
-and validation rules used for the paper ablations.
-
-## Files
-
-- `matrix.yaml`: canonical ablation matrix, common overrides, benchmark splits,
-  token/output caps, and invalid-run rules.
-- `launch_commands.sh`: exact launcher commands for the valid run roots.
-- `validation.md`: monitoring, result extraction, and invalidation checklist.
-
-## Source Of Truth
-
-Use the matrix launcher:
-
-```bash
-/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python scripts/run_ablation_matrix.py
-```
-
-The launcher builds runs from the same defaults and values recorded in
-`matrix.yaml`. It skips completed runs by checking `summary.json` and skips
-active runs by checking `env.out_root` in active `scripts/train.py` processes.
-
-Do not manually rerun a completed run into the same `env.out_root`. If a run is
-invalid, archive or remove its output directory first, then let the launcher
-start it cleanly.
-
-## Current Correct Run Roots
-
-- SearchQA / SpreadsheetBench original ablations:
-  `outputs/ablation_20260502_040604_unique48`
-- SearchQA / SpreadsheetBench batch-size ablations:
-  `outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run`
-- LiveMathBench / ALFWorld clean ablations:
-  `outputs/ablation_livemath_alfworld_clean_20260503_155155_run`
-- DocVQA ablations:
-  `outputs/ablation_docvqa_20260503_160225_run`
-
-Archived, superseded, misaligned, dry-run, or pre-fix directories must not be
-used for paper tables.
-
-## End-To-End Runbook
-
-### Environment
-
-Run from the repository root:
-
-```bash
-cd /home/azureuser/workspace-gzy/SkillReflection
-```
-
-Always use:
-
-```bash
-PY=/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
-export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
-```
-
-Default model/auth settings are generated by `scripts/run_ablation_matrix.py`:
-
-```text
-teacher=gpt-5.5
-student=gpt-5.5
-teacher_backend=openai_chat
-student_backend=openai_chat
-reasoning_effort=medium
-teacher/student endpoint=https://t2vgoaigpt4o3.openai.azure.com/
-teacher/student api_version=2024-12-01-preview
-teacher/student auth_mode=azure_cli
-```
-
-Core training settings:
-
-```text
-train.num_epochs=4
-train.train_size=0
-train.batch_size=40
-train.accumulation=1
-train.seed=42
-gradient.minibatch_size=8
-gradient.merge_batch_size=8
-gradient.analyst_workers=16
-gradient.use_deep_reflect=false
-optimizer.learning_rate=4
-optimizer.min_learning_rate=2
-optimizer.lr_scheduler=cosine
-optimizer.lr_control_mode=fixed
-optimizer.use_slow_update=true
-optimizer.slow_update_samples=20
-optimizer.use_meta_skill=true
-optimizer.use_meta_reflect=false
-optimizer.longitudinal_pair_policy=mixed
-evaluation.use_gate=true
-evaluation.eval_test=true
-env.split_mode=split_dir
-```
-
-`train.train_size=0` is intentional. The dataloader derives the train size from
-the fixed split. Batch-size ablations rely on the default `ceil(train_size /
-batch_size)` behavior; the last batch can be smaller than `train.batch_size`.
-
-### Fixed Splits
-
-Default split directories:
-
-```text
-searchqa: data/ablation_splits/searchqa/2-1-7_seed42
-spreadsheetbench: data/ablation_splits/spreadsheetbench/2-1-7_seed42
-livemathematicianbench: data/ablation_splits/livemathematicianbench/2-1-7_seed42
-alfworld: data/ablation_splits/alfworld/2-1-7_seed42
-docvqa: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
-```
-
-Default train/val/test sizes:
-
-| Benchmark | Train | Val | Test |
-| --- | ---: | ---: | ---: |
-| SearchQA | 400 | 200 | 1400 |
-| SpreadsheetBench | 80 | 40 | 280 |
-| LiveMathBench | 35 | 18 | 124 |
-| ALFWorld | 39 | 18 | 134 |
-| DocVQA | 1070 | 535 | 3744 |
-
-DocVQA images are not copied. The valid setup uses:
-
-```text
-data/docvqa_images -> /home/azureuser/zisu/SkillReflection/data/docvqa_images
-```
-
-2026-05-05 DocVQA data correction: all DocVQA final reruns should use the zisu
-10% split above and a fresh output root such as
-`outputs/ablation_docvqa_zisu10pct_20260505_run`. The older local
-`data/ablation_splits/docvqa/2-1-7_seed42` contains the same 5349 questionId
-pool but a different train/val/test assignment, so its completed summaries are
-historical only.
-
-### Matrix Groups
-
-Use these group names with `scripts/run_ablation_matrix.py`:
-
-```text
-default split batch mbs lr sched slown mod smodel longpair lrctrl
-```
-
-`longpair` is the slow-update/meta-skill comparison-example ablation. It keeps
-all prompts and training settings unchanged and only overrides:
-
-```text
-optimizer.longitudinal_pair_policy=changed
-optimizer.longitudinal_pair_policy=unchanged
-```
-
-The default paper setting remains `mixed`.
-
-`lrctrl` contains the two learning-rate-control baselines:
-
-```text
-optimizer.lr_control_mode=autonomous
-optimizer.lr_control_mode=none + optimizer.skill_update_mode=full_rewrite_minibatch
-```
-
-The autonomous run logs the chosen integer per step in `lr_decision.json` and
-`lr_history.jsonl`. The full-rewrite run removes the LR/edit-selection concept:
-each minibatch analyst produces a complete skill candidate, and aggregate/merge
-produces the candidate skill directly.
-
-Batch-size values are:
-
-```text
-8 / 24 / 40 / 56 / full
-```
-
-`40` is the default point. `full` expands to the benchmark train size.
-
-### Launch Commands Used In This Session
-
-The exact commands are recorded in `launch_commands.sh` and in
-`docs/ablation_plan.md`. The important current policy is:
-
-- SearchQA / SpreadsheetBench batch-only matrix can run at `--max-parallel 8`.
-- DocVQA matrix can run with its launcher at `--max-parallel 8`; later top-up used `--max-parallel 16` only because completed runs were skipped and active roots were checked.
-- LiveMathBench is safe as API-only benchmark after the token cap fix.
-- ALFWorld must not be mixed into a 24-way run on this shared machine. Use `--bench alfworld --max-parallel 1` only after memory is available.
-
-### Token And Timeout Fixes
-
-LiveMathBench must use a large student completion cap:
-
-```text
-max_completion_tokens=16384
-timeout=300
-```
-
-The old 768/512 cap produced many empty visible responses because hidden
-reasoning consumed the budget.
-
-ALFWorld must use:
-
-```text
-max_completion_tokens=2048
-empty response fallback -> <action>look</action>
-missing action fallback -> <action>look</action>
-```
-
-### Invalid Runs
-
-Never fill paper tables from these archive directories:
-
-```text
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_livemath_token768_20260504_022258/
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_token512_20260504_021417/
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_empty_action_20260504_025311/
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_prefallback_20260504_025402/
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_oom_partial_20260504_050517/
-outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_serial_lowmem_20260504_1300/
-```
-
-### Current Resource Notes
-
-ALFWorld model calls are API calls. The ablation branch now creates local
-ALFWorld/TextWorld environments through multiprocessing workers ported from
-`skillopt_final_zzw`, not through Ray actors. Old Ray-based archived runs are
-not valid for table fill. The observed historical failure mode in this session
-was system RAM pressure and Ray OOM prevention, not model GPU memory.
-
-GPU memory currently shown by `nvidia-smi` came from unrelated Ray Serve visual
-models under:
-
-```text
-/home/azureuser/workspace-gzy/zyf/gca-skill
-```
-
-Those processes are `GroundingDINOModel` / `DA3Model`, not the
-SkillReflection ablation ALFWorld run.
-
-There are also unrelated ALFWorld jobs under:
-
-```text
-/home/azureuser/zisu/skill_distill
-```
-
-Do not confuse those with this repository's ablation outputs.
-
-### Monitoring
-
-Active run and duplicate output-root check:
-
-```bash
-$PY - <<'PY'
-import subprocess, re, collections, time
-try:
-    raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
-except subprocess.CalledProcessError:
-    raw = ""
-roots = []
-for line in raw.splitlines():
-    m = re.search(r"env\.out_root=([^\s]+)", line)
-    if m:
-        roots.append(m.group(1))
-ctr = collections.Counter(roots)
-print("time", time.strftime("%F %T"))
-print("active_count", len(roots))
-print("duplicates", [r.rsplit("/", 1)[-1] for r, c in ctr.items() if c > 1])
-for root in sorted(roots):
-    print(root.rsplit("/", 1)[-1])
-PY
-```
-
-Error scan:
-
-```bash
-rg -n "Traceback|ERROR|Error code|AuthenticationError|BadRequest|RateLimit|content_filter|Killed|OutOfMemory|CUDA out of memory|\\[FAIL\\]|LLM call failed" \
-  outputs/ablation_docvqa_20260503_160225_run/logs \
-  outputs/ablation_livemath_alfworld_clean_20260503_155155_run/logs \
-  outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/logs \
-  -g '*.log' | tail -100 || true
-```
-
-Resource checks:
-
-```bash
-free -h | sed -n '1,3p'
-df -h /tmp
-du -sh /tmp/ray 2>/dev/null || true
-nvidia-smi
-```
-
-### Filling Tables
-
-Only use top-level `summary.json` from valid run roots. Fill
-`docs/ablation_paper_tables.md` from:
-
-```text
-best_selection_hard
-baseline_test_hard
-test_hard
-test_delta_hard
-token_summary._total.total_tokens
-```

configs/ablation_study/launch_commands.sh

@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-cd /home/azureuser/workspace-gzy/SkillReflection
-
-PY=/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
-export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
-
-# Original SearchQA / SpreadsheetBench full matrix reproduction command.
-# Do not run this into the existing root unless intentionally reproducing from
-# scratch; the current valid root is already populated:
-# outputs/ablation_20260502_040604_unique48
-#
-# setsid "$PY" scripts/run_ablation_matrix.py \
-#   --groups default split mbs lr sched slown mod smodel \
-#   --bench searchqa spreadsheetbench \
-#   --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_20260502_040604_unique48 \
-#   --max-parallel 24 \
-#   --execute \
-#   > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_20260502_040604_unique48/launcher_reproduce_full_matrix.log 2>&1 < /dev/null &
-#
-# SearchQA / SpreadsheetBench batch-size ablations only.
-# Original non-batch SearchQA/SpreadsheetBench ablations live in:
-# outputs/ablation_20260502_040604_unique48
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups batch \
-  --bench searchqa spreadsheetbench \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run \
-  --max-parallel 8 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/launcher_parallel8.log 2>&1 < /dev/null &
-
-# DocVQA full matrix.
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups default split batch mbs lr sched slown mod smodel \
-  --bench docvqa \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_docvqa_20260503_160225_run \
-  --max-parallel 8 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_docvqa_20260503_160225_run/launcher_parallel8.log 2>&1 < /dev/null &
-
-# LiveMathBench clean matrix. ALFWorld should be launched separately at lower
-# concurrency because Ray OOM occurred when many ALFWorld runs were mixed into a
-# 24-way run.
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups default split batch mbs lr sched slown mod smodel \
-  --bench livemathematicianbench \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run \
-  --max-parallel 8 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_livemath_parallel8.log 2>&1 < /dev/null &
-
-# ALFWorld clean matrix. Increase to 2 only after checking memory, /tmp/ray,
-# and that no other ALFWorld run is active. Do not use 8/16/24 for ALFWorld on
-# the current shared machine unless resources are explicitly reserved.
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups default split batch mbs lr sched slown mod smodel \
-  --bench alfworld \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run \
-  --max-parallel 1 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_alfworld_parallel1.log 2>&1 < /dev/null &
-
-# Longitudinal comparison-example policy ablations. This intentionally excludes
-# ALFWorld. The only varied setting is optimizer.longitudinal_pair_policy.
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups longpair \
-  --bench searchqa spreadsheetbench livemathematicianbench docvqa \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_longpair_20260504_run \
-  --max-parallel 8 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_longpair_20260504_run/launcher_longpair_parallel8.log 2>&1 < /dev/null &
-
-# Learning-rate-control baselines. This intentionally excludes ALFWorld.
-setsid "$PY" scripts/run_ablation_matrix.py \
-  --groups lrctrl \
-  --bench searchqa spreadsheetbench livemathematicianbench docvqa \
-  --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_lrctrl_20260504_run \
-  --max-parallel 8 \
-  --execute \
-  > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_lrctrl_20260504_run/launcher_lrctrl_parallel8.log 2>&1 < /dev/null &

configs/ablation_study/matrix.yaml

@@ -1,257 +0,0 @@
-version: 2026-05-04
-purpose: "Canonical paper ablation settings matching the valid current runs."
-
-launcher:
-  script: scripts/run_ablation_matrix.py
-  python: /home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
-  skip_completed_by: summary.json
-  skip_active_by: "active scripts/train.py env.out_root"
-
-environment:
-  working_directory: /home/azureuser/workspace-gzy/SkillReflection
-  required_env:
-    ALFWORLD_DATA: /home/azureuser/.cache/alfworld
-  docvqa_images_symlink: "data/docvqa_images -> /home/azureuser/zisu/SkillReflection/data/docvqa_images"
-
-common_overrides:
-  model.teacher_backend: openai_chat
-  model.student_backend: openai_chat
-  model.teacher: gpt-5.5
-  model.student: gpt-5.5
-  model.teacher_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
-  model.teacher_azure_openai_api_version: 2024-12-01-preview
-  model.teacher_azure_openai_auth_mode: azure_cli
-  model.student_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
-  model.student_azure_openai_api_version: 2024-12-01-preview
-  model.student_azure_openai_auth_mode: azure_cli
-  model.reasoning_effort: medium
-  train.num_epochs: 4
-  train.train_size: 0
-  train.batch_size: 40
-  train.accumulation: 1
-  train.seed: 42
-  gradient.minibatch_size: 8
-  gradient.merge_batch_size: 8
-  gradient.analyst_workers: 16
-  gradient.use_deep_reflect: false
-  optimizer.learning_rate: 4
-  optimizer.min_learning_rate: 2
-  optimizer.lr_scheduler: cosine
-  optimizer.skill_update_mode: patch
-  optimizer.use_slow_update: true
-  optimizer.slow_update_samples: 20
-  optimizer.use_meta_skill: true
-  optimizer.use_meta_reflect: false
-  evaluation.use_gate: true
-  evaluation.eval_test: true
-  env.split_mode: split_dir
-
-benchmarks:
-  searchqa:
-    config: configs/searchqa/default.yaml
-    run_roots:
-      original_matrix: outputs/ablation_20260502_040604_unique48
-      batch_matrix: outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run
-    default_split: data/ablation_splits/searchqa/2-1-7_seed42
-    train: 400
-    val: 200
-    test: 1400
-    student_rollout:
-      function: reflact/envs/searchqa/rollout.py::chat_student
-      max_completion_tokens:
-        first_turn: 512
-        refinement: 512
-      rationale: "Short-answer QA; sampled empties are low and not LiveMath-like."
-
-  spreadsheetbench:
-    config: configs/spreadsheetbench/default.yaml
-    run_roots:
-      original_matrix: outputs/ablation_20260502_040604_unique48
-      batch_matrix: outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run
-    default_split: data/ablation_splits/spreadsheetbench/2-1-7_seed42
-    train: 80
-    val: 40
-    test: 280
-    student_rollout:
-      function: reflact/envs/spreadsheetbench/codegen_agent.py::run_multi
-      max_output_tokens: 16384
-      result_note: "results.jsonl stores execution fields, not a response field."
-
-  livemathematicianbench:
-    config: configs/livemathematicianbench/default.yaml
-    run_roots:
-      clean_matrix: outputs/ablation_livemath_alfworld_clean_20260503_155155_run
-    default_split: data/ablation_splits/livemathematicianbench/2-1-7_seed42
-    train: 35
-    val: 18
-    test: 124
-    student_rollout:
-      function: reflact/envs/livemathematicianbench/rollout.py::chat_student
-      max_completion_tokens:
-        first_turn: 16384
-        refinement: 16384
-      timeout_seconds: 300
-      invalid_old_caps:
-        first_turn: 768
-        refinement: 512
-      invalid_archive: outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_livemath_token768_20260504_022258
-      rationale: "GPT-5 reasoning consumed small budgets and produced many empty visible responses."
-
-  alfworld:
-    config: configs/alfworld/default.yaml
-    run_roots:
-      clean_matrix: outputs/ablation_livemath_alfworld_clean_20260503_155155_run
-    default_split: data/ablation_splits/alfworld/2-1-7_seed42
-    train: 39
-    val: 18
-    test: 134
-    student_rollout:
-      function: reflact/envs/alfworld/rollout.py::chat_student
-      max_completion_tokens: 2048
-      timeout_seconds: 120
-      max_steps: 50
-      fallback_action: look
-      invalid_old_cap: 512
-      invalid_archives:
-        - outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_token512_20260504_021417
-        - outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_empty_action_20260504_025311
-        - outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_prefallback_20260504_025402
-        - outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_oom_partial_20260504_050517
-      concurrency_note: "Do not mix many ALFWorld runs into 24-way total concurrency; Ray OOM occurred. Prefer 1-2 ALFWorld runs at a time unless resources are clearly free."
-
-  docvqa:
-    config: configs/docvqa/default.yaml
-    run_roots:
-      matrix: outputs/ablation_docvqa_zisu10pct_20260505_run
-    default_split: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
-    train: 1070
-    val: 535
-    test: 3744
-    data_note: "2026-05-05: use zisu-provided 5349-item DocVQA split directly; previous local 2-1-7_seed42 used the same item pool but a different train/val/test assignment and must not be used for final DocVQA reruns."
-    student_rollout:
-      function: reflact/envs/docvqa/rollout.py::chat_student_messages
-      max_completion_tokens:
-        first_turn: 768
-        refinement: 512
-      rationale: "Short-answer VQA output; preserve current setting for alignment unless explicitly rerunning all affected DocVQA."
-
-splits:
-  tags:
-    1shot:
-      extra_overrides:
-        optimizer.slow_update_samples: 1
-    1-1-8: {}
-    2-1-7:
-      default: true
-    4-1-5: {}
-  paths:
-    searchqa:
-      1shot: data/ablation_splits/searchqa/1shot_seed42
-      1-1-8: data/ablation_splits/searchqa/1-1-8_seed42
-      2-1-7: data/ablation_splits/searchqa/2-1-7_seed42
-      4-1-5: data/ablation_splits/searchqa/4-1-5_seed42
-    spreadsheetbench:
-      1shot: data/ablation_splits/spreadsheetbench/1shot_seed42
-      1-1-8: data/ablation_splits/spreadsheetbench/1-1-8_seed42
-      2-1-7: data/ablation_splits/spreadsheetbench/2-1-7_seed42
-      4-1-5: data/ablation_splits/spreadsheetbench/4-1-5_seed42
-    livemathematicianbench:
-      1shot: data/ablation_splits/livemathematicianbench/1shot_seed42
-      1-1-8: data/ablation_splits/livemathematicianbench/1-1-8_seed42
-      2-1-7: data/ablation_splits/livemathematicianbench/2-1-7_seed42
-      4-1-5: data/ablation_splits/livemathematicianbench/4-1-5_seed42
-    alfworld:
-      1shot: data/ablation_splits/alfworld/1shot_seed42
-      1-1-8: data/ablation_splits/alfworld/1-1-8_seed42
-      2-1-7: data/ablation_splits/alfworld/2-1-7_seed42
-      4-1-5: data/ablation_splits/alfworld/4-1-5_seed42
-    docvqa:
-      1shot: data/ablation_splits/docvqa/1shot_seed42
-      1-1-8: data/ablation_splits/docvqa/1-1-8_seed42
-      2-1-7: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
-      4-1-5: data/ablation_splits/docvqa/4-1-5_seed42
-
-groups:
-  default:
-    run_id: "DEFAULT-{benchmark}-5.5"
-    overrides: {}
-  split:
-    values: [1shot, 1-1-8, 4-1-5]
-    skip_default_2_1_7: true
-    override_template: "env.split_dir={split_path}"
-  batch:
-    values: [8, 24, 56, full]
-    default_value_reused: 40
-    full_values:
-      searchqa: 400
-      spreadsheetbench: 80
-      livemathematicianbench: 35
-      alfworld: 39
-      docvqa: 1070
-    fixed_overrides:
-      gradient.minibatch_size: 8
-  mbs:
-    values: [1, 2, 4, 16, 32]
-    default_value_reused: 8
-    override_template: "gradient.minibatch_size={value}"
-  lr:
-    values: [1, 2, 4, 8, 16]
-    fixed_overrides:
-      optimizer.lr_scheduler: constant
-      optimizer.min_learning_rate: 1
-    override_template: "optimizer.learning_rate={value}"
-  sched:
-    values: [constant, linear]
-    default_value_reused: cosine
-    override_template: "optimizer.lr_scheduler={value}"
-  slown:
-    values: [5, 10, 40]
-    default_value_reused: 20
-    override_template: "optimizer.slow_update_samples={value}"
-  mod:
-    values:
-      slow-only:
-        optimizer.use_slow_update: true
-        optimizer.use_meta_skill: false
-      meta-only:
-        optimizer.use_slow_update: false
-        optimizer.use_meta_skill: true
-      none:
-        optimizer.use_slow_update: false
-        optimizer.use_meta_skill: false
-    default_value_reused: slow-meta
-  longpair:
-    values: [changed, unchanged]
-    default_value_reused: mixed
-    override_template: "optimizer.longitudinal_pair_policy={value}"
-    note: "Only changes slow-update/meta-skill comparison examples; prompts and other settings remain unchanged."
-  lrctrl:
-    values:
-      autonomous:
-        optimizer.lr_control_mode: autonomous
-      full-rewrite:
-        optimizer.lr_control_mode: none
-        optimizer.skill_update_mode: full_rewrite_minibatch
-    default_value_reused: "fixed patch learning_rate=4"
-    note: "autonomous records lr_decision.json/lr_history.jsonl; full-rewrite removes LR/select/apply-edit and uses full skill candidates."
-  smodel:
-    values:
-      "5.4":
-        model.student: gpt-5.4-pro
-        model.student_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
-        model.student_azure_openai_api_version: 2025-03-01-preview
-        model.student_azure_openai_auth_mode: azure_cli
-      "5.4-mini":
-        model.student: gpt-5.4-mini
-        model.student_azure_openai_endpoint: https://searchagent5.cognitiveservices.azure.com/
-        model.student_azure_openai_api_version: 2024-12-01-preview
-        model.student_azure_openai_auth_mode: azure_cli
-    default_value_reused: "5.5"
-
-validity_rules:
-  use_for_tables:
-    - "Only runs with summary.json in valid run roots."
-    - "Do not use archive, archived, MISALIGNED, SUPERSEDED, dryrun, smoke, or debug directories."
-    - "Do not use ALFWorld runs started before empty/missing-action fallback."
-    - "Do not use old LiveMath runs with 768/512 token caps."
-  rerun_rule: "Archive or remove invalid out_root before relaunch; never write a rerun into a polluted output directory."

configs/ablation_study/validation.md

@@ -1,141 +0,0 @@
-# Ablation Validation Checklist
-
-Use this checklist before launch, during monitoring, and before filling
-`docs/ablation_paper_tables.md`.
-
-## Before Launch
-
-Run from repo root:
-
-```bash
-cd /home/azureuser/workspace-gzy/SkillReflection
-export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
-```
-
-Verify syntax for edited files:
-
-```bash
-/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python -m py_compile \
-  scripts/run_ablation_matrix.py \
-  scripts/train.py \
-  reflact/model/azure_openai.py \
-  reflact/envs/searchqa/rollout.py \
-  reflact/envs/spreadsheetbench/rollout.py \
-  reflact/envs/livemathematicianbench/rollout.py \
-  reflact/envs/alfworld/rollout.py \
-  reflact/envs/docvqa/rollout.py
-```
-
-Check active runs and duplicate `env.out_root` before starting more:
-
-```bash
-/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
-import subprocess, re, collections
-try:
-    raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
-except subprocess.CalledProcessError:
-    raw = ""
-roots = []
-for line in raw.splitlines():
-    m = re.search(r"env\.out_root=([^\s]+)", line)
-    if m:
-        roots.append(m.group(1))
-ctr = collections.Counter(roots)
-print("train_count", len(roots))
-print("duplicate_roots", [r.rsplit("/", 1)[-1] for r, c in ctr.items() if c > 1])
-for root in sorted(roots):
-    print(root.rsplit("/", 1)[-1])
-PY
-```
-
-## During Monitoring
-
-Check launchers:
-
-```bash
-pgrep -af 'scripts/run_ablation_matrix.py' || true
-tail -80 outputs/ablation_docvqa_20260503_160225_run/launcher_parallel8.log 2>/dev/null || true
-tail -80 outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_livemath_parallel8.log 2>/dev/null || true
-tail -80 outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_alfworld_parallel1.log 2>/dev/null || true
-```
-
-Scan current logs for new hard failures:
-
-```bash
-rg -n "Traceback|ERROR|Error code|AuthenticationError|BadRequest|RateLimit|content_filter|Killed|OutOfMemory|\\[FAIL\\]|\\[RETRY\\]" \
-  outputs/ablation_docvqa_20260503_160225_run/logs \
-  outputs/ablation_livemath_alfworld_clean_20260503_155155_run/logs \
-  outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/logs \
-  -g '*.log' | tail -160 || true
-```
-
-Check resource pressure:
-
-```bash
-df -h /tmp
-du -sh /tmp/ray 2>/dev/null || true
-free -h | sed -n '1,3p'
-```
-
-## Quality Checks
-
-LiveMathBench current valid runs should not look like old 768/512 runs:
-
-```bash
-/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
-import json, pathlib
-root = pathlib.Path("outputs/ablation_livemath_alfworld_clean_20260503_155155_run")
-for run in sorted(root.glob("*livemathematicianbench*")):
-    if not run.is_dir() or "archive" in str(run):
-        continue
-    for rel in ["test_eval_baseline/results.jsonl", "test_eval/results.jsonl"]:
-        p = run / rel
-        if not p.exists():
-            continue
-        rows = [json.loads(l) for l in p.open(errors="ignore") if l.strip()]
-        empty = sum(1 for r in rows if not str(r.get("response", "")).strip())
-        answer = sum(1 for r in rows if "<answer>" in str(r.get("response", "")).lower())
-        if empty:
-            print(run.name, rel, "empty", empty, "answer", answer, "n", len(rows))
-PY
-```
-
-ALFWorld valid runs must not contain empty action or missing action:
-
-```bash
-/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
-import json, pathlib
-root = pathlib.Path("outputs/ablation_livemath_alfworld_clean_20260503_155155_run")
-for run in sorted(root.glob("*alfworld*")):
-    if not run.is_dir() or "archive" in str(run):
-        continue
-    bad = []
-    fallback = 0
-    for c in run.glob("**/conversation.json"):
-        data = json.load(c.open(errors="ignore"))
-        for step in data:
-            if step.get("step") is None:
-                continue
-            if not step.get("action"):
-                bad.append(str(c.relative_to(run)))
-                break
-            mr = str(step.get("model_response", ""))
-            if "empty model response" in mr or "missing action tag" in mr:
-                fallback += 1
-    print(run.name, "bad_action_files", len(bad), "fallback", fallback)
-PY
-```
-
-## Filling Tables
-
-Use only `summary.json` fields:
-
-- `best_selection_hard` -> Best Sel
-- `baseline_test_hard` -> Base Test
-- `test_hard` -> Best Test
-- `test_delta_hard` -> Delta
-- `total_accepts` -> Accept
-- `total_rejects` -> Reject
-- `token_summary._total.total_tokens` -> Tokens
-
-Do not fill table rows from logs alone.

configs/alfworld/default.yaml

@@ -10,7 +10,6 @@ gradient:
 
 optimizer:
   learning_rate: 4
-  use_meta_reflect: false
 
 evaluation:
   sel_env_num: 0
@@ -18,13 +17,13 @@ evaluation:
 
 env:
   name: alfworld
-  skill_init: reflact/envs/alfworld/skills/initial.md
+  skill_init: skillopt/envs/alfworld/skills/initial.md
   split_mode: split_dir
-  split_ratio: "2:1:7"
-  split_dir: data/ablation_splits/alfworld/2-1-7_seed42
+  split_dir: data/alfworld_path_split
   data_path: ""
   split_output_dir: ""
   max_steps: 50
+  max_completion_tokens: 16384
   workers: 8
   max_api_workers: 8
   limit: 0

configs/alfworld/meta_reflect.yaml

@@ -1,4 +0,0 @@
-_base_: default.yaml
-
-optimizer:
-  use_meta_reflect: true

configs/babyvision/default.yaml

@@ -1,21 +0,0 @@
-_base_: ../_base_/default.yaml
-
-train:
-  batch_size: 64
-  accumulation: 1
-
-env:
-  name: babyvision
-  skill_init: reflact/envs/babyvision/skills/initial.md
-  split_mode: ratio
-  split_ratio: "2:1:7"
-  split_dir: ""
-  data_path: ""
-  split_output_dir: ""
-  max_turns: 1
-  workers: 16
-  limit: 0
-  image_detail: auto
-  judge_model: gpt-5.4
-  judge_max_completion_tokens: 256
-  judge_retries: 5

configs/docvqa/default.yaml

@@ -16,13 +16,13 @@ optimizer:
 
 env:
   name: docvqa
-  skill_init: reflact/envs/docvqa/skills/initial.md
+  skill_init: skillopt/envs/docvqa/skills/initial.md
   split_mode: split_dir
-  split_ratio: "2:1:7"
-  split_dir: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
+  split_dir: data/docvqa/splits
   data_path: ""
   split_output_dir: ""
   max_turns: 1
+  max_completion_tokens: 16384
   workers: 16
   image_detail: auto
   limit: 0

configs/features/soft_gate.yaml

@@ -0,0 +1,47 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# Feature: soft / mixed validation-gate metric (community-contributed, PR #25)
+# ─────────────────────────────────────────────────────────────────────────────
+#
+# This is NOT a default SkillOpt setting and was NOT used to produce the
+# numbers reported in the paper. It is provided as a reference for users
+# who encounter a specific scenario where the default `hard` gate is too
+# coarse to drive training.
+#
+# When to consider this:
+#   - You are running on a custom environment.
+#   - Your held-out *selection* split has very few items (e.g. ≤ ~10).
+#   - Your reward function is continuous / partial-credit (e.g. F1, BLEU,
+#     soft match) rather than purely binary 0/1.
+#
+# Symptom this addresses:
+#   With a small selection split + continuous rewards, candidate skills
+#   often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but
+#   never flip the discrete hard outcome. The default `hard` gate then
+#   rejects every candidate and training stalls. Switching the gate to
+#   `soft` or `mixed` lets these partial improvements be accepted.
+#
+# When NOT to use this:
+#   - When reproducing the paper. The paper-reported numbers were obtained
+#     under the default `hard` gate.
+#   - When your selection split is large (dozens+ items) and / or your
+#     reward is already binary — `hard` is the more conservative choice
+#     and matches the design described in the paper.
+#
+# To use: inherit your env config from this file, e.g.
+#   _base_: ../features/soft_gate.yaml
+# or copy the `evaluation:` block below into your config.
+# ─────────────────────────────────────────────────────────────────────────────
+
+_base_: ../_base_/default.yaml
+
+evaluation:
+  # Three options:
+  #   'hard'  — default; exact-match accuracy. Use this to reproduce the paper.
+  #   'soft'  — per-item soft / partial-credit score (recommended for the
+  #             small-split + continuous-reward scenario described above).
+  #   'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by
+  #             `gate_mixed_weight` below.
+  gate_metric: soft
+
+  # Only used when gate_metric == 'mixed'. Ignored otherwise.
+  gate_mixed_weight: 0.5

configs/livemathematicianbench/default.yaml

@@ -7,13 +7,13 @@ train:
 
 env:
   name: livemathematicianbench
-  skill_init: reflact/envs/livemathematicianbench/skills/initial.md
+  skill_init: skillopt/envs/livemathematicianbench/skills/initial.md
   split_mode: split_dir
-  split_ratio: "2:1:7"
-  split_dir: data/ablation_splits/livemathematicianbench/2-1-7_seed42
+  split_dir: data/livemathematicianbench_split
   data_path: ""
   split_output_dir: ""
   max_turns: 1
+  max_completion_tokens: 16384
   exec_timeout: 300
   workers: 64
   limit: 0

configs/mathverse/default.yaml

@@ -1,23 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  codex_exec_sandbox: danger-full-access
-
-train:
-  batch_size: 64
-  accumulation: 1
-
-env:
-  name: mathverse
-  skill_init: reflact/envs/mathverse/skills/initial.md
-  split_dir: ""
-  data_root: data/MathVerse
-  problem_version: Text Lite
-  use_text_dominant_reference: false
-  max_turns: 1
-  workers: 16
-  limit: 0
-  image_detail: auto
-  judge_model: gpt-5.4
-  judge_max_completion_tokens: 256
-  judge_retries: 5

configs/mmrb/default.yaml

@@ -1,18 +0,0 @@
-_base_: ../_base_/default.yaml
-
-train:
-  batch_size: 128
-  accumulation: 1
-
-env:
-  name: mmrb
-  skill_init: reflact/envs/mmrb/skills/initial.md
-  split_mode: ratio
-  split_ratio: "2:1:7"
-  split_dir: ""
-  data_path: ""
-  split_output_dir: ""
-  max_turns: 1
-  workers: 16
-  limit: 0
-  image_detail: auto

configs/officeqa/default.yaml

@@ -16,10 +16,19 @@ optimizer:
 
 env:
   name: officeqa
-  skill_init: reflact/envs/officeqa/skills/initial.md
+  skill_init: skillopt/envs/officeqa/skills/initial.md
+  split_mode: split_dir
   split_dir: data/officeqa_split
   data_dirs:
     - data/officeqa_docs_official
   workers: 4
   max_tool_turns: 24
+  max_completion_tokens: 16384
+  search_mode: offline
+  max_queries_per_turn: 4
+  search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search
+  search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH
+  search_provider: duckduckgo
+  search_max_num_results: 4
+  search_timeout_seconds: 20
   limit: 0

configs/sealqa/default.yaml

@@ -1,23 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  batch_size: 10
-  accumulation: 1
-
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-env:
-  name: sealqa
-  skill_init: reflact/envs/sealqa/skills/initial.md
-  split_dir: data/sealqa_split
-  workers: 4
-  max_tool_turns: 12
-  limit: 0

configs/searchqa/default.yaml

@@ -21,12 +21,12 @@ evaluation:
 
 env:
   name: searchqa
-  skill_init: reflact/envs/searchqa/skills/initial.md
+  skill_init: skillopt/envs/searchqa/skills/initial.md
   split_mode: split_dir
-  split_ratio: "2:1:7"
   split_dir: data/searchqa_split
   data_path: ""
   split_output_dir: ""
   max_turns: 1
+  max_completion_tokens: 16384
   workers: 24
   limit: 0

configs/spreadsheetbench/default.yaml

@@ -21,14 +21,14 @@ evaluation:
 
 env:
   name: spreadsheetbench
-  skill_init: reflact/envs/spreadsheetbench/skills/initial.md
+  skill_init: skillopt/envs/spreadsheetbench/skills/initial.md
   split_mode: split_dir
-  split_ratio: "2:1:7"
   split_dir: data/spreadsheetbench_split
   data_path: ""
   split_output_dir: ""
   data_root: data/spreadsheetbench_verified_400
   mode: multi
   max_turns: 30
+  max_completion_tokens: 16384
   exec_timeout: 600
   workers: 24

configs/swebench/default.yaml

@@ -1,36 +0,0 @@
-_base_: ../_base_/default.yaml
-
-model:
-  reasoning_effort: medium
-
-train:
-  batch_size: 20
-  accumulation: 1
-
-gradient:
-  minibatch_size: 4
-  merge_batch_size: 8
-
-optimizer:
-  learning_rate: 4
-
-evaluation:
-  sel_env_num: 0
-  test_env_num: 0
-
-env:
-  name: swebench
-  skill_init: reflact/envs/swebench/skills/initial.md
-  split_mode: ratio
-  split_ratio: "2:1:7"
-  split_dir: ""
-  data_path: ""
-  split_output_dir: ""
-  dataset_name: lite
-  hf_split: test
-  workers: 8
-  eval_workers: 8
-  step_limit: 50
-  cost_limit: 3.0
-  timeout_per_instance: 600
-  limit: 0

data/README.md

@@ -0,0 +1,237 @@
+# Data Manifests
+
+This directory releases lightweight split manifests for the SkillOpt paper
+splits. These manifests are not full runnable benchmark payloads. To evaluate a
+benchmark, first materialize the full examples from the raw data source when
+needed, then point `--split_dir` at the split directory listed below.
+
+In this README, "coverage" describes which part of the upstream benchmark the
+manifest references. It does not mean the released manifest directory contains
+the full runnable examples.
+
+## Layout
+
+Every released manifest directory uses the same file layout:
+
+```text
+data/<benchmark>_<manifest_type>/
+|-- split_manifest.json
+|-- train/items.json
+|-- val/items.json
+`-- test/items.json
+```
+
+`split_manifest.json` records source metadata, split counts, and item fields.
+Each `items.json` contains only stable IDs or source-path hints.
+
+## Released Splits
+
+| Manifest directory | Benchmark | Counts | Coverage | Raw data source | `split_dir` |
+|---|---|---:|---|---|---|
+| `searchqa_id_split/` | SearchQA | 400 / 200 / 1400 | Official HF dataset IDs | [lucadiliello/searchqa](https://huggingface.co/datasets/lucadiliello/searchqa) | `data/searchqa_split` |
+| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 18 / 124 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` |
+| `docvqa_id_split/` | DocVQA | 107 / 53 / 374 | 10% subset of validation | [lmms-lab/DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA) | `data/docvqa/splits` |
+| `officeqa_id_split/` | OfficeQA | 50 / 24 / 172 | OfficeQA Full | [databricks/officeqa](https://huggingface.co/datasets/databricks/officeqa) | `data/officeqa_split` |
+| `spreadsheetbench_id_split/` | SpreadsheetBench | 80 / 40 / 280 | SpreadsheetBench Verified 400 | [KAKA22/SpreadsheetBench](https://huggingface.co/datasets/KAKA22/SpreadsheetBench) | `data/spreadsheetbench_split` |
+| `alfworld_path_split/` | ALFWorld | 39 / 18 / 134 | ALFWorld `json_2.1.1` paths | [alfworld/alfworld](https://github.com/alfworld/alfworld) | `data/alfworld_path_split` |
+
+Counts are ordered as train / val / test.
+
+## Direct Use
+
+Only `alfworld_path_split/` can be used directly as `--split_dir` from this
+release, because the ALFWorld loader reads `gamefile` and `task_type` from the
+split items.
+
+This does not mean the ALFWorld raw data is included. You still need to
+download ALFWorld separately with `alfworld-download` and set `$ALFWORLD_DATA`
+to the data root containing `json_2.1.1`.
+
+The other manifest directories are lookup manifests. They intentionally omit
+full example fields such as questions, answers, contexts, images, or task
+instructions. Materialize those benchmarks into the `split_dir` paths listed
+above before running SkillOpt.
+
+## Lookup Keys
+
+The manifests are sufficient to locate the corresponding raw examples after
+the raw data has been downloaded or otherwise made available:
+
+| Benchmark | Manifest lookup key |
+|---|---|
+| SearchQA | Match `items.json[].id` to the `key` field in `lucadiliello/searchqa`. |
+| LiveMathematicianBench | Open `source_file`, then match `no`; the manifest `id` is `<month>:<no>`. |
+| DocVQA | Match `questionId` within the official DocVQA `validation` split; `image_path` records the expected local image path. |
+| OfficeQA | Match `uid` in `officeqa_full.csv`; `source_files` and `source_docs` identify the supporting document. |
+| SpreadsheetBench | Match `id`; `spreadsheet_path` identifies the referenced spreadsheet directory. |
+| ALFWorld | Resolve `gamefile` relative to `$ALFWORLD_DATA`. |
+
+## Manifest Item Examples
+
+SearchQA:
+
+```json
+{
+  "id": "221c83e6630f4e7983da48fa28da1882"
+}
+```
+
+LiveMathematicianBench:
+
+```json
+{
+  "id": "202602:22",
+  "month": "202602",
+  "no": 22,
+  "paper_link": "http://arxiv.org/abs/2602.10700v1",
+  "source_file": "data/202602/qa_202602_final.json"
+}
+```
+
+DocVQA:
+
+```json
+{
+  "id": "50877",
+  "questionId": "50877",
+  "docId": "14724",
+  "image_path": "data/docvqa_images/q50877_d14724.png",
+  "source_split": "validation"
+}
+```
+
+OfficeQA:
+
+```json
+{
+  "id": "UID0002",
+  "uid": "UID0002",
+  "category": "easy",
+  "source_files": "treasury_bulletin_1944_01.txt"
+}
+```
+
+SpreadsheetBench:
+
+```json
+{
+  "id": "32438",
+  "spreadsheet_path": "spreadsheet/32438",
+  "instruction_type": "Cell-Level Manipulation"
+}
+```
+
+ALFWorld:
+
+```json
+{
+  "id": "train:0000",
+  "gamefile": "json_2.1.1/train/.../game.tw-pddl",
+  "task_type": "look_at_obj_in_light"
+}
+```
+
+## Benchmark Notes
+
+### SearchQA
+
+`searchqa_id_split/` is an ID-only manifest. Each released `id` exactly matches
+the `key` field in `lucadiliello/searchqa`.
+
+To materialize the runnable SearchQA split used by
+`configs/searchqa/default.yaml`, install the optional dependency and run:
+
+```bash
+python -m pip install 'skillopt[searchqa]'
+python scripts/materialize_searchqa.py
+```
+
+This writes full examples to:
+
+```text
+data/searchqa_split
+```
+
+Materialized examples must include the fields consumed by the SearchQA
+environment, including:
+
+```text
+question
+context
+answers
+```
+
+### LiveMathematicianBench
+
+`livemathematicianbench_id_split/` was generated from these raw files:
+
+```text
+data/202511/qa_202511_final.json
+data/202512/qa_202512_final.json
+data/202601/qa_202601_final.json
+data/202602/qa_202602_final.json
+```
+
+The manifest stores IDs in the loader format:
+
+```text
+<month>:<no>
+```
+
+Materialized examples must include:
+
+```text
+question
+choices
+correct_choice
+theorem_type
+theorem
+sketch
+paper_link
+```
+
+### DocVQA
+
+`docvqa_id_split/` records `docvqa_validation_10pct`: a 10% subset sampled from
+the official DocVQA `validation` split.
+
+```text
+source_split: validation
+docvqa_validation_10pct: train=107, val=53, test=374
+```
+
+Each manifest item contains question/document IDs plus image location metadata.
+Materialized examples must provide `question`, `answer` or `ground_truth`, and
+an `image_path` that resolves locally.
+
+### OfficeQA
+
+`officeqa_id_split/` records the split over OfficeQA Full
+(`officeqa_full.csv`). The official OfficeQA CSVs are gated on Hugging Face, so
+materialization requires authorized access.
+
+Each manifest item contains `uid`, `category`, `source_files`, and
+`source_docs` hints. Materialized examples must include `question` and
+`ground_truth` or `answer`.
+
+### SpreadsheetBench
+
+`spreadsheetbench_id_split/` records the split over SpreadsheetBench Verified
+400, from `spreadsheetbench_verified_400.tar.gz`.
+
+Each manifest item contains task identity metadata such as `id`,
+`spreadsheet_path`, and `instruction_type`. Materialization must also place the
+referenced spreadsheet directories at:
+
+```text
+data/spreadsheetbench_verified_400
+```
+
+### ALFWorld
+
+`alfworld_path_split/` records `gamefile` paths relative to `$ALFWORLD_DATA`.
+The source payload is `json_2.1.1`, which must be downloaded separately with
+`alfworld-download`.
+
+This manifest can be used directly as `--split_dir` after `$ALFWORLD_DATA`
+points to the local ALFWorld data root containing `json_2.1.1`.

data/alfworld_path_split/split_manifest.json

@@ -0,0 +1,29 @@
+{
+  "benchmark": "ALFWorld",
+  "manifest_type": "path_split",
+  "source_repo": "alfworld/alfworld",
+  "source_repo_type": "repository",
+  "source_url": "https://github.com/alfworld/alfworld",
+  "source_file": "json_2.1.1",
+  "source_method": "generated by alfworld-download",
+  "source_split_files": [
+    "split_train.json",
+    "split_val.json",
+    "split_test.json"
+  ],
+  "counts": {
+    "train": 39,
+    "val": 18,
+    "test": 134
+  },
+  "item_fields": [
+    "id",
+    "gamefile",
+    "task_type"
+  ],
+  "path_root": "$ALFWORLD_DATA",
+  "notes": [
+    "This is a path manifest, not the ALFWorld game payload.",
+    "The gamefile field is relative to ALFWORLD_DATA and must be expanded before direct use as split_dir data."
+  ]
+}

data/alfworld_path_split/test/items.json

@@ -0,0 +1,672 @@
+[
+  {
+    "id": "test:0000",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222917_366542/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0001",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222933_607649/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0002",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222951_616606/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0003",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_020029_636862/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0004",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_020048_814402/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0005",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_144951_587345/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0006",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133919_856963/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0007",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133935_066606/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0008",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133953_562557/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0009",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_141942_810052/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0010",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_141958_463362/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0011",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_142046_281296/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0012",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_161733_213242/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0013",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_201421_021646/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0014",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_201444_037645/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0015",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220545_153480/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0016",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220604_010430/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0017",
+    "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220656_510400/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "test:0018",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190908_125200_737896/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0019",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190909_203041_433487/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0020",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190909_210238_431966/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0021",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_121952_610012/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0022",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_122024_052056/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0023",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_122154_042763/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0024",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190906_184021_215264/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0025",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190918_154326_823501/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0026",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190918_154424_844749/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0027",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191429_743650/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0028",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191445_723170/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0029",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191501_563086/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0030",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021613_077537/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0031",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021650_880235/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0032",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021728_339782/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0033",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004321_405868/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0034",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004351_281384/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0035",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004404_604165/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0036",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205204_244321/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0037",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205221_748352/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0038",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205246_776817/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0039",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074524_006355/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0040",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074556_124850/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0041",
+    "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074643_810052/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "test:0042",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061130_844814/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0043",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061158_110530/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0044",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061232_368489/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0045",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-Cabinet-424/trial_T20190908_022321_380927/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0046",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-Cabinet-424/trial_T20190908_022436_073995/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0047",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-CounterTop-424/trial_T20190908_100632_546757/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0048",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-CounterTop-424/trial_T20190908_114340_674467/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0049",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120554_888709/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0050",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120632_691361/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0051",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120712_273910/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0052",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110347_624008/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0053",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110445_675754/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0054",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110531_148235/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0055",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221208_560499/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0056",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221300_362511/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0057",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221355_558505/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0058",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032434_013084/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0059",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032518_891433/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0060",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032543_712058/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0061",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213356_017769/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0062",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213420_728917/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0063",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213533_897289/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0064",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_214926_337906/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0065",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_214946_567644/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0066",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_215019_162873/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0067",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074045_109439/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0068",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074106_050405/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0069",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074124_966890/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0070",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080730_211959/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0071",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080800_275989/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0072",
+    "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080825_222432/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "test:0073",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091747_866951/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0074",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091811_414150/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0075",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091835_825830/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0076",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_123133_763972/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0077",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_174807_646433/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0078",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_174840_771703/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0079",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121559_082363/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0080",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121635_622676/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0081",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121710_650938/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0082",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183715_299073/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0083",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183807_477267/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0084",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183853_958104/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0085",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114545_244903/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0086",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114622_738670/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0087",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114656_768805/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0088",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033157_424297/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0089",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033228_194678/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0090",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033306_962974/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0091",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102608_318800/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0092",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102644_926781/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0093",
+    "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102710_795182/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "test:0094",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182259_116320/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0095",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182353_418140/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0096",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182435_622538/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0097",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145050_918567/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0098",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145143_820541/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0099",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145356_918528/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0100",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083346_800823/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0101",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083429_887065/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0102",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083507_594820/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0103",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113432_673307/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0104",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113523_123938/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0105",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113610_425142/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0106",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021100_341887/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0107",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021200_669381/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0108",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021247_306737/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0109",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171806_406231/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0110",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171850_960211/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0111",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171933_349922/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0112",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Potato-None-GarbageCan-10/trial_T20190907_161745_664033/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0113",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Potato-None-GarbageCan-10/trial_T20190907_161853_945788/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0114",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225046_020282/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0115",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225359_617900/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0116",
+    "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225453_272533/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "test:0117",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_050942_897916/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0118",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_051013_060265/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0119",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_051056_585414/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0120",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-KeyChain-None-Safe-219/trial_T20190909_011803_423115/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0121",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-KeyChain-None-Safe-219/trial_T20190909_012027_782483/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0122",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190908_010306_215435/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0123",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190912_221016_460197/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0124",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190912_221141_608117/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0125",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163240_345855/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0126",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163327_486300/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0127",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163408_914117/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0128",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-Cabinet-424/trial_T20190909_081720_491733/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0129",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-Cabinet-424/trial_T20190909_081746_857594/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0130",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064053_839817/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0131",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064221_368939/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0132",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064309_357168/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "test:0133",
+    "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-ToiletPaper-None-Cabinet-424/trial_T20190906_202926_527010/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  }
+]

data/alfworld_path_split/train/items.json

@@ -0,0 +1,197 @@
+[
+  {
+    "id": "train:0000",
+    "gamefile": "json_2.1.1/train/look_at_obj_in_light-AlarmClock-None-DeskLamp-305/trial_T20190908_082736_108723/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "train:0001",
+    "gamefile": "json_2.1.1/train/look_at_obj_in_light-CD-None-DeskLamp-304/trial_T20190907_185649_782438/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "train:0002",
+    "gamefile": "json_2.1.1/train/look_at_obj_in_light-CD-None-DeskLamp-320/trial_T20190907_224439_174735/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "train:0003",
+    "gamefile": "json_2.1.1/train/look_at_obj_in_light-Pillow-None-DeskLamp-316/trial_T20190908_232421_645610/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "train:0004",
+    "gamefile": "json_2.1.1/train/look_at_obj_in_light-Statue-None-DeskLamp-319/trial_T20190907_035546_167548/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "train:0005",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-CellPhone-None-Shelf-313/trial_T20190908_123725_452958/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0006",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-Newspaper-None-Sofa-211/trial_T20190906_175004_203092/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0007",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-Pencil-None-Desk-302/trial_T20190908_032836_462632/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0008",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-SoapBar-None-GarbageCan-416/trial_T20190908_020839_714699/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0009",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-Statue-None-CoffeeTable-222/trial_T20190907_131249_788749/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0010",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-ToiletPaper-None-ToiletPaperHanger-406/trial_T20190908_122807_136741/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0011",
+    "gamefile": "json_2.1.1/train/pick_and_place_simple-ToiletPaper-None-ToiletPaperHanger-415/trial_T20190908_050443_333939/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "train:0012",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Apple-None-DiningTable-4/trial_T20190908_104413_450768/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0013",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-DishSponge-None-Shelf-20/trial_T20190907_222429_992578/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0014",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-DishSponge-None-Shelf-401/trial_T20190908_072225_397518/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0015",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Kettle-None-Cabinet-2/trial_T20190909_043103_418752/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0016",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Knife-None-Drawer-22/trial_T20190907_224827_746945/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0017",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Lettuce-None-DiningTable-20/trial_T20190906_191148_519826/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0018",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Lettuce-None-Fridge-13/trial_T20190908_203022_601787/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0019",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Plate-None-Fridge-5/trial_T20190909_112954_869911/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0020",
+    "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Spoon-None-DiningTable-18/trial_T20190909_102159_277894/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "train:0021",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-1/trial_T20190908_212439_711334/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0022",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-15/trial_T20190909_085448_256298/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0023",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-16/trial_T20190908_143948_082471/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0024",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Pan-None-StoveBurner-27/trial_T20190906_212619_469871/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0025",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Plate-None-DiningTable-17/trial_T20190909_122939_032098/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0026",
+    "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Pot-None-CounterTop-1/trial_T20190909_124252_504581/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "train:0027",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Apple-None-Fridge-20/trial_T20190908_013911_274341/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0028",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Egg-None-CounterTop-12/trial_T20190908_215527_416490/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0029",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-1/trial_T20190907_222924_821086/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0030",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-28/trial_T20190908_062730_537428/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0031",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Plate-None-Cabinet-13/trial_T20190907_062749_759882/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0032",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Potato-None-Fridge-2/trial_T20190909_030845_198194/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0033",
+    "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Tomato-None-CounterTop-26/trial_T20190907_005525_499114/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "train:0034",
+    "gamefile": "json_2.1.1/train/pick_two_obj_and_place-CD-None-Drawer-319/trial_T20190907_145515_348252/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "train:0035",
+    "gamefile": "json_2.1.1/train/pick_two_obj_and_place-Candle-None-Drawer-427/trial_T20190909_043917_251333/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "train:0036",
+    "gamefile": "json_2.1.1/train/pick_two_obj_and_place-KeyChain-None-ArmChair-222/trial_T20190909_100312_677332/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "train:0037",
+    "gamefile": "json_2.1.1/train/pick_two_obj_and_place-Newspaper-None-Sofa-212/trial_T20190908_112632_208041/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "train:0038",
+    "gamefile": "json_2.1.1/train/pick_two_obj_and_place-SaltShaker-None-SideTable-21/trial_T20190909_041626_844806/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  }
+]

data/alfworld_path_split/val/items.json

@@ -0,0 +1,92 @@
+[
+  {
+    "id": "val:0000",
+    "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-AlarmClock-None-DeskLamp-323/trial_T20190909_044715_250790/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "val:0001",
+    "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-Bowl-None-DeskLamp-301/trial_T20190909_150719_492274/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "val:0002",
+    "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-Pillow-None-DeskLamp-323/trial_T20190908_053153_077977/game.tw-pddl",
+    "task_type": "look_at_obj_in_light"
+  },
+  {
+    "id": "val:0003",
+    "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Mug-None-SideTable-329/trial_T20190909_032318_169393/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "val:0004",
+    "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Mug-None-SideTable-329/trial_T20190909_032340_274147/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "val:0005",
+    "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Pencil-None-Desk-310/trial_T20190909_113054_894334/game.tw-pddl",
+    "task_type": "pick_and_place_simple"
+  },
+  {
+    "id": "val:0006",
+    "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-ButterKnife-None-Drawer-30/trial_T20190908_052007_212776/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "val:0007",
+    "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-ButterKnife-None-Drawer-8/trial_T20190909_124425_112757/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "val:0008",
+    "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-402/trial_T20190908_055221_984342/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "val:0009",
+    "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-SoapBar-None-Toilet-410/trial_T20190906_201106_979461/game.tw-pddl",
+    "task_type": "pick_clean_then_place_in_recep"
+  },
+  {
+    "id": "val:0010",
+    "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Apple-None-Microwave-19/trial_T20190906_210937_878489/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "val:0011",
+    "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Plate-None-CounterTop-1/trial_T20190906_205324_559361/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "val:0012",
+    "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Tomato-None-Microwave-18/trial_T20190909_012524_159092/game.tw-pddl",
+    "task_type": "pick_cool_then_place_in_recep"
+  },
+  {
+    "id": "val:0013",
+    "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Apple-None-DiningTable-26/trial_T20190907_060234_011675/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "val:0014",
+    "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Tomato-None-Fridge-15/trial_T20190909_020200_054379/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "val:0015",
+    "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Tomato-None-Fridge-23/trial_T20190909_082320_103350/game.tw-pddl",
+    "task_type": "pick_heat_then_place_in_recep"
+  },
+  {
+    "id": "val:0016",
+    "gamefile": "json_2.1.1/valid_seen/pick_two_obj_and_place-Book-None-Desk-313/trial_T20190908_125930_920681/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  },
+  {
+    "id": "val:0017",
+    "gamefile": "json_2.1.1/valid_seen/pick_two_obj_and_place-CreditCard-None-Safe-323/trial_T20190907_001129_214240/game.tw-pddl",
+    "task_type": "pick_two_obj_and_place"
+  }
+]

data/docvqa_id_split/split_manifest.json

@@ -0,0 +1,36 @@
+{
+  "benchmark": "DocVQA",
+  "manifest_type": "id_split",
+  "source_repo": "lmms-lab/DocVQA",
+  "source_repo_type": "dataset",
+  "source_url": "https://huggingface.co/datasets/lmms-lab/DocVQA",
+  "source_revision": "539088ef8a8ada01ac8e2e6d4e372586748a265e",
+  "source_config": "DocVQA",
+  "source_split": "validation",
+  "source_split_name": "docvqa_validation_10pct",
+  "split_method": "10% subset sampled from the DocVQA validation split",
+  "counts": {
+    "train": 107,
+    "val": 53,
+    "test": 374
+  },
+  "item_fields": [
+    "id",
+    "questionId",
+    "docId",
+    "image_path",
+    "ucsf_document_id",
+    "ucsf_document_page_no",
+    "topic",
+    "source_dataset",
+    "source_config",
+    "source_split",
+    "sample_seed"
+  ],
+  "notes": [
+    "This is a split manifest, not the full DocVQA payload.",
+    "Materialize full CSV rows and image files before evaluation.",
+    "This manifest corresponds to docvqa_validation_10pct.",
+    "All released train/val/test items originate from a 10% subset of the official DocVQA validation split."
+  ]
+}

data/docvqa_id_split/val/items.json

@@ -0,0 +1,691 @@
+[
+  {
+    "id": "62409",
+    "questionId": "62409",
+    "docId": "8554",
+    "image_path": "data/docvqa_images/q62409_d8554.png",
+    "ucsf_document_id": "pgjw0227",
+    "ucsf_document_page_no": "5",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "50961",
+    "questionId": "50961",
+    "docId": "549",
+    "image_path": "data/docvqa_images/q50961_d549.png",
+    "ucsf_document_id": "qtjf0226",
+    "ucsf_document_page_no": "2",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "46461",
+    "questionId": "46461",
+    "docId": "13361",
+    "image_path": "data/docvqa_images/q46461_d13361.png",
+    "ucsf_document_id": "ysbw0217",
+    "ucsf_document_page_no": "5",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "3041",
+    "questionId": "3041",
+    "docId": "1204",
+    "image_path": "data/docvqa_images/q3041_d1204.png",
+    "ucsf_document_id": "xfjv0228",
+    "ucsf_document_page_no": "3",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "41716",
+    "questionId": "41716",
+    "docId": "11835",
+    "image_path": "data/docvqa_images/q41716_d11835.png",
+    "ucsf_document_id": "qjgn0226",
+    "ucsf_document_page_no": "131",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "61123",
+    "questionId": "61123",
+    "docId": "7374",
+    "image_path": "data/docvqa_images/q61123_d7374.png",
+    "ucsf_document_id": "mldg0227",
+    "ucsf_document_page_no": "5",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "43068",
+    "questionId": "43068",
+    "docId": "12393",
+    "image_path": "data/docvqa_images/q43068_d12393.png",
+    "ucsf_document_id": "rmwn0226",
+    "ucsf_document_page_no": "52",
+    "topic": "figure/diagram",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "51221",
+    "questionId": "51221",
+    "docId": "764",
+    "image_path": "data/docvqa_images/q51221_d764.png",
+    "ucsf_document_id": "kzbn0226",
+    "ucsf_document_page_no": "14",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "6397",
+    "questionId": "6397",
+    "docId": "2242",
+    "image_path": "data/docvqa_images/q6397_d2242.png",
+    "ucsf_document_id": "jkcn0000",
+    "ucsf_document_page_no": "2",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "57428",
+    "questionId": "57428",
+    "docId": "4779",
+    "image_path": "data/docvqa_images/q57428_d4779.png",
+    "ucsf_document_id": "rnbx0223",
+    "ucsf_document_page_no": "208",
+    "topic": "Image/Photo",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "3135",
+    "questionId": "3135",
+    "docId": "1221",
+    "image_path": "data/docvqa_images/q3135_d1221.png",
+    "ucsf_document_id": "ngph0227",
+    "ucsf_document_page_no": "5",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "18819",
+    "questionId": "18819",
+    "docId": "5749",
+    "image_path": "data/docvqa_images/q18819_d5749.png",
+    "ucsf_document_id": "jhfd0079",
+    "ucsf_document_page_no": "9",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "15382",
+    "questionId": "15382",
+    "docId": "4890",
+    "image_path": "data/docvqa_images/q15382_d4890.png",
+    "ucsf_document_id": "kjvw0217",
+    "ucsf_document_page_no": "3",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "5772",
+    "questionId": "5772",
+    "docId": "1940",
+    "image_path": "data/docvqa_images/q5772_d1940.png",
+    "ucsf_document_id": "pzyw0224",
+    "ucsf_document_page_no": "10",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "49077",
+    "questionId": "49077",
+    "docId": "14179",
+    "image_path": "data/docvqa_images/q49077_d14179.png",
+    "ucsf_document_id": "nrxb0228",
+    "ucsf_document_page_no": "3",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "58519",
+    "questionId": "58519",
+    "docId": "5347",
+    "image_path": "data/docvqa_images/q58519_d5347.png",
+    "ucsf_document_id": "sjbw0217",
+    "ucsf_document_page_no": "11",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "50720",
+    "questionId": "50720",
+    "docId": "281",
+    "image_path": "data/docvqa_images/q50720_d281.png",
+    "ucsf_document_id": "nrcj0037",
+    "ucsf_document_page_no": "7",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "56785",
+    "questionId": "56785",
+    "docId": "14289",
+    "image_path": "data/docvqa_images/q56785_d14289.png",
+    "ucsf_document_id": "xkbv0228",
+    "ucsf_document_page_no": "1",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "59653",
+    "questionId": "59653",
+    "docId": "6579",
+    "image_path": "data/docvqa_images/q59653_d6579.png",
+    "ucsf_document_id": "mzbx0227",
+    "ucsf_document_page_no": "2",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "61791",
+    "questionId": "61791",
+    "docId": "8072",
+    "image_path": "data/docvqa_images/q61791_d8072.png",
+    "ucsf_document_id": "hfmf0227",
+    "ucsf_document_page_no": "1",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "37229",
+    "questionId": "37229",
+    "docId": "10742",
+    "image_path": "data/docvqa_images/q37229_d10742.png",
+    "ucsf_document_id": "nkcd0227",
+    "ucsf_document_page_no": "2",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "60407",
+    "questionId": "60407",
+    "docId": "7135",
+    "image_path": "data/docvqa_images/q60407_d7135.png",
+    "ucsf_document_id": "gkpk0226",
+    "ucsf_document_page_no": "1",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "64420",
+    "questionId": "64420",
+    "docId": "10230",
+    "image_path": "data/docvqa_images/q64420_d10230.png",
+    "ucsf_document_id": "jnjm0223",
+    "ucsf_document_page_no": "107",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "47365",
+    "questionId": "47365",
+    "docId": "13813",
+    "image_path": "data/docvqa_images/q47365_d13813.png",
+    "ucsf_document_id": "nxym0227",
+    "ucsf_document_page_no": "28",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "47458",
+    "questionId": "47458",
+    "docId": "13639",
+    "image_path": "data/docvqa_images/q47458_d13639.png",
+    "ucsf_document_id": "skdv0228",
+    "ucsf_document_page_no": "5",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "7621",
+    "questionId": "7621",
+    "docId": "2668",
+    "image_path": "data/docvqa_images/q7621_d2668.png",
+    "ucsf_document_id": "flxn0020",
+    "ucsf_document_page_no": "1",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "53575",
+    "questionId": "53575",
+    "docId": "2766",
+    "image_path": "data/docvqa_images/q53575_d2766.png",
+    "ucsf_document_id": "hsfn0020",
+    "ucsf_document_page_no": "2",
+    "topic": "free_text|table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "60913",
+    "questionId": "60913",
+    "docId": "7349",
+    "image_path": "data/docvqa_images/q60913_d7349.png",
+    "ucsf_document_id": "jzhd0227",
+    "ucsf_document_page_no": "61",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "60454",
+    "questionId": "60454",
+    "docId": "7163",
+    "image_path": "data/docvqa_images/q60454_d7163.png",
+    "ucsf_document_id": "jgyk0226",
+    "ucsf_document_page_no": "1",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "57978",
+    "questionId": "57978",
+    "docId": "4920",
+    "image_path": "data/docvqa_images/q57978_d4920.png",
+    "ucsf_document_id": "lkvw0217",
+    "ucsf_document_page_no": "2",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "64547",
+    "questionId": "64547",
+    "docId": "10361",
+    "image_path": "data/docvqa_images/q64547_d10361.png",
+    "ucsf_document_id": "lpdl0226",
+    "ucsf_document_page_no": "32",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "59481",
+    "questionId": "59481",
+    "docId": "6243",
+    "image_path": "data/docvqa_images/q59481_d6243.png",
+    "ucsf_document_id": "psgv0228",
+    "ucsf_document_page_no": "5",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "61472",
+    "questionId": "61472",
+    "docId": "7757",
+    "image_path": "data/docvqa_images/q61472_d7757.png",
+    "ucsf_document_id": "ymkp0227",
+    "ucsf_document_page_no": "13",
+    "topic": "handwritten|table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "5673",
+    "questionId": "5673",
+    "docId": "1908",
+    "image_path": "data/docvqa_images/q5673_d1908.png",
+    "ucsf_document_id": "lldj0224",
+    "ucsf_document_page_no": "2",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "49109",
+    "questionId": "49109",
+    "docId": "13644",
+    "image_path": "data/docvqa_images/q49109_d13644.png",
+    "ucsf_document_id": "mzdv0228",
+    "ucsf_document_page_no": "1",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "46123",
+    "questionId": "46123",
+    "docId": "13503",
+    "image_path": "data/docvqa_images/q46123_d13503.png",
+    "ucsf_document_id": "xmww0217",
+    "ucsf_document_page_no": "17",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "48158",
+    "questionId": "48158",
+    "docId": "13976",
+    "image_path": "data/docvqa_images/q48158_d13976.png",
+    "ucsf_document_id": "zqhm0227",
+    "ucsf_document_page_no": "1",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "1955",
+    "questionId": "1955",
+    "docId": "892",
+    "image_path": "data/docvqa_images/q1955_d892.png",
+    "ucsf_document_id": "jsbn0226",
+    "ucsf_document_page_no": "2",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "8127",
+    "questionId": "8127",
+    "docId": "2754",
+    "image_path": "data/docvqa_images/q8127_d2754.png",
+    "ucsf_document_id": "xtvn0020",
+    "ucsf_document_page_no": "2",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "57431",
+    "questionId": "57431",
+    "docId": "4779",
+    "image_path": "data/docvqa_images/q57431_d4779.png",
+    "ucsf_document_id": "rnbx0223",
+    "ucsf_document_page_no": "208",
+    "topic": "Image/Photo",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "64306",
+    "questionId": "64306",
+    "docId": "10149",
+    "image_path": "data/docvqa_images/q64306_d10149.png",
+    "ucsf_document_id": "lpjm0223",
+    "ucsf_document_page_no": "23",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "64887",
+    "questionId": "64887",
+    "docId": "9754",
+    "image_path": "data/docvqa_images/q64887_d9754.png",
+    "ucsf_document_id": "szpg0227",
+    "ucsf_document_page_no": "9",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "58680",
+    "questionId": "58680",
+    "docId": "5545",
+    "image_path": "data/docvqa_images/q58680_d5545.png",
+    "ucsf_document_id": "hhwh0078",
+    "ucsf_document_page_no": "1",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "5287",
+    "questionId": "5287",
+    "docId": "1785",
+    "image_path": "data/docvqa_images/q5287_d1785.png",
+    "ucsf_document_id": "mtnh0227",
+    "ucsf_document_page_no": "10",
+    "topic": "form",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "55471",
+    "questionId": "55471",
+    "docId": "4340",
+    "image_path": "data/docvqa_images/q55471_d4340.png",
+    "ucsf_document_id": "fsgj0223",
+    "ucsf_document_page_no": "96",
+    "topic": "free_text",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "53095",
+    "questionId": "53095",
+    "docId": "296",
+    "image_path": "data/docvqa_images/q53095_d296.png",
+    "ucsf_document_id": "qhxj0037",
+    "ucsf_document_page_no": "3",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "53726",
+    "questionId": "53726",
+    "docId": "2008",
+    "image_path": "data/docvqa_images/q53726_d2008.png",
+    "ucsf_document_id": "hhnf0094",
+    "ucsf_document_page_no": "5",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "57321",
+    "questionId": "57321",
+    "docId": "4722",
+    "image_path": "data/docvqa_images/q57321_d4722.png",
+    "ucsf_document_id": "xybx0223",
+    "ucsf_document_page_no": "32",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "26659",
+    "questionId": "26659",
+    "docId": "7470",
+    "image_path": "data/docvqa_images/q26659_d7470.png",
+    "ucsf_document_id": "lhmg0227",
+    "ucsf_document_page_no": "1",
+    "topic": "layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "38920",
+    "questionId": "38920",
+    "docId": "11157",
+    "image_path": "data/docvqa_images/q38920_d11157.png",
+    "ucsf_document_id": "klnf0227",
+    "ucsf_document_page_no": "1",
+    "topic": "table/list|layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "50837",
+    "questionId": "50837",
+    "docId": "14742",
+    "image_path": "data/docvqa_images/q50837_d14742.png",
+    "ucsf_document_id": "ysmc0228",
+    "ucsf_document_page_no": "4",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "59615",
+    "questionId": "59615",
+    "docId": "6569",
+    "image_path": "data/docvqa_images/q59615_d6569.png",
+    "ucsf_document_id": "hnnp0227",
+    "ucsf_document_page_no": "45",
+    "topic": "handwritten|table/list|layout",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  },
+  {
+    "id": "58687",
+    "questionId": "58687",
+    "docId": "5545",
+    "image_path": "data/docvqa_images/q58687_d5545.png",
+    "ucsf_document_id": "hhwh0078",
+    "ucsf_document_page_no": "1",
+    "topic": "table/list",
+    "source_dataset": "lmms-lab/DocVQA",
+    "source_config": "DocVQA",
+    "source_split": "validation",
+    "sample_seed": "full_validation_5349"
+  }
+]

data/livemathematicianbench_id_split/split_manifest.json

@@ -0,0 +1,34 @@
+{
+  "benchmark": "LiveMathematicianBench",
+  "manifest_type": "id_split",
+  "source_repo": "LiveMathematicianBench/LiveMathematicianBench",
+  "source_repo_type": "dataset",
+  "source_url": "https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench",
+  "source_revision": "b72450f6ce96c26158d64d945a5d31ef7727be41",
+  "source_files": [
+    "data/202511/qa_202511_final.json",
+    "data/202512/qa_202512_final.json",
+    "data/202601/qa_202601_final.json",
+    "data/202602/qa_202602_final.json"
+  ],
+  "split_mode": "ratio",
+  "split_ratio": "2:1:7",
+  "split_seed": 42,
+  "counts": {
+    "train": 35,
+    "val": 18,
+    "test": 124
+  },
+  "item_fields": [
+    "id",
+    "month",
+    "no",
+    "paper_link",
+    "source_file"
+  ],
+  "id_format": "<month>:<no>",
+  "notes": [
+    "This is an ID split manifest, not the full LiveMathematicianBench payload.",
+    "Materialize full split items from the official LiveMathematicianBench raw qa_*_final.json files before evaluation."
+  ]
+}

data/livemathematicianbench_id_split/test/items.json

@@ -0,0 +1,870 @@
+[
+  {
+    "id": "202602:12",
+    "month": "202602",
+    "no": 12,
+    "paper_link": "http://arxiv.org/abs/2602.07171v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:3",
+    "month": "202601",
+    "no": 3,
+    "paper_link": "http://arxiv.org/abs/2601.01447v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:4",
+    "month": "202511",
+    "no": 4,
+    "paper_link": "http://arxiv.org/abs/2511.23123v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:20",
+    "month": "202601",
+    "no": 20,
+    "paper_link": "http://arxiv.org/abs/2601.13212v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:42",
+    "month": "202601",
+    "no": 42,
+    "paper_link": "http://arxiv.org/abs/2601.09348v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:38",
+    "month": "202512",
+    "no": 38,
+    "paper_link": "http://arxiv.org/abs/2512.19831v2",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:4",
+    "month": "202512",
+    "no": 4,
+    "paper_link": "http://arxiv.org/abs/2512.03141v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:4",
+    "month": "202602",
+    "no": 4,
+    "paper_link": "http://arxiv.org/abs/2602.14368v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202511:15",
+    "month": "202511",
+    "no": 15,
+    "paper_link": "http://arxiv.org/abs/2511.17325v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:32",
+    "month": "202602",
+    "no": 32,
+    "paper_link": "http://arxiv.org/abs/2602.14817v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:51",
+    "month": "202512",
+    "no": 51,
+    "paper_link": "http://arxiv.org/abs/2512.14581v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:26",
+    "month": "202512",
+    "no": 26,
+    "paper_link": "http://arxiv.org/abs/2512.19586v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:13",
+    "month": "202601",
+    "no": 13,
+    "paper_link": "http://arxiv.org/abs/2601.10017v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:1",
+    "month": "202602",
+    "no": 1,
+    "paper_link": "http://arxiv.org/abs/2602.23137v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202511:18",
+    "month": "202511",
+    "no": 18,
+    "paper_link": "http://arxiv.org/abs/2511.10795v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:5",
+    "month": "202512",
+    "no": 5,
+    "paper_link": "http://arxiv.org/abs/2512.00348v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:19",
+    "month": "202511",
+    "no": 19,
+    "paper_link": "http://arxiv.org/abs/2511.06951v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:40",
+    "month": "202602",
+    "no": 40,
+    "paper_link": "http://arxiv.org/abs/2602.20462v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:29",
+    "month": "202602",
+    "no": 29,
+    "paper_link": "http://arxiv.org/abs/2602.10676v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:35",
+    "month": "202512",
+    "no": 35,
+    "paper_link": "http://arxiv.org/abs/2512.08840v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:48",
+    "month": "202512",
+    "no": 48,
+    "paper_link": "http://arxiv.org/abs/2512.03482v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:52",
+    "month": "202512",
+    "no": 52,
+    "paper_link": "http://arxiv.org/abs/2512.11246v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:44",
+    "month": "202512",
+    "no": 44,
+    "paper_link": "http://arxiv.org/abs/2512.10385v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:28",
+    "month": "202511",
+    "no": 28,
+    "paper_link": "http://arxiv.org/abs/2511.03812v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:43",
+    "month": "202601",
+    "no": 43,
+    "paper_link": "http://arxiv.org/abs/2601.22555v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:9",
+    "month": "202602",
+    "no": 9,
+    "paper_link": "http://arxiv.org/abs/2602.19882v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:23",
+    "month": "202512",
+    "no": 23,
+    "paper_link": "http://arxiv.org/abs/2512.09180v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:21",
+    "month": "202602",
+    "no": 21,
+    "paper_link": "http://arxiv.org/abs/2602.10509v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202511:5",
+    "month": "202511",
+    "no": 5,
+    "paper_link": "http://arxiv.org/abs/2511.20164v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:35",
+    "month": "202601",
+    "no": 35,
+    "paper_link": "http://arxiv.org/abs/2601.15606v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:50",
+    "month": "202602",
+    "no": 50,
+    "paper_link": "http://arxiv.org/abs/2602.05652v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:13",
+    "month": "202512",
+    "no": 13,
+    "paper_link": "http://arxiv.org/abs/2512.22861v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:49",
+    "month": "202602",
+    "no": 49,
+    "paper_link": "http://arxiv.org/abs/2602.07167v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:18",
+    "month": "202602",
+    "no": 18,
+    "paper_link": "http://arxiv.org/abs/2602.20124v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:15",
+    "month": "202601",
+    "no": 15,
+    "paper_link": "http://arxiv.org/abs/2601.05327v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:21",
+    "month": "202601",
+    "no": 21,
+    "paper_link": "http://arxiv.org/abs/2601.04994v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:32",
+    "month": "202601",
+    "no": 32,
+    "paper_link": "http://arxiv.org/abs/2601.09183v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:34",
+    "month": "202602",
+    "no": 34,
+    "paper_link": "http://arxiv.org/abs/2602.21118v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:20",
+    "month": "202602",
+    "no": 20,
+    "paper_link": "http://arxiv.org/abs/2602.16506v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:5",
+    "month": "202602",
+    "no": 5,
+    "paper_link": "http://arxiv.org/abs/2602.09806v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:40",
+    "month": "202512",
+    "no": 40,
+    "paper_link": "http://arxiv.org/abs/2512.16535v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:22",
+    "month": "202511",
+    "no": 22,
+    "paper_link": "http://arxiv.org/abs/2511.07607v2",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:36",
+    "month": "202601",
+    "no": 36,
+    "paper_link": "http://arxiv.org/abs/2601.12457v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:49",
+    "month": "202512",
+    "no": 49,
+    "paper_link": "http://arxiv.org/abs/2512.21565v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:10",
+    "month": "202511",
+    "no": 10,
+    "paper_link": "http://arxiv.org/abs/2511.06484v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:2",
+    "month": "202601",
+    "no": 2,
+    "paper_link": "http://arxiv.org/abs/2601.07068v4",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:19",
+    "month": "202602",
+    "no": 19,
+    "paper_link": "http://arxiv.org/abs/2602.18179v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:9",
+    "month": "202601",
+    "no": 9,
+    "paper_link": "http://arxiv.org/abs/2601.17765v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:6",
+    "month": "202512",
+    "no": 6,
+    "paper_link": "http://arxiv.org/abs/2512.23079v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:5",
+    "month": "202601",
+    "no": 5,
+    "paper_link": "http://arxiv.org/abs/2601.20344v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:14",
+    "month": "202602",
+    "no": 14,
+    "paper_link": "http://arxiv.org/abs/2602.09177v2",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:17",
+    "month": "202512",
+    "no": 17,
+    "paper_link": "http://arxiv.org/abs/2512.11657v2",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:19",
+    "month": "202512",
+    "no": 19,
+    "paper_link": "http://arxiv.org/abs/2512.16655v2",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:33",
+    "month": "202602",
+    "no": 33,
+    "paper_link": "http://arxiv.org/abs/2602.13734v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:18",
+    "month": "202512",
+    "no": 18,
+    "paper_link": "http://arxiv.org/abs/2512.22960v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:26",
+    "month": "202601",
+    "no": 26,
+    "paper_link": "http://arxiv.org/abs/2601.06814v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:1",
+    "month": "202601",
+    "no": 1,
+    "paper_link": "http://arxiv.org/abs/2601.18276v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:30",
+    "month": "202512",
+    "no": 30,
+    "paper_link": "http://arxiv.org/abs/2512.07260v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:44",
+    "month": "202602",
+    "no": 44,
+    "paper_link": "http://arxiv.org/abs/2602.01138v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:20",
+    "month": "202512",
+    "no": 20,
+    "paper_link": "http://arxiv.org/abs/2512.14575v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:13",
+    "month": "202511",
+    "no": 13,
+    "paper_link": "http://arxiv.org/abs/2511.16910v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:30",
+    "month": "202601",
+    "no": 30,
+    "paper_link": "http://arxiv.org/abs/2601.12140v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:40",
+    "month": "202601",
+    "no": 40,
+    "paper_link": "http://arxiv.org/abs/2601.05146v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:29",
+    "month": "202601",
+    "no": 29,
+    "paper_link": "http://arxiv.org/abs/2601.12846v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:11",
+    "month": "202511",
+    "no": 11,
+    "paper_link": "http://arxiv.org/abs/2511.17548v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:9",
+    "month": "202512",
+    "no": 9,
+    "paper_link": "http://arxiv.org/abs/2512.08817v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:18",
+    "month": "202601",
+    "no": 18,
+    "paper_link": "http://arxiv.org/abs/2601.01797v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:1",
+    "month": "202512",
+    "no": 1,
+    "paper_link": "http://arxiv.org/abs/2512.20055v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:4",
+    "month": "202601",
+    "no": 4,
+    "paper_link": "http://arxiv.org/abs/2601.21223v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:6",
+    "month": "202511",
+    "no": 6,
+    "paper_link": "http://arxiv.org/abs/2511.14959v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:38",
+    "month": "202602",
+    "no": 38,
+    "paper_link": "http://arxiv.org/abs/2602.08398v2",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:10",
+    "month": "202601",
+    "no": 10,
+    "paper_link": "http://arxiv.org/abs/2601.15524v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:11",
+    "month": "202602",
+    "no": 11,
+    "paper_link": "http://arxiv.org/abs/2602.11045v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:45",
+    "month": "202512",
+    "no": 45,
+    "paper_link": "http://arxiv.org/abs/2512.08395v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:12",
+    "month": "202601",
+    "no": 12,
+    "paper_link": "http://arxiv.org/abs/2601.11877v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:47",
+    "month": "202512",
+    "no": 47,
+    "paper_link": "http://arxiv.org/abs/2512.09683v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:21",
+    "month": "202511",
+    "no": 21,
+    "paper_link": "http://arxiv.org/abs/2511.21288v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:16",
+    "month": "202601",
+    "no": 16,
+    "paper_link": "http://arxiv.org/abs/2601.05008v2",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:3",
+    "month": "202512",
+    "no": 3,
+    "paper_link": "http://arxiv.org/abs/2512.13450v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:37",
+    "month": "202601",
+    "no": 37,
+    "paper_link": "http://arxiv.org/abs/2601.09443v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:12",
+    "month": "202511",
+    "no": 12,
+    "paper_link": "http://arxiv.org/abs/2511.04978v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:39",
+    "month": "202512",
+    "no": 39,
+    "paper_link": "http://arxiv.org/abs/2512.19003v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:8",
+    "month": "202601",
+    "no": 8,
+    "paper_link": "http://arxiv.org/abs/2601.19754v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:11",
+    "month": "202601",
+    "no": 11,
+    "paper_link": "http://arxiv.org/abs/2601.13552v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:25",
+    "month": "202511",
+    "no": 25,
+    "paper_link": "http://arxiv.org/abs/2511.10548v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:17",
+    "month": "202601",
+    "no": 17,
+    "paper_link": "http://arxiv.org/abs/2601.02655v2",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:36",
+    "month": "202602",
+    "no": 36,
+    "paper_link": "http://arxiv.org/abs/2602.13001v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:43",
+    "month": "202602",
+    "no": 43,
+    "paper_link": "http://arxiv.org/abs/2602.06897v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:6",
+    "month": "202601",
+    "no": 6,
+    "paper_link": "http://arxiv.org/abs/2601.04747v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:35",
+    "month": "202602",
+    "no": 35,
+    "paper_link": "http://arxiv.org/abs/2602.20938v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:11",
+    "month": "202512",
+    "no": 11,
+    "paper_link": "http://arxiv.org/abs/2512.03294v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:23",
+    "month": "202602",
+    "no": 23,
+    "paper_link": "http://arxiv.org/abs/2602.09201v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:7",
+    "month": "202601",
+    "no": 7,
+    "paper_link": "http://arxiv.org/abs/2601.02859v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:39",
+    "month": "202602",
+    "no": 39,
+    "paper_link": "http://arxiv.org/abs/2602.21659v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:12",
+    "month": "202512",
+    "no": 12,
+    "paper_link": "http://arxiv.org/abs/2512.00690v3",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:2",
+    "month": "202511",
+    "no": 2,
+    "paper_link": "http://arxiv.org/abs/2511.19681v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:43",
+    "month": "202512",
+    "no": 43,
+    "paper_link": "http://arxiv.org/abs/2512.10820v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:24",
+    "month": "202602",
+    "no": 24,
+    "paper_link": "http://arxiv.org/abs/2602.08680v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:34",
+    "month": "202601",
+    "no": 34,
+    "paper_link": "http://arxiv.org/abs/2601.07318v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:28",
+    "month": "202512",
+    "no": 28,
+    "paper_link": "http://arxiv.org/abs/2512.11294v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:27",
+    "month": "202601",
+    "no": 27,
+    "paper_link": "http://arxiv.org/abs/2601.05692v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:42",
+    "month": "202602",
+    "no": 42,
+    "paper_link": "http://arxiv.org/abs/2602.09749v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:22",
+    "month": "202512",
+    "no": 22,
+    "paper_link": "http://arxiv.org/abs/2512.11658v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:17",
+    "month": "202602",
+    "no": 17,
+    "paper_link": "http://arxiv.org/abs/2602.22504v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:48",
+    "month": "202602",
+    "no": 48,
+    "paper_link": "http://arxiv.org/abs/2602.08760v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:28",
+    "month": "202602",
+    "no": 28,
+    "paper_link": "http://arxiv.org/abs/2602.11595v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:3",
+    "month": "202602",
+    "no": 3,
+    "paper_link": "http://arxiv.org/abs/2602.17369v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:31",
+    "month": "202512",
+    "no": 31,
+    "paper_link": "http://arxiv.org/abs/2512.23668v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:27",
+    "month": "202512",
+    "no": 27,
+    "paper_link": "http://arxiv.org/abs/2512.16505v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:24",
+    "month": "202511",
+    "no": 24,
+    "paper_link": "http://arxiv.org/abs/2511.12549v2",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202511:8",
+    "month": "202511",
+    "no": 8,
+    "paper_link": "http://arxiv.org/abs/2511.12657v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202511:9",
+    "month": "202511",
+    "no": 9,
+    "paper_link": "http://arxiv.org/abs/2511.09015v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:28",
+    "month": "202601",
+    "no": 28,
+    "paper_link": "http://arxiv.org/abs/2601.14825v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:25",
+    "month": "202602",
+    "no": 25,
+    "paper_link": "http://arxiv.org/abs/2602.16048v3",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202511:23",
+    "month": "202511",
+    "no": 23,
+    "paper_link": "http://arxiv.org/abs/2511.06595v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:13",
+    "month": "202602",
+    "no": 13,
+    "paper_link": "http://arxiv.org/abs/2602.12261v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202511:27",
+    "month": "202511",
+    "no": 27,
+    "paper_link": "http://arxiv.org/abs/2511.04407v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:7",
+    "month": "202512",
+    "no": 7,
+    "paper_link": "http://arxiv.org/abs/2512.09490v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:29",
+    "month": "202512",
+    "no": 29,
+    "paper_link": "http://arxiv.org/abs/2512.08562v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:34",
+    "month": "202512",
+    "no": 34,
+    "paper_link": "http://arxiv.org/abs/2512.09598v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:42",
+    "month": "202512",
+    "no": 42,
+    "paper_link": "http://arxiv.org/abs/2512.10845v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:7",
+    "month": "202511",
+    "no": 7,
+    "paper_link": "http://arxiv.org/abs/2511.13976v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202511:29",
+    "month": "202511",
+    "no": 29,
+    "paper_link": "http://arxiv.org/abs/2511.03722v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:37",
+    "month": "202602",
+    "no": 37,
+    "paper_link": "http://arxiv.org/abs/2602.08644v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  }
+]

data/livemathematicianbench_id_split/train/items.json

@@ -0,0 +1,247 @@
+[
+  {
+    "id": "202602:22",
+    "month": "202602",
+    "no": 22,
+    "paper_link": "http://arxiv.org/abs/2602.10700v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:8",
+    "month": "202512",
+    "no": 8,
+    "paper_link": "http://arxiv.org/abs/2512.08863v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:16",
+    "month": "202511",
+    "no": 16,
+    "paper_link": "http://arxiv.org/abs/2511.15668v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:44",
+    "month": "202601",
+    "no": 44,
+    "paper_link": "http://arxiv.org/abs/2601.21267v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:14",
+    "month": "202511",
+    "no": 14,
+    "paper_link": "http://arxiv.org/abs/2511.13447v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202602:30",
+    "month": "202602",
+    "no": 30,
+    "paper_link": "http://arxiv.org/abs/2602.16692v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:2",
+    "month": "202602",
+    "no": 2,
+    "paper_link": "http://arxiv.org/abs/2602.22933v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202601:41",
+    "month": "202601",
+    "no": 41,
+    "paper_link": "http://arxiv.org/abs/2601.01164v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:23",
+    "month": "202601",
+    "no": 23,
+    "paper_link": "http://arxiv.org/abs/2601.02528v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:20",
+    "month": "202511",
+    "no": 20,
+    "paper_link": "http://arxiv.org/abs/2511.02963v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:22",
+    "month": "202601",
+    "no": 22,
+    "paper_link": "http://arxiv.org/abs/2601.03984v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:14",
+    "month": "202512",
+    "no": 14,
+    "paper_link": "http://arxiv.org/abs/2512.22459v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:26",
+    "month": "202511",
+    "no": 26,
+    "paper_link": "http://arxiv.org/abs/2511.07817v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202511:3",
+    "month": "202511",
+    "no": 3,
+    "paper_link": "http://arxiv.org/abs/2511.11409v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:33",
+    "month": "202601",
+    "no": 33,
+    "paper_link": "http://arxiv.org/abs/2601.07747v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202602:7",
+    "month": "202602",
+    "no": 7,
+    "paper_link": "http://arxiv.org/abs/2602.22912v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:27",
+    "month": "202602",
+    "no": 27,
+    "paper_link": "http://arxiv.org/abs/2602.13968v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:31",
+    "month": "202602",
+    "no": 31,
+    "paper_link": "http://arxiv.org/abs/2602.15528v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:41",
+    "month": "202602",
+    "no": 41,
+    "paper_link": "http://arxiv.org/abs/2602.10707v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:25",
+    "month": "202512",
+    "no": 25,
+    "paper_link": "http://arxiv.org/abs/2512.04531v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:10",
+    "month": "202602",
+    "no": 10,
+    "paper_link": "http://arxiv.org/abs/2602.17863v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:16",
+    "month": "202602",
+    "no": 16,
+    "paper_link": "http://arxiv.org/abs/2602.02723v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:16",
+    "month": "202512",
+    "no": 16,
+    "paper_link": "http://arxiv.org/abs/2512.11601v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:2",
+    "month": "202512",
+    "no": 2,
+    "paper_link": "http://arxiv.org/abs/2512.16120v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:24",
+    "month": "202512",
+    "no": 24,
+    "paper_link": "http://arxiv.org/abs/2512.08391v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:32",
+    "month": "202512",
+    "no": 32,
+    "paper_link": "http://arxiv.org/abs/2512.23224v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:47",
+    "month": "202602",
+    "no": 47,
+    "paper_link": "http://arxiv.org/abs/2602.10391v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:46",
+    "month": "202602",
+    "no": 46,
+    "paper_link": "http://arxiv.org/abs/2602.13727v2",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:21",
+    "month": "202512",
+    "no": 21,
+    "paper_link": "http://arxiv.org/abs/2512.12835v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:33",
+    "month": "202512",
+    "no": 33,
+    "paper_link": "http://arxiv.org/abs/2512.19500v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:45",
+    "month": "202602",
+    "no": 45,
+    "paper_link": "http://arxiv.org/abs/2602.23912v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:26",
+    "month": "202602",
+    "no": 26,
+    "paper_link": "http://arxiv.org/abs/2602.14658v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:41",
+    "month": "202512",
+    "no": 41,
+    "paper_link": "http://arxiv.org/abs/2512.15177v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:38",
+    "month": "202601",
+    "no": 38,
+    "paper_link": "http://arxiv.org/abs/2601.07817v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:14",
+    "month": "202601",
+    "no": 14,
+    "paper_link": "http://arxiv.org/abs/2601.08704v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  }
+]

data/livemathematicianbench_id_split/val/items.json

@@ -0,0 +1,128 @@
+[
+  {
+    "id": "202602:8",
+    "month": "202602",
+    "no": 8,
+    "paper_link": "http://arxiv.org/abs/2602.19529v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:50",
+    "month": "202512",
+    "no": 50,
+    "paper_link": "http://arxiv.org/abs/2512.15277v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202512:36",
+    "month": "202512",
+    "no": 36,
+    "paper_link": "http://arxiv.org/abs/2512.06696v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202511:1",
+    "month": "202511",
+    "no": 1,
+    "paper_link": "http://arxiv.org/abs/2511.04651v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202601:31",
+    "month": "202601",
+    "no": 31,
+    "paper_link": "http://arxiv.org/abs/2601.10298v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202511:17",
+    "month": "202511",
+    "no": 17,
+    "paper_link": "http://arxiv.org/abs/2511.13215v1",
+    "source_file": "data/202511/qa_202511_final.json"
+  },
+  {
+    "id": "202512:37",
+    "month": "202512",
+    "no": 37,
+    "paper_link": "http://arxiv.org/abs/2512.20498v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:39",
+    "month": "202601",
+    "no": 39,
+    "paper_link": "http://arxiv.org/abs/2601.06601v2",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:25",
+    "month": "202601",
+    "no": 25,
+    "paper_link": "http://arxiv.org/abs/2601.10996v3",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:24",
+    "month": "202601",
+    "no": 24,
+    "paper_link": "http://arxiv.org/abs/2601.12250v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:45",
+    "month": "202601",
+    "no": 45,
+    "paper_link": "http://arxiv.org/abs/2601.12113v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202601:19",
+    "month": "202601",
+    "no": 19,
+    "paper_link": "http://arxiv.org/abs/2601.00779v1",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:10",
+    "month": "202512",
+    "no": 10,
+    "paper_link": "http://arxiv.org/abs/2512.07073v2",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202601:46",
+    "month": "202601",
+    "no": 46,
+    "paper_link": "http://arxiv.org/abs/2601.07793v2",
+    "source_file": "data/202601/qa_202601_final.json"
+  },
+  {
+    "id": "202512:15",
+    "month": "202512",
+    "no": 15,
+    "paper_link": "http://arxiv.org/abs/2512.16165v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  },
+  {
+    "id": "202602:15",
+    "month": "202602",
+    "no": 15,
+    "paper_link": "http://arxiv.org/abs/2602.05303v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202602:6",
+    "month": "202602",
+    "no": 6,
+    "paper_link": "http://arxiv.org/abs/2602.01571v1",
+    "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:46",
+    "month": "202512",
+    "no": 46,
+    "paper_link": "http://arxiv.org/abs/2512.05945v1",
+    "source_file": "data/202512/qa_202512_final.json"
+  }
+]

data/officeqa_id_split/split_manifest.json

@@ -0,0 +1,27 @@
+{
+  "benchmark": "OfficeQA",
+  "manifest_type": "id_split",
+  "source_repo": "databricks/officeqa",
+  "source_repo_type": "dataset",
+  "source_url": "https://huggingface.co/datasets/databricks/officeqa",
+  "source_revision": "8ecbf18d3833daf4750a903d14963e4c4c1d4cd8",
+  "source_file": "officeqa_full.csv",
+  "source_split_name": "officeqa_split",
+  "counts": {
+    "train": 50,
+    "val": 24,
+    "test": 172
+  },
+  "item_fields": [
+    "id",
+    "uid",
+    "category",
+    "source_files",
+    "source_docs",
+    "source_split"
+  ],
+  "notes": [
+    "This is a split manifest, not the full OfficeQA payload.",
+    "The official OfficeQA CSV is gated on Hugging Face; materialization requires authorized access."
+  ]
+}

data/officeqa_id_split/train/items.json

@@ -0,0 +1,402 @@
+[
+  {
+    "id": "UID0002",
+    "uid": "UID0002",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1944_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1944-6565?page=18",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0007",
+    "uid": "UID0007",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1950_02.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1950-6637?page=15",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0014",
+    "uid": "UID0014",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1942_07.txt\r\ntreasury_bulletin_2001_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1942-6547?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2001-7108?page=17&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0017",
+    "uid": "UID0017",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1982_08.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1982-7028?page=13",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0018",
+    "uid": "UID0018",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1985_03.txt\r\ntreasury_bulletin_1986_03.txt\r\ntreasury_bulletin_1987_03.txt\r\ntreasury_bulletin_1988_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1985-7040?page=22\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1986-7045?page=26\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1987-7049?page=24\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1988-7052?page=36",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0019",
+    "uid": "UID0019",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2016_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2016-533966?page=54\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2016-533966?page=58",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0026",
+    "uid": "UID0026",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1963_01.txt\r\ntreasury_bulletin_1962_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1963-6793?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1962-6781?page=82&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0028",
+    "uid": "UID0028",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1970_06.txt\r\ntreasury_bulletin_1964_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=89&deep=true\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1964-6816?page=25&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0030",
+    "uid": "UID0030",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1990_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1990-7062?page=19&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0031",
+    "uid": "UID0031",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1992_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1992-7068?page=158&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0033",
+    "uid": "UID0033",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1977_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1977-6964?page=9",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0034",
+    "uid": "UID0034",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1992_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1992-7069?page=32",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0044",
+    "uid": "UID0044",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1939_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1939-6506?page=61",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0046",
+    "uid": "UID0046",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1988_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1988-7054?page=37",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0049",
+    "uid": "UID0049",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1942_02.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1942-6542?page=19&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0056",
+    "uid": "UID0056",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1991_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1991-7066?page=30&deep=true",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0063",
+    "uid": "UID0063",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1990_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1990-7061?page=127",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0065",
+    "uid": "UID0065",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1998_06.txt\r\ntreasury_bulletin_1995_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1998-7094?page=7\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1995-7084?page=16",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0073",
+    "uid": "UID0073",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1982_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=24",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0079",
+    "uid": "UID0079",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2013_12.txt\r\ntreasury_bulletin_2015_12.txt\r\ntreasury_bulletin_2017_12.txt\r\ntreasury_bulletin_2019_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=25\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=24\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2015-519209?page=23\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2017-575188?page=23\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=22",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0083",
+    "uid": "UID0083",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1981_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1981-7020?page=24",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0085",
+    "uid": "UID0085",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2019_12.txt\r\ntreasury_bulletin_2018_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=23\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2018-581283?page=22",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0087",
+    "uid": "UID0087",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2013_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=17",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0092",
+    "uid": "UID0092",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1987_12.txt\r\ntreasury_bulletin_1992_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1987-7051?page=69\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1992-7071?page=84",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0098",
+    "uid": "UID0098",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2020_12.txt\r\ntreasury_bulletin_2024_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=21\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2024-679984?page=22",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0101",
+    "uid": "UID0101",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2019_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=25\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=22",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0110",
+    "uid": "UID0110",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2020_03.txt\r\ntreasury_bulletin_2016_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2020-587316?page=10\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2016-527290?page=9",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0115",
+    "uid": "UID0115",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1980_02.txt\r\ntreasury_bulletin_1981_02.txt\r\ntreasury_bulletin_1982_02.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1980-6998?page=27\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1981-7010?page=38\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1982-7022?page=31",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0122",
+    "uid": "UID0122",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2001_03.txt\r\ntreasury_bulletin_2002_03.txt\r\ntreasury_bulletin_2003_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2001-7105?page=112\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2002-7110?page=115\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2003-7109?page=113",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0123",
+    "uid": "UID0123",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1941_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1941-6540?page=91",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0133",
+    "uid": "UID0133",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2004_09.txt\r\ntreasury_bulletin_2013_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2004-7119?page=63\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=68",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0141",
+    "uid": "UID0141",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1962_04.txt\r\ntreasury_bulletin_1963_04.txt\r\ntreasury_bulletin_1964_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1962-6784?page=75\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1963-6796?page=79\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1964-6808?page=82",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0144",
+    "uid": "UID0144",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1980_11.txt\r\ntreasury_bulletin_1981_11.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1980-7007?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1981-7019?page=67",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0145",
+    "uid": "UID0145",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1943_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1943-6553?page=41",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0150",
+    "uid": "UID0150",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1972_04.txt\r\ntreasury_bulletin_1973_04.txt\r\ntreasury_bulletin_1974_04.txt\r\ntreasury_bulletin_1975_04.txt\r\ntreasury_bulletin_1976_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1972-6905?page=89\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1973-6916?page=91\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1974-6927?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1975-6940?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1976-6952?page=104",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0151",
+    "uid": "UID0151",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1953_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1953-6674?page=54",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0162",
+    "uid": "UID0162",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2011_06.txt\r\ntreasury_bulletin_2012_06.txt\r\ntreasury_bulletin_2013_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2011-7129?page=105\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2012-7151?page=105\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2013-7153?page=104",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0163",
+    "uid": "UID0163",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1981_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1981-7020?page=28",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0165",
+    "uid": "UID0165",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2010_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2010-7143?page=49",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0166",
+    "uid": "UID0166",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1943_03.txt\r\ntreasury_bulletin_1944_03.txt\r\ntreasury_bulletin_1945_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1943-6555?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1944-6567?page=83\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1945-6577?page=71",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0169",
+    "uid": "UID0169",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1982_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=73",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0189",
+    "uid": "UID0189",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1970_08.txt\r\ntreasury_bulletin_1970_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1970-6884?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1970-6885?page=70",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0195",
+    "uid": "UID0195",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1956_08.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1956-6715?page=59",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0202",
+    "uid": "UID0202",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1939_07.txt\r\ntreasury_bulletin_1939_08.txt\r\ntreasury_bulletin_1939_09.txt\r\ntreasury_bulletin_1939_10.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1939-6509?page=99\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=107\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1939-6511?page=60\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=62",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0212",
+    "uid": "UID0212",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1964_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1964-6805?page=99",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0222",
+    "uid": "UID0222",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2001_06.txt\r\ntreasury_bulletin_2006_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2001-7106?page=50\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2006-7126?page=50",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0228",
+    "uid": "UID0228",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1956_03.txt\r\ntreasury_bulletin_1956_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1956-6710?page=22\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1956-6711?page=22",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0229",
+    "uid": "UID0229",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2005_03.txt\r\ntreasury_bulletin_2006_03.txt\r\ntreasury_bulletin_2007_03.txt\r\ntreasury_bulletin_2008_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2005-7121?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2006-7125?page=106\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2007-7130?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2008-7134?page=107",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0238",
+    "uid": "UID0238",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1982_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=44",
+    "source_split": "train"
+  },
+  {
+    "id": "UID0241",
+    "uid": "UID0241",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1963_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1963-6798?page=13",
+    "source_split": "train"
+  }
+]

data/officeqa_id_split/val/items.json

@@ -0,0 +1,194 @@
+[
+  {
+    "id": "UID0001",
+    "uid": "UID0001",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1941_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=15",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0027",
+    "uid": "UID0027",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1970_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=89&deep=true",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0039",
+    "uid": "UID0039",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2004_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=20\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=21&deep=true",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0041",
+    "uid": "UID0041",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1970_10.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1970-6886?page=35",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0052",
+    "uid": "UID0052",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2000_06.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=56",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0070",
+    "uid": "UID0070",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1939_01.txt\r\ntreasury_bulletin_1939_02.txt\r\ntreasury_bulletin_1939_03.txt\r\ntreasury_bulletin_1939_04.txt\r\ntreasury_bulletin_1939_05.txt\r\ntreasury_bulletin_1939_06.txt\r\ntreasury_bulletin_1939_07.txt\r\ntreasury_bulletin_1939_08.txt\r\ntreasury_bulletin_1939_09.txt\r\ntreasury_bulletin_1939_10.txt\r\ntreasury_bulletin_1939_11.txt\r\ntreasury_bulletin_1939_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1939-6518?page=81\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1939-6505?page=111\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1939-6519?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1939-6506?page=95\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1939-6507?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1939-6508?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1939-6509?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1939-6511?page=66&deep=true \r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1939-6512?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1939-6513?page=72",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0072",
+    "uid": "UID0072",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2016_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=58\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2016-535293?page=57",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0086",
+    "uid": "UID0086",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2022_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2022-627778?page=86",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0091",
+    "uid": "UID0091",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1940_12.txt\r\ntreasury_bulletin_1941_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1940-6528?page=21\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1941-6540?page=64",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0109",
+    "uid": "UID0109",
+    "category": "hard",
+    "source_files": "treasury_bulletin_2015_12.txt\r\ntreasury_bulletin_2020_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2015-519209?page=21\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=24",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0142",
+    "uid": "UID0142",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1944_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1944-6567?page=93",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0154",
+    "uid": "UID0154",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1977_03.txt\r\ntreasury_bulletin_1978_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1977-6963?page=83\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1978-6975?page=84",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0159",
+    "uid": "UID0159",
+    "category": "easy",
+    "source_files": "treasury_bulletin_2000_09.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2000-7103?page=109",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0161",
+    "uid": "UID0161",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1980_03.txt\r\ntreasury_bulletin_1985_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1980-6999?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1985-7040?page=48",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0170",
+    "uid": "UID0170",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1960_03.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1960-6759?page=64",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0190",
+    "uid": "UID0190",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1939_10.txt\r\ntreasury_bulletin_1939_11.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=14\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1939-6512?page=14",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0213",
+    "uid": "UID0213",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1947_04.txt\r\ntreasury_bulletin_1948_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1947-6603?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=18",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0217",
+    "uid": "UID0217",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1963_10.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1963-6802?page=15",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0220",
+    "uid": "UID0220",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1939_02.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1939-6505?page=25",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0233",
+    "uid": "UID0233",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1948_04.txt\r\ntreasury_bulletin_1958_04.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=42\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1958-6735?page=54",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0234",
+    "uid": "UID0234",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1958_01.txt\r\ntreasury_bulletin_1958_02.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1958-6732?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1958-6733?page=32",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0235",
+    "uid": "UID0235",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1948_04.txt\r\ntreasury_bulletin_1948_05.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=27\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1948-6616?page=27",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0239",
+    "uid": "UID0239",
+    "category": "easy",
+    "source_files": "treasury_bulletin_1953_01.txt\r\ntreasury_bulletin_1954_01.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1953-6672?page=62\r\nhttp://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1954-6684?page=51",
+    "source_split": "val"
+  },
+  {
+    "id": "UID0240",
+    "uid": "UID0240",
+    "category": "hard",
+    "source_files": "treasury_bulletin_1957_12.txt",
+    "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1957-6731?page=26",
+    "source_split": "val"
+  }
+]

data/searchqa_id_split/split_manifest.json

@@ -0,0 +1,21 @@
+{
+  "benchmark": "SearchQA",
+  "manifest_type": "id_split",
+  "source_repo": "lucadiliello/searchqa",
+  "source_repo_type": "dataset",
+  "source_url": "https://huggingface.co/datasets/lucadiliello/searchqa",
+  "source_id_field": "key",
+  "counts": {
+    "train": 400,
+    "val": 200,
+    "test": 1400
+  },
+  "item_fields": [
+    "id"
+  ],
+  "notes": [
+    "This is a split manifest, not the full SearchQA payload.",
+    "Materialize full split items from lucadiliello/searchqa before evaluation.",
+    "The IDs in items.json exactly match the key field in lucadiliello/searchqa."
+  ]
+}

data/searchqa_id_split/val/items.json

@@ -0,0 +1,602 @@
+[
+  {
+    "id": "1758dc50625e46ee814e44a6061f091d"
+  },
+  {
+    "id": "52967b20d9564d5b94756bc0b3d113d6"
+  },
+  {
+    "id": "e77dbfc0d45e476b9ed5b2204b6d9fc8"
+  },
+  {
+    "id": "675f01b5343e46599626104968c0f45a"
+  },
+  {
+    "id": "dccd0f04cd3b41bc8c4182e41a413b3e"
+  },
+  {
+    "id": "efde5820a29949488feaa18ce7e429f9"
+  },
+  {
+    "id": "98773558971e490d8de41088e41a3207"
+  },
+  {
+    "id": "70e3e1e1d41f4795ba45678d5f4b6524"
+  },
+  {
+    "id": "206a9ffb8ed24686a24db1492c9f10dc"
+  },
+  {
+    "id": "e91f1225cdb84ee7a2d773b5177945c1"
+  },
+  {
+    "id": "b68a14a7b43842a0a50a3965d3bbffbb"
+  },
+  {
+    "id": "e2e679989b544e94bf8dbc3466e49003"
+  },
+  {
+    "id": "c51c26c07b384bbe865976b81a86736d"
+  },
+  {
+    "id": "83928d21f3504843aa6294ce9d9c8da1"
+  },
+  {
+    "id": "f822cde36d864c508b426f3e2b5245b2"
+  },
+  {
+    "id": "f03757766e034c539bf2595428c03626"
+  },
+  {
+    "id": "349be67be2e141438287d21cb6a94131"
+  },
+  {
+    "id": "7f3e18272c6d43dd8c6385c4758b685e"
+  },
+  {
+    "id": "66dc8aee931142b1834da63363511491"
+  },
+  {
+    "id": "52ac78645e144b4598958d67920617dc"
+  },
+  {
+    "id": "32a66c67a8d7490b8572387d8f1d762a"
+  },
+  {
+    "id": "b028bea99d2641268532cd2a32d4ad2d"
+  },
+  {
+    "id": "c9f3a971abe64853aca3e0bb1fea6962"
+  },
+  {
+    "id": "5324041c25d04316a800ccba2d79a2c7"
+  },
+  {
+    "id": "9960a3f1ac954282a7ea1d7f7c8a86f9"
+  },
+  {
+    "id": "03bd8fc86a9d465e9370dcdaedb3c422"
+  },
+  {
+    "id": "a1d0d03fcac54483a1e88542b767afd5"
+  },
+  {
+    "id": "c21a41745ab9402da8b1deea67055a8d"
+  },
+  {
+    "id": "d5c48bd600824c8a9328df80a8baf964"
+  },
+  {
+    "id": "746806ac0b1c48cb80d7ab5ad0a63517"
+  },
+  {
+    "id": "e3b96fc1b99247ebada09a84fa5fd205"
+  },
+  {
+    "id": "bbeeec20466a4dab9e4d73d86c964cba"
+  },
+  {
+    "id": "2b85b097d130404daeb26a3349bdc297"
+  },
+  {
+    "id": "fbaae95260474ed58ccbb05ac482089b"
+  },
+  {
+    "id": "28b837ac304845bd961a46cedc9246af"
+  },
+  {
+    "id": "691e3a24e5994c749ff11c6bc43b2d1c"
+  },
+  {
+    "id": "12d5b51c793a43b1aa3b7e1bf5f4bc73"
+  },
+  {
+    "id": "a3a3892296c74fc4b23b2475156901b1"
+  },
+  {
+    "id": "8065f2e8639848f5b2f16a4f2877b7c2"
+  },
+  {
+    "id": "b06e0d2be0a5465ea12ff2bf2b27d387"
+  },
+  {
+    "id": "e50328dc70634863ac39580ef6f8dd5a"
+  },
+  {
+    "id": "17b441db9b444435951d859e6e081f3b"
+  },
+  {
+    "id": "0fc37a4bd8424ecf8eede56e956edb3a"
+  },
+  {
+    "id": "09c769c7262e43eb95a52e874042c02a"
+  },
+  {
+    "id": "2417ce75564b4b8eab1ddd755b10d962"
+  },
+  {
+    "id": "8f4beaa8a7704eba93f753a0e7040165"
+  },
+  {
+    "id": "6d8e99531264435badb67bf6ac209075"
+  },
+  {
+    "id": "940ddc06648f4c9bac6b3bf5df402e5a"
+  },
+  {
+    "id": "fa0fcf36e99644d0b51be95869d87adc"
+  },
+  {
+    "id": "1882f20b777146608474ac0056d54a5e"
+  },
+  {
+    "id": "439e908aa7c7422daf5632af7e0440ec"
+  },
+  {
+    "id": "2d90c36ab1dd4724b1c108b2507316e4"
+  },
+  {
+    "id": "efa5cc2f750c421a807bc4315db0dc1e"
+  },
+  {
+    "id": "3b1c596e1720499ebe790d14f34a67af"
+  },
+  {
+    "id": "c87234a3a1074b5ea5500114ad9aa16d"
+  },
+  {
+    "id": "9d08ada791844519b7a4bb247d50c89c"
+  },
+  {
+    "id": "74c45e35e5a740798fbd528ff08a70e3"
+  },
+  {
+    "id": "650b683d39dd45c99ad7435fc7d33fe3"
+  },
+  {
+    "id": "191ce59c97e143f7b4c33400913d3570"
+  },
+  {
+    "id": "74441a0f7dae42ca9ee19d8be8c9b318"
+  },
+  {
+    "id": "baef3cef84f14ed98655ac73293a56db"
+  },
+  {
+    "id": "099a1ae148184bd5a93c9bc9889306e7"
+  },
+  {
+    "id": "4fb4332a7a0e44e4b5655de1d7c2e5ac"
+  },
+  {
+    "id": "cdf86d1982f64cb49d6595680b1c53b8"
+  },
+  {
+    "id": "fa4cbba4ffed4f4e8ed4c148711cbc78"
+  },
+  {
+    "id": "41c691d3b7e94ffd993647c7e531e94e"
+  },
+  {
+    "id": "df43ea868a6d409ea67eb632fc9f285c"
+  },
+  {
+    "id": "746448dcf441410db997a59f7137a2c8"
+  },
+  {
+    "id": "3d0cad58c3c145f3b98d51c06a65a0c9"
+  },
+  {
+    "id": "9cb4317831024d589622e2a3cf544360"
+  },
+  {
+    "id": "802015dbc9f5406d8df3b468497f6553"
+  },
+  {
+    "id": "5463f1d3fc124cadb2ad016076a699f5"
+  },
+  {
+    "id": "8c77419b4e864778b965753f411bb469"
+  },
+  {
+    "id": "2e112a7c9d97482fa0deb079c6c09172"
+  },
+  {
+    "id": "f0c26f4f2e9f4d1d84b4f13cb201cbeb"
+  },
+  {
+    "id": "231d7d9955d54f60a1f1ff4f0bf1c5b6"
+  },
+  {
+    "id": "9b56a920d67a48e8b33932802d17d374"
+  },
+  {
+    "id": "092a8aab433c4184bde9c8aa08789464"
+  },
+  {
+    "id": "266a4337fa5742c8bbcc5aa79e571e85"
+  },
+  {
+    "id": "d67cecf38f954800a7c651cca70bdc66"
+  },
+  {
+    "id": "e634fa14bd4941ef96e1d21e796a9cd3"
+  },
+  {
+    "id": "dbc29aef324e4c4f9f19082dcb1b5cb1"
+  },
+  {
+    "id": "e688c5081a84488694b111a11a50ef7e"
+  },
+  {
+    "id": "80b114d03dbb4598b5da648bbe174d0f"
+  },
+  {
+    "id": "ce42d55ff5874c6380e018896b16e128"
+  },
+  {
+    "id": "ef2b37212c98453390c9dca445d69de1"
+  },
+  {
+    "id": "a66639dbb5ed4dfb9e65fe359ed8dc03"
+  },
+  {
+    "id": "10eacaf72a4e4ad49cc43d3a710ee24a"
+  },
+  {
+    "id": "b7e3d45f77bd4a4db768505fc4726592"
+  },
+  {
+    "id": "0d1592f640f04ecb9c7add7164f87c5a"
+  },
+  {
+    "id": "296a678ada814b09bdd0a53629f7f3f9"
+  },
+  {
+    "id": "6e6fd6d62ce54031a4e36f88c2d5b63d"
+  },
+  {
+    "id": "8d0dd71f94cc4abba255f3cdd6c2d940"
+  },
+  {
+    "id": "7c26258ff99c4adeb4b9b35001005453"
+  },
+  {
+    "id": "d13c24c99c704e31bbb3aaa585884d56"
+  },
+  {
+    "id": "dbfb81f100dc474f8267e6f9ab0ae51c"
+  },
+  {
+    "id": "3b4a5a2f617b43c8aa37b0890f576749"
+  },
+  {
+    "id": "c49a8bf2d0ad4afd9b0bb0d03c1b9d96"
+  },
+  {
+    "id": "fbfb4851dd494e44b9a21475d7e6dbaa"
+  },
+  {
+    "id": "e0ba941f8e2c4fa6a0dea2b562b0e4be"
+  },
+  {
+    "id": "7c9ec2b1cd8d4a8eaa6e1154e6fb40c4"
+  },
+  {
+    "id": "4ae645d2721c4751b1e71c83870bbe16"
+  },
+  {
+    "id": "c7bd32aee95f485d85d0b6749d542c51"
+  },
+  {
+    "id": "20e140b44e0f4a6484e9cf776b09679b"
+  },
+  {
+    "id": "5deb6bfcdd8c486d9f5f630309968763"
+  },
+  {
+    "id": "18d1fcc3ed394b23ada799aad3beca90"
+  },
+  {
+    "id": "74a655bcaead4857b92a49ec5f93ee23"
+  },
+  {
+    "id": "bfec094ae9da4aaebd91713b39d3008e"
+  },
+  {
+    "id": "80ebe13e36eb413a9acaace84648b36d"
+  },
+  {
+    "id": "982e414b371248788c72732975949957"
+  },
+  {
+    "id": "963ad06f3e314d3b99e0b6b9eaeaf2d8"
+  },
+  {
+    "id": "90b6478bd7ae4145976c474dc42da80f"
+  },
+  {
+    "id": "ea475dcd08d54da6896f56e330fa2a1c"
+  },
+  {
+    "id": "64899cac91e5423fa3617bf576e6df4d"
+  },
+  {
+    "id": "6231d57d040143e3bc56cafaabaa0975"
+  },
+  {
+    "id": "3fdb66ab2b0a49318b053edda6633237"
+  },
+  {
+    "id": "7dff080db1b8438e9ed737df1d7c32e3"
+  },
+  {
+    "id": "878108ec7e9742a090175bd33390e2c5"
+  },
+  {
+    "id": "3b2acfd62bdf4361aa4e51895d12df8b"
+  },
+  {
+    "id": "0335847f4a44451db5a1b775cc019765"
+  },
+  {
+    "id": "b6bfa34339c649398ae3b4540ed95fcf"
+  },
+  {
+    "id": "82fd2c61b9a64b5f8971b28a1cfd8c1b"
+  },
+  {
+    "id": "a9385b35a3a5492283aa461b93fdb361"
+  },
+  {
+    "id": "847f117d4638487d92539d52eb07a999"
+  },
+  {
+    "id": "de103586389245e7858b26b9b9a1ba57"
+  },
+  {
+    "id": "82d95f6b3df64bb48668f7e98ddf5b9c"
+  },
+  {
+    "id": "b1f6907b8f0f41ec82dc2eae7f84666b"
+  },
+  {
+    "id": "1f232b1de1f34793b6557c8688bf4d9c"
+  },
+  {
+    "id": "c16f718bd6554e998a95fe1d07623a13"
+  },
+  {
+    "id": "dc2add9004fd40e69fcbd2eea80c513c"
+  },
+  {
+    "id": "464e204e258340b69eca2b90451f06b8"
+  },
+  {
+    "id": "c12fd75ea3db450bbfcabeb4f62f7e8f"
+  },
+  {
+    "id": "5dbcadd728fa4956a4f739c1168fbf18"
+  },
+  {
+    "id": "1730220302104d54ba2bb781ca219e52"
+  },
+  {
+    "id": "ffcf25fe8cce4198bd483f2faaa17889"
+  },
+  {
+    "id": "98ed5d627d0746e0aecb42eae39c14dc"
+  },
+  {
+    "id": "30718db44c7f44a5b4b0c68842b1053b"
+  },
+  {
+    "id": "4e90a93f4548458bb5d35ac9c767233b"
+  },
+  {
+    "id": "dabccc32402a43a7ab9c82408f3603a1"
+  },
+  {
+    "id": "80a8a1ea275944228e71fd0dfc74fb74"
+  },
+  {
+    "id": "687434d75186430896ea636fec156e6d"
+  },
+  {
+    "id": "28e5233352ba44ea9f7a8d4cf7f6b86a"
+  },
+  {
+    "id": "b11499ee2a1f4a2b8fa8c9f561136430"
+  },
+  {
+    "id": "6efcba5974464dcbaf5d9c5e9322bd9c"
+  },
+  {
+    "id": "a1d460152c3840d98ed3bc45bf80d5c1"
+  },
+  {
+    "id": "0d482f0e297544dcb2ce1e87e4c3b4b4"
+  },
+  {
+    "id": "1a5baeb7a51d453d892ba70a9de2146d"
+  },
+  {
+    "id": "ab84169c66b2415180563269295f53be"
+  },
+  {
+    "id": "08c0e9678f1548aebeada27ab7b954e1"
+  },
+  {
+    "id": "efd2d367aeec4cc4ae72cd3eaa040a0f"
+  },
+  {
+    "id": "ae54626131ed42eaa55755f10ee64a76"
+  },
+  {
+    "id": "1787cbe3e160427ca4e61931daf66e51"
+  },
+  {
+    "id": "24143f01e7dd455e86c34a4a6f4ed3b9"
+  },
+  {
+    "id": "5e92b869dd0b4ecbaf8694041891eb75"
+  },
+  {
+    "id": "8d130f744b214055ad0c4cdce459a4b9"
+  },
+  {
+    "id": "87913f6be2004e938ba60cda550bc675"
+  },
+  {
+    "id": "68c7f892cbd94fc59ebe4ef1e1ad0c89"
+  },
+  {
+    "id": "f7bf36628c7540da907dda84b48aac82"
+  },
+  {
+    "id": "357af030b55046fc856e03fdac36032a"
+  },
+  {
+    "id": "3a90f09909344dfdbb1a73f80c983a87"
+  },
+  {
+    "id": "44fd76ce5f864cc2aef99c75cfd4904e"
+  },
+  {
+    "id": "95e54b962ce1420d82c716ff428d84fe"
+  },
+  {
+    "id": "8e22ba3b6fc34886a86d60dd1a1db025"
+  },
+  {
+    "id": "162917ea12b24d0e874540feecf9b445"
+  },
+  {
+    "id": "c8e87f5da41c47c7a3116bd1ed0cc34d"
+  },
+  {
+    "id": "a3458bd73564491585df515468475a66"
+  },
+  {
+    "id": "e27a4c1fb25c48809feb3cc438b5eb3c"
+  },
+  {
+    "id": "f36338c39e2846959491ea48cbce07f6"
+  },
+  {
+    "id": "eae01f064523415c95e57ff1e8dadc43"
+  },
+  {
+    "id": "8b845d50f998467a898367107377c64b"
+  },
+  {
+    "id": "a50ae9e7cb6a4f1e8dad07a9bbace810"
+  },
+  {
+    "id": "5c2009e4f1384b27b4b4fe9422f048ed"
+  },
+  {
+    "id": "0baaa0fe4fde4bae99bc6d2be230d525"
+  },
+  {
+    "id": "d2563a0ca702457497995a6959ff1c61"
+  },
+  {
+    "id": "b07871d2991d47e899cfa68a91f16b52"
+  },
+  {
+    "id": "b745738e5b3c482aa2ae94ad5688246b"
+  },
+  {
+    "id": "d66f1f0158054f638ff1b8acf3e18a8a"
+  },
+  {
+    "id": "ede0705d606d4435a300e06beab2322f"
+  },
+  {
+    "id": "e9d415f928644b59b5ed19b7011f627d"
+  },
+  {
+    "id": "1a78a8cf476c4a0dbf551558da1d879b"
+  },
+  {
+    "id": "ea7c66d5dae241eda155763541749cd1"
+  },
+  {
+    "id": "78ae271549144680a42139693d1117e9"
+  },
+  {
+    "id": "7eb9a69a23b8469b9f20f1915abff3be"
+  },
+  {
+    "id": "db73ca2786684b9e92deb1fcc796effd"
+  },
+  {
+    "id": "a876fb1accf64f4eb1b20b4a0ee089db"
+  },
+  {
+    "id": "71d6dabd553c471b98debe941341a4c0"
+  },
+  {
+    "id": "1c6add1cbd464f34b6e90485780f1c3e"
+  },
+  {
+    "id": "5e9b0b44825e44d89ba0501c3b819685"
+  },
+  {
+    "id": "4cfc841cb81e4c90822cfb7c0ec9f86e"
+  },
+  {
+    "id": "2933df89dd6146c791eac1a85f0d02c8"
+  },
+  {
+    "id": "5af06dca120a46889964109e94fff0a0"
+  },
+  {
+    "id": "77970683cb6444388e5cfdd743784db6"
+  },
+  {
+    "id": "3e40d5d91615427d976a56932be3c955"
+  },
+  {
+    "id": "70bc65e0c17041729a4eabe86ff36d53"
+  },
+  {
+    "id": "b913e55670474b69a70736d314822155"
+  },
+  {
+    "id": "ada9cbe2dd8e48deb55171645e47a24a"
+  },
+  {
+    "id": "fba813b8837249cbb39a1996dc657ef0"
+  },
+  {
+    "id": "f7ecc32a212e4c3b8471ce26c32f3135"
+  },
+  {
+    "id": "015641bf588b43aebba1aee07a8afde0"
+  },
+  {
+    "id": "bcecb67a14f34fc6896bc5d65c58405a"
+  }
+]

data/spreadsheetbench_id_split/split_manifest.json

@@ -0,0 +1,24 @@
+{
+  "benchmark": "SpreadsheetBench",
+  "manifest_type": "id_split",
+  "source_repo": "KAKA22/SpreadsheetBench",
+  "source_repo_type": "dataset",
+  "source_url": "https://huggingface.co/datasets/KAKA22/SpreadsheetBench",
+  "source_revision": "ab0b742b0fc95b946f212d80ac7771b5531272e4",
+  "source_file": "spreadsheetbench_verified_400.tar.gz",
+  "source_split_name": "spreadsheetbench_split",
+  "counts": {
+    "train": 80,
+    "val": 40,
+    "test": 280
+  },
+  "item_fields": [
+    "id",
+    "spreadsheet_path",
+    "instruction_type"
+  ],
+  "notes": [
+    "This is a split manifest, not the full SpreadsheetBench payload.",
+    "Materialize full task JSON rows plus spreadsheet files from SpreadsheetBench Verified 400 before evaluation."
+  ]
+}

data/spreadsheetbench_id_split/train/items.json

@@ -0,0 +1,402 @@
+[
+  {
+    "id": "32438",
+    "spreadsheet_path": "spreadsheet/32438",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "398-14",
+    "spreadsheet_path": "spreadsheet/398-14",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "47766",
+    "spreadsheet_path": "spreadsheet/47766",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48365",
+    "spreadsheet_path": "spreadsheet/48365",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "32255",
+    "spreadsheet_path": "spreadsheet/32255",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "10747",
+    "spreadsheet_path": "spreadsheet/10747",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "50916",
+    "spreadsheet_path": "spreadsheet/50916",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "577-40",
+    "spreadsheet_path": "spreadsheet/577-40",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "35742",
+    "spreadsheet_path": "spreadsheet/35742",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "46121",
+    "spreadsheet_path": "spreadsheet/46121",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "51090",
+    "spreadsheet_path": "spreadsheet/51090",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "51249",
+    "spreadsheet_path": "spreadsheet/51249",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "82-30",
+    "spreadsheet_path": "spreadsheet/82-30",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "56274",
+    "spreadsheet_path": "spreadsheet/56274",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "57445",
+    "spreadsheet_path": "spreadsheet/57445",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "46646",
+    "spreadsheet_path": "spreadsheet/46646",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "105-24",
+    "spreadsheet_path": "spreadsheet/105-24",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "6239",
+    "spreadsheet_path": "spreadsheet/6239",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "414-20",
+    "spreadsheet_path": "spreadsheet/414-20",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "165-23",
+    "spreadsheet_path": "spreadsheet/165-23",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "40892",
+    "spreadsheet_path": "spreadsheet/40892",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48745",
+    "spreadsheet_path": "spreadsheet/48745",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "32612",
+    "spreadsheet_path": "spreadsheet/32612",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "325-44",
+    "spreadsheet_path": "spreadsheet/325-44",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "262-17",
+    "spreadsheet_path": "spreadsheet/262-17",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "141-20",
+    "spreadsheet_path": "spreadsheet/141-20",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "52216",
+    "spreadsheet_path": "spreadsheet/52216",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "22-47",
+    "spreadsheet_path": "spreadsheet/22-47",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "55421",
+    "spreadsheet_path": "spreadsheet/55421",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "56427",
+    "spreadsheet_path": "spreadsheet/56427",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "36097",
+    "spreadsheet_path": "spreadsheet/36097",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "32902",
+    "spreadsheet_path": "spreadsheet/32902",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "32023",
+    "spreadsheet_path": "spreadsheet/32023",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "1818",
+    "spreadsheet_path": "spreadsheet/1818",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "170-13",
+    "spreadsheet_path": "spreadsheet/170-13",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "66-24",
+    "spreadsheet_path": "spreadsheet/66-24",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "58949",
+    "spreadsheet_path": "spreadsheet/58949",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "42354",
+    "spreadsheet_path": "spreadsheet/42354",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "194-19",
+    "spreadsheet_path": "spreadsheet/194-19",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "31915",
+    "spreadsheet_path": "spreadsheet/31915",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "58499",
+    "spreadsheet_path": "spreadsheet/58499",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "45372",
+    "spreadsheet_path": "spreadsheet/45372",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "11842",
+    "spreadsheet_path": "spreadsheet/11842",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "57558",
+    "spreadsheet_path": "spreadsheet/57558",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "472-15",
+    "spreadsheet_path": "spreadsheet/472-15",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "55060",
+    "spreadsheet_path": "spreadsheet/55060",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "31011",
+    "spreadsheet_path": "spreadsheet/31011",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "408-39",
+    "spreadsheet_path": "spreadsheet/408-39",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "54085",
+    "spreadsheet_path": "spreadsheet/54085",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "39903",
+    "spreadsheet_path": "spreadsheet/39903",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48983",
+    "spreadsheet_path": "spreadsheet/48983",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "108-24",
+    "spreadsheet_path": "spreadsheet/108-24",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "58484",
+    "spreadsheet_path": "spreadsheet/58484",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "118-50",
+    "spreadsheet_path": "spreadsheet/118-50",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "10452",
+    "spreadsheet_path": "spreadsheet/10452",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "39931",
+    "spreadsheet_path": "spreadsheet/39931",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "3413",
+    "spreadsheet_path": "spreadsheet/3413",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "247-24",
+    "spreadsheet_path": "spreadsheet/247-24",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "56786",
+    "spreadsheet_path": "spreadsheet/56786",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "55965",
+    "spreadsheet_path": "spreadsheet/55965",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "379-36",
+    "spreadsheet_path": "spreadsheet/379-36",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "58109",
+    "spreadsheet_path": "spreadsheet/58109",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "433-47",
+    "spreadsheet_path": "spreadsheet/433-47",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "192-22",
+    "spreadsheet_path": "spreadsheet/192-22",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "49333",
+    "spreadsheet_path": "spreadsheet/49333",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "493-18",
+    "spreadsheet_path": "spreadsheet/493-18",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "54638",
+    "spreadsheet_path": "spreadsheet/54638",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "34033",
+    "spreadsheet_path": "spreadsheet/34033",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "30930",
+    "spreadsheet_path": "spreadsheet/30930",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "585-41",
+    "spreadsheet_path": "spreadsheet/585-41",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "32337",
+    "spreadsheet_path": "spreadsheet/32337",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "55427",
+    "spreadsheet_path": "spreadsheet/55427",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "263-1",
+    "spreadsheet_path": "spreadsheet/263-1",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "254-34",
+    "spreadsheet_path": "spreadsheet/254-34",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "57113",
+    "spreadsheet_path": "spreadsheet/57113",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "57743",
+    "spreadsheet_path": "spreadsheet/57743",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "43589",
+    "spreadsheet_path": "spreadsheet/43589",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "250-20",
+    "spreadsheet_path": "spreadsheet/250-20",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "48080",
+    "spreadsheet_path": "spreadsheet/48080",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "370-43",
+    "spreadsheet_path": "spreadsheet/370-43",
+    "instruction_type": "Sheet-Level Manipulation"
+  }
+]

data/spreadsheetbench_id_split/val/items.json

@@ -0,0 +1,202 @@
+[
+  {
+    "id": "45635",
+    "spreadsheet_path": "spreadsheet/45635",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "560-12",
+    "spreadsheet_path": "spreadsheet/560-12",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "55049",
+    "spreadsheet_path": "spreadsheet/55049",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "9569",
+    "spreadsheet_path": "spreadsheet/9569",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "7902",
+    "spreadsheet_path": "spreadsheet/7902",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "227-40",
+    "spreadsheet_path": "spreadsheet/227-40",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "463-17",
+    "spreadsheet_path": "spreadsheet/463-17",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "54144",
+    "spreadsheet_path": "spreadsheet/54144",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "80-42",
+    "spreadsheet_path": "spreadsheet/80-42",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "2768",
+    "spreadsheet_path": "spreadsheet/2768",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "37456",
+    "spreadsheet_path": "spreadsheet/37456",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "12864",
+    "spreadsheet_path": "spreadsheet/12864",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "55979",
+    "spreadsheet_path": "spreadsheet/55979",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48620",
+    "spreadsheet_path": "spreadsheet/48620",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48588",
+    "spreadsheet_path": "spreadsheet/48588",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "395-36",
+    "spreadsheet_path": "spreadsheet/395-36",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "382-10",
+    "spreadsheet_path": "spreadsheet/382-10",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "59595",
+    "spreadsheet_path": "spreadsheet/59595",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "53383",
+    "spreadsheet_path": "spreadsheet/53383",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48921",
+    "spreadsheet_path": "spreadsheet/48921",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "416-15",
+    "spreadsheet_path": "spreadsheet/416-15",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "47798",
+    "spreadsheet_path": "spreadsheet/47798",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "56563",
+    "spreadsheet_path": "spreadsheet/56563",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "46897",
+    "spreadsheet_path": "spreadsheet/46897",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "9726",
+    "spreadsheet_path": "spreadsheet/9726",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "50768",
+    "spreadsheet_path": "spreadsheet/50768",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "51-12",
+    "spreadsheet_path": "spreadsheet/51-12",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "31628",
+    "spreadsheet_path": "spreadsheet/31628",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "39046",
+    "spreadsheet_path": "spreadsheet/39046",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "8942",
+    "spreadsheet_path": "spreadsheet/8942",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "48527",
+    "spreadsheet_path": "spreadsheet/48527",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "59196",
+    "spreadsheet_path": "spreadsheet/59196",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "6698",
+    "spreadsheet_path": "spreadsheet/6698",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "43436",
+    "spreadsheet_path": "spreadsheet/43436",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "38462",
+    "spreadsheet_path": "spreadsheet/38462",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "402-43",
+    "spreadsheet_path": "spreadsheet/402-43",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "267-18",
+    "spreadsheet_path": "spreadsheet/267-18",
+    "instruction_type": "Sheet-Level Manipulation"
+  },
+  {
+    "id": "37378",
+    "spreadsheet_path": "spreadsheet/37378",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "53647",
+    "spreadsheet_path": "spreadsheet/53647",
+    "instruction_type": "Cell-Level Manipulation"
+  },
+  {
+    "id": "142-12",
+    "spreadsheet_path": "spreadsheet/142-12",
+    "instruction_type": "Sheet-Level Manipulation"
+  }
+]

docs/contributing.md

@@ -0,0 +1,78 @@
+# Contributing to SkillOpt
+
+Thank you for your interest in contributing to SkillOpt! This guide covers how to get started.
+
+## Development Setup
+
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+pip install -e ".[dev]"
+```
+
+## Ways to Contribute
+
+### 🐛 Bug Reports
+
+Open an issue with:
+
+- Steps to reproduce
+- Expected vs actual behavior
+- Config file used (sanitize API keys)
+- Python version and OS
+
+### 🔧 New Benchmark
+
+See [Add a New Benchmark](guide/new-benchmark.md) for the implementation guide.
+
+**Checklist:**
+
+- [ ] Data loader in `skillopt/envs/<benchmark>/dataloader.py`
+- [ ] Scored rollout implementation in `skillopt/envs/<benchmark>/rollout.py`
+- [ ] Per-item `predictions/<id>/conversation.json` artifacts for shared reflection
+- [ ] Environment adapter in `skillopt/envs/<benchmark>/adapter.py`
+- [ ] Config file in `configs/<benchmark>/default.yaml`
+- [ ] Lazy registration in `scripts/train.py` and `scripts/eval_only.py`
+- [ ] Focused tests and an optional seed skill referenced by `env.skill_init`
+- [ ] Documentation update
+
+### 🤖 New Model Backend
+
+See [Add a New Model Backend](guide/new-backend.md) for the implementation guide.
+
+**Checklist:**
+
+- [ ] Function-based backend module in `skillopt/model/<name>_backend.py`
+- [ ] Alias and default model in `skillopt/model/common.py`
+- [ ] Optimizer/target whitelist entries in `skillopt/model/backend_config.py`
+- [ ] Dispatch, token tracking, and setter forwarding in `skillopt/model/__init__.py`
+- [ ] YAML/CLI wiring when the backend exposes structured config fields
+- [ ] Focused routing, configuration, tool-call, and token-accounting tests
+- [ ] `.env.example` and backend/configuration reference updates
+
+### 📝 Documentation
+
+Documentation is built with MkDocs Material:
+
+```bash
+pip install -e ".[docs]"
+mkdocs serve  # Preview at http://localhost:8000
+```
+
+## Code Style
+
+- Follow existing patterns in the codebase
+- Use type hints for function signatures
+- Keep docstrings concise
+
+## Pull Request Process
+
+1. Fork the repository
+2. Create a feature branch: `git switch -c feature/my-benchmark`
+3. Make your changes
+4. Run focused tests, the full test suite, and `mkdocs build --strict` when docs change
+5. Submit a PR with a clear description
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the MIT License.

docs/guide/configuration.md

@@ -0,0 +1,212 @@
+# Configuration Guide
+
+SkillOpt uses YAML configuration files with a hierarchical override system.
+
+## Config Structure
+
+```
+configs/
+├── _base_/
+│   └── default.yaml          # Global defaults
+├── searchqa/
+│   └── default.yaml          # SearchQA overrides
+├── docvqa/
+│   └── default.yaml          # DocVQA overrides
+└── alfworld/
+    └── default.yaml          # ALFWorld overrides
+```
+
+Benchmark configs inherit from `_base_/default.yaml` and override specific values.
+
+## Key Parameters
+
+### Model Backends
+
+`optimizer_backend` controls reflection and skill editing;
+`target_backend` controls task rollout. The legacy `backend` field remains for
+backward compatibility, but explicit role fields are the clearest configuration.
+
+```yaml
+model:
+  backend: azure_openai          # High-level compatibility label
+  optimizer_backend: openai_chat
+  target_backend: openai_chat
+  optimizer: gpt-5.5             # Optimizer deployment/model
+  target: gpt-5.5                # Target deployment/model
+  azure_openai_auth_mode: api_key
+```
+
+| Backend | Optimizer | Target | Configuration |
+|---|:---:|:---:|---|
+| `openai_chat` | ✓ | ✓ | Azure OpenAI, or its explicit compatibility auth mode |
+| `openai_compatible` | ✓ | ✓ | Generic OpenAI Chat Completions endpoint |
+| `claude_chat` | ✓ | ✓ | Claude Code CLI (`claude -p`) |
+| `qwen_chat` | ✓ | ✓ | Qwen served through an OpenAI-compatible local endpoint |
+| `minimax_chat` | ✓ | ✓ | MiniMax API |
+| `codex_exec` | — | ✓ | Codex CLI execution harness |
+| `claude_code_exec` | — | ✓ | Claude Code CLI execution harness |
+
+The current MiniMax adapter has one shared deployment. Set
+`model.minimax_model` when MiniMax is the target; a mixed-backend run cannot
+independently select a MiniMax optimizer model and a different target model.
+
+For a generic compatible provider, select the role backends explicitly rather
+than relying on a high-level shorthand:
+
+```yaml
+model:
+  optimizer_backend: openai_compatible
+  target_backend: openai_compatible
+  optimizer: deepseek-chat
+  target: deepseek-chat
+```
+
+The train/eval entry points apply `model.optimizer` and `model.target` after
+backend initialization. For the selected roles, these YAML values override
+`OPENAI_COMPATIBLE_MODEL`, `QWEN_CHAT_MODEL`, and their per-role environment
+forms. The environment model variables mainly seed direct library use; always
+set the role models in a training or evaluation config.
+
+### Training
+
+```yaml
+train:
+  num_epochs: 4                  # Number of training epochs
+  batch_size: 40                 # Tasks per step (batch size)
+  accumulation: 1                # Gradient accumulation
+  seed: 42
+```
+
+### Gradient (Reflection)
+
+```yaml
+gradient:
+  minibatch_size: 8              # Reflect minibatch size
+  analyst_workers: 16            # Parallel reflection workers
+  max_analyst_rounds: 3          # Max rounds of analyst reflection
+  failure_only: false            # Only reflect on failures
+```
+
+### Optimizer
+
+```yaml
+optimizer:
+  learning_rate: 4               # Max edits per step (edit budget)
+  min_learning_rate: 2           # Min edits for decay schedulers
+  lr_scheduler: cosine           # constant | linear | cosine | autonomous
+  use_slow_update: true          # Momentum-like blending at epoch boundary
+  slow_update_samples: 20        # Samples for slow update evaluation
+  use_meta_skill: true           # Cross-epoch strategy memory
+```
+
+### Skill-Aware Reflection (optional, off by default)
+
+EmbodiSkill-style failure routing: the failure analyst classifies each
+failure pattern as **SKILL_DEFECT** (the rule is wrong or missing → normal
+gated body edit) or **EXECUTION_LAPSE** (a valid rule exists but was not
+followed → a short reminder appended to a protected appendix region inside
+the skill that step-level edits can never modify).
+
+```yaml
+optimizer:
+  use_skill_aware_reflection: false    # Master switch (default off = baseline-identical)
+  skill_aware_appendix_source: both    # both | failure_only (paper-faithful S_app)
+  skill_aware_consolidate_threshold: 0 # >0: LLM-compact the appendix past N notes (experimental)
+```
+
+Notes:
+
+- The switch is resolved process-wide from the config
+  (`configure_skill_aware_reflection`), so it applies to every benchmark
+  with no per-adapter wiring.
+- `failure_only` restricts appendix notes to the failure analyst, matching
+  the original S_app formulation; `both` additionally lets the success
+  analyst re-emphasize existing rules.
+- Appendix notes bypass the validation gate by design and accumulate with
+  order-preserving dedup; lapse-only steps (no body edits) still flush
+  their notes.
+- Not supported together with `skill_update_mode=rewrite_from_suggestions`
+  or the full-rewrite modes: whole-document rewrites can drop the appendix
+  region.
+
+### Evaluation
+
+```yaml
+evaluation:
+  use_gate: true                 # Validation gating (accept/reject updates)
+  gate_metric: hard              # hard | soft | mixed
+  gate_mixed_weight: 0.5         # Soft-score weight when metric=mixed
+  use_semantic_density: false    # Optional instruction-density bonus
+  eval_test: true                # Run test evaluation after training
+```
+
+The default and paper-style setting is `use_gate: true`. Setting it to `false`
+still records selection scores but force-accepts every candidate, so it changes
+the optimization semantics and should be reported explicitly.
+
+### Environment (Data)
+
+```yaml
+env:
+  name: searchqa                 # Benchmark name
+  split_mode: ratio              # ratio | split_dir
+  split_ratio: "2:1:7"           # train:val:test ratio
+  data_path: ""                  # Path to dataset
+  exec_timeout: 120              # Per-task timeout (seconds)
+```
+
+## CLI Overrides
+
+Override any config value from the command line:
+
+```bash
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --cfg-options \
+    optimizer.learning_rate=16 \
+    optimizer.lr_scheduler=linear \
+    gradient.analyst_workers=8
+```
+
+## Environment Variables
+
+Model credentials are loaded from environment variables:
+
+| Variable | Backend | Description |
+|---|---|---|
+| `AZURE_OPENAI_ENDPOINT` | `openai_chat` | Azure resource URL, or compatibility-mode base URL |
+| `AZURE_OPENAI_API_VERSION` | `openai_chat` | Azure API version |
+| `AZURE_OPENAI_AUTH_MODE` | `openai_chat` | `api_key`, `azure_cli`, `managed_identity`, or `openai_compatible` |
+| `AZURE_OPENAI_API_KEY` | `openai_chat` | Required when auth mode is `api_key` or `openai_compatible` |
+| `OPENAI_COMPATIBLE_BASE_URL` | `openai_compatible` | Generic Chat Completions base URL |
+| `OPENAI_COMPATIBLE_API_KEY` | `openai_compatible` | Provider API key; optional for local servers |
+| `OPENAI_COMPATIBLE_MODEL` | `openai_compatible` | Shared provider model ID for direct library use; train/eval YAML role models take precedence |
+| `CLAUDE_CLI_BIN` | `claude_chat` | Optional path to the `claude` executable; defaults to `claude` |
+| `ANTHROPIC_API_KEY` | `claude_chat` | Optional authentication method understood by the Claude CLI, not a direct SkillOpt API client |
+| `QWEN_CHAT_BASE_URL` | `qwen_chat` | Local Qwen/vLLM endpoint |
+| `QWEN_CHAT_MODEL` | `qwen_chat` | Served model name for direct library use; train/eval YAML role models take precedence |
+| `MINIMAX_BASE_URL` | `minimax_chat` | MiniMax-compatible base URL |
+| `MINIMAX_API_KEY` | `minimax_chat` | MiniMax API key |
+
+`OPTIMIZER_` and `TARGET_` prefixes provide per-role overrides for the
+Azure, OpenAI-compatible, and Qwen variable families. See the
+[Configuration Reference](../reference/config.md) for exact names.
+
+`claude_chat` launches the installed Claude Code CLI with `claude -p`; install
+and authenticate that CLI before use. Setting `ANTHROPIC_API_KEY` is one way
+the CLI may authenticate, but SkillOpt does not call the Anthropic API
+directly through this backend.
+
+### Three OpenAI-compatible paths
+
+- Research, generic provider: select `openai_compatible` and use
+  `OPENAI_COMPATIBLE_*`.
+- Research, Azure-family compatibility mode: keep `openai_chat`, set
+  `AZURE_OPENAI_AUTH_MODE=openai_compatible`, and use `AZURE_OPENAI_*`.
+- SkillOpt-Sleep: run with `--backend azure_openai` and use the same
+  compatibility-mode `AZURE_OPENAI_*` variables. Sleep does not read the
+  research backend's role-specific variables.
+
+## Full Reference
+
+See [Configuration Reference](../reference/config.md) for the complete parameter list.

docs/guide/dl-analogy.md

@@ -0,0 +1,53 @@
+# Deep Learning ↔ SkillOpt Analogy
+
+SkillOpt is designed around a core insight: **optimizing natural-language prompts follows the same structure as training neural networks**. This page maps every DL concept to its SkillOpt counterpart.
+
+## Complete Mapping
+
+| Deep Learning | SkillOpt | Description |
+|---|---|---|
+| **Model weights** | Skill document (Markdown) | The thing being optimized |
+| **Forward pass** | Rollout | Target executes tasks using current skill |
+| **Loss function** | Task evaluator | Scores task execution quality |
+| **Backpropagation** | Reflect | Optimizer analyzes failures → edit patches |
+| **Gradients** | Edit patches | Proposed changes to the skill |
+| **Gradient aggregation** | Patch aggregation | Merge similar edits |
+| **Gradient clipping** | Edit selection | Cap max edits per step |
+| **Learning rate** | `learning_rate` | Max number of edits applied per step |
+| **LR scheduler** | `lr_scheduler` | Edit-budget schedule: cosine, linear, constant, or autonomous |
+| **SGD step** | Skill update | Apply selected patches to document |
+| **Validation set** | Selection split | Gate checks improvement before accepting |
+| **Training step** | Step | One rollout → reflect → update cycle |
+| **Epoch** | Epoch | Full pass with slow update + meta memory |
+| **Momentum** | Slow update | Longitudinal comparison at epoch boundary |
+| **Meta-learning** | Meta skill | Cross-epoch optimizer strategy memory |
+| **Batch size** | `batch_size` | Tasks sampled per rollout |
+| **Data parallelism** | `analyst_workers` | Parallel reflection workers |
+| **Training set** | Train split | Items used for rollout |
+| **Test set** | Test split | Held-out final evaluation |
+| **Warm-up** | (implicit) | High LR early steps explore broadly |
+| **Checkpointing** | Skill snapshots | Saved after each accepted step |
+| **Transfer learning** | Seed skill / cross-benchmark init | Start from pre-trained skill |
+
+## Why This Analogy Matters
+
+1. **Familiar mental model**: ML practitioners immediately understand how to tune SkillOpt
+2. **Principled hyperparameter search**: Grid search over `learning_rate` × `lr_scheduler` works just like in DL
+3. **Reusable mechanisms**: Gating provides validation-based model selection, while slow update plays a momentum-like role across epochs
+
+The gate is a per-candidate accept/reject decision. SkillOpt does not implement
+a gate-patience counter or stop training after a run of rejected candidates.
+
+## Hyperparameter Transfer Rules
+
+From our experiments, these DL intuitions transfer well:
+
+!!! success "What transfers"
+    - **Cosine schedule > constant** — same as in DL, cosine annealing helps convergence
+    - **Moderate LR (4-16) > very high/low** — too few edits = slow learning, too many = noisy
+    - **Slow update helps** — longitudinal comparison prevents catastrophic forgetting across epochs
+    - **Meta skill memory improves reflection** — optimizer benefits from cross-epoch strategy notes
+
+!!! warning "What doesn't transfer"
+    - **Batch size ≠ better** — larger rollout batches have diminishing returns due to API costs
+    - **More epochs ≠ better** — skills converge faster than neural networks (2-4 epochs usually enough)

docs/guide/first-experiment.md

@@ -0,0 +1,137 @@
+# Your First Experiment
+
+This guide walks through running a complete SkillOpt training on SearchQA.
+
+## 1. Choose a Benchmark
+
+SkillOpt includes ready-to-use configs for several benchmarks. End-to-end
+runtime depends on the chosen models, provider latency, worker limits, and
+dataset size, so the project does not promise fixed wall-clock estimates.
+
+| Benchmark | Modality | Additional setup |
+|---|---|---|
+| SearchQA | Text QA | Materialize the released ID manifest |
+| DocVQA | Document/image QA | Obtain and materialize images and examples |
+| ALFWorld | Embodied agent | Install ALFWorld and download its assets |
+
+We'll use **SearchQA** because it is the simplest text-only walkthrough.
+
+## 2. Install and Materialize SearchQA
+
+The repository contains a stable SearchQA ID manifest, not the full runnable
+examples. From a source checkout, install the data extra and materialize the
+split once:
+
+```bash
+python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+```
+
+By default, the materializer reads `data/searchqa_id_split/` and writes the
+train/validation/test payloads expected by the config to
+`data/searchqa_split/`; both paths have command-line overrides.
+
+## 3. Configure
+
+Configure and export one model backend as described in
+[Installation](installation.md#environment-variables). For example:
+
+```bash
+cp .env.example .env
+# Edit .env, choose one authentication mode, then export it:
+set -a; source .env; set +a
+```
+
+Review the config file:
+
+```bash
+cat configs/searchqa/default.yaml
+```
+
+Key parameters (deep learning analogy in parentheses):
+
+```yaml
+train:
+  num_epochs: 4           # (epochs)
+  batch_size: 40          # (batch size)
+
+optimizer:
+  learning_rate: 4        # (max edits per step)
+  lr_scheduler: cosine    # (learning rate schedule)
+  use_slow_update: true   # (momentum at epoch boundary)
+  use_meta_skill: true    # (cross-epoch optimizer memory)
+
+gradient:
+  analyst_workers: 16     # (parallel reflection workers)
+
+evaluation:
+  use_gate: true          # (validation gating)
+```
+
+## 4. Train
+
+```bash
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_first_run
+```
+
+The command prints the resolved backend/data configuration, per-step rollout
+and gate progress, and the generated output directory.
+
+## 5. Monitor
+
+The explicit `--out_root` above creates this run directory:
+
+```
+outputs/searchqa_first_run/
+├── config.json
+├── runtime_state.json
+├── history.json
+├── best_skill.md
+├── skills/
+│   └── skill_vXXXX.md
+├── steps/
+│   └── step_XXXX/
+│       ├── candidate_skill.md
+│       ├── step_record.json
+│       └── trajectory_digest.json
+├── slow_update/
+│   └── epoch_XX/
+└── meta_skill/
+    └── epoch_XX/
+```
+
+## 6. Evaluate
+
+Evaluate the best skill on the test split:
+
+```bash
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill outputs/searchqa_first_run/best_skill.md \
+  --split valid_unseen
+```
+
+The `--skill` path above is the training artifact. Evaluation writes
+`eval_summary.json` to its own timestamped `outputs/eval_.../` directory unless
+you pass an explicit `--out_root`; it does not overwrite the training run.
+
+## WebUI
+
+Prefer a graphical interface? Launch the WebUI:
+
+```bash
+pip install -e ".[webui]"
+python -m skillopt_webui.app
+```
+
+Then open `http://localhost:7860` in your browser to configure parameters and
+launch training. The default host is `0.0.0.0`; pass `--host 127.0.0.1` for a
+local-only dashboard.
+
+## Next Steps
+
+- [Understand the training loop](training-loop.md)
+- [Configuration reference](../reference/config.md)
+- [Add a new benchmark](new-benchmark.md)

docs/guide/installation.md

@@ -0,0 +1,164 @@
+# Installation
+
+## Requirements
+
+- Python ≥ 3.10
+- For research training/evaluation, access to at least one configured model
+  backend (hosted API, local server, or an installed execution CLI)
+- The SkillOpt-Sleep `mock` backend needs no credentials
+
+## Choose an Install
+
+### PyPI
+
+Use PyPI for the Python packages and installed commands:
+
+```bash
+python -m pip install skillopt
+skillopt-sleep --help
+```
+
+This installs `skillopt-train`, `skillopt-eval`, and `skillopt-sleep`. The wheel
+does not include the repository's benchmark configs, data materializers,
+agent-integration shells/MCP servers, or development tests; use a source
+checkout for those files.
+
+!!! important "PyPI versus `main`"
+    These docs track the latest `main`. The current PyPI release is `0.2.0`.
+    The generic research `openai_compatible` backend, SkillOpt-Sleep handoff,
+    Sleep support for non-Azure OpenAI-compatible endpoints, and the Sleep
+    `--preferences` flag landed after that release and require a source install
+    from `main` until the next release.
+
+### Source checkout
+
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e .
+```
+
+Use the source checkout for paper reproduction, built-in benchmark configs,
+and contributions.
+
+## Optional Dependencies
+
+Install extras for specific benchmarks or backends:
+
+=== "ALFWorld"
+
+    ```bash
+    python -m pip install -e ".[alfworld]"
+    ```
+
+=== "Claude agent SDK (optional)"
+
+    ```bash
+    python -m pip install -e ".[claude]"
+    ```
+
+    This extra does not install the `claude` executable. The research
+    `claude_chat` backend launches `claude -p`, so install and authenticate the
+    Claude Code CLI separately. The SDK extra is only needed when selecting an
+    SDK-backed Claude Code exec path.
+
+=== "Qwen (Local)"
+
+    ```bash
+    python -m pip install -e ".[qwen]"
+    ```
+
+=== "SearchQA data"
+
+    ```bash
+    python -m pip install -e ".[searchqa]"
+    ```
+
+=== "WebUI"
+
+    ```bash
+    python -m pip install -e ".[webui]"
+    ```
+
+=== "Development"
+
+    ```bash
+    python -m pip install -e ".[dev]"
+    ```
+
+=== "All"
+
+    ```bash
+    python -m pip install -e ".[alfworld,claude,qwen,searchqa,webui,docs,dev]"
+    ```
+
+## Environment Variables
+
+From a source checkout, copy the template and fill in only the backend you
+will use:
+
+```bash
+cp .env.example .env
+```
+
+SkillOpt does not automatically load `.env`; export it into the current shell
+before running commands:
+
+```bash
+set -a
+source .env
+set +a
+```
+
+For Azure OpenAI with API-key authentication, the minimum settings are:
+
+```ini
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+AZURE_OPENAI_API_KEY=your-key
+AZURE_OPENAI_AUTH_MODE=api_key
+```
+
+Use `AZURE_OPENAI_AUTH_MODE=azure_cli` for Azure CLI credentials, or
+`managed_identity` with an optional
+`AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID`.
+
+The research `claude_chat` backend is a Claude Code CLI adapter, not a direct
+Anthropic API client. Install and authenticate `claude`, and set
+`CLAUDE_CLI_BIN` only if the executable is not available as `claude` on
+`PATH`. `ANTHROPIC_API_KEY` is one authentication option the CLI may consume.
+
+OpenAI-compatible servers have three distinct entry points:
+
+1. The research engine's generic `openai_compatible` backend uses
+   `OPENAI_COMPATIBLE_BASE_URL`, `OPENAI_COMPATIBLE_API_KEY`, and
+   `OPENAI_COMPATIBLE_MODEL`.
+2. The research `openai_chat` backend can use
+   `AZURE_OPENAI_AUTH_MODE=openai_compatible` with
+   `AZURE_OPENAI_ENDPOINT` and `AZURE_OPENAI_API_KEY`.
+3. SkillOpt-Sleep uses the same Azure-family variables as item 2 with
+   `skillopt-sleep run --backend azure_openai`.
+
+For research train/eval commands, `model.optimizer` and `model.target` in the
+YAML config are applied after backend initialization. They override model-name
+environment variables such as `OPENAI_COMPATIBLE_MODEL` and
+`QWEN_CHAT_MODEL`; set both role models explicitly when selecting those
+backends.
+
+!!! tip
+    You only need to configure the backend you plan to use. See
+    [Configuration](configuration.md#model-backends) for exact backend names
+    and role-specific overrides.
+
+## Verify Installation
+
+```bash
+python -c "import skillopt; print('SkillOpt ready!')"
+skillopt-train --help
+skillopt-eval --help
+skillopt-sleep --help
+```
+
+## Next Steps
+
+→ [Run your first experiment](first-experiment.md)

docs/guide/local-env-smoke.md

@@ -0,0 +1,149 @@
+# Local Environment Smoke Tests
+
+This guide describes a lightweight pattern for testing a custom SkillOpt environment before connecting it to expensive model calls or a full benchmark dataset.
+
+The goal is to validate the training loop plumbing first:
+
+- config loading
+- adapter construction
+- dataloader splits
+- rollout output shape
+- reflection patch shape
+- merge/rank/update control flow
+- artifact creation under `out_root`
+
+Once those are stable, you can switch the same environment to real model calls and larger evaluation splits.
+
+## 1. Add a tiny fixture split
+
+Start with a handful of deterministic examples that cover the expected pass/fail cases for your environment. Keep them small enough that a single training step can run locally.
+
+A minimal fixture item usually needs:
+
+```json
+{
+  "id": "example-1",
+  "split": "train",
+  "question": "...",
+  "expected": "..."
+}
+```
+
+Use the split names your adapter maps to SkillOpt phases:
+
+- `train` for optimization rollouts
+- `val` or `valid_seen` for selection/gating
+- `test` or `valid_unseen` for final evaluation
+
+## 2. Support a genuinely offline mock mode
+
+Add a configuration flag such as `mock: true` to your adapter. In mock mode,
+`rollout()` should return deterministic responses without calling external
+model APIs. The inherited `EnvAdapter.reflect()` does call the configured
+optimizer backend, so a no-credential smoke test must also override
+`reflect()` in mock mode to return a small, schema-valid deterministic patch
+(and delegate to `super().reflect(...)` otherwise).
+
+This lets you verify the SkillOpt loop with a fast command such as:
+
+```bash
+python scripts/train.py \
+  --config configs/myenv/tiny_mock.yaml
+```
+
+Mock mode should still exercise the trainer's normal artifact paths, including:
+
+- `config.json`, `runtime_state.json`, and `history.json`
+- `skills/skill_vXXXX.md`
+- `steps/step_XXXX/ranked_edits.json`
+- `steps/step_XXXX/candidate_skill.md`
+- `steps/step_XXXX/step_record.json`
+- the final `summary.json`
+
+## 3. Keep the smoke config tiny
+
+A CI-friendly smoke config should run a single small step:
+
+```yaml
+train:
+  num_epochs: 1
+  train_size: 3
+  batch_size: 3
+
+gradient:
+  minibatch_size: 1
+  merge_batch_size: 2
+  analyst_workers: 1
+  max_analyst_rounds: 1
+
+optimizer:
+  learning_rate: 1
+  min_learning_rate: 1
+  lr_scheduler: constant
+  skill_update_mode: patch
+  use_slow_update: false
+
+evaluation:
+  use_gate: true
+  sel_env_num: 2
+  test_env_num: 2
+  eval_test: false
+
+env:
+  name: myenv
+  out_root: outputs/myenv_tiny_mock
+  mock: true
+```
+
+Prefer a mock config that runs without credentials. That makes it useful for contributors and CI.
+
+## 4. Validate optimizer JSON before returning it
+
+If your environment or extension asks an LLM to merge or rank skill edits, validate the returned JSON before passing it back into SkillOpt. This avoids silent fallbacks from empty, malformed, or out-of-range responses.
+
+Useful checks for edit payloads:
+
+- response is a JSON object
+- `edits` is a non-empty list
+- every edit is an object
+- every edit has an allowed operation
+- required fields such as `content` or `target` are present for that operation
+
+Useful checks for ranking payloads:
+
+- `selected_indices` exists
+- indices are integers
+- indices are unique
+- indices are within the candidate edit range
+- selected count does not exceed the edit budget
+
+On failure, retry with a compact prompt that includes the schema error. If retries fail, raise an explicit error instead of silently accepting malformed output.
+
+## 5. Run progressively stronger checks
+
+A good development sequence is:
+
+```bash
+python -m py_compile scripts/train.py skillopt/envs/myenv/adapter.py
+python scripts/train.py --config configs/myenv/tiny_mock.yaml
+python scripts/train.py --config configs/myenv/tiny.yaml
+```
+
+For the real tiny run, verify that:
+
+- the run completes
+- `summary.json` is written
+- the step directory's `ranked_edits.json` contains the expected ranking metadata
+- any optimizer bridge log marks the response schema as valid
+- no generated files are written outside `out_root`
+
+## 6. Keep custom environments isolated
+
+When adding a custom environment to the registry, avoid side effects for existing benchmarks:
+
+- lazy-import optional dependencies
+- install environment-specific hooks only when `cfg["env"]` matches your environment
+- keep mock behavior behind an explicit config flag
+- write generated artifacts only under `out_root`
+
+This makes it easier to review and test a custom integration without affecting the built-in benchmarks.

docs/guide/new-backend.md

@@ -0,0 +1,200 @@
+# Add a New Model Backend
+
+SkillOpt's model layer is function-based: each chat backend is a Python module
+that exposes the call, token-tracking, and deployment-setting functions used by
+`skillopt.model`. There is no backend base class or registry object to subclass.
+
+## Built-in: the generic OpenAI-compatible backend
+
+!!! note "Version requirement"
+    This backend landed after v0.2.0. Install from the latest `main` until it is
+    included in the next release.
+
+Before writing a new backend, check whether your provider already speaks the
+OpenAI Chat Completions protocol. Most do, in which case you can use the
+built-in **`openai_compatible`** backend
+(`skillopt/model/openai_compatible_backend.py`) with no code changes.
+
+A single `base_url` + `api_key` pair lets you point SkillOpt at, for example:
+
+| Provider | `base_url` | Example model |
+|---|---|---|
+| DeepSeek | `https://api.deepseek.com/v1` | `deepseek-chat` |
+| Groq | `https://api.groq.com/openai/v1` | `llama-3.3-70b-versatile` |
+| Together AI | `https://api.together.xyz/v1` | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
+| Ollama (local) | `http://localhost:11434/v1` | `qwen2.5:7b` |
+| vLLM / SGLang / TGI | `http://localhost:8000/v1` | your served model |
+| LiteLLM proxy | `http://localhost:4000` | any proxied model |
+| OpenRouter / Fireworks / xAI / … | provider base URL | provider model id |
+
+### Python API
+
+Select and configure the backend directly when embedding SkillOpt as a Python
+library:
+
+```python
+import skillopt.model as model
+
+# Use the generic backend for both optimizer and target calls.
+model.set_backend("openai_compatible")
+model.configure_openai_compatible(
+    base_url="https://api.deepseek.com/v1",
+    api_key="sk-...",
+    model="deepseek-chat",
+)
+```
+
+`configure_openai_compatible()` also accepts `optimizer_*` and `target_*`
+arguments when the two roles use different endpoints or models.
+
+### Environment variables
+
+The shared variables below configure both roles. Role-specific
+`OPTIMIZER_OPENAI_COMPATIBLE_*` and `TARGET_OPENAI_COMPATIBLE_*` variables take
+precedence:
+
+```bash
+export OPENAI_COMPATIBLE_BASE_URL="https://api.groq.com/openai/v1"
+export OPENAI_COMPATIBLE_API_KEY="gsk_..."
+export OPENAI_COMPATIBLE_MODEL="llama-3.3-70b-versatile"
+# Optional: OPENAI_COMPATIBLE_TEMPERATURE, _MAX_TOKENS, _TIMEOUT_SECONDS
+```
+
+For direct library use, `OPTIMIZER_BACKEND=openai_compatible` and/or
+`TARGET_BACKEND=openai_compatible` select the role. The training and evaluation
+scripts resolve backend selection from their config, so set the split fields
+explicitly there:
+
+```yaml
+model:
+  optimizer_backend: openai_compatible
+  target_backend: openai_compatible
+  optimizer: llama-3.3-70b-versatile
+  target: llama-3.3-70b-versatile
+```
+
+Equivalently, override those fields on the command line:
+
+```bash
+python scripts/train.py --config configs/searchqa/default.yaml \
+  --cfg-options \
+  model.optimizer_backend=openai_compatible \
+  model.target_backend=openai_compatible \
+  model.optimizer=llama-3.3-70b-versatile \
+  model.target=llama-3.3-70b-versatile
+```
+
+Do not rely on the legacy high-level `model.backend` label to replace the two
+role-specific fields in a structured config.
+
+The generic backend uses the official `openai` SDK and the Chat Completions
+API. It records token usage through the shared tracker, supports provider tool
+calling through `chat_*_messages(..., tools=...)`, and exposes `count_tokens()`
+(tiktoken when available, with a character-based fallback). Provider-specific
+Responses API features are outside this backend's contract.
+
+Only write a new backend when the provider is not compatible with this surface
+or requires behavior that cannot be expressed by its configuration.
+
+## Backend architecture
+
+The active split optimizer/target dispatcher is the public
+`skillopt/model/__init__.py` module:
+
+```text
+skillopt/model/
+├── common.py                       # aliases, default models, token/response helpers
+├── backend_config.py               # optimizer/target whitelists and runtime selection
+├── __init__.py                     # public API and split-role dispatch
+├── openai_compatible_backend.py    # generic Chat Completions example
+├── qwen_backend.py                 # raw-HTTP chat example with per-role config
+├── minimax_backend.py              # compact raw-HTTP chat example
+├── codex_harness.py                # target-only exec harnesses
+└── router.py                       # legacy single-backend compatibility surface
+```
+
+`router.py` is not the dispatcher used by the current training loop. Update it
+only if the new backend must also be exposed through that legacy single-backend
+API.
+
+## Step 1: implement the module contract
+
+Create a module such as `skillopt/model/your_backend.py`. Copy the signatures
+from `openai_compatible_backend.py` or `qwen_backend.py`; model calls in the
+current framework are synchronous.
+
+For a chat backend that supports both roles, the public module surface is:
+
+| Function | Purpose |
+|---|---|
+| `chat_optimizer(...)` | Optimizer system/user call; returns `(text, usage)` |
+| `chat_target(...)` | Target system/user call; returns `(text, usage)` |
+| `chat_optimizer_messages(...)` | Optimizer message-list call, including optional tools |
+| `chat_target_messages(...)` | Target message-list call, including optional tools |
+| `get_token_summary()` | Return per-stage counters plus `_total` |
+| `reset_token_tracker()` | Clear this backend's counters |
+| `set_optimizer_deployment(name)` | Change the optimizer model at runtime |
+| `set_target_deployment(name)` | Change the target model at runtime |
+| `set_reasoning_effort(effort)` | Apply or safely ignore the shared reasoning setting |
+
+Every call returns a usage dict with `prompt_tokens`, `completion_tokens`, and
+`total_tokens`. Use `TokenTracker` from `skillopt.model.common` and record each
+call exactly once. Message-list calls that accept tools should return the
+compatibility message objects from `common.py` when `return_message=True`.
+
+Provider-specific configuration helpers and `count_tokens()` are optional, but
+their state must be safe to update while calls may run concurrently. Keep
+credentials out of logs and persisted artifacts.
+
+Exec-style targets do not implement this chat contract. They are target-only
+and are integrated through `codex_harness.py` plus environment-specific rollout
+code.
+
+## Step 2: register and route the backend
+
+A new backend normally requires all of the following:
+
+1. Add its canonical name, aliases, and default model to
+   `skillopt/model/common.py`.
+2. Add the canonical name to the appropriate optimizer and/or target whitelist
+   in `skillopt/model/backend_config.py`. Do not advertise a role the module
+   cannot execute.
+3. Import the module in `skillopt/model/__init__.py` and add dispatch branches
+   for every supported call surface.
+4. Include its counters in `get_token_summary()` / `reset_token_tracker()` and
+   forward the shared deployment/reasoning setters where applicable.
+5. If it has YAML settings, add structured-to-flat mappings in
+   `skillopt/config.py`, wire them through `scripts/train.py` and
+   `scripts/eval_only.py`, and document their precedence over environment
+   variables.
+6. Update `router.py` only when legacy single-backend compatibility is part of
+   the intended feature.
+
+Backend selection in `scripts/train.py` must use
+`model.optimizer_backend` and `model.target_backend`. A high-level
+`model.backend` alias alone is not a substitute for this explicit split.
+
+## Step 3: test the integration
+
+Add focused tests under `tests/` that do not call a live provider. At minimum,
+cover:
+
+- optimizer and target whitelist validation;
+- routing for text and message-list calls;
+- role-specific configuration precedence;
+- tool-call compatibility, if supported;
+- deployment/reasoning setters;
+- token accounting, including a single correct `_total`;
+- actionable errors for missing credentials or invalid responses.
+
+Then run the focused test, the full suite, and the documentation build:
+
+```bash
+python -m pytest tests/test_your_backend.py -q
+python -m pytest tests/ -q
+mkdocs build --strict
+```
+
+Also update `.env.example`, the configuration reference, and the backend table
+in the API reference. Add an optional dependency extra only when the backend
+requires a package that is not already a core dependency.

docs/guide/new-benchmark.md

@@ -0,0 +1,408 @@
+# Add a New Benchmark
+
+Extend SkillOpt with your own benchmark in ~200 lines of code. We will use
+a tiny worked example, `docfaithful`, that scores a target model on
+how faithfully it answers questions grounded in a small reference doc.
+
+> **Working reference.** The easiest way to copy-cargo-cult a new env is
+> to read [`skillopt/envs/officeqa/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs/officeqa).
+> Everything below is the same shape, simplified.
+
+## What you need to build
+
+To add a benchmark you implement four things:
+
+1. **A `SplitDataLoader` subclass** — knows how to load train / val / test
+   item dicts from disk.
+2. **A rollout helper** — runs the target model on a batch of items, scores
+   each prediction, and persists the per-item conversation consumed by the
+   shared reflection stage.
+3. **An `EnvAdapter` subclass** — wires the loader + rollout helper into
+   SkillOpt's lifecycle (`build_*_env`, `rollout`, and `get_task_types`).
+   The shared `reflect()` implementation is inherited unless the benchmark
+   needs custom reflection logic.
+4. **A YAML config** — references your env name plus the standard
+   train / optimizer / gradient knobs.
+
+Then lazy registration in the training and evaluation scripts makes it
+discoverable without importing optional dependencies at startup.
+
+---
+
+## Step 1 — Create the package
+
+```bash
+mkdir -p skillopt/envs/docfaithful
+touch skillopt/envs/docfaithful/__init__.py
+```
+
+## Step 2 — Implement the data loader
+
+`skillopt/envs/docfaithful/dataloader.py`:
+
+```python
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+def _normalize(raw: dict) -> dict:
+    """Make sure every item has an ``id``. Other keys are env-specific."""
+    return {
+        "id": str(raw["uid"]),
+        "question": raw["question"],
+        "ground_truth": raw["answer"],
+        "reference_text": raw.get("reference", ""),
+        "task_type": raw.get("category", "docfaithful"),
+    }
+
+
+class DocFaithfulDataLoader(SplitDataLoader):
+    """Load DocFaithful items from JSON files inside each split dir."""
+
+    def load_split_items(self, split_path: str) -> list[dict]:
+        # split_path is e.g. data/docfaithful_split/train/
+        json_files = sorted(Path(split_path).glob("*.json"))
+        if not json_files:
+            raise FileNotFoundError(f"No .json file found in {split_path}")
+        with json_files[0].open(encoding="utf-8") as f:
+            raw = json.load(f)
+        return [_normalize(item) for item in raw]
+```
+
+Only `load_split_items()` is mandatory. If you also want to support
+`split_mode="ratio"` (auto-split a single raw file into train/val/test),
+override `load_raw_items(data_path)` as well — see
+`skillopt/datasets/base.py` docstrings.
+
+## Step 3 — Write the rollout helper
+
+`skillopt/envs/docfaithful/rollout.py`:
+
+```python
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from skillopt.model import chat_target
+
+
+def _score(prediction: str, ground_truth: str) -> tuple[int, float]:
+    """Trivial exact-match scorer. Replace with F1 / ROUGE / LLM-judge."""
+    p = (prediction or "").strip().lower()
+    g = (ground_truth or "").strip().lower()
+    hard = int(p == g and bool(g))
+    soft = 1.0 if hard else 0.0
+    return hard, soft
+
+
+def _rollout_one(item: dict, skill_content: str, *, prediction_dir: Path,
+                 max_completion_tokens: int) -> dict:
+    system = skill_content
+    user = (
+        f"Question: {item['question']}\n\n"
+        f"Reference:\n{item.get('reference_text', '')}\n\n"
+        "Answer:"
+    )
+    prediction, _usage = chat_target(
+        system=system,
+        user=user,
+        max_completion_tokens=max_completion_tokens,
+    )
+    hard, soft = _score(prediction, item.get("ground_truth", ""))
+
+    # EnvAdapter.reflect() reads this exact trajectory path. Keep item IDs
+    # unique and filesystem-safe.
+    task_dir = prediction_dir / str(item["id"])
+    task_dir.mkdir(parents=True, exist_ok=True)
+    conversation = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+        {"role": "assistant", "content": prediction},
+    ]
+    (task_dir / "conversation.json").write_text(
+        json.dumps(conversation, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    return {
+        "id": str(item["id"]),
+        "hard": hard,
+        "soft": soft,
+        "predicted_answer": prediction,
+        "task_description": item.get("question", ""),
+        "question": item.get("question", ""),
+        "reference_text": item.get("reference_text", ""),
+        "task_type": item.get("task_type", "docfaithful"),
+        "target_system_prompt": system,
+        "target_user_prompt": user,
+        "n_turns": 1,
+    }
+
+
+def run_batch(*, items: list[dict], skill_content: str, out_root: str,
+              workers: int = 4, max_completion_tokens: int = 4096) -> list[dict]:
+    """Run a batch of episodes sequentially or with a thread pool."""
+    os.makedirs(out_root, exist_ok=True)
+    prediction_dir = Path(out_root, "predictions")
+    # For brevity we go sequentially — swap in concurrent.futures.ThreadPoolExecutor
+    # when network / model latency dominates.
+    results = [
+        _rollout_one(item, skill_content,
+                     prediction_dir=prediction_dir,
+                     max_completion_tokens=max_completion_tokens)
+        for item in items
+    ]
+    Path(out_root, "rollouts.json").write_text(
+        json.dumps(results, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    return results
+```
+
+Two design points worth flagging:
+
+- **Scoring lives here, not in `EnvAdapter`.** There is no `evaluate()`
+  method on the ABC. Whatever signal you put in `hard` (0/1, or a float
+  in [0, 1] for smoothed reward) and `soft` (float in [0, 1]) is what
+  the optimizer reads.
+- **Use `skillopt.model.chat_target`**, not raw OpenAI/Claude calls.
+  That routes through whichever **chat** target backend the user
+  configured (`openai_chat` / `claude_chat` / `qwen_chat` /
+  `minimax_chat` / `openai_compatible`) without your adapter caring.
+  Exec-style backends (`codex_exec`, `claude_code_exec`) need
+  environment-specific rollout code —
+  see `skillopt/model/codex_harness.py` together with the rollout modules in
+  `skillopt/envs/searchqa/`, `skillopt/envs/docvqa/`, or
+  `skillopt/envs/officeqa/` for working examples.
+- **Persist a conversation for reflection.** The shared `EnvAdapter.reflect()`
+  looks under `<rollout_dir>/predictions/<result-id>/conversation.json` and
+  skips results whose trajectory is absent or empty. Returning `hard`/`soft`
+  scores alone is sufficient for evaluation, but it cannot produce learning
+  patches.
+
+## Step 4 — Implement the environment adapter
+
+`skillopt/envs/docfaithful/adapter.py`:
+
+```python
+from __future__ import annotations
+
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs.base import EnvAdapter
+from skillopt.envs.docfaithful.dataloader import DocFaithfulDataLoader
+from skillopt.envs.docfaithful.rollout import run_batch
+
+
+class DocFaithfulAdapter(EnvAdapter):
+    """SkillOpt adapter for the DocFaithful benchmark."""
+
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "split_dir",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        workers: int = 4,
+        analyst_workers: int = 4,
+        failure_only: bool = False,
+        minibatch_size: int = 8,
+        edit_budget: int = 4,
+        seed: int = 42,
+        limit: int = 0,
+        max_completion_tokens: int = 4096,
+    ) -> None:
+        self.workers = workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.max_completion_tokens = int(max_completion_tokens)
+        self.dataloader = DocFaithfulDataLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+
+    # ── Lifecycle ───────────────────────────────────────────────────────
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    # ── Env construction ────────────────────────────────────────────────
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        # For dataset-backed envs the "manager" is just the items list.
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(
+            batch_size=batch_size, seed=seed, **kwargs
+        )
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(
+            env_num=env_num, split=split, seed=seed, **kwargs
+        )
+        return self.build_env_from_batch(batch, **kwargs)
+
+    # ── The rollout method (reflect is inherited) ───────────────────────
+
+    def rollout(self, env_manager, skill_content: str,
+                out_dir: str, **kwargs) -> list[dict]:
+        items: list[dict] = env_manager
+        return run_batch(
+            items=items,
+            skill_content=skill_content,
+            out_root=out_dir,
+            workers=self.workers,
+            max_completion_tokens=self.max_completion_tokens,
+        )
+
+    # reflect() is inherited from EnvAdapter — it delegates to
+    # run_minibatch_reflect with your analyst_error_* / analyst_success_*
+    # prompts. Override it only if you need custom reflection logic.
+
+    def get_task_types(self) -> list[str]:
+        seen: list[str] = []
+        for item in (
+            self.dataloader.train_items
+            + self.dataloader.val_items
+            + self.dataloader.test_items
+        ):
+            tt = str(item.get("task_type") or "docfaithful")
+            if tt not in seen:
+                seen.append(tt)
+        return seen or ["docfaithful"]
+```
+
+### What the rollout actually does
+
+Look back at `run_batch` from Step 3 — it sends each `item["question"]`
+to the target model with `skill_content` as the system prompt, scores
+the answer against `item["ground_truth"]`, and returns a list of dicts:
+
+```python
+[
+    {"id": "ex_001", "hard": 1, "soft": 0.92,
+     "predicted_answer": "...", "question": "...",
+     "reference_text": item["reference_text"]},
+    {"id": "ex_002", "hard": 0, "soft": 0.13, "fail_reason": "...", ...},
+    ...
+]
+```
+
+The trainer requires `id`, `hard`, and `soft` for scoring. The remaining fields
+are preserved on `RolloutResult.extras` (see `skillopt/types.py`). The shared
+reflection implementation combines those fields with each persisted
+`predictions/<id>/conversation.json`; without that file the result is omitted
+from reflection.
+
+## Step 5 — Register the adapter
+
+Edit [`scripts/train.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/train.py)
+and add to `_register_builtins()`:
+
+```python
+    try:
+        from skillopt.envs.docfaithful.adapter import DocFaithfulAdapter
+        _ENV_REGISTRY["docfaithful"] = DocFaithfulAdapter
+    except ImportError:
+        pass  # docfaithful deps not installed — skip
+```
+
+Mirror the same lazy registration in
+[`scripts/eval_only.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/eval_only.py)
+so standalone evaluation can resolve the environment too. There is **no
+`BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`**; both entry points
+keep a small lazy registry so optional dependencies do not break `--help`.
+
+## Step 6 — Create the YAML config
+
+`configs/docfaithful/default.yaml`:
+
+```yaml
+_base_: ../_base_/default.yaml      # NOTE: string, not list
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 16
+  accumulation: 1
+  num_epochs: 4
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: docfaithful
+  # Point to an existing Markdown file. Use an empty file to start blank.
+  skill_init: skillopt/envs/docfaithful/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/docfaithful_split
+  workers: 4
+  max_completion_tokens: 4096
+  limit: 0
+```
+
+> ⚠️ `_base_` is currently parsed as a **string path**, not a list. Write
+> `_base_: ../_base_/default.yaml`, not `_base_: ['../_base_/default.yaml']`.
+> See [`skillopt/config.py`](https://github.com/microsoft/SkillOpt/blob/main/skillopt/config.py)
+> if you want to add list-form inheritance.
+
+## Step 7 — Run
+
+```bash
+# Create the file referenced by env.skill_init before the first run:
+#   mkdir -p skillopt/envs/docfaithful/skills
+#   echo "# DocFaithful initial skill" > skillopt/envs/docfaithful/skills/initial.md
+
+python scripts/train.py --config configs/docfaithful/default.yaml
+```
+
+If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`,
+you forgot Step 5.
+
+If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`,
+you forgot to implement one of the four abstract methods on `EnvAdapter`:
+`build_train_env`, `build_eval_env`, `rollout`, `get_task_types`.
+
+## Tips
+
+- Start with `train.batch_size: 4` and `limit: 10` while debugging.
+- The `evaluate` half lives **inside your `rollout`**, not as a separate
+  method — there is no `evaluate()` in the `EnvAdapter` ABC. Score the
+  prediction in `run_batch` and put the score on each result dict's
+  `hard` / `soft`.
+- Noisy scoring kills the optimizer. Spend time on `run_batch`'s scoring
+  before you spend time on prompts.
+- If training repeatedly reports `skip_no_patches`, first verify that every
+  rollout result has a non-empty
+  `rollout/predictions/<id>/conversation.json` using the same `id` string.
+- If your benchmark needs heavy optional deps (selenium, vllm, ...),
+  wrap both registration blocks with `try / except ImportError` (Step 5)
+  so people without those deps can still `--help`.
+- Copy `skillopt/envs/_template/` as a starting skeleton — it now
+  implements the real abstract methods.

docs/guide/skill-document.md

@@ -0,0 +1,101 @@
+# Skill Document
+
+A **skill document** is a Markdown file that serves as the "prompt weights" of your agent. SkillOpt trains this document through iterative optimization.
+
+## What is a Skill Document?
+
+A skill document is a structured set of instructions that tells a language model **how** to approach a specific type of task. It's analogous to learned weights in a neural network — encoding task-specific knowledge in natural language rather than floating-point parameters.
+
+## Structure
+
+A typical skill document contains:
+
+```markdown
+# Task Strategy
+
+## General Approach
+- Break complex problems into sub-steps
+- Always verify intermediate results
+
+## Common Patterns
+- When you see X, try approach Y
+- Avoid Z because it leads to errors
+
+## Edge Cases
+- If the input contains A, handle it specially by...
+- Watch out for B — it requires C
+
+## Output Format
+- Always include reasoning before the answer
+- Format numbers with proper units
+```
+
+## How It Evolves
+
+During training, the skill document is modified by **edit patches**:
+
+1. **Additions**: New rules or strategies discovered from failed trajectories
+2. **Modifications**: Refining existing rules that are partially correct
+3. **Deletions**: Removing rules that consistently lead to errors
+
+Selected edits are applied together to produce a candidate skill. With the
+validation gate enabled, that candidate replaces the current skill only when
+its score on the selection split strictly improves.
+
+SkillOpt may maintain two protected, machine-managed regions:
+
+```markdown
+<!-- SLOW_UPDATE_START -->
+... epoch-level longitudinal guidance ...
+<!-- SLOW_UPDATE_END -->
+
+<!-- APPENDIX_START -->
+... skill-aware execution reminders ...
+<!-- APPENDIX_END -->
+```
+
+Normal edit patches cannot modify either region. Slow update owns the first;
+optional skill-aware reflection owns the second. Preserve these markers when
+copying or manually inspecting a trained skill.
+
+## Initial Skill
+
+You can start training with:
+
+- **Empty skill**: Point `env.skill_init` to an empty Markdown file
+- **Seed skill**: Provide initial instructions to bootstrap training
+- **Pre-trained skill**: Transfer a skill from a related benchmark
+
+Configure the initial skill in your YAML:
+
+```yaml
+env:
+  skill_init: path/to/initial_skill.md
+```
+
+To start from scratch, create an empty Markdown file and use its path. A missing
+path currently also starts blank, so using an explicit file avoids silently
+treating a typo as an empty skill.
+
+## Skill Quality Metrics
+
+Track your skill's evolution through:
+
+- **Validation score**: Primary metric on the selection split
+- **Test score**: Final metric on held-out test data
+- **Skill length**: Total tokens in the document
+- **Candidate acceptance rate**: Fraction of candidate skill updates that pass
+  gating; multiple proposed edits can be combined into one candidate
+
+## Best Practices
+
+!!! tip "Tips for better skills"
+    1. **Start with a seed skill** (`env.skill_init`) if you have domain knowledge — it converges faster
+    2. **Use cosine LR schedule** — aggressive early exploration + careful late refinement
+    3. **Enable slow update** (`optimizer.use_slow_update: true`) to counter forgetting across epochs
+    4. **Enable meta skill** (`optimizer.use_meta_skill: true`) so the optimizer accumulates strategy memory
+
+## Next Steps
+
+- [Deep Learning Analogy](dl-analogy.md)
+- [Configuration Reference](../reference/config.md)

docs/guide/training-loop.md

@@ -0,0 +1,105 @@
+# The Training Loop
+
+SkillOpt's core insight: **optimizing natural-language skill documents follows the same structure as training neural networks**.
+
+## Overview
+
+```
+┌─────────────────────────────────────────────────────────┐
+│                    Training Loop                         │
+│                                                         │
+│  for epoch in epochs:                                   │
+│    for step in steps:                                   │
+│      1. Rollout   — Target executes tasks              │
+│      2. Reflect   — Optimizer analyzes trajectories       │
+│      3. Aggregate — Hierarchical merge of patches       │
+│      4. Select    — Rank & clip edits (learning rate)   │
+│      5. Update    — Apply patches to skill doc          │
+│      6. Gate      — Validate & accept/reject            │
+│                                                         │
+│    Epoch Boundary:                                       │
+│      • Slow Update (longitudinal comparison & guidance) │
+│      • Meta Skill  (cross-epoch strategy memory)        │
+└─────────────────────────────────────────────────────────┘
+```
+
+## Stage Details
+
+### 1. Rollout (Forward Pass)
+
+The **target** model executes tasks using the current skill document as its prompt. Each task produces a trajectory and a score.
+
+```python
+# Analogy: forward pass through the network
+predictions = model(input, skill_document)
+scores = evaluate(predictions, ground_truth)
+```
+
+### 2. Reflect (Backward Pass)
+
+The **optimizer** model analyzes trajectory minibatches and produces **edit
+patches** — structured suggestions for improving the skill document. Failure
+minibatches are always eligible for analysis; successful trajectories are also
+analyzed unless `gradient.failure_only` is enabled. Independent minibatches can
+run concurrently according to `gradient.analyst_workers`.
+
+```python
+# Analogy: computing gradients
+gradients = loss.backward()  # → edit patches
+```
+
+### 3. Aggregate
+
+Semantically similar edit patches are merged to avoid redundant edits.
+
+### 4. Select (Gradient Clipping)
+
+Edits are ranked by relevance score. The `learning_rate` parameter caps how many edits are applied per step — just like gradient clipping prevents overshooting.
+
+```python
+# Analogy: gradient clipping + optimizer step size
+selected = top_k(edits, k=learning_rate)
+```
+
+The `lr_scheduler` adjusts this over training:
+
+- **cosine**: Start aggressive, taper smoothly
+- **linear**: Linear decay
+- **constant**: Fixed rate
+
+### 5. Update (Parameter Update)
+
+Selected edits are applied to the skill document, producing a new version.
+
+### 6. Gate (Validation)
+
+The updated skill is evaluated on a **selection split** (analogous to a
+validation set). With the gate enabled, the candidate is accepted only when its
+configured gate score (`hard`, `soft`, or `mixed`) is strictly higher than the
+current skill's score. With `evaluation.use_gate: false`, validation is still
+recorded but candidates are force-accepted.
+
+## Epoch Boundary Mechanisms
+
+### Slow Update
+
+At the end of each epoch (starting from epoch 2), the system performs a
+**longitudinal comparison**: it rolls out both the previous epoch's skill and
+the current skill on the same samples, categorizes items as
+improved/regressed/persistent-fail/stable-success, then generates high-level
+**guidance** for the skill document. Depending on
+`optimizer.slow_update_gate_with_selection`, that guidance is either checked on
+the selection split or applied unconditionally. Its purpose is to counter
+cross-epoch forgetting.
+
+### Meta Skill
+
+A **meta-skill memory** accumulates high-level strategy notes across the training
+run. Starting at the end of epoch 2, the optimizer compares the previous and
+current epoch, writes a compact memory, and provides the prior epoch's memory as
+additional context during later reflection and update stages.
+
+## Next Steps
+
+- [Understand Skill Documents](skill-document.md)
+- [DL ↔ SkillOpt analogy table](dl-analogy.md)

docs/guideline.html

@@ -0,0 +1,550 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>SkillOpt — Documentation &amp; Reproduction Guide</title>
+<meta name="description" content="Accurate entry points for installing, configuring, running, and extending SkillOpt and SkillOpt-Sleep.">
+<link rel="icon" type="image/svg+xml" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 23 23'%3E%3Crect width='10' height='10' fill='%23F25022'/%3E%3Crect x='13' width='10' height='10' fill='%237FBA00'/%3E%3Crect y='13' width='10' height='10' fill='%2300A4EF'/%3E%3Crect x='13' y='13' width='10' height='10' fill='%23FFB900'/%3E%3C/svg%3E">
+<style>
+  :root {
+    --bg: #fff;
+    --soft: #f7f8fb;
+    --ink: #1f2733;
+    --muted: #5b6675;
+    --quiet: #7c8797;
+    --line: #e2e7ef;
+    --brand: #4f46e5;
+    --brand-soft: #eef0ff;
+    --green: #047857;
+    --amber: #a16207;
+    --code-bg: #0f172a;
+    --code-ink: #e2e8f0;
+    --mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+    --sans: Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+  }
+  * { box-sizing: border-box; }
+  html { scroll-behavior: smooth; }
+  body {
+    margin: 0;
+    color: var(--ink);
+    background: var(--bg);
+    font: 15px/1.65 var(--sans);
+    -webkit-font-smoothing: antialiased;
+  }
+  header {
+    position: sticky;
+    top: 0;
+    z-index: 10;
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    height: 58px;
+    padding: 0 24px;
+    background: rgba(255,255,255,.94);
+    border-bottom: 1px solid var(--line);
+    backdrop-filter: blur(8px);
+  }
+  header svg { width: 22px; }
+  header strong { letter-spacing: -.01em; }
+  header strong span { color: var(--brand); }
+  header .spacer { flex: 1; }
+  header a {
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 13px;
+    font-weight: 600;
+  }
+  header a:hover { color: var(--brand); }
+  .layout {
+    display: grid;
+    grid-template-columns: 240px minmax(0, 880px);
+    justify-content: center;
+    align-items: start;
+  }
+  nav {
+    position: sticky;
+    top: 58px;
+    height: calc(100vh - 58px);
+    padding: 32px 26px;
+    overflow-y: auto;
+    border-right: 1px solid var(--line);
+  }
+  nav strong {
+    display: block;
+    margin: 18px 0 6px;
+    color: var(--quiet);
+    font-size: 11px;
+    letter-spacing: .08em;
+    text-transform: uppercase;
+  }
+  nav strong:first-child { margin-top: 0; }
+  nav a {
+    display: block;
+    padding: 4px 0;
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 13px;
+  }
+  nav a:hover { color: var(--brand); }
+  main { padding: 46px 54px 100px; min-width: 0; }
+  section { scroll-margin-top: 76px; }
+  h1 {
+    margin: 4px 0 10px;
+    font-size: clamp(30px, 4vw, 42px);
+    line-height: 1.12;
+    letter-spacing: -.035em;
+  }
+  h2 {
+    margin: 54px 0 14px;
+    padding-bottom: 9px;
+    border-bottom: 1px solid var(--line);
+    font-size: 24px;
+    letter-spacing: -.02em;
+  }
+  h3 { margin: 28px 0 8px; font-size: 18px; }
+  p { margin: 10px 0; }
+  a { color: var(--brand); }
+  code {
+    padding: 2px 5px;
+    border-radius: 5px;
+    background: #eef1f6;
+    color: #9d174d;
+    font: .88em var(--mono);
+  }
+  pre {
+    overflow-x: auto;
+    margin: 14px 0;
+    padding: 16px 18px;
+    border: 1px solid #1e293b;
+    border-radius: 11px;
+    background: var(--code-bg);
+    color: var(--code-ink);
+    font: 13px/1.6 var(--mono);
+  }
+  pre code { padding: 0; background: transparent; color: inherit; font-size: inherit; }
+  ul, ol { padding-left: 23px; }
+  li { margin: 5px 0; }
+  .eyebrow {
+    color: var(--brand);
+    font-size: 12px;
+    font-weight: 750;
+    letter-spacing: .09em;
+    text-transform: uppercase;
+  }
+  .lead { color: var(--muted); font-size: 17px; }
+  .notice {
+    margin: 22px 0;
+    padding: 14px 17px;
+    border: 1px solid #c7d2fe;
+    border-radius: 10px;
+    background: var(--brand-soft);
+  }
+  .notice.warn { border-color: #fde68a; background: #fffbeb; }
+  .notice strong { display: block; color: var(--brand); }
+  .notice.warn strong { color: var(--amber); }
+  .cards {
+    display: grid;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+    gap: 14px;
+    margin: 18px 0;
+  }
+  .card {
+    padding: 17px;
+    border: 1px solid var(--line);
+    border-radius: 11px;
+    background: var(--soft);
+  }
+  .card h3 { margin: 0 0 6px; font-size: 17px; }
+  .card p { margin: 0; color: var(--muted); font-size: 14px; }
+  .table {
+    margin: 16px 0;
+    overflow-x: auto;
+    border: 1px solid var(--line);
+    border-radius: 11px;
+  }
+  table { width: 100%; border-collapse: collapse; font-size: 13.5px; }
+  th, td { padding: 9px 12px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
+  th { background: var(--soft); white-space: nowrap; }
+  tr:last-child td { border-bottom: 0; }
+  footer {
+    margin-top: 60px;
+    padding-top: 20px;
+    border-top: 1px solid var(--line);
+    color: var(--quiet);
+    font-size: 13px;
+  }
+  @media (max-width: 850px) {
+    header { padding: 0 16px; }
+    header .optional { display: none; }
+    .layout { display: block; }
+    nav { display: none; }
+    main { padding: 34px 20px 80px; }
+    .cards { grid-template-columns: 1fr; }
+  }
+</style>
+</head>
+<body>
+<header>
+  <svg viewBox="0 0 23 23" aria-hidden="true"><rect width="10" height="10" fill="#F25022"/><rect x="13" width="10" height="10" fill="#7FBA00"/><rect y="13" width="10" height="10" fill="#00A4EF"/><rect x="13" y="13" width="10" height="10" fill="#FFB900"/></svg>
+  <strong>Skill<span>Opt</span></strong>
+  <span class="spacer"></span>
+  <a class="optional" href="https://arxiv.org/abs/2605.23904">Paper</a>
+  <a class="optional" href="https://microsoft.github.io/SkillOpt/blog/">Blog</a>
+  <a href="https://github.com/microsoft/SkillOpt">GitHub</a>
+</header>
+
+<div class="layout">
+<nav aria-label="Guide sections">
+  <strong>Start here</strong>
+  <a href="#overview">Overview</a>
+  <a href="#choose">Choose a workflow</a>
+  <a href="#install">Install</a>
+  <a href="#credentials">Credentials</a>
+
+  <strong>Research engine</strong>
+  <a href="#research">First experiment</a>
+  <a href="#backends">Model backends</a>
+  <a href="#research-docs">Reference map</a>
+
+  <strong>SkillOpt-Sleep</strong>
+  <a href="#sleep">Safe first run</a>
+  <a href="#sleep-plugins">Agent integrations</a>
+  <a href="#sleep-replay">Advanced controls</a>
+  <a href="#safety">Data and safety</a>
+
+  <strong>Project</strong>
+  <a href="#contributing">Contributing</a>
+</nav>
+
+<main>
+  <span class="eyebrow">Microsoft Research · documentation hub</span>
+  <h1>SkillOpt Documentation &amp; Reproduction Guide</h1>
+  <p class="lead">Improve frozen agents by optimizing the Markdown skills that guide them—using reflective updates and held-out validation instead of weight training.</p>
+
+  <div class="notice">
+    <strong>How this guide stays accurate</strong>
+    This page is a stable, concise entry point. Detailed commands, defaults,
+    and APIs live in the versioned
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md">Markdown documentation</a>
+    beside the code. For exact behavior in a checkout, the command's
+    <code>--help</code>, the selected YAML config, and the code are authoritative.
+  </div>
+
+  <section id="overview">
+    <h2>Overview</h2>
+    <p><strong>SkillOpt</strong> treats a natural-language skill document as the
+    trainable state of an agent. A target model executes tasks, an optimizer
+    reflects on the resulting trajectories, bounded edits form a candidate
+    skill, and a validation gate decides whether to keep it.</p>
+    <div class="cards">
+      <div class="card">
+        <h3>Research engine</h3>
+        <p>Run reproducible training and evaluation over benchmark splits.
+        Six released benchmark configurations cover QA, document QA, embodied
+        agents, math, spreadsheets, and tool-augmented QA.</p>
+      </div>
+      <div class="card">
+        <h3>SkillOpt-Sleep preview</h3>
+        <p>Harvest supported coding-agent sessions, mine replayable tasks, and
+        stage proposed memory or skill updates for review. It is a separate,
+        evolving deployment companion—not the paper's benchmark runner.</p>
+      </div>
+    </div>
+    <p>The optimizer and target are separate roles and may use different
+    backends. Validation gating is the research default and the paper-style
+    setting; deliberately disabling it force-accepts candidates and changes the
+    experiment semantics. SkillOpt-Sleep stages updates by default; automatic
+    adoption is opt-in.</p>
+  </section>
+
+  <section id="choose">
+    <h2>Choose the right workflow</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Goal</th><th>Start with</th></tr></thead>
+        <tbody>
+          <tr><td>Reproduce paper-style benchmark training</td><td><a href="#research">Research first experiment</a></td></tr>
+          <tr><td>Evaluate an existing skill without training</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/cli.md">Evaluation CLI reference</a></td></tr>
+          <tr><td>Add a benchmark adapter</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-benchmark.md">New benchmark guide</a></td></tr>
+          <tr><td>Connect another model provider</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-backend.md">Backend guide</a></td></tr>
+          <tr><td>Improve a coding-agent skill from local sessions</td><td><a href="#sleep">SkillOpt-Sleep</a></td></tr>
+        </tbody>
+      </table>
+    </div>
+  </section>
+
+  <section id="install">
+    <h2>Install</h2>
+    <p>SkillOpt requires Python 3.10 or newer.</p>
+<pre><code># Published package
+python -m pip install skillopt
+
+# Latest source and development workflow
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e .
+
+# Install only the extras you need
+python -m pip install -e ".[searchqa]"  # SearchQA materialization
+python -m pip install -e ".[alfworld]"  # ALFWorld
+python -m pip install -e ".[claude]"    # optional Claude agent SDK support
+python -m pip install -e ".[webui]"     # Gradio dashboard
+python -m pip install -e ".[dev]"       # tests and linting</code></pre>
+    <div class="notice warn">
+      <strong>Release boundary</strong>
+      This guide tracks <code>main</code>. PyPI currently serves 0.2.0; the
+      generic research <code>openai_compatible</code> backend, Sleep handoff,
+      SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, and the
+      Sleep <code>--preferences</code> flag require a source install from
+      <code>main</code> until the next release.
+    </div>
+    <p>See the <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/installation.md">installation guide</a>
+    for platform notes and dependency boundaries.</p>
+  </section>
+
+  <section id="credentials">
+    <h2>Credentials and endpoint families</h2>
+    <p>Copy <code>.env.example</code>, fill only the backend you use, and load
+    it into your shell. Do not commit the resulting <code>.env</code>.</p>
+<pre><code>cp .env.example .env
+set -a
+source .env
+set +a</code></pre>
+
+    <h3>Azure OpenAI</h3>
+<pre><code>export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_API_VERSION="2024-12-01-preview"
+export AZURE_OPENAI_AUTH_MODE="api_key"
+export AZURE_OPENAI_API_KEY="your-key"</code></pre>
+    <p>For keyless Azure authentication, use <code>azure_cli</code> or
+    <code>managed_identity</code> and follow the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/configuration.md">configuration guide</a>.
+    Setting an API key without <code>AZURE_OPENAI_AUTH_MODE=api_key</code> does
+    not change the default authentication mode.</p>
+
+    <h3>Generic OpenAI-compatible research backend</h3>
+<pre><code>export OPENAI_COMPATIBLE_BASE_URL="https://api.example.com/v1"
+export OPENAI_COMPATIBLE_API_KEY="your-key"
+export OPENAI_COMPATIBLE_MODEL="provider-model"
+
+python scripts/train.py --config configs/searchqa/default.yaml &#92;
+  --cfg-options &#92;
+  model.optimizer_backend=openai_compatible &#92;
+  model.target_backend=openai_compatible &#92;
+  model.optimizer=provider-model &#92;
+  model.target=provider-model</code></pre>
+    <p>This provider-neutral backend is distinct from Azure OpenAI. Per-role
+    overrides use <code>OPTIMIZER_OPENAI_COMPATIBLE_*</code> and
+    <code>TARGET_OPENAI_COMPATIBLE_*</code>. Train/eval applies the YAML role
+    models after backend initialization, so they override model-name
+    environment variables.</p>
+
+    <h3>OpenAI-compatible endpoints in SkillOpt-Sleep</h3>
+    <p>The Sleep CLI exposes this compatibility path through its
+    <code>azure_openai</code> backend for backward compatibility, so it uses a
+    different environment-variable family:</p>
+<pre><code>export AZURE_OPENAI_ENDPOINT="https://api.example.com/v1"
+export AZURE_OPENAI_API_KEY="your-key"
+export AZURE_OPENAI_AUTH_MODE="openai_compatible"
+
+skillopt-sleep run --backend azure_openai --model provider-model</code></pre>
+    <p>Do not mix this mode with Azure CLI or managed-identity settings. See
+    the dedicated
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/openai-compatible-endpoints.md">Sleep endpoint guide</a>.</p>
+  </section>
+
+  <section id="research">
+    <h2>Research engine: first experiment</h2>
+    <p>The repository ships deterministic ID manifests, not the benchmark
+    examples themselves. Materialize the SearchQA examples once, then run its
+    checked-in config:</p>
+<pre><code>python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+
+# Load model credentials first, then:
+python scripts/train.py --config configs/searchqa/default.yaml</code></pre>
+    <p>The run directory contains <code>best_skill.md</code>,
+    <code>runtime_state.json</code>, <code>history.json</code>, versioned files
+    under <code>skills/</code>, and step-level artifacts. Re-running with the
+    same output root resumes from persisted state.</p>
+<pre><code>python scripts/eval_only.py &#92;
+  --config configs/searchqa/default.yaml &#92;
+  --skill outputs/&lt;run&gt;/best_skill.md &#92;
+  --split valid_unseen</code></pre>
+    <div class="notice warn">
+      <strong>Reproduction boundary</strong>
+      Use the released train/validation/test manifests and record the exact
+      model deployment, config, seed, and source revision. Provider behavior
+      can change independently of this repository.
+    </div>
+    <p>Continue with the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/first-experiment.md">first-experiment guide</a>
+    and <a href="https://github.com/microsoft/SkillOpt/blob/main/data/README.md">dataset manifest documentation</a>.</p>
+  </section>
+
+  <section id="backends">
+    <h2>Research model backends</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Backend</th><th>Optimizer</th><th>Target</th><th>Notes</th></tr></thead>
+        <tbody>
+          <tr><td><code>openai_chat</code></td><td>Yes</td><td>Yes</td><td>Azure OpenAI plus its explicit authentication modes.</td></tr>
+          <tr><td><code>openai_compatible</code></td><td>Yes</td><td>Yes</td><td>Provider-neutral chat-completions endpoint.</td></tr>
+          <tr><td><code>claude_chat</code></td><td>Yes</td><td>Yes</td><td>Runs an installed, authenticated Claude Code CLI via <code>claude -p</code>; not a direct Anthropic API client.</td></tr>
+          <tr><td><code>qwen_chat</code></td><td>Yes</td><td>Yes</td><td>Local or hosted Qwen-compatible server.</td></tr>
+          <tr><td><code>minimax_chat</code></td><td>Yes</td><td>Yes</td><td>MiniMax chat endpoint.</td></tr>
+          <tr><td><code>codex_exec</code></td><td>Yes</td><td>Supported adapters only</td><td>Executes Codex for optimizer calls and as a target agent where supported.</td></tr>
+          <tr><td><code>claude_code_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Claude Code as a target agent.</td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>Prefer the structured <code>model.optimizer_backend</code> and
+    <code>model.target_backend</code> settings. Legacy <code>--backend</code>
+    aliases do not expose every role-specific combination. Exec backends are
+    not generic chat replacements and require adapter support.</p>
+  </section>
+
+  <section id="research-docs">
+    <h2>Research documentation map</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Reference</th><th>Use it for</th></tr></thead>
+        <tbody>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/configuration.md">Configuration</a></td><td>Authentication, structured YAML, role-specific backends, and overrides.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/cli.md">CLI</a></td><td>Current train/eval entry points and exact output paths.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/config.md">Config reference</a></td><td>Supported sections, defaults, and validation constraints.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/training-loop.md">Training loop</a></td><td>Rollout, reflection, edit selection, gating, slow update, and meta skill.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/skill-document.md">Skill document</a></td><td>Skill structure and protected regions.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/api.md">Python API</a></td><td>Stable public imports and low-level/internal boundaries.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/CHANGELOG.md">Changelog</a></td><td>Recently merged capabilities, fixes, and contributor credits.</td></tr>
+        </tbody>
+      </table>
+    </div>
+  </section>
+
+  <section id="sleep">
+    <h2>SkillOpt-Sleep: safe first run</h2>
+    <p>SkillOpt-Sleep is a preview deployment companion. Its default
+    <code>mock</code> backend is useful for testing control flow without API
+    spend; it is not evidence that a real model's quality improved.</p>
+<pre><code># Deterministic engine proof; no model credentials required
+python -m skillopt_sleep.experiments.run_experiment &#92;
+  --persona researcher --assert-improves
+
+# Inspect local session handling without adopting any update
+skillopt-sleep dry-run &#92;
+  --project "$PWD" &#92;
+  --source auto &#92;
+  --backend mock
+
+# A real run: explicitly identify the skill to evolve
+skillopt-sleep run &#92;
+  --project "$PWD" &#92;
+  --target-skill-path path/to/SKILL.md &#92;
+  --source auto &#92;
+  --backend claude
+
+skillopt-sleep status --project "$PWD"
+skillopt-sleep adopt --project "$PWD"</code></pre>
+    <p><code>--project</code> scopes collection but does not automatically
+    choose a project's skill file. Use <code>--target-skill-path</code> when
+    you intend to evolve a particular <code>SKILL.md</code>. Transcript source
+    (<code>claude</code>, <code>codex</code>, or <code>auto</code>) and replay
+    backend are independent settings.</p>
+    <p>For subscription-based workflows that should not launch an API or model
+    subprocess, use <code>--backend handoff</code> and follow the generated
+    prompt/answer loop. Read the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/README.md">complete Sleep guide</a>
+    before a real run.</p>
+  </section>
+
+  <section id="sleep-plugins">
+    <h2>Agent integrations</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Agent</th><th>Integration status</th><th>Guide</th></tr></thead>
+        <tbody>
+          <tr><td>Claude Code</td><td>Shared-engine plugin and handoff command</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/claude-code/README.md">README</a></td></tr>
+          <tr><td>Codex</td><td>Shared-engine skill shell</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/codex/README.md">README</a></td></tr>
+          <tr><td>GitHub Copilot</td><td>Shared-engine Sleep MCP plus a separate research MCP</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/copilot/README.md">README</a></td></tr>
+          <tr><td>Devin</td><td>Shared-engine MCP with Devin transcript conversion</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/devin/README.md">README</a></td></tr>
+          <tr><td>OpenClaw</td><td>Independent community/reference adaptation; review locally before use</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/openclaw/README.md">README</a></td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>The <a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/README.md">plugin overview</a>
+    records which integrations use the shared engine and which require local
+    adaptation.</p>
+  </section>
+
+  <section id="sleep-replay">
+    <h2>Advanced Sleep controls</h2>
+    <p>The main CLI exposes project/source selection, backend/model selection,
+    bounded task and edit counts, preferences, reviewed task files, and
+    staged adoption. Additional JSON configuration fields include:</p>
+    <div class="table">
+      <table>
+        <thead><tr><th>Field</th><th>Default</th><th>Status</th></tr></thead>
+        <tbody>
+          <tr><td><code>dream_rollouts</code></td><td>1</td><td>Single rollout by default; values above 1 enable experimental contrastive replay.</td></tr>
+          <tr><td><code>dream_factor</code></td><td>0</td><td>Synthetic task variants are off by default.</td></tr>
+          <tr><td><code>recall_k</code></td><td>0</td><td>Historical associative recall is off by default.</td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>These are configuration fields, not current <code>skillopt-sleep run</code>
+    flags. Treat multi-rollout, recall, synthetic dreaming, and experimental
+    reward/budget controls as advanced features that require task-specific
+    validation. The reported experiments and their exact settings are in
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/RESULTS.md">RESULTS.md</a>.</p>
+  </section>
+
+  <section id="safety">
+    <h2>Data, privacy, and adoption safety</h2>
+    <ul>
+      <li>Real Sleep backends may send session-derived prompts, mined tasks,
+      trajectories, and candidate edits to the selected provider. Review the
+      source data and provider policy before use.</li>
+      <li>Secret redaction for persisted diagnostics is defense in depth; it
+      is not a guarantee that every outbound model prompt is free of sensitive
+      content. In particular, do not treat raw coding-agent transcripts as
+      pre-sanitized.</li>
+      <li>Updates are staged for review by default. Use
+      <code>--auto-adopt</code> only when you have an independent rollback and
+      validation process.</li>
+      <li>A held-out gate reduces regressions on its measured tasks; it is not
+      a security boundary or a proof of general improvement.</li>
+      <li>Use a temporary clone and synthetic transcripts when validating a
+      new backend or plugin integration.</li>
+    </ul>
+  </section>
+
+  <section id="contributing">
+    <h2>Contributing and extending</h2>
+    <p>Before proposing a change, run the focused tests for the affected area,
+    then the full suite where practical. Documentation changes should pass a
+    strict MkDocs build and should be checked against actual CLI
+    <code>--help</code> output.</p>
+<pre><code>python -m pip install -e ".[dev,docs]"
+python -m pytest -q
+python -m mkdocs build --strict</code></pre>
+    <p>See <a href="https://github.com/microsoft/SkillOpt/blob/main/CONTRIBUTING.md">CONTRIBUTING.md</a>,
+    the <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/contributing.md">documentation workflow</a>,
+    and the focused guides for
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-benchmark.md">benchmarks</a>
+    and <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-backend.md">model backends</a>.</p>
+  </section>
+
+  <footer>
+    SkillOpt · <a href="https://github.com/microsoft/SkillOpt">github.com/microsoft/SkillOpt</a>
+    · <a href="https://microsoft.github.io/SkillOpt/blog/">Technical Blog</a>
+    · <a href="https://arxiv.org/abs/2605.23904">arXiv:2605.23904</a><br>
+    This public overview intentionally avoids duplicating the complete,
+    fast-changing configuration surface. Follow the linked versioned
+    references for details.
+  </footer>
+</main>
+</div>
+</body>
+</html>

docs/index.md

@@ -0,0 +1,215 @@
+---
+hide:
+  - navigation
+---
+
+<div class="hero" markdown>
+
+# SkillOpt
+
+### Train Agent Skills Like Neural Networks
+
+*Optimize natural-language skill documents through iterative rollout, reflection, and gated validation — with epochs, learning rates, and validation gates — without touching model weights.*
+
+[Get Started :material-rocket-launch:](guide/installation.md){ .md-button .md-button--primary }
+[View on GitHub :material-github:](https://github.com/microsoft/SkillOpt){ .md-button }
+
+</div>
+
+---
+
+## Two Complementary Workflows
+
+| Workflow | Package / command | Use it for |
+|---|---|---|
+| **Research engine** | `skillopt`, `skillopt-train`, `skillopt-eval` | Train and evaluate skill documents on explicit benchmark splits. |
+| **SkillOpt-Sleep (preview)** | `skillopt_sleep`, `skillopt-sleep` | Review supported coding-agent sessions and stage proposed memory/skill updates for human adoption. |
+
+They share the idea of bounded text updates and validation, but they are
+separate entry points with different configs and safety boundaries. Start with
+the [SkillOpt-Sleep overview](sleep/README.md) before using real session data.
+
+---
+
+## How It Works
+
+<div class="pipeline-container" markdown>
+<div class="pipeline-wrapper">
+
+<div class="pipeline-stage" id="stage-rollout">
+<div class="stage-icon">🎯</div>
+<div class="stage-label">Rollout</div>
+<div class="stage-desc">Target executes tasks</div>
+</div>
+
+<div class="pipeline-arrow"><div class="flow-line"></div></div>
+
+<div class="pipeline-stage" id="stage-reflect">
+<div class="stage-icon">🔍</div>
+<div class="stage-label">Reflect</div>
+<div class="stage-desc">Optimizer analyzes trajectories</div>
+</div>
+
+<div class="pipeline-arrow"><div class="flow-line"></div></div>
+
+<div class="pipeline-stage" id="stage-aggregate">
+<div class="stage-icon">🔗</div>
+<div class="stage-label">Aggregate</div>
+<div class="stage-desc">Merge edit patches</div>
+</div>
+
+<div class="pipeline-arrow"><div class="flow-line"></div></div>
+
+<div class="pipeline-stage" id="stage-select">
+<div class="stage-icon">✂️</div>
+<div class="stage-label">Select</div>
+<div class="stage-desc">Rank & clip edits</div>
+</div>
+
+<div class="pipeline-arrow"><div class="flow-line"></div></div>
+
+<div class="pipeline-stage" id="stage-update">
+<div class="stage-icon">📝</div>
+<div class="stage-label">Update</div>
+<div class="stage-desc">Apply to skill doc</div>
+</div>
+
+<div class="pipeline-arrow"><div class="flow-line"></div></div>
+
+<div class="pipeline-stage" id="stage-gate">
+<div class="stage-icon">🚦</div>
+<div class="stage-label">Gate</div>
+<div class="stage-desc">Validate & accept</div>
+</div>
+
+</div>
+
+<div class="pipeline-epoch-bar">
+<div class="epoch-mechanism">🔄 Slow Update</div>
+<div class="epoch-mechanism">🧠 Meta Skill</div>
+<div class="epoch-label">Epoch Boundary</div>
+</div>
+
+</div>
+
+---
+
+## Deep Learning Analogy
+
+SkillOpt brings the familiar deep-learning training paradigm to agentic prompt optimization:
+
+| Deep Learning | SkillOpt | 
+|---|---|
+| Model weights | Skill document (Markdown) |
+| Forward pass | Rollout (target executes tasks) |
+| Loss / gradient | Reflect (optimizer produces edit patches) |
+| Gradient clipping | Edit selection (`learning_rate` = max edits) |
+| SGD step | Patch application to skill |
+| Validation set | Gated evaluation on selection split |
+| LR schedule | `lr_scheduler`: cosine, linear, constant |
+| Epochs | Multi-epoch with slow update & meta skill memory |
+
+---
+
+## Supported Benchmarks
+
+| Benchmark | Type | Config |
+|---|---|---|
+| **DocVQA** | Document QA | `configs/docvqa/` |
+| **ALFWorld** | Embodied AI | `configs/alfworld/` |
+| **OfficeQA** | Enterprise QA | `configs/officeqa/` |
+| **SearchQA** | Open-domain QA | `configs/searchqa/` |
+| **LiveMathematicianBench** | Math reasoning | `configs/livemathematicianbench/` |
+| **SpreadsheetBench** | Spreadsheet editing | `configs/spreadsheetbench/` |
+
+---
+
+## Model Backends
+
+Optimizer and target roles are configured separately. Chat backends include
+Azure OpenAI (`openai_chat`), the provider-neutral
+`openai_compatible` backend, the Claude Code CLI (`claude_chat`), Qwen, and
+MiniMax. Codex and Claude Code exec harnesses are target-only and require
+adapter support. Despite its name, `claude_chat` launches `claude -p`; it is
+not a direct Anthropic API client.
+
+If a provider implements OpenAI Chat Completions, begin with the
+[built-in compatible backend](guide/new-backend.md#built-in-the-generic-openai-compatible-backend)
+instead of adding a new integration. See [Configuration](guide/configuration.md)
+for authentication and per-role overrides.
+
+---
+
+## Quick Example
+
+```bash
+# Clone and install the research checkout plus the SearchQA data extra
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e ".[searchqa]"
+
+# Configure credentials (choose one auth mode in .env)
+cp .env.example .env
+set -a; source .env; set +a
+
+# Materialize the runnable split from the checked-in ID manifest
+python scripts/materialize_searchqa.py
+
+# Train on SearchQA into a predictable output directory
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_quickstart
+
+# Evaluate best skill
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill outputs/searchqa_quickstart/best_skill.md \
+  --split valid_unseen
+```
+
+---
+
+<div class="grid cards" markdown>
+
+-   :material-book-open-variant:{ .lg .middle } **Getting Started**
+
+    ---
+
+    Install SkillOpt, configure your API keys, and run your first experiment in 5 minutes.
+
+    [:octicons-arrow-right-24: Installation](guide/installation.md)
+
+-   :material-puzzle:{ .lg .middle } **Add a Benchmark**
+
+    ---
+
+    Extend SkillOpt with your own benchmark in ~100 lines of code.
+
+    [:octicons-arrow-right-24: Extension Guide](guide/new-benchmark.md)
+
+-   :material-cog:{ .lg .middle } **Configuration**
+
+    ---
+
+    Full reference for all hyperparameters with deep learning analogies.
+
+    [:octicons-arrow-right-24: Config Reference](reference/config.md)
+
+-   :material-monitor-dashboard:{ .lg .middle } **WebUI**
+
+    ---
+
+    Configure, launch, and monitor training from your browser.
+
+    [:octicons-arrow-right-24: WebUI Guide](guide/first-experiment.md#webui)
+
+-   :material-weather-night:{ .lg .middle } **SkillOpt-Sleep**
+
+    ---
+
+    Test the deployment companion with the no-provider mock path, then review
+    its data boundary before selecting a real backend.
+
+    [:octicons-arrow-right-24: Sleep Overview](sleep/README.md)
+
+</div>

docs/reference/api.md

@@ -0,0 +1,198 @@
+# API Reference
+
+This page documents the public Python API SkillOpt exposes for **extending the
+framework** with new environments / benchmarks. For ready-made adapters,
+browse [`skillopt/envs/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs).
+
+> **Source of truth.** The classes below are real Python ABCs defined in
+> `skillopt/envs/base.py`, `skillopt/datasets/base.py`, `skillopt/types.py`,
+> and `skillopt/evaluation/gate.py`. If this page ever drifts, the code
+> wins — please open an issue.
+
+---
+
+## Core Classes
+
+### `EnvAdapter`
+
+`skillopt/envs/base.py` — abstract adapter that connects the SkillOpt
+trainer to an environment (benchmark, simulator, REST API, ...).
+Subclasses **must** implement the four abstract methods below. Reflection has a
+shared default implementation and only needs to be overridden for
+environment-specific behavior.
+
+```python
+from abc import ABC, abstractmethod
+from skillopt.datasets.base import BaseDataLoader, BatchSpec
+
+class EnvAdapter(ABC):
+
+    # ── Lifecycle hooks (have defaults; override only if needed) ────────
+
+    def setup(self, cfg: dict) -> None: ...
+    def get_dataloader(self) -> BaseDataLoader | None: ...
+    def requires_ray(self) -> bool: ...                 # default False
+    def reflect(self, results: list[dict], skill_content: str,
+                out_dir: str, **kwargs) -> list[dict | None]:
+        """Delegate to the shared minibatch reflection pipeline."""
+        ...
+
+    # ── Abstract methods (subclasses MUST implement) ────────────────────
+
+    @abstractmethod
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        """Return an environment-manager object to be passed to rollout()."""
+
+    @abstractmethod
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        """Like build_train_env() but for a fixed eval split."""
+
+    @abstractmethod
+    def rollout(self, env_manager, skill_content: str,
+                out_dir: str, **kwargs) -> list[dict]:
+        """Run a batch of episodes with the current skill.
+
+        Each returned dict MUST contain:
+          - "id":   str        episode/task identifier
+          - "hard": int (0|1)  pass/fail (may be float 0.0-1.0 if smoothed)
+          - "soft": float      partial-credit score in [0.0, 1.0]
+        It MAY contain env-specific extra keys (parsed into RolloutResult.extras).
+        """
+
+    @abstractmethod
+    def get_task_types(self) -> list[str]:
+        """Distinct task-type strings used for stratified sampling."""
+```
+
+The default `reflect()` delegates to `run_minibatch_reflect` and returns raw
+patch dicts with a `patch` payload plus a `failure` or `success` source type.
+It expects each rollout to persist a non-empty trajectory at
+`<rollout_dir>/predictions/<result-id>/conversation.json`; results without that
+file can be scored but are skipped during reflection.
+The trainer also calls several default-implemented helpers on every adapter:
+`build_reference_text`, `get_reference_metadata`, `attach_reference_context`,
+`select_representative_items`, and `build_env_from_batch`. Read the docstrings
+in `skillopt/envs/base.py` if you need to override any of these — most
+benchmarks do not.
+
+### `BaseDataLoader` / `SplitDataLoader`
+
+`skillopt/datasets/base.py` — episode-planning loaders.
+
+```python
+class BaseDataLoader(ABC):
+    def setup(self, cfg: dict) -> None: ...
+    @abstractmethod
+    def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: ...
+    @abstractmethod
+    def build_eval_batch(self, env_num: int, split: str, seed: int, **kwargs) -> BatchSpec: ...
+
+class SplitDataLoader(BaseDataLoader):
+    """Concrete base for dataset-backed envs with on-disk train/val/test splits.
+
+    Subclasses only need to implement load_split_items() (and optionally
+    load_raw_items() if you also want ``split_mode='ratio'``).
+    """
+    def load_split_items(self, split_path: str) -> list[dict]: ...
+    def load_raw_items(self, data_path: str) -> list[dict]: ...   # optional
+```
+
+`SplitDataLoader` handles two layout modes:
+
+| `split_mode` | What it expects |
+|---|---|
+| `"split_dir"` | A directory with `train/`, `val/`, `test/` subdirs already split. |
+| `"ratio"` | A raw dataset path + `split_ratio: "2:1:7"` style string. |
+
+In either case the items returned by `load_split_items()` are plain
+`dict` objects with at minimum an `"id"` key.
+
+### `BatchSpec`
+
+`skillopt/datasets/base.py` — a slotted dataclass describing one batch
+request the trainer hands to the adapter.
+
+```python
+@dataclass(slots=True)
+class BatchSpec:
+    phase: str                 # "train" | "eval"
+    split: str                 # "train" | "val" | "test" | "valid_seen" | ...
+    seed: int
+    batch_size: int
+    payload: object | None = None     # what the loader produced (e.g. list[dict])
+    metadata: dict = field(default_factory=dict)
+```
+
+### `Edit` / `Patch`
+
+`skillopt/types.py` — the I/O types Reflect / Aggregate / Update produce
+and consume.
+
+```python
+EditOp = Literal["append", "insert_after", "replace", "delete"]
+
+@dataclass
+class Edit:
+    op: EditOp
+    content: str = ""
+    target: str = ""
+    support_count: int | None = None
+    source_type: Literal["failure", "success"] | None = None
+    merge_level: int | None = None
+    update_origin: str = ""
+    update_target: str = ""
+
+@dataclass
+class Patch:
+    edits: list[Edit] = field(default_factory=list)
+    reasoning: str = ""
+    ranking_details: dict[str, Any] | None = None
+```
+
+Both types support `to_dict()` / `from_dict()` for serialization.
+
+### `RolloutResult`
+
+`skillopt/types.py` — the normalised rollout return type. The trainer
+calls `RolloutResult.from_dict(...)` on each dict returned from
+`EnvAdapter.rollout()`, so the only **hard** requirement on those dicts is
+the three keys above (`id`, `hard`, `soft`). Extra fields are preserved
+into `RolloutResult.extras`.
+
+### `GateResult` / `GateAction`
+
+`skillopt/evaluation/gate.py` — the validation-gate decision types
+returned for each candidate optimization step, and optionally for a separate
+epoch-end slow-update candidate.
+
+---
+
+## Registering an environment
+
+Environments are not registered via decorators or a `BENCHMARK_REGISTRY`
+dict. The training and standalone-evaluation entry points each keep a lazy
+`_ENV_REGISTRY`, populated by `_register_builtins()` in `scripts/train.py` and
+`scripts/eval_only.py`. Add the environment to both entry points. See
+[Add a New Benchmark](../guide/new-benchmark.md) for the full step-by-step.
+
+---
+
+## Backends (model layer)
+
+The model layer lives under `skillopt.model.*`. Backends are selected
+via `model.optimizer_backend` and `model.target_backend` in the config —
+not via a base class subclass. Supported values (as of this writing):
+
+| Backend | Optimizer? | Target? |
+|---|---|---|
+| `openai_chat` | ✓ | ✓ |
+| `claude_chat` | ✓ | ✓ |
+| `qwen_chat` | ✓ | ✓ |
+| `minimax_chat` | ✓ | ✓ |
+| `openai_compatible` | ✓ | ✓ |
+| `codex_exec` | ✓ | ✓ |
+| `claude_code_exec` | — | ✓ |
+
+See `skillopt/model/backend_config.py` for the live whitelist and
+[`docs/reference/config.md`](./config.md) for the per-backend
+configuration keys.

docs/reference/cli.md

@@ -0,0 +1,137 @@
+# CLI Reference
+
+> **Version note.** This reference tracks `main`. PyPI 0.2.0 does not yet
+> include the generic research `openai_compatible` backend, Sleep handoff,
+> Sleep support for non-Azure OpenAI-compatible endpoints, or the Sleep
+> `--preferences` flag; use a source install from `main` for those features
+> until the next release.
+
+## Training
+
+```bash
+python scripts/train.py --config <config.yaml> [overrides...]
+# Installed equivalent:
+skillopt-train --config <config.yaml> [overrides...]
+```
+
+### Arguments
+
+| Argument | Description |
+|---|---|
+| `--config` | Path to YAML config file (required) |
+| `--cfg-options key=value [...]` | Override structured config parameters |
+
+### Examples
+
+```bash
+# Basic training
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_run
+
+# With overrides
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --cfg-options optimizer.learning_rate=16 optimizer.lr_scheduler=linear
+
+# With custom initial skill
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --cfg-options env.skill_init=skills/my_seed.md
+```
+
+## Evaluation
+
+```bash
+python scripts/eval_only.py --config <config.yaml> --skill <skill.md>
+# Installed equivalent:
+skillopt-eval --config <config.yaml> --skill <skill.md>
+```
+
+### Arguments
+
+| Argument | Description |
+|---|---|
+| `--config` | Path to YAML config file (required) |
+| `--skill` | Path to skill document to evaluate (required) |
+| `--split` | `train`, `valid_seen`, `valid_unseen`, or `all` (default) |
+| `--cfg-options` | One or more `section.key=value` overrides |
+
+### Examples
+
+```bash
+# Evaluate best skill on test set
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill outputs/searchqa_run/best_skill.md \
+  --split valid_unseen
+
+# Evaluate on validation set
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill outputs/searchqa_run/best_skill.md \
+  --split valid_seen
+```
+
+`--skill` consumes the artifact produced by training. Unless `--out_root` is
+set for evaluation, `eval_only.py` creates a separate timestamped
+`outputs/eval_<env>_<model>_<timestamp>/` directory and writes
+`eval_summary.json` there; it does not modify the training run directory.
+
+For the generic OpenAI-compatible research backend, select the role backends
+explicitly:
+
+```bash
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --cfg-options \
+    model.optimizer_backend=openai_compatible \
+    model.target_backend=openai_compatible \
+    model.optimizer=deepseek-chat \
+    model.target=deepseek-chat
+```
+
+## SkillOpt-Sleep
+
+```bash
+skillopt-sleep <action> [options]
+# Equivalent from a source checkout:
+python -m skillopt_sleep <action> [options]
+```
+
+Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and
+`unschedule`. Common options include:
+
+| Argument | Description |
+|---|---|
+| `--project PATH` | Project to evolve (default: current directory) |
+| `--scope invoked\|all` | Harvest this project or all projects |
+| `--source claude\|codex\|auto` | Transcript source |
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | Replay/optimizer backend |
+| `--model NAME` | Backend-specific model override |
+| `--preferences TEXT` | House rules supplied to reflection |
+| `--lookback-hours N` | Initial transcript lookback; `0` scans all history |
+| `--max-sessions N` / `--max-tasks N` | Bound the harvested workload |
+| `--target-skill-path PATH` | Explicit skill document to stage/adopt |
+| `--tasks-file PATH` | Replay a reviewed task JSON file instead of harvesting |
+| `--edit-budget N` | Maximum bounded edits for the night |
+| `--progress` / `--json` | Progress or machine-readable output |
+| `--auto-adopt` | Apply an accepted staged proposal automatically |
+
+Backend-specific setup for compatible endpoints is documented in
+[OpenAI-compatible endpoints for SkillOpt-Sleep](../sleep/openai-compatible-endpoints.md).
+
+## WebUI
+
+```bash
+python -m skillopt_webui.app [--port PORT] [--share]
+```
+
+| Argument | Default | Description |
+|---|---|---|
+| `--port` | 7860 | Port number |
+| `--host` | `0.0.0.0` | Server bind address |
+| `--share` | false | Create public Gradio link |
+
+The default host binds every network interface. Use `--host 127.0.0.1` when
+the dashboard should be reachable only from the local machine.

docs/reference/config.md

@@ -0,0 +1,174 @@
+# Configuration Reference
+
+SkillOpt loads structured YAML, resolves `_base_` inheritance, and flattens
+the result for the trainer. Shipped defaults live in
+`configs/_base_/default.yaml`; benchmark configs override them.
+
+## Model and Backend Selection
+
+Use explicit optimizer and target backends when the two roles differ or when
+selecting the generic OpenAI-compatible backend.
+
+| Backend | Optimizer | Target |
+|---|:---:|:---:|
+| `openai_chat` | ✓ | ✓ |
+| `openai_compatible` | ✓ | ✓ |
+| `claude_chat` | ✓ | ✓ |
+| `qwen_chat` | ✓ | ✓ |
+| `minimax_chat` | ✓ | ✓ |
+| `codex_exec` | ✓ | ✓ |
+| `claude_code_exec` | — | ✓ |
+
+MiniMax currently has one shared deployment. `model.minimax_model` is applied
+when MiniMax is the target; mixed-backend runs cannot independently choose a
+MiniMax optimizer model and a different target model.
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `model.backend` | str | `azure_openai` | Backward-compatible high-level run label |
+| `model.optimizer` | str | `gpt-5.5` | Optimizer deployment/model |
+| `model.target` | str | `gpt-5.5` | Target deployment/model |
+| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends plus `codex_exec` |
+| `model.target_backend` | str | `openai_chat` | Target client path; chat or exec backend |
+| `model.reasoning_effort` | str | `medium` | Shared reasoning effort |
+| `model.rewrite_reasoning_effort` | str | empty | Optional full-rewrite effort override |
+| `model.rewrite_max_completion_tokens` | int | `64000` | Full-rewrite output cap |
+
+### Azure/OpenAI `openai_chat`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `model.azure_openai_endpoint` | empty | Shared Azure resource URL or compatibility-mode base URL |
+| `model.azure_openai_api_version` | `2024-12-01-preview` | Azure API version |
+| `model.azure_openai_api_key` | empty | Key for `api_key` or compatibility auth |
+| `model.azure_openai_auth_mode` | empty | Config value; empty falls back to env, whose default is `azure_cli` |
+| `model.azure_openai_ad_scope` | Azure Cognitive Services scope | AAD token scope |
+| `model.azure_openai_managed_identity_client_id` | empty | Optional user-assigned identity client ID |
+
+Every shared key also has an `optimizer_azure_openai_*` and
+`target_azure_openai_*` form.
+
+### Claude `claude_chat`
+
+`claude_chat` launches an installed, authenticated Claude Code CLI with
+`claude -p`; it does not instantiate an Anthropic API client. The executable
+defaults to `claude` and can be overridden with `CLAUDE_CLI_BIN`.
+`ANTHROPIC_API_KEY` is one authentication option understood by the CLI.
+
+### Qwen, MiniMax, and Exec Backends
+
+| Parameter family | Description |
+|---|---|
+| `model.qwen_chat_*` | Shared `base_url`, `api_key`, `temperature`, `timeout_seconds`, `max_tokens`, and `enable_thinking` |
+| `model.optimizer_qwen_chat_*` / `model.target_qwen_chat_*` | Per-role Qwen overrides |
+| `model.minimax_*` | MiniMax `base_url`, `api_key`, shared `minimax_model`, `temperature`, `max_tokens`, and `enable_thinking`; `minimax_model` applies when MiniMax is the target |
+| `model.codex_exec_*` | Codex path, sandbox, profile, SDK mode, reasoning, network/search, and approval policy |
+| `model.claude_code_exec_*` | Claude path, profile, SDK mode, effort, and thinking-token cap |
+
+## Training (`train`)
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `train.num_epochs` | int | `4` | Training epochs |
+| `train.train_size` | int | `0` | `0` derives the size from the dataset split |
+| `train.steps_per_epoch` | int | derived | Runtime field recomputed from train size, batch size, and accumulation; configured values are overwritten |
+| `train.batch_size` | int | `40` | Tasks sampled per step |
+| `train.accumulation` | int | `1` | Accumulation rounds per step |
+| `train.seed` | int | `42` | Random seed |
+
+## Gradient / Reflection (`gradient`)
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `gradient.minibatch_size` | int | `8` | Reflect minibatch size |
+| `gradient.merge_batch_size` | int | `8` | Patch merge batch size |
+| `gradient.analyst_workers` | int | `16` | Parallel reflection workers |
+| `gradient.max_analyst_rounds` | int | `3` | Maximum analyst rounds |
+| `gradient.failure_only` | bool | `false` | Reflect only on failures |
+
+## Optimizer (`optimizer`)
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `optimizer.learning_rate` | int | `4` | Maximum edit patches per step |
+| `optimizer.min_learning_rate` | int | `2` | Floor for decaying schedules |
+| `optimizer.lr_scheduler` | str | `cosine` | `constant`, `linear`, `cosine`, or `autonomous` |
+| `optimizer.lr_control_mode` | str | `fixed` | `fixed`, `autonomous`, or `none` |
+| `optimizer.skill_update_mode` | str | `patch` | `patch`, `rewrite_from_suggestions`, or `full_rewrite_minibatch` |
+| `optimizer.use_slow_update` | bool | `true` | Epoch-boundary longitudinal update |
+| `optimizer.slow_update_samples` | int | `20` | Longitudinal evaluation samples |
+| `optimizer.slow_update_gate_with_selection` | bool | `false` | Gate slow-update guidance on the selection split |
+| `optimizer.longitudinal_pair_policy` | str | `mixed` | `mixed`, `changed`, or `unchanged` |
+| `optimizer.use_meta_skill` | bool | `true` | Cross-epoch optimizer memory |
+| `optimizer.use_skill_aware_reflection` | bool | `false` | Enable skill-defect vs execution-lapse routing |
+| `optimizer.skill_aware_appendix_source` | str | `both` | `both` or `failure_only` |
+| `optimizer.skill_aware_consolidate_threshold` | int | `0` | Appendix compaction threshold; `0` disables it |
+
+## Evaluation (`evaluation`)
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `evaluation.use_gate` | bool | `true` | Accept only improvements when enabled; `false` records validation but force-accepts each candidate |
+| `evaluation.gate_metric` | str | `hard` | `hard`, `soft`, or `mixed` |
+| `evaluation.gate_mixed_weight` | float | `0.5` | Soft-score weight for `mixed` |
+| `evaluation.use_semantic_density` | bool | `false` | Add the optional instruction-density bonus |
+| `evaluation.semantic_density_weight` | float | `0.05` | Density bonus weight |
+| `evaluation.leading_words` | list/str | built in | Optional custom high-influence words |
+| `evaluation.sel_env_num` | int | `0` | Selection size; `0` uses the full split |
+| `evaluation.test_env_num` | int | `0` | Test size; `0` uses the full split |
+| `evaluation.eval_test` | bool | `true` | Run final test evaluation |
+
+## Environment (`env`)
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `env.name` | str | empty | Benchmark name |
+| `env.skill_init` | str | empty | Initial skill document |
+| `env.split_mode` | str | `ratio` | `ratio` or `split_dir` |
+| `env.split_ratio` | str | benchmark/default | Train:validation:test ratio |
+| `env.split_seed` | int | `42` | Deterministic split seed |
+| `env.split_dir` | str | empty | Materialized train/val/test directory |
+| `env.data_path` | str | empty | Raw data path for ratio mode |
+| `env.split_output_dir` | str | empty | Optional materialized split output |
+| `env.exec_timeout` | int | `120` | Per-task timeout in seconds |
+| `env.out_root` | str | generated by the train/eval CLIs | Output directory |
+
+Benchmark-specific `env` keys are passed through to the adapter.
+
+## Credential Environment Variables
+
+### Azure-family backend
+
+| Variable | Description |
+|---|---|
+| `AZURE_OPENAI_ENDPOINT` | Shared Azure endpoint or compatibility base URL |
+| `AZURE_OPENAI_API_VERSION` | Azure API version |
+| `AZURE_OPENAI_AUTH_MODE` | `api_key`, `azure_cli`, `managed_identity`, or `openai_compatible` |
+| `AZURE_OPENAI_API_KEY` | Key for `api_key` or `openai_compatible` mode |
+| `AZURE_OPENAI_AD_SCOPE` | Optional AAD scope |
+| `AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID` | Optional managed-identity client ID |
+
+Use `OPTIMIZER_AZURE_OPENAI_*` and `TARGET_AZURE_OPENAI_*` for role-specific
+overrides.
+
+### Generic OpenAI-compatible backend
+
+| Variable suffix | Shared / per-role forms |
+|---|---|
+| `BASE_URL` | `OPENAI_COMPATIBLE_BASE_URL`, `OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL`, `TARGET_OPENAI_COMPATIBLE_BASE_URL` |
+| `API_KEY` | Corresponding shared/optimizer/target `*_API_KEY` names |
+| `MODEL` | Corresponding shared/optimizer/target `*_MODEL` names |
+| `TEMPERATURE` | Corresponding shared/optimizer/target `*_TEMPERATURE` names |
+| `MAX_TOKENS` | Corresponding shared/optimizer/target `*_MAX_TOKENS` names |
+| `TIMEOUT_SECONDS` | Corresponding shared/optimizer/target `*_TIMEOUT_SECONDS` names |
+
+The train/eval entry points set deployments from YAML `model.optimizer` and
+`model.target` after backend initialization. For selected OpenAI-compatible or
+Qwen roles, those values override the corresponding `*_MODEL` environment
+variables; the environment model names mainly seed direct library use.
+
+Other backend families use the authenticated Claude CLI (`CLAUDE_CLI_BIN`;
+optionally `ANTHROPIC_API_KEY`), `QWEN_CHAT_*`, and `MINIMAX_*`.
+SkillOpt-Sleep's compatible endpoint uses `AZURE_OPENAI_*`, not the research
+backend's `OPENAI_COMPATIBLE_*`; see
+[the Sleep endpoint guide](../sleep/openai-compatible-endpoints.md).

docs/sleep/README.md

@@ -0,0 +1,132 @@
+# SkillOpt-Sleep 😴 — deployment-time companion (preview)
+
+**SkillOpt-Sleep** applies SkillOpt's discipline to your *own daily usage*. It gives a
+local coding agent a nightly **sleep cycle** that reviews your past sessions, replays
+your recurring tasks on your own API budget, and consolidates what it learns into
+**validated** long-term memory and skills — behind a held-out gate, staged for your
+review. It requires **no weight training** and adds no separate optimization loop to
+normal agent requests.
+
+> **Preview.** This is an early preview we are actively iterating on; interfaces and
+> defaults may change. The engine lives in the top-level [`skillopt_sleep/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt_sleep)
+> package with **zero dependency** on the paper's `skillopt/` code (the validation gate
+> is vendored).
+
+## How it works
+
+One "night":
+
+```
+harvest Claude Code / Codex transcripts → mine recurring tasks → replay offline
+   → consolidate (reflect → bounded edit → GATE on real held-out tasks)
+   → stage proposal → (you) adopt
+```
+
+It synthesizes **SkillOpt** (validation-gated bounded text edits), **Claude Dreams**
+(offline consolidation; review-then-adopt), and the **agent-sleep** idea (short-term
+experience → long-term competence).
+
+> **Data boundary.** Harvesting is local and read-only. The `mock` backend makes no
+> provider calls. A real backend, however, sends truncated excerpts from harvested
+> sessions and derived tasks to the provider you select for mining, replay, judging,
+> and reflection. Outbound prompts are not currently guaranteed to be secret-free;
+> review your transcript source and provider policy before running on sensitive
+> projects. For a reviewable workflow, harvest to a task file, inspect/redact it, mark
+> it `"reviewed": true`, and then replay that file with the real backend.
+
+## How to use it
+
+### Quickest path: the `skillopt-sleep` CLI (pip)
+
+```bash
+pip install skillopt        # installs the engine + the `skillopt-sleep` command
+skillopt-sleep dry-run      # harvest + mine + replay, report only; stages nothing
+skillopt-sleep run          # a full nightly cycle; the proposal is staged for review
+skillopt-sleep status       # show state + the latest staged proposal
+skillopt-sleep adopt        # apply the latest staged proposal
+skillopt-sleep schedule     # install a nightly cron entry for this project
+```
+
+> **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base
+> commands above. Sleep handoff, non-Azure OpenAI-compatible endpoints, and
+> `--preferences` landed later and require a source install from `main` until
+> the next release.
+
+The per-agent integrations below still come from the repo; the CLI above is the
+standalone, pip-only way to run a cycle. Claude Code, Codex, Copilot, and Devin wrap
+the shared engine. OpenClaw is a separate reference adaptation and has its own setup.
+
+One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins)):
+
+| Platform | Folder | Install |
+|---|---|---|
+| **Claude Code** | [`plugins/claude-code`](https://github.com/microsoft/SkillOpt/tree/main/plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/skillopt-sleep` |
+| **Codex** | [`plugins/codex`](https://github.com/microsoft/SkillOpt/tree/main/plugins/codex) | `bash plugins/codex/install.sh` → `skillopt-sleep` skill |
+| **Copilot** | [`plugins/copilot`](https://github.com/microsoft/SkillOpt/tree/main/plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server |
+| **Devin** | [`plugins/devin`](https://github.com/microsoft/SkillOpt/tree/main/plugins/devin) | register `plugins/devin/mcp_server.py` as an MCP server |
+| **OpenClaw** | [`plugins/openclaw`](https://github.com/microsoft/SkillOpt/tree/main/plugins/openclaw) | adapt the reference wrapper and paths for your installation |
+
+To use DeepSeek, vLLM, Ollama, or another Chat Completions server, see
+**[OpenAI-compatible endpoints](openai-compatible-endpoints.md)**. That guide also
+documents the separate HTTPS-only boundary for Azure managed-identity credentials.
+
+Deterministic proof (no API key):
+`python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves`.
+
+### Opt-in: experience replay & dream rollouts
+
+Two consolidation mechanisms, both default **off** (behavior is unchanged unless you
+enable them). They strengthen the nightly update when your tasks have a clean
+correctness signal; the validation gate still governs what ships.
+
+| Config knob | Default | Effect |
+|---|---|---|
+| `dream_rollouts` | `1` | Run each task K times → learn from the good-vs-bad contrast (contrastive reflection). |
+| `recall_k` | `0` | Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream. |
+| `dream_factor` | `0` | Add N lightweight synthetic variants of each task. |
+
+## Results
+
+> 📊 **More results & analysis — the gate-safety stress test, experience-replay
+> scaling, and the dream-diversity ablation — are in
+> [`docs/sleep/RESULTS.md`](RESULTS.md).** The highlights:
+
+**Controlled experiment recipe (not the shipping CLI defaults).** 5 nights × 10 new
+real "today" tasks per night; the full held-out **test** split is scored before night
+1 (baseline) and after night 5 (after); optimizer = GPT-5.5; single seed (42). The
+experiments use the shipped consolidation and gate components, while the nightly CLI
+and benchmark harnesses remain separate entry points. Numbers are absolute held-out
+accuracy; **Δ** = `after − baseline` in percentage points.
+
+**(a) End-to-end on real agents — [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`.**
+Deficient seed skills go **0.00 → 1.00** on the held-out set with **both Claude Code
+and Codex** as the target agent (all 4 seeds, including a real tool-use loop).
+
+**(b) Experience replay scales the gain — SearchQA** (1,400-item held-out test,
+SQuAD exact-match; target = GPT-5.5; **validation-gated**):
+
+| Replay config (`dream_rollouts=5`) | Baseline → After | Δ (pts) |
+|---|---|---|
+| `recall_k=10` | 0.802 → 0.834 | +3.1 |
+| `recall_k=20` | 0.803 → 0.848 | **+4.5** |
+| full-history replay *(reference, not a shipping default)* | 0.796 → 0.851 | +5.6 |
+| `recall_k=10`, `dream_rollouts=8` *(more dreaming, same recall)* | 0.798 → 0.835 | +3.7 |
+
+The gain rises monotonically with how much relevant past experience is recalled. The
+same SearchQA cell **without** the gate (`recall_k=10`) is 0.808 → 0.839 (+3.1).
+
+**(c) Second benchmark — SpreadsheetBench** (280-item held-out test; the agent's
+generated openpyxl code is executed and compared cell-by-cell to a golden workbook;
+target = GPT-5.4-nano; gate-free + the output-contract guardrail): 0.279 → 0.314 (**+3.6**).
+
+**(d) Honest scope.** These gains hold where tasks recur and have a checkable
+correctness signal. On saturated or noisy benchmarks (e.g. a strong model already
+near ceiling) the effect is **flat within run-to-run noise** — single-seed baseline
+variance here is ±1–2 pts, so treat sub-~1.5 pt differences as noise. The validation
+gate keeps the worst case bounded; keep it **on** by default.
+
+## Learn more
+
+See the [SkillOpt documentation index](../index.md), the
+[CLI reference](../reference/cli.md), and the integration-specific READMEs under
+[`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins).

docs/sleep/RESULTS.md

@@ -0,0 +1,191 @@
+# SkillOpt-Sleep — results & analysis
+
+This is the evidence behind SkillOpt-Sleep: does a nightly, offline sleep cycle
+actually make a *deployed* agent better, and is it safe to run unattended? We
+answer with a controlled deployment-scale study built from the same shipped
+consolidation and gate components. Its multi-night benchmark recipe is an
+experiment configuration, not the default configuration of the nightly CLI.
+
+## Setup
+
+**Protocol (identical for every cell unless stated).** 5 nights; each night adds
+**10 new real "today" tasks**; the skill carries over and is refined night to
+night. The full held-out **test** split is scored before night 1 (*baseline*) and
+after night 5 (*after*); **Δ = after − baseline** in percentage points. Optimizer
+model = **GPT-5.5**; single seed (42). The measurements use the shipped replay,
+consolidation, and gate implementations. The nightly CLI and the checked-in
+benchmark convenience harnesses are separate entry points and do not all call one
+shared wrapper function.
+
+**Benchmarks** (real evaluators, not format heuristics):
+
+| Benchmark | Held-out test | Scoring |
+|---|---|---|
+| SearchQA | 1,400 items | SQuAD exact-match vs gold |
+| LiveMathematicianBench | 124 items | multiple-choice label (choices shuffled per item) |
+| SpreadsheetBench | 280 items | the agent's generated openpyxl code is **executed**, output workbook compared cell-by-cell to a golden file |
+
+**Targets:** GPT-5.5, GPT-5.4-mini, GPT-5.4-nano. **Modes:** validation-gated
+(default) and gate-free.
+
+---
+
+## 1. The headline — the validation gate is what makes nightly self-evolution *safe*
+
+Self-evolution is easy to build and easy to ruin: an optimizer that accepts its
+own "lessons" unconditionally can adopt a plausible-but-wrong rule and an obedient
+model will follow it off a cliff. We reproduced exactly that failure, then showed
+the gate prevents it.
+
+Stress case — **GPT-5.4-nano on SearchQA**, weak model on a single-sample (degraded)
+reflection signal, same nights, same candidate edits, gate **off** vs **on**:
+
+| | Night 0 → Night 5 | Δ |
+|---|---|---|
+| **no gate** | 0.554 → **0.026** | **−52.8** |
+| **with gate (default)** | 0.570 → 0.570 | 0.0 |
+
+Ungated, the optimizer learned "answer with the document-title string, verbatim";
+the model complied and accuracy collapsed night after night
+(0.554 → 0.490 → 0.325 → 0.031 → 0.034 → 0.026). The gated twin **rejected every one
+of those edits** and never lost a point. This single experiment is the core
+argument for SkillOpt-Sleep's design, and why the gate ships **on by default**.
+
+---
+
+## 2. Cross-model scaling — bigger gains where there's headroom
+
+The same protocol on a weaker target model (**GPT-5.4-nano**, optimizer = GPT-5.5)
+produces substantially larger gains — because the weaker model has more room to
+learn. This is the realistic "cheap deployed agent, strong overnight optimizer"
+scenario:
+
+| Config (SearchQA, nano, gated) | Baseline → After | Δ | Night-by-night |
+|---|---|---|---|
+| **cumulative replay, nights=5** | 0.560 → **0.679** | **+11.9** | 0.560 → 0.626 → 0.665 → 0.665 → 0.665 → 0.679 |
+| recall_k=20, nights=5 | 0.566 → 0.681 | +11.5 | 0.566 → 0.659 → 0.685 → 0.685 → 0.681 → 0.681 |
+| cumulative, nights=8 | 0.562 → 0.657 | +9.5 | saturates after night 5 |
+
+Both replay strategies (cumulative and recall) agree within 0.4 pt — the gain is
+robust across configurations.
+
+**Compared to GPT-5.5 on the same benchmark (SearchQA, gated):**
+
+| Target model | Best Δ | Baseline | Headroom |
+|---|---|---|---|
+| GPT-5.4-nano | **+11.9** | 0.560 | 44 pt |
+| GPT-5.5 | +6.0 | 0.798 | 20 pt |
+
+The story: **SkillOpt-Sleep helps most where there's the most to learn** — weaker
+deployed models benefit ~2× as much from the same nightly optimization. This is
+also the economical deployment pattern (cheap inference model + one strong
+overnight optimizer call).
+
+---
+
+## 3. Experience replay turns a one-time bump into a climb
+
+The plugin's two opt-in knobs (`recall_k`, `dream_rollouts`) are what produce the
+gains. On **SearchQA, GPT-5.5, gated** — the gain rises monotonically with how
+much relevant past experience is recalled:
+
+| Replay (`dream_rollouts=5`) | Baseline → After | Δ |
+|---|---|---|
+| `recall_k=10` | 0.802 → 0.834 | +3.1 |
+| `recall_k=20` | 0.803 → 0.848 | **+4.5** |
+| full-history (reference, not a default) | 0.796 → 0.851 | +5.6 |
+
+And the curve genuinely **climbs across nights** rather than jumping once and
+plateauing — full-history replay, gated, night by night:
+
+```
+0.798 → 0.814 → 0.854 → 0.854 → 0.854 → 0.858
+```
+
+The gate accepts a new, better skill as late as **night 5** (0.854 → 0.858).
+Replay-policy ablation (SearchQA, GPT-5.5):
+
+| Replay policy | Gate-free Δ | Gated Δ |
+|---|---|---|
+| none (tonight's tasks only) | +3.9 | +2.0 |
+| **recall k=10 (opt-in experiment)** | +5.1 | +4.4 |
+| cumulative (full history) | +4.8 | +6.0 |
+
+Recall captures most of cumulative's benefit at a fraction of the per-night cost.
+
+---
+
+## 4. Sensitivity around the experiment recipe
+
+We swept `dream_factor`, `rollouts`, `per_night`, and `nights` on the nano cell
+(SearchQA, gated) around the study recipe: `dream_factor=2`, `rollouts=5`,
+`per_night=10`, and `nights=5`. These are **experiment values**, not the shipping
+defaults (`dream_factor=0`, `dream_rollouts=1`, and `recall_k=0`):
+
+| Variant | Δ | vs experiment baseline (+11.9) |
+|---|---|---|
+| dream_factor=4 (baseline 2) | +8.8 | −3.1 |
+| rollouts=10 (baseline 5) | +9.5 | −2.4 |
+| per_night=15 (baseline 10) | +2.7 | −9.2 |
+| nights=8 (baseline 5) | +9.5 | −2.4 |
+
+Every tested direction away from that baseline reduced the measured gain in this
+cell. The result supports that particular study recipe; it does not establish a
+universal optimum. Shipping stays conservative, and users must opt in to additional
+dream rollouts or recall after considering task quality and provider cost.
+
+---
+
+## 5. Why these gains exist — the dream-diversity fix (and a rigor note)
+
+Reflection learns from the **contrast** between good and bad rollouts of the same
+task, which requires the K dream rollouts to be *independent samples*. An early
+version of the engine collapsed them to one cached sample, so contrastive
+reflection never fired. Fixing that, then adding recall, is what produces the
+gains in Sections 1–2. Measured across an 18-cell deployment sweep (3 benchmarks ×
+3 targets × 2 modes), under three engine configurations:
+
+| Engine configuration | mean Δ | worst-cell Δ | cells > +0.5 | cells < −0.5 |
+|---|---|---|---|---|
+| single-sample reflection (degraded) | −2.66 | **−52.8** | 7 / 18 | 5 / 18 |
+| diverse rollouts (K=5), no recall | +0.24 | −4.0 | 6 / 18 | 7 / 18 |
+| **diverse rollouts + recall (experiment recipe)** | **+0.53** | **−2.4** | 7 / 18 | 7 / 18 |
+
+The catastrophic −52.8 is removed **at its source** by diverse rollouts: the same
+gate-free nano-SearchQA cell goes 0.554 → **0.586 (+2.7)** with no gate at all once
+the dream is fixed. Recall then lifts the grid mean and tightens the worst case.
+This is **defense in depth, each layer measured**: diverse rollouts propose better
+edits, recall remembers relevant experience, and the gate catches whatever still
+slips through.
+
+---
+
+## 6. End-to-end on real agents
+
+On the public [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`
+benchmark — designed for exactly this learnable-gap setting — deficient seed skills
+go **0.00 → 1.00** on the held-out set with **both Claude Code and Codex** as the
+target agent (all 4 seeds, including a real tool-use loop), and the two agents
+cross-verify each other's consolidated skills.
+
+---
+
+## 7. Honest scope & limitations
+
+- **Where it helps:** recurring tasks with a checkable correctness signal and real
+  headroom. That is the plugin's actual use case (your repeated daily tasks and
+  house rules the agent keeps missing).
+- **Where it's flat:** saturated tasks on strong models, or noisy tasks with a weak
+  learning signal — within run-to-run noise.
+- **Single seed.** Cells aggregate one seed per config; treat sub-~1.5 pt
+  differences as noise. Spot seed-robustness check on the one flagged cell
+  (nano SearchQA gated): seeds 42/43/44 give −1.9 / +3.6 / +4.7 (3-seed mean
+  **+2.1**), i.e. the tabled −1.9 is a pessimistic draw, not the typical outcome.
+- **Keep the gate on.** It is the difference between bounded downside (−2.4) and a
+  −52.8 collapse. Gate-free mode is for users who cannot hold out a validation set
+  and is additionally protected by the output-contract guardrail.
+
+---
+
+Back to the module overview: [`docs/sleep/README.md`](README.md) ·
+documentation index: [SkillOpt documentation](../index.md).

docs/sleep/examples/runner.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Reference launcher for running SkillOpt-Sleep against an OpenAI-compatible
+endpoint (DeepSeek shown here), plus an Antigravity `session-end` hook.
+
+This is a *sanitized example*, not a supported entry point. Adapt the paths and
+provider details to your environment. No API keys are hardcoded — the key is read
+from an .env file or the process environment.
+
+Usage:
+    python runner.py run           # run a full sleep cycle against DeepSeek
+    python runner.py dry-run       # harvest + replay, report only
+    python runner.py session-end   # Antigravity Stop-hook: append rollout evidence
+"""
+import os
+import re
+import sys
+import json
+import subprocess
+import datetime
+from pathlib import Path
+
+# --- Configure these for your environment -----------------------------------
+# Path to a file containing your provider key as `sk-...` (kept out of source).
+PROVIDER_ENV_FILE = Path(os.environ.get("SKILLOPT_PROVIDER_ENV_FILE", "provider.env"))
+# Endpoint + model for the OpenAI-compatible provider.
+PROVIDER_ENDPOINT = os.environ.get("SKILLOPT_PROVIDER_ENDPOINT", "https://api.deepseek.com")
+PROVIDER_MODEL = os.environ.get("SKILLOPT_PROVIDER_MODEL", "deepseek-v4-pro")
+# Project whose SKILL.md files the sleep cycle should evolve.
+PROJECT_DIR = os.environ.get("SKILLOPT_PROJECT_DIR", os.getcwd())
+# Where the session-end hook appends rollout evidence.
+ROLLOUT_LOG = Path(os.environ.get("SKILLOPT_ROLLOUT_LOG", "brain/rollout-evidence.jsonl"))
+# ----------------------------------------------------------------------------
+
+
+def load_provider_key(env: dict) -> None:
+    """Ensure DEEPSEEK_API_KEY is set, reading it from PROVIDER_ENV_FILE if needed."""
+    if env.get("DEEPSEEK_API_KEY"):
+        return
+    try:
+        text = PROVIDER_ENV_FILE.read_text(encoding="utf-8")
+    except OSError:
+        return
+    m = re.search(r"sk-[A-Za-z0-9]+", text)
+    if m:
+        env["DEEPSEEK_API_KEY"] = m.group(0)
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: runner.py [dry-run|run|status|adopt|session-end]")
+        sys.exit(1)
+
+    command = sys.argv[1]
+
+    # Antigravity Stop-hook: enrich future nights with task-outcome metadata.
+    if command == "session-end":
+        ROLLOUT_LOG.parent.mkdir(parents=True, exist_ok=True)
+        outcome = {
+            "timestamp": datetime.datetime.now().isoformat(),
+            "event": "SessionEnd",
+            "metadata": "Appended task outcome metadata",
+        }
+        with open(ROLLOUT_LOG, "a", encoding="utf-8") as f:
+            f.write(json.dumps(outcome) + "\n")
+        print("Rollout evidence metadata appended.")
+        return
+
+    env = os.environ.copy()
+    load_provider_key(env)
+
+    if env.get("DEEPSEEK_API_KEY"):
+        # OpenAI-compatible path — see docs/sleep/openai-compatible-endpoints.md
+        backend = "azure_openai"
+        env["PYTHONIOENCODING"] = "utf-8"
+        env["AZURE_OPENAI_AUTH_MODE"] = "openai_compatible"
+        env["AZURE_OPENAI_ENDPOINT"] = PROVIDER_ENDPOINT
+        env["AZURE_OPENAI_API_KEY"] = env["DEEPSEEK_API_KEY"]
+        # Provider-specific request fields are opt-in, never inferred from the
+        # model name. For DeepSeek reasoning models, enable the thinking channel:
+        env.setdefault("SKILLOPT_SLEEP_CHAT_EXTRA_BODY",
+                       json.dumps({"thinking": {"type": "enabled"}}))
+        env.setdefault("SKILLOPT_SLEEP_COMPAT_MAX_TOKENS", "8192")
+    else:
+        # OPTIONAL, UNVERIFIED fallback: route the `claude` CLI backend through a
+        # local Anthropic-compatible proxy (e.g. LiteLLM) to reach Gemini. There
+        # is no native Gemini backend; this path was not validated. See the doc.
+        backend = "claude"
+        if "ANTHROPIC_API_KEY" not in env and "GEMINI_API_KEY" in env:
+            env["ANTHROPIC_API_KEY"] = env["GEMINI_API_KEY"]
+        env.setdefault("ANTHROPIC_BASE_URL", "http://127.0.0.1:4000")
+
+    args = ["skillopt-sleep", command]
+    if command in ("run", "dry-run"):
+        args = ["skillopt-sleep", command, "--backend", backend,
+                "--model", PROVIDER_MODEL, "--project", PROJECT_DIR]
+
+    print(f"Running: {' '.join(args)}")
+    # Propagate the child's exit code so supervisors (watchdog.py, systemd,
+    # Task Scheduler) see a failed sleep run as a failure, not a success.
+    proc = subprocess.run(args, env=env, check=False)
+    sys.exit(proc.returncode)
+
+
+if __name__ == "__main__":
+    main()

docs/sleep/examples/watchdog.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Minimal supervisor that runs the SkillOpt-Sleep cycle on a fixed interval.
+
+Sanitized example (see docs/sleep/openai-compatible-endpoints.md). On Windows,
+register this under a Scheduled Task so it survives logout; on Linux/macOS a
+systemd timer or cron entry serves the same purpose and is usually preferable to
+a long-lived process.
+"""
+import os
+import sys
+import time
+import subprocess
+import datetime
+import traceback
+
+INTERVAL_SECONDS = int(os.environ.get("SKILLOPT_WATCHDOG_INTERVAL", str(4 * 3600)))
+RUNNER = os.environ.get("SKILLOPT_RUNNER", os.path.join(os.path.dirname(__file__), "runner.py"))
+LOG_FILE = os.environ.get("SKILLOPT_WATCHDOG_LOG", "brain/watchdog.log")
+
+
+def log(msg: str) -> None:
+    os.makedirs(os.path.dirname(LOG_FILE) or ".", exist_ok=True)
+    line = f"[{datetime.datetime.now().isoformat()}] {msg}"
+    with open(LOG_FILE, "a", encoding="utf-8") as f:
+        f.write(line + "\n")
+    print(line)
+
+
+def run_once() -> None:
+    log("Invoking skillopt-sleep run via runner.py...")
+    try:
+        result = subprocess.run([sys.executable, RUNNER, "run"],
+                                capture_output=True, text=True)
+        if result.returncode == 0:
+            log("Successfully completed run.")
+        else:
+            log(f"Run failed (exit {result.returncode}).")
+            log(f"STDERR: {result.stderr}")
+    except Exception as e:
+        log(f"Exception while running skillopt: {e}")
+        log(traceback.format_exc())
+
+
+def main() -> None:
+    log(f"Watchdog started. Interval: {INTERVAL_SECONDS}s.")
+    while True:
+        try:
+            run_once()
+        except Exception as e:
+            log(f"Unexpected error in watchdog loop: {e}")
+        log(f"Sleeping for {INTERVAL_SECONDS}s...")
+        time.sleep(INTERVAL_SECONDS)
+
+
+if __name__ == "__main__":
+    main()

docs/sleep/openai-compatible-endpoints.md

@@ -0,0 +1,173 @@
+# OpenAI-compatible endpoints for SkillOpt-Sleep (DeepSeek, local vLLM, …)
+
+This document describes the `azure_openai` backend in
+`skillopt_sleep/backend.py`, which can drive servers that implement the expected
+OpenAI-compatible Chat Completions request shape — for example DeepSeek's hosted
+API or a self-hosted vLLM/Ollama server — in addition to native Azure OpenAI
+deployments. The included runner is a sanitized unattended-launch example that
+was originally used alongside Antigravity; it is not an Antigravity transcript
+integration.
+
+> **Version requirement.** This capability landed after v0.2.0. Until the next
+> release, install SkillOpt from the latest `main`; the current PyPI 0.2.0
+> package does not provide this compatible-endpoint path.
+
+## What changed
+
+All changes are backward-compatible — the default managed-identity Azure path
+is unchanged:
+
+1. **CLI acceptance.** `skillopt-sleep run --backend azure_openai` is now an
+   accepted choice in `skillopt_sleep/__main__.py` (it was previously rejected
+   by argparse even though `get_backend()` understood the name).
+
+2. **Endpoint resolution honors `AZURE_OPENAI_ENDPOINT`.**
+   `AzureOpenAIBackend.__init__` resolves the endpoint as `explicit arg` →
+   `AZURE_OPENAI_ENDPOINT` env → the built-in `_AZURE_ENDPOINTS` table.
+   Previously a non-Azure endpoint could not be supplied at all.
+
+3. **`openai_compatible` auth mode.** When
+   `AZURE_OPENAI_AUTH_MODE=openai_compatible` (also accepts `compat`/`openai`),
+   `_get_client()` builds a plain `openai.OpenAI(base_url=…)` client with
+   `AZURE_OPENAI_API_KEY` instead of an `AzureOpenAI` client. This mirrors the
+   auth mode already supported by the sibling `skillopt/model/azure_openai.py`
+   module. (The `AzureOpenAI` client rewrites request URLs with Azure-only
+   `?api-version=…` query params and deployment path segments, which non-Azure
+   servers reject with `404 Resource not found` — the sleep cycle then scores
+   every rollout `0.0` with no diagnostic.)
+
+4. **Managed-identity credential guard.** The managed-identity path attaches an
+   Azure AD bearer token to every request. It therefore accepts only an **HTTPS**
+   endpoint whose hostname ends in `*.openai.azure.com` or
+   `*.cognitiveservices.azure.com`. An HTTP endpoint — even one with an
+   Azure-looking hostname — and any host outside those suffixes are rejected
+   before a credential-bearing client is created.
+
+5. **Provider-neutral request shape.** In compat mode the backend sends only the
+   standard OpenAI-compatible contract (`model`, `messages`, `max_tokens`).
+   Provider-specific request fields are **opt-in** via environment variables
+   (below) and are attached only in compat mode — nothing is inferred from
+   model-name substrings, and the native Azure request remains unchanged.
+
+6. **Reliable error state.** `_call()` records the last exception in
+   `self.last_call_error` (surfaced in `diagnostics.json`), clears it when a
+   retry recovers, and sets an explicit `"empty response on all N attempts"`
+   diagnostic when every attempt returns empty text.
+
+## Configuration reference
+
+SkillOpt-Sleep's `azure_openai` backend reads these environment variables
+(unprefixed only — the `OPTIMIZER_*`/`TARGET_*` dual-role variables belong to
+the separate `skillopt.model.azure_openai` module and are **not** used by the
+sleep cycle):
+
+| Variable | Meaning |
+|---|---|
+| `AZURE_OPENAI_AUTH_MODE` | `openai_compatible` (or `compat`/`openai`) selects the plain OpenAI client. Unset/other = Azure managed identity (default). |
+| `AZURE_OPENAI_ENDPOINT` | Base URL of the server, e.g. `https://api.deepseek.com`. Azure managed identity requires HTTPS plus an approved Azure hostname. |
+| `AZURE_OPENAI_API_KEY` | API key sent by the compat client to the configured base URL. |
+| `SKILLOPT_SLEEP_COMPAT_MAX_TOKENS` | Optional int (default `8192`): `max_tokens` sent in compat mode. |
+| `SKILLOPT_SLEEP_CHAT_EXTRA_BODY` | Optional JSON object passed as `extra_body` for provider-specific fields in compat mode only. It is ignored in native Azure mode. |
+
+## Data and transport boundaries
+
+- Harvesting reads local transcripts without modifying them, and the `mock`
+  backend makes no provider calls. A real backend sends **truncated transcript
+  excerpts and derived task content** to the selected provider for mining,
+  replay, judging, and reflection.
+- Outbound prompts are not currently guaranteed to be free of secrets. Review
+  the provider's data policy and avoid a third-party endpoint for sensitive
+  transcripts unless you have first inspected and redacted the task material.
+  One reviewable path is `skillopt-sleep harvest --output tasks.json`, followed
+  by a reviewed `--tasks-file` run.
+- Use HTTPS for every remote compatible provider. Plain HTTP is appropriate only
+  for an explicitly trusted loopback development server such as
+  `http://127.0.0.1:8000/v1`; the compat client sends its API key to the configured
+  URL.
+- Azure managed-identity credentials have the stricter invariant described
+  above: HTTPS **and** an approved Azure hostname are both mandatory.
+
+## How to use it
+
+```bash
+export AZURE_OPENAI_AUTH_MODE=openai_compatible
+export AZURE_OPENAI_ENDPOINT=https://api.deepseek.com   # DeepSeek base URL
+export AZURE_OPENAI_API_KEY=sk-...                       # your provider key
+
+# DeepSeek reasoning models: enable the thinking channel (opt-in, not inferred)
+export SKILLOPT_SLEEP_CHAT_EXTRA_BODY='{"thinking": {"type": "enabled"}}'
+export SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=8192
+
+skillopt-sleep run \
+  --backend azure_openai \
+  --model deepseek-v4-pro \
+  --project /path/to/your/project
+```
+
+The same pattern works for a server that implements this Chat Completions
+contract: point `AZURE_OPENAI_ENDPOINT` at the provider-specific base URL, set a
+matching `--model`, and omit `SKILLOPT_SLEEP_CHAT_EXTRA_BODY` unless the provider
+needs extra request fields. Self-hosted vLLM and Ollama commonly use a `/v1` base
+path, for example `http://127.0.0.1:8000/v1` or
+`http://127.0.0.1:11434/v1`.
+
+`--project` selects the project/transcript scope and the project `CLAUDE.md`; it
+does **not** by itself select an arbitrary project `SKILL.md`. Pass
+`--target-skill-path path/to/SKILL.md` when a specific skill is the optimization
+target. Without that flag, SkillOpt-Sleep uses its configured managed skill.
+
+## Unattended runner example (originally used with Antigravity)
+
+The [`examples/`](https://github.com/microsoft/SkillOpt/tree/main/docs/sleep/examples) directory contains a sanitized reference for running
+the compatible backend unattended:
+
+- **`examples/runner.py`** — a thin launcher that loads a provider key from an
+  `.env` file, exports the variables above, invokes `skillopt-sleep run` with
+  the DeepSeek backend, and **exits with the child's return code** so
+  supervisors see failures as failures. Its `session-end` action writes a small
+  local rollout-evidence event as an example hook target.
+- **`examples/watchdog.py`** — a minimal supervisor loop that invokes the runner
+  on a fixed interval (e.g. every 4 hours) and logs non-zero exits as failures.
+  On Windows this is registered as a Scheduled Task so it survives logout; on
+  Linux/macOS a `systemd` timer or cron entry serves the same role.
+
+The current engine does **not** read `brain/rollout-evidence.jsonl`, and it does
+not harvest Antigravity transcripts. That hook output is illustrative metadata,
+not additional training evidence. A real run must use a supported Claude
+Code/Codex transcript source or a reviewed task file converted by the operator.
+
+### Contributor-reported validation
+
+The contributor reported the following results from a private Windows 11 setup
+driving the cycle against `deepseek-v4-pro` in `openai_compatible` mode. They are
+useful integration evidence, but the private session set is not a reproducible
+benchmark bundled with this repository:
+
+- A direct backend smoke test returns a live completion (no `404`,
+  `last_call_error` empty, client type `OpenAI`).
+- A full nightly cycle using the configured session source moved the held-out
+  validation gate from `0.250 → 1.000`, **accepting** a DeepSeek-authored
+  skill edit (`accept_new_best`). `diagnostics.json` for that night reports
+  `"backend": "azure_openai"` with a non-empty token count and an empty
+  `call_error` — i.e. a genuine optimization night, versus the prior all-`0.0`
+  nights that the endpoint bug produced.
+- A subsequent unattended night triggered by the watchdog completed the full
+  chain (watchdog → runner → `skillopt-sleep` → DeepSeek) and the gate correctly
+  **rejected** a non-improving proposal (`0.3 → 0.3`), confirming the validation
+  gate behaves normally on the new backend.
+
+Deterministic no-network coverage for the new behavior lives in
+`tests/test_azure_openai_compat.py` (CLI acceptance, client selection,
+endpoint/auth guard, request kwargs, retry error-state, empty-response
+diagnostics, and runner exit-code propagation).
+
+## Unsupported Gemini proxy branch in the example
+
+`examples/runner.py` still contains an illustrative branch that routes the
+**`claude` CLI backend** through a loopback Anthropic-compatible proxy such as
+[LiteLLM](https://github.com/BerriAI/litellm). It is not a native Gemini backend,
+has no validated model mapping in this example, and is not part of the supported
+path documented here. The sample currently enters that branch whenever no
+DeepSeek key is found, so a production adaptation should remove it or replace it
+with an explicit opt-in, a separately configured model, and a trusted isolated
+loopback proxy. Do not treat this branch as tested Gemini support.

docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md

@@ -0,0 +1,244 @@
+# SkillOpt Sleep — Claude Code self-evolving plugin (design)
+
+> **Historical design proposal.** This document records the June 2026 design
+> target and includes planned controls that are not part of the current nightly
+> CLI. It is not an installation or configuration reference. For implemented
+> behavior, flags, defaults, and data boundaries, use
+> [`docs/sleep/README.md`](../../sleep/README.md),
+> [`docs/reference/cli.md`](../../reference/cli.md), and
+> [`plugins/README.md`](https://github.com/microsoft/SkillOpt/blob/main/plugins/README.md).
+
+**Status:** approved-for-build (autonomous offline session, 2026-06-07)
+**Author:** generated for Yifan Yang, executed autonomously while user is asleep
+**Branch:** `feat/claude-code-sleep-plugin` (worktree `my_repo/SkillOpt-sleep`)
+
+---
+
+## 1. One-paragraph summary
+
+`skillopt-sleep` is a Claude Code plugin that gives a user's local Claude
+agent a nightly **sleep cycle**. While the user is offline, it (1) **harvests**
+the day's real Claude Code session transcripts from `~/.claude`, (2) **mines**
+them into discrete *task records* with checkable outcomes, (3) **replays /
+"dreams"** those tasks offline using the user's own API budget, and (4) runs
+the **SkillOpt optimizer loop** (reflect → bounded edit → held-out gate) to
+consolidate short-term experience into long-term **memory** (`CLAUDE.md`) and
+**skills** (`SKILL.md`). Only changes that pass a validation gate are kept, and
+every change is written to a **review staging area** the user approves before it
+touches live config — mirroring Claude Dream's "input store is never modified"
+safety contract. The result: an agent that measurably gets better at *this
+user's* recurring work, every night, with zero model-weight training.
+
+## 2. Why this is the right synthesis of the three ingredients
+
+| Ingredient | What we take from it | Where it lives in this design |
+|---|---|---|
+| **SkillOpt** (your paper/code) | Skill = trainable text state; bounded add/delete/replace edits under a textual learning rate; **held-out validation gate**; rejected-edit buffer; epoch-wise slow/meta update. | The `consolidate` stage *is* a single SkillOpt epoch, reusing `skillopt.optimizer.*` and `skillopt.evaluation.gate`. |
+| **Claude Dreams** | Async offline job: read a memory store + 1–100 session transcripts → emit a **new, separate** reorganized memory store (dedup / merge / resolve contradictions / surface insights). Input never mutated; output reviewed then adopted or discarded. | The `harvest` + `consolidate-memory` stages and the **staging/adopt** safety model are modeled directly on Dreams. |
+| **Agent Sleep paper** (2605.26099) | Agents need periodic offline consolidation: short-term experience buffer → synthetic replay/self-generated data → self-update; "sleep" turns episodes into durable competence. | The whole nightly schedule, the `replay` step, and the short-term→long-term framing. |
+
+The key novel claim this enables for the project (and a future paper section):
+**SkillOpt's validation-gated bounded-edit optimizer is the missing "safe
+update rule" for Dream-style memory consolidation.** Dreams reorganize memory
+but don't *prove* the reorganization helps; the Sleep paper consolidates but
+assumes weight updates. SkillOpt-Sleep consolidates **text** (memory + skills)
+and **gates each change on replayed task performance**, so nightly evolution is
+both weight-free and regression-protected.
+
+## 3. Goals / non-goals
+
+**Goals**
+1. A working Claude Code plugin: scheduled (nightly/cron) **and** user-triggered (`/sleep`).
+2. Look back over the user's real past prompts & trajectories from local `~/.claude` records.
+3. Offline "dream training": re-run mined tasks (mock-env or fresh retry) on the user's budget.
+4. Continuous evolution of **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`) via the SkillOpt gate.
+5. A reproducible experiment that answers: *does the nightly loop actually improve a held-out score?*
+6. Safety: never silently overwrite user config; stage → user approves → adopt.
+
+**Non-goals (now)**
+- Codex version (explicitly deferred by user; architecture keeps it pluggable).
+- Anthropic managed Dreams API integration (we *emulate* Dreams locally; managed API is a future backend).
+- Model fine-tuning / weight updates (out of scope by design — text-only).
+- Fully unattended auto-adopt by default (opt-in; default is review-gated).
+
+## 4. The local data we read (verified on this machine)
+
+- **Prompt history:** `~/.claude/history.jsonl` — one JSON/line: `{display, pastedContents, timestamp, project}`. The cross-session list of every prompt the user typed, with project path + epoch-ms timestamp.
+- **Full transcripts:** `~/.claude/projects/<path-slug>/<sessionId>.jsonl` — one record/line. Record `type` ∈ {`user`,`assistant`,`mode`,`permission-mode`,`attachment`,`file-history-snapshot`,`last-prompt`,…}. User/assistant records carry `message` (role+content blocks), plus `cwd`, `gitBranch`, `timestamp`, `sessionId`, `version`, `userType`. ~215k transcripts present on this box.
+- **Deployment targets we may evolve:**
+  - Project memory: `<project>/CLAUDE.md` (and `~/.claude/CLAUDE.md` global).
+  - User skills: `~/.claude/skills/<name>/SKILL.md` (frontmatter: `name`, `description`, optional `allowed-tools`, `argument-hint`).
+  - Plugin skills under `~/.claude/plugins/...`.
+
+Everything stays **on-disk and local**; the only network calls are the LLM
+optimizer/replay calls the user already pays for.
+
+## 5. Architecture
+
+### 5.1 The nightly Sleep Cycle (stages)
+
+```
+            ┌────────────────────────── SLEEP CYCLE (one "night") ──────────────────────────┐
+            │                                                                                │
+ trigger →  │  1.HARVEST     2.MINE          3.REPLAY            4.CONSOLIDATE      5.STAGE   │ → wake report
+ (cron or   │  read ~/.claude scan sessions  re-run tasks        SkillOpt epoch:   write to   │
+  /sleep)   │  transcripts → → task records   offline (mock or    reflect→edit→     .skillopt-│
+            │  + history     w/ outcomes &    fresh retry) under  GATE on held-out  sleep/    │
+            │                checkable refs   current skill/mem    replay split      staging/ │
+            │                                                                          ↓      │
+            │                                              6.ADOPT (opt-in / user-approved)   │
+            └────────────────────────────────────────────────────────────────────────────────┘
+```
+
+**1. Harvest** (`harvest.py`)
+Read `history.jsonl` + per-project transcript JSONLs for a time window
+(default: since last sleep, fallback last 24–72h). Group by project (`cwd` /
+`project`). Emit normalized `SessionDigest` objects: ordered user prompts,
+assistant final texts, tool-call summary, files touched (from
+`file-history-snapshot`), git branch, errors seen, and **user-feedback signals**
+(e.g. "still broken", "that's wrong", "perfect", re-asks of the same thing).
+
+**2. Mine** (`mine.py`)
+Turn digests into `TaskRecord`s — the unit the optimizer trains on. A task is a
+self-contained intent (the user's request) plus an *outcome label* and, where
+possible, a **checkable reference**:
+- *Explicit success/failure* from feedback signals ("works now" after N retries → the early attempts are failures, the fix is the success exemplar).
+- *Self-consistency check*: re-derivable answers (math, lookups) get a reference; open-ended ones get an LLM-judge rubric instead.
+- Each TaskRecord: `{id, project, intent, context_excerpt, attempted_solution, outcome ∈ {success,fail,mixed}, reference_kind ∈ {exact, rubric, none}, reference, tags}`.
+Mining is itself an LLM call (the **miner**), prompt-tunable, with a deterministic regex/heuristic fallback for offline/no-key runs.
+
+**3. Replay / "Dream"** (`replay.py`)
+For mined tasks, re-run the intent **offline** under the *current* skill+memory
+to get a fresh trajectory & score. Two modes:
+- `mock` (default, safe): reconstruct a sandboxed prompt from the task's captured context (no live repo mutation, no network side effects) and run the target model. Deterministic, cheap, safe to run unattended.
+- `fresh` (opt-in): actually re-attempt in a throwaway git worktree of the project. Higher fidelity, heavier, never touches the user's working tree.
+Scoring: exact-match / substring for `exact` refs; LLM-judge (0–1) for `rubric` refs; this yields the `hard`/`soft` scores SkillOpt already expects.
+
+**4. Consolidate** (`consolidate.py`) — *this is one SkillOpt epoch*
+Reuse the existing optimizer pieces rather than reinventing:
+- `reflect`: partition replayed tasks into failure/success minibatches → propose add/delete/replace edits to **skill** and a parallel proposer for **memory** (`CLAUDE.md`). (Memory consolidation also does Dream-style dedup/merge/contradiction-resolution over existing `CLAUDE.md` lines.)
+- `aggregate` + `rank_and_select` under an **edit budget** (textual learning rate).
+- `apply_patch_with_report` → candidate skill / candidate memory.
+- **GATE** (`skillopt.evaluation.gate.evaluate_gate`): replay a *held-out* slice of tasks with the candidate; accept only if it strictly beats current. Rejected edits go to the rejected-edit buffer (negative feedback) exactly as in the paper.
+- A **slow/meta** pass across nights (not just within one night) carries durable, cross-session lessons — the literal "short-term experience → long-term knowledge" of the Sleep paper. Per-night state persists in `~/.skillopt-sleep/state.json`.
+
+**5. Stage** (`staging/`)
+Write `proposed_CLAUDE.md`, `proposed_SKILL.md`, a unified diff, and a
+`sleep_report.md` (what changed, why, gate deltas, token cost) into
+`<project>/.skillopt-sleep/staging/<date>/`. **Nothing live is modified.**
+
+**6. Adopt**
+`/sleep adopt` (or `auto_adopt: true` in config for power users) copies staged
+files over the live `CLAUDE.md` / `SKILL.md`, after a `git`-style backup. This
+is the only stage that mutates user-facing config, and it is explicit by default
+— the Dreams "review the output, then adopt or discard" contract.
+
+### 5.2 Components & boundaries (each independently testable)
+
+```
+skillopt/sleep/
+  __init__.py
+  types.py         # SessionDigest, TaskRecord, ReplayResult, SleepConfig, SleepReport (dataclasses)
+  harvest.py       # ~/.claude transcripts + history.jsonl  ->  list[SessionDigest]
+  mine.py          # list[SessionDigest]  ->  list[TaskRecord]   (LLM miner + heuristic fallback)
+  replay.py        # TaskRecord + skill + memory  ->  ReplayResult (hard/soft)   (mock | fresh)
+  consolidate.py   # ReplayResults -> candidate skill+memory -> GATE -> accepted artifacts
+  memory.py        # CLAUDE.md read/merge/dedup/diff (Dream-style) + protected-region markers
+  state.py         # ~/.skillopt-sleep/state.json: last_sleep, night counter, slow/meta memory
+  staging.py       # write/adopt staging dir, backups
+  cli.py           # `python -m skillopt.sleep {run|status|adopt|harvest|dry-run}`
+  config.py        # SleepConfig load/merge (defaults + ~/.skillopt-sleep/config.yaml)
+  optimizer_backend.py  # thin: route reflect/judge to a chosen backend; mock backend for tests
+
+skillopt-sleep-plugin/            # the Claude Code plugin surface
+  .claude-plugin/plugin.json
+  commands/sleep.md               # /sleep [run|status|adopt|dry-run]
+  commands/sleep-status.md
+  skills/skillopt-sleep/SKILL.md  # so Claude knows how to drive the engine
+  hooks/hooks.json                # optional: schedule + on-session-end harvest
+  scripts/*                       # shims that call `python -m skillopt.sleep ...`
+```
+
+**Reuse, don't fork:** `consolidate.py` calls into existing
+`skillopt.optimizer.clip.rank_and_select`, `skillopt.gradient.aggregate.merge_patches`,
+`skillopt.optimizer.skill.apply_patch_with_report`, and
+`skillopt.evaluation.gate.evaluate_gate`. The sleep layer is an **EnvAdapter-shaped
+shim** over the user's own life, not a new optimizer.
+
+### 5.3 Data flow (one task, end to end)
+
+```
+history.jsonl + <session>.jsonl
+   └─harvest→ SessionDigest{prompts, finals, tools, feedback}
+        └─mine→ TaskRecord{intent, attempted, outcome, reference}
+             └─replay(current skill+mem)→ ReplayResult{hard, soft, trajectory}
+                  └─reflect→ edits(skill), edits(memory)
+                       └─rank/clip(edit_budget)→ candidate
+                            └─GATE(replay held-out)→ accept? → staging/  → (adopt) live CLAUDE.md/SKILL.md
+```
+
+## 6. Scheduling & triggering
+
+- **Cron/scheduled:** documented `crontab` line + an optional Claude Code hook; default `0 3 * * *` (3am local; pick an off-:00 minute in practice). The engine is a plain CLI so it works under cron, systemd-timer, or the Claude Code scheduler.
+- **User-triggered:** `/sleep run` (full cycle), `/sleep dry-run` (harvest+mine+replay, no edits), `/sleep status`, `/sleep adopt`.
+- **On-session-end harvest (optional hook):** cheaply append the just-finished session to the night's buffer so the 3am run has fresh data without a full rescan.
+
+## 7. Safety model (hard requirements)
+
+1. **Never mutate live `CLAUDE.md`/`SKILL.md` except via explicit `adopt`** (or opt-in `auto_adopt`). Default = staged + reviewed (Dreams contract).
+2. **Backups:** every adopt snapshots the prior file to `staging/<date>/backup/`.
+3. **Read-only harvest:** transcripts are read, never written.
+4. **`fresh` replay runs only in throwaway worktrees**, never the user's checkout; no `rm -rf`, no force-push, network off unless `replay.network: true`.
+5. **Budget cap:** `max_tokens_per_night` + `max_tasks_per_night`; stop early when hit, log what was skipped (no silent truncation).
+6. **Secret hygiene:** redact obvious secrets from digests before they enter prompts (reuse `_redact_*` ideas from trainer).
+7. **PII/scope:** only harvest projects on an allowlist (default: the project the plugin is invoked in) or `projects: all` opt-in.
+
+## 8. Validation experiment — "does it actually improve?"
+
+A self-contained, **deterministic-by-default** experiment lives in
+`skillopt/sleep/experiments/` and is the acceptance test for the whole idea.
+
+**Setup:** a synthetic "user persona" (e.g. *researcher who keeps asking for
+arXiv-id extraction in a fixed format*, or *programmer who keeps mis-formatting
+git commit messages*). We ship 12–20 tiny tasks with **exact checkable
+references**, split into `replay` (train) and `holdout` (test).
+
+**Procedure:**
+1. Score the holdout with an **empty** skill+memory → `baseline`.
+2. Run `N` sleep nights (each: replay train slice → reflect → gated edit).
+3. Score holdout with the evolved skill+memory → `after`.
+4. Report `after − baseline`, accept/reject counts, edit count, tokens.
+
+**Two backends:**
+- `mock` (default, **no API key, fully deterministic**): a scripted optimizer that proposes the known-good rule on failure and a scripted judge. Proves the *plumbing* (harvest→mine→replay→gate→adopt) monotonically improves the score and the gate blocks regressions. This is the CI-able acceptance test.
+- `anthropic` (opt-in, uses `ANTHROPIC_API_KEY`): the real optimizer/judge, to demonstrate genuine lift on the persona tasks.
+
+**Success criteria:**
+- Mock: `after > baseline`, gate rejects an injected harmful edit, adopt+backup works, re-run is reproducible. (Hard gate in CI.)
+- Anthropic (when run): `after ≥ baseline` on holdout with ≥1 accepted, human-readable edit; documented in the wake-up report.
+
+## 9. Personas (the user's framing) → concrete recurring-task families
+
+- **Programmer:** commit-message conventions, repo-specific build/test commands, "always run X before Y", framework gotchas → consolidated into project `CLAUDE.md` + a `repo-workflow` skill.
+- **Researcher:** citation/format preferences, experiment-logging habits, paper-section style, dataset-path memory → `research-prefs` skill + memory.
+- **Finance/analyst:** report formatting, recurring data-pull recipes, terminology → `report-style` skill + memory.
+The engine is domain-agnostic; the persona only changes which tasks get mined.
+
+## 10. Phased delivery
+
+- **Phase 0 — scaffold + types + harvest** (read-only, no API). Provable on this box's real `~/.claude`.
+- **Phase 1 — mine + replay(mock) + consolidate + gate + staging**, with the **mock** optimizer backend and the deterministic experiment green. *(primary deliverable of the offline session)*
+- **Phase 2 — plugin surface** (`/sleep`, skill, hooks, plugin.json) wired to the CLI.
+- **Phase 3 — real Anthropic backend** for miner/reflect/judge + `fresh` replay in worktrees.
+- **Phase 4 — slow/meta cross-night memory**, adopt automation, multi-project, polish + docs.
+
+This session targets **Phase 0 + Phase 1 fully**, **Phase 2 scaffolded**, and the
+**deterministic experiment passing**, all committed (not pushed) for review.
+
+## 11. Open questions for the user (answer when awake)
+
+1. **Adopt policy:** keep default *review-gated*, or do you want `auto_adopt` for your own machine?
+2. **Scope:** harvest only the invoked project, or all projects in `~/.claude/projects`?
+3. **Real-API demo:** want me to spend live `ANTHROPIC_API_KEY` budget on the persona demo, or keep everything mock until you say go?
+4. **Skill target:** evolve a *new* dedicated `skillopt-sleep`-managed skill, or also edit your existing hand-written skills in `~/.claude/skills`?
+5. **Paper:** should this become a section/figure in the SkillOpt arXiv (Dream+Sleep framing as "deployment-time continual skill optimization")?

mkdocs.yml

@@ -0,0 +1,83 @@
+site_name: SkillOpt Documentation
+site_url: https://microsoft.github.io/SkillOpt
+site_description: "SkillOpt: Agentic Skill Optimization via Reflective Training Loops"
+repo_url: https://github.com/microsoft/SkillOpt
+repo_name: microsoft/SkillOpt
+
+theme:
+  name: material
+  palette:
+    - scheme: default
+      primary: indigo
+      accent: deep purple
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    - scheme: slate
+      primary: indigo
+      accent: deep purple
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+  features:
+    - navigation.instant
+    - navigation.tracking
+    - navigation.sections
+    - navigation.expand
+    - navigation.top
+    - content.code.copy
+    - content.tabs.link
+    - search.suggest
+    - search.highlight
+  icon:
+    repo: fontawesome/brands/github
+  font:
+    text: Inter
+    code: JetBrains Mono
+
+
+
+nav:
+  - Home: index.md
+  - Technical Blog: https://microsoft.github.io/SkillOpt/blog/
+  - Getting Started:
+    - Installation: guide/installation.md
+    - First Experiment: guide/first-experiment.md
+    - Configuration: guide/configuration.md
+  - Core Concepts:
+    - Training Loop: guide/training-loop.md
+    - Skill Document: guide/skill-document.md
+    - Deep Learning Analogy: guide/dl-analogy.md
+  - SkillOpt-Sleep:
+    - Overview: sleep/README.md
+    - OpenAI-compatible Endpoints: sleep/openai-compatible-endpoints.md
+    - Results: sleep/RESULTS.md
+  - Extension Guides:
+    - Add a New Benchmark: guide/new-benchmark.md
+    - Local Environment Smoke Tests: guide/local-env-smoke.md
+    - Add a New Model Backend: guide/new-backend.md
+  - Reference:
+    - Configuration Reference: reference/config.md
+    - CLI Reference: reference/cli.md
+    - API Reference: reference/api.md
+  - Contributing: contributing.md
+
+markdown_extensions:
+  - admonition
+  - pymdownx.details
+  - pymdownx.superfences
+  - pymdownx.tabbed:
+      alternate_style: true
+  - pymdownx.highlight:
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+  - attr_list
+  - md_in_html
+  - toc:
+      permalink: true
+
+plugins:
+  - search

plugins/README.md

@@ -0,0 +1,178 @@
+# SkillOpt-Sleep integrations
+
+**SkillOpt-Sleep** reviews recent agent sessions, mines recurring tasks, replays
+them, and proposes bounded updates to memory and skills. A held-out validation
+gate decides whether a proposal is worth staging, and nothing live changes until
+the user explicitly adopts it.
+
+The shared engine lives in [`skillopt_sleep/`](../skillopt_sleep) and has no
+runtime dependency on the paper's `skillopt/` experiment package.
+
+## Available integrations
+
+Four integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate
+reference adaptation with its own backend and setup assumptions.
+
+| Platform | Folder | Mechanism | Status |
+|---|---|---|---|
+| **Claude Code** | [`claude-code/`](claude-code) | marketplace plugin, commands, skill, and hooks | installable shared-engine integration |
+| **Codex** | [`codex/`](codex) | user-level skill and shared runner | installable shared-engine integration |
+| **GitHub Copilot** | [`copilot/`](copilot) | MCP server exposing seven `sleep_*` tools | shared-engine MCP integration |
+| **Devin** | [`devin/`](devin) | MCP server plus Devin transcript conversion | shared-engine MCP integration |
+| **OpenClaw** | [`openclaw/`](openclaw) | custom DeepSeek/Ollama wrapper | independent reference adaptation; review and adapt before use |
+
+## Install
+
+Clone the repository first unless an installed `skillopt-sleep` CLI is sufficient
+for your workflow.
+
+| Platform | Install | Then |
+|---|---|---|
+| **Claude Code** | from the repository root, `/plugin marketplace add ./plugins/claude-code`, then `/plugin install skillopt-sleep@skillopt-sleep` | `/skillopt-sleep status` |
+| **Codex** | `bash plugins/codex/install.sh` | ask Codex to use the `skillopt-sleep` skill |
+| **Copilot** | register `plugins/copilot/mcp_server.py` using its example MCP config | ask Copilot to run `sleep_status` |
+| **Devin** | register `plugins/devin/mcp_server.py` using its example MCP config | ask Devin to run `sleep_status` |
+| **OpenClaw** | follow and adapt [`openclaw/README.md`](openclaw/README.md) | validate paths, credentials, and tasks locally |
+
+Python 3.10 or newer is required. Real CLI backends also require the selected
+agent CLI to be installed and authenticated.
+
+The shared [`run-sleep.sh`](run-sleep.sh) supports both source checkouts and
+installed packages. If it cannot find the repository, it tries the
+`skillopt-sleep` executable on `PATH` (including `uv tool`/`pipx` installs), then
+an importable `skillopt_sleep` module. Install with `uv tool install skillopt` or
+`pip install skillopt` when using that fallback.
+
+> **Version note.** This integration reference tracks `main`. PyPI 0.2.0
+> supports the base Sleep CLI, while handoff, Sleep support for non-Azure
+> OpenAI-compatible endpoints, and `--preferences` require a source checkout
+> from `main` until the next release.
+
+## One sleep cycle
+
+```text
+harvest supported local sessions → mine recurring tasks → replay tasks
+  → reflect and propose bounded edits → validate on held-out real tasks
+  → stage proposal → (you) review and adopt
+```
+
+The default backend is `mock`: it makes no provider calls and is useful for
+checking plumbing. A real backend is required for model-driven mining and genuine
+optimization.
+
+## Data boundary
+
+- Harvesting is local and read-only. The `mock` backend has no model-provider
+  data path and no API spend.
+- A real backend sends truncated transcript excerpts and derived task content to
+  the provider selected for mining, replay, judging, and reflection.
+- Outbound prompts are not currently guaranteed to be free of secrets. Do not
+  use a third-party provider on sensitive transcripts without reviewing the data
+  source and the provider's retention policy.
+- For a reviewable workflow, export tasks first, inspect and redact the JSON, set
+  its top-level `"reviewed"` field to `true`, and then use the task file with a
+  real backend:
+
+  ```bash
+  python -m skillopt_sleep harvest --project "$(pwd)" --output reviewed-tasks.json
+  python -m skillopt_sleep dry-run --project "$(pwd)" --backend codex \
+    --tasks-file reviewed-tasks.json --progress
+  ```
+
+  Real backends reject task files that are still marked unreviewed.
+
+For the separate API-key and Azure managed-identity transport boundaries, see
+[OpenAI-compatible endpoints](../docs/sleep/openai-compatible-endpoints.md).
+
+## Supported CLI surface
+
+Actions:
+
+| Action | Behavior |
+|---|---|
+| `status` | show state and the latest staged proposal |
+| `dry-run` | harvest, mine, replay, and report; stage nothing |
+| `run` | run the full cycle and stage a proposal |
+| `adopt` | apply the latest staged proposal, with backups |
+| `harvest` | inspect or export mined tasks |
+| `schedule` / `unschedule` | install or remove the managed nightly cron entry |
+
+Common implemented flags include:
+
+| Flag | Default | Purpose |
+|---|---|---|
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls |
+| `--model NAME` | backend default | select a backend-specific model |
+| `--source claude\|codex\|auto` | `claude` | select the transcript source |
+| `--project PATH` | current directory | select the project and invoked harvest scope |
+| `--scope invoked\|all` | `invoked` | limit transcript harvesting |
+| `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt |
+| `--tasks-file PATH` | none | replay a reviewed task file instead of harvesting |
+| `--max-sessions N` / `--max-tasks N` | unset → `3 × tasks` / `40` tasks | bound harvested work; these are not hard token or wall-clock budgets |
+| `--edit-budget N` | `4` | cap bounded edits per cycle |
+| `--preferences "..."` | empty | add house rules to the reflection prior |
+| `--progress` | off | print phase progress to stderr |
+| `--auto-adopt` | off | adopt an accepted proposal without a separate command |
+| `--json` | off | emit machine-readable output where supported |
+
+The nightly CLI does **not** currently expose `--gate`, `--rollouts-k`,
+`--optimizer-model`, `--target-model`, `--budget-tokens`, or `--budget-minutes`.
+Do not pass experiment-harness flags to the main CLI.
+
+### Preferences
+
+`--preferences` is the main user-facing steering knob:
+
+```bash
+python -m skillopt_sleep run --backend codex --project "$(pwd)" \
+  --preferences "Prefer pytest. Keep commit subjects imperative and concise."
+```
+
+Preferences guide reflection but remain subject to the validation gate.
+
+### Advanced config
+
+The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys,
+including `gate_mode`, `gate_metric`, `dream_rollouts`, `dream_factor`, `recall_k`,
+`evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the
+unsupported CLI flags listed above. Shipping defaults are conservative:
+`gate_mode="on"`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`.
+
+### Handoff backend
+
+`--backend handoff` keeps model subprocesses out of the engine. It writes pending
+model calls to `.skillopt-sleep-handoff/PROMPTS.md` and `pending.json`, exits with
+code 3, and resumes after answers are placed in `answers/<id>.md`:
+
+```bash
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"
+# answer each prompt in a fresh context, then run the same command again
+```
+
+Answering held-out prompts from a context that has already seen their references
+contaminates the validation gate. Claude Code's `/skillopt-sleep-handoff` command
+automates the loop with isolated fresh-context subagents.
+
+## Validation
+
+The deterministic no-provider check exercises consolidation and the gate:
+
+```bash
+python -m skillopt_sleep.experiments.run_experiment \
+  --persona researcher --assert-improves
+```
+
+Real-model benchmark results and their limitations are documented in
+[`docs/sleep/RESULTS.md`](../docs/sleep/RESULTS.md). The benchmark recipes are not
+the shipping CLI defaults.
+
+## Safety summary
+
+- Session harvesting is read-only.
+- `mock` replay makes no provider calls.
+- `run` stages proposals; `adopt` is the normal live-change boundary.
+- Adoption backs up existing target files.
+- `--max-sessions` and `--max-tasks` bound work, but the main CLI does not yet
+  enforce a hard token or elapsed-time budget.
+- Treat real-backend transcript excerpts as data shared with the selected
+  provider.

plugins/claude-code/.claude-plugin/marketplace.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
+  "name": "skillopt-sleep",
+  "description": "SkillOpt-Sleep: give your local Claude agent a nightly sleep cycle that reviews past sessions and consolidates validated memory + skills.",
+  "owner": {
+    "name": "Yifan Yang",
+    "email": "yifanyang@microsoft.com"
+  },
+  "plugins": [
+    {
+      "name": "skillopt-sleep",
+      "description": "Nightly offline self-evolution: harvest your past Claude Code sessions, replay recurring tasks on your own API budget, and consolidate what the agent learns into validated CLAUDE.md memory and SKILL.md skills, behind a held-out gate, staged for your review. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.",
+      "author": {
+        "name": "Yifan Yang"
+      },
+      "category": "productivity",
+      "source": {
+        "source": "git-subdir",
+        "url": "https://github.com/microsoft/SkillOpt.git",
+        "path": "plugins/claude-code",
+        "ref": "main"
+      },
+      "homepage": "https://github.com/microsoft/SkillOpt"
+    }
+  ]
+}

plugins/claude-code/.claude-plugin/plugin.json

@@ -0,0 +1,22 @@
+{
+  "name": "skillopt-sleep",
+  "description": "Give your local Claude agent a nightly 'sleep cycle': it reviews your past sessions offline, replays recurring tasks on your own API budget, and consolidates what it learns into validated memory (CLAUDE.md) and skills (SKILL.md) so it gets better the more you use it. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.",
+  "version": "0.1.0",
+  "author": {
+    "name": "Yifan Yang",
+    "email": "yifanyang@microsoft.com"
+  },
+  "homepage": "https://github.com/microsoft/SkillOpt",
+  "repository": "https://github.com/microsoft/SkillOpt",
+  "license": "MIT",
+  "keywords": [
+    "skillopt",
+    "self-improvement",
+    "memory-consolidation",
+    "dreams",
+    "sleep",
+    "skills",
+    "continual-learning",
+    "offline-optimization"
+  ]
+}

plugins/claude-code/README.md

@@ -0,0 +1,187 @@
+# SkillOpt-Sleep (Claude Code plugin)
+
+> Give your local Claude agent a **sleep cycle**. Every night it reviews your
+> past sessions offline, replays your recurring tasks on your own API budget,
+> and consolidates what it learns into **validated** memory (`CLAUDE.md`) and
+> skills (`SKILL.md`). Your agent gets better the more you use it — no
+> model-weight training.
+
+SkillOpt-Sleep is the **deployment-time** companion to
+[SkillOpt](https://github.com/microsoft/SkillOpt). SkillOpt trains a skill
+offline on a benchmark; SkillOpt-Sleep applies the same discipline to *your own
+daily usage*: bounded text edits, accepted only through a held-out validation
+gate, with rejected candidates recorded in the cycle report for review.
+
+It synthesizes three ideas:
+
+| Idea | Contribution |
+|---|---|
+| **SkillOpt** | skill/memory = trainable text; bounded add/delete/replace edits; **held-out gate** keeps only changes that help. |
+| **Claude Dreams** | offline consolidation over past sessions; input never mutated; output **reviewed then adopted**. |
+| **Agent sleep** | periodic offline replay turns short-term episodes into long-term skill. |
+
+## What it does (one "night")
+
+```
+harvest ~/.claude transcripts → mine recurring tasks → replay offline
+   → consolidate (reflect → bounded edit → GATE) → stage proposal → (you) adopt
+```
+
+Nothing live is modified until **you** run `/skillopt-sleep adopt` (the Dreams "review,
+then adopt or discard" contract). Every adopt backs up the prior file first.
+
+## Install
+
+**Requirements:** Python ≥ 3.10. A real CLI backend additionally requires its
+corresponding `claude` or `codex` executable on `PATH` and authenticated.
+
+```bash
+# 1) get the code (the plugin ships inside the SkillOpt repo)
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+
+# 2) add the plugin to Claude Code as a local marketplace
+/plugin marketplace add ./plugins/claude-code
+/plugin install skillopt-sleep@skillopt-sleep
+
+# 3) verify
+/skillopt-sleep status
+```
+
+The plugin's bundled runner (`scripts/sleep.sh`) auto-selects a Python ≥ 3.10
+interpreter and calls the `skillopt_sleep` engine. A source checkout needs no
+`pip install`. If the marketplace cache does not contain a usable source tree,
+the shared runner falls back first to a `skillopt-sleep` executable on `PATH`
+(including `uv tool`/`pipx` installs), then to an importable Python module. Use
+`uv tool install skillopt` or `pip install skillopt` for that fallback.
+
+> **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base Sleep
+> CLI, but handoff mode and `--preferences` require a source checkout from
+> `main` until the next release.
+
+## Quick start
+
+```bash
+# from inside any project you use with Claude Code:
+/skillopt-sleep dry-run     # preview what it would learn; no changes staged
+/skillopt-sleep run         # full cycle: stages a reviewed proposal (still no live edits)
+/skillopt-sleep status      # see history + the latest staged proposal
+/skillopt-sleep adopt       # apply the staged proposal to CLAUDE.md / SKILL.md (with backup)
+
+/skillopt-sleep-handoff run # same cycle, but THIS session answers the model calls
+                            # (no claude -p subprocess, no API key — subscription-friendly)
+```
+
+Or call the engine directly (Python ≥ 3.10):
+
+```bash
+python -m skillopt_sleep run --project "$(pwd)" --scope invoked --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --backend claude   # real lift via Claude
+python -m skillopt_sleep run --project "$(pwd)" --backend codex    # real lift via Codex
+```
+
+Default backend is **`mock`** — deterministic, no API spend — so you can try the
+plumbing for free. Switch to `--backend claude` or `--backend codex` for
+model-driven mining and optimization on your own budget; an accepted gain is
+task- and model-dependent, not guaranteed.
+
+### Data boundary for real backends
+
+Harvesting `~/.claude` is local and read-only, and the `mock` backend makes no
+provider calls. A real backend sends truncated transcript excerpts and derived
+tasks to the selected provider for mining, replay, judging, and reflection.
+Outbound prompts are not currently guaranteed to be secret-free. Review your
+session data and provider policy before using a real backend on a sensitive
+project; the [shared integration guide](../README.md#data-boundary) describes a
+reviewed task-file workflow.
+
+### Handoff mode (session answers the model calls)
+
+`--backend handoff` runs the cycle without any model subprocess: the engine
+executes the deterministic stages and writes every model call it needs to
+`.skillopt-sleep-handoff/PROMPTS.md` + `pending.json` (exit code 3). You (or
+the `/skillopt-sleep-handoff` command, which automates the loop with isolated
+fresh-context subagents) write each raw answer to `answers/<id>.md` and re-run
+the same command; it resumes from the answers and either finishes or stages
+the next batch. Typically 3–6 rounds per night.
+
+```bash
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"
+# ... answer .skillopt-sleep-handoff/PROMPTS.md into answers/<id>.md ...
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"   # resume
+```
+
+Answer every prompt in a **fresh context** — a session that has already seen
+the mined tasks and their references would contaminate the held-out gate.
+Details: [the plugins README](../README.md#handoff-backend).
+
+## Does it actually improve? (real models, public benchmark)
+
+SkillOpt-Sleep is validated against [gbrain-evals](https://github.com/garrytan/gbrain-evals)'
+public `skillopt-v1` suite — the same benchmark gbrain scores its own skill
+optimizer against. We take a deliberately **deficient** skill and run one sleep
+night; held-out scoring is done by a local rule judge (no judge-API, no way to
+grade its own homework).
+
+| Backend | Seed | Held-out before → after | Nights |
+|---|---|---|---|
+| **Claude (Haiku 4.5)** | brief-writer | **0.00 → 1.00** | 1 |
+| **Codex** | brief-writer | **0.00 → 1.00** | 2 |
+
+Both took a brief-writer with no risks section / no confidence level and, within
+1–2 nights, proposed gated edits that lifted the held-out score to perfect —
+into the protected `LEARNED` block, nothing else touched. The Codex 2-night
+trace even shows the optimizer **diagnosing its own residual failure** and
+adding a meta-rule to fix it. See the recorded results and limitations in
+[`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+Reproduce:
+
+```bash
+git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals
+python -m skillopt_sleep.experiments.run_gbrain --backend claude --model haiku \
+  --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 1 --limit-replay 3 --limit-holdout 3
+python -m skillopt_sleep.experiments.run_gbrain --backend codex \
+  --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 1 --limit-replay 3 --limit-holdout 3
+```
+
+## Deterministic proof (no API, no keys)
+
+```bash
+python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
+python -m skillopt_sleep.experiments.run_experiment --persona programmer  --assert-improves
+```
+
+Each prints the held-out score rising from baseline toward 1.0 as the gate
+accepts the general rules your tasks need, and confirms the gate **rejects** an
+injected harmful edit. Context for the measured experiments is in
+[`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+## Schedule it nightly
+
+```bash
+/skillopt-sleep schedule --hour 3 --minute 17
+/skillopt-sleep unschedule
+```
+
+The built-in scheduler creates a managed cron entry and logs under the project.
+The scheduled run stages proposals unless `--auto-adopt` is explicitly selected.
+
+## Safety
+
+- **Read-only** harvest of `~/.claude`. `mock` replay has no side effects.
+- Proposals are **staged**, never auto-applied (unless you opt in with `--auto-adopt`).
+- Every adopt writes a backup under the staging dir's `backup/`.
+- `--max-sessions` and `--max-tasks` bound work, but the main CLI does not enforce
+  a hard token or wall-clock budget.
+- Real backends share truncated session/task content with the selected provider;
+  do not assume outbound prompts have been fully redacted.
+
+## Status
+
+The engine, deterministic experiment, Claude/Codex CLI backends, handoff mode,
+and staged adoption flow are implemented. Advanced experiment-harness flags are
+not automatically available on the nightly CLI; see the
+[shared integration reference](../README.md#supported-cli-surface).

plugins/claude-code/commands/skillopt-sleep-handoff.md

@@ -0,0 +1,74 @@
+---
+description: Run the SkillOpt-Sleep cycle with the handoff backend — no API subprocess; this session answers the engine's model calls via prompt/answer files, in isolated fresh-context subagents
+argument-hint: "[run | dry-run] [--preferences \"...\"] (default: run)"
+allowed-tools: Bash, Read, Write, Task
+---
+
+# /skillopt-sleep-handoff — session-executed sleep cycle
+
+You are driving **SkillOpt-Sleep in handoff mode**: the Python engine runs
+every deterministic stage (harvest → mine → replay scoring → gate → stage)
+and outsources each model call (attempt / judge / reflect) to YOU via
+prompt files. No `claude -p` subprocess, no API key — the model work runs
+on this session's budget, but each prompt MUST be answered in a fresh,
+isolated context so the validation gate stays honest.
+
+## Requested action: $ARGUMENTS
+
+(If `$ARGUMENTS` is empty, treat it as `run`.)
+
+## The loop
+
+Repeat until the engine exits 0 (done) — at most 8 rounds:
+
+1. **Run the engine** via the bundled runner. Split `$ARGUMENTS` into the action
+   and remaining options, and preserve those options on every resumed round:
+
+   ```bash
+   "${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" <action> --backend handoff --project "$(pwd)" --scope invoked <remaining options>
+   ```
+
+   - exit 0 → the night is complete; go to "Finish" below.
+   - exit 3 → pending model calls; continue with step 2.
+   - anything else → stop and show the user the error output.
+
+2. **Read the batch**: `Read` `.skillopt-sleep-handoff/pending.json` in the
+   project. Each entry has `id`, `prompt`, `max_tokens`, `answer_file`.
+
+3. **Answer each prompt in ISOLATION** — this is the integrity rule:
+   - For each entry, launch a subagent (Task tool) whose ENTIRE input is
+     the `prompt` text verbatim. Add nothing: no summary of this session,
+     no mention of SkillOpt, no other prompts from the batch.
+   - Take the subagent's reply and `Write` the raw answer text (no
+     commentary, no code fences) to the entry's `answer_file`.
+   - NEVER answer from this session's own context — you have seen the
+     mined tasks and their references, so inline answers would contaminate
+     the held-out gate and fake the improvement score.
+
+4. **Re-run the same engine command** — it resumes from the answers
+   directory and either finishes or stages the next batch.
+
+## Finish
+
+- For `run`, if the engine prints a staging directory, `Read` its `report.md`
+  and show the user: held-out baseline → candidate score, the gate decision,
+  the proposed edits, and where the proposal is staged. If an accepted proposal
+  was staged, tell the user nothing live changed and offer
+  `/skillopt-sleep adopt`.
+- For `dry-run`, no staging directory or `report.md` is created; summarize the
+  final stdout instead.
+- The engine archives `.skillopt-sleep-handoff/` on a completed real run;
+  do not delete it yourself.
+
+## Safety reminders
+
+- **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only `adopt` does
+  that, with a backup.
+- Mined tasks are pinned to `.skillopt-sleep-handoff/tasks.json` on round
+  one, so sessions created while answering prompts cannot shift the task
+  set. Do not edit that file.
+- If a batch looks like it contains secrets or content the user would not
+  want re-processed, stop and ask before answering.
+- Handoff files apply pattern-based secret redaction, but that is not a
+  guarantee that prompts are free of sensitive data. Treat the pending batch as
+  private user data and do not copy it into chat, logs, or commits.

plugins/claude-code/commands/skillopt-sleep.md

@@ -0,0 +1,82 @@
+---
+description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks through a selected backend, consolidate validated memory + skills, or schedule nightly runs)
+argument-hint: "[run | dry-run | status | adopt | harvest | schedule | unschedule] (default: status)"
+allowed-tools: Bash, Read
+---
+
+# /skillopt-sleep — SkillOpt-Sleep nightly self-evolution
+
+You are driving **SkillOpt-Sleep**: a tool that lets this user's Claude agent
+improve from past usage by reviewing sessions, replaying recurring tasks, and
+consolidating what it learns into **validated** memory (`CLAUDE.md`) and skills
+(`SKILL.md`). With the default gate enabled, a change is kept only if it improves
+a held-out replay score. Nothing live is modified until adoption unless the
+user explicitly requests `--auto-adopt`.
+
+## Requested action: $ARGUMENTS
+
+(If `$ARGUMENTS` is empty, treat it as `status`.)
+
+## How to run it
+
+The engine is the `skillopt_sleep` Python package in this repo. Split
+`$ARGUMENTS` into the first action token and its remaining options, then use the
+**plugin's bundled runner** so the right interpreter and repo are on the path.
+Preserve the user's remaining options (for example `--preferences`, `--backend`,
+or `--target-skill-path`) instead of silently dropping them:
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" <action> --project "$(pwd)" --scope invoked <remaining options>
+```
+
+`<action>` is one of:
+
+| action       | what it does |
+|--------------|--------------|
+| `status`     | show how many nights have run + the latest staged proposal (READ-ONLY) |
+| `dry-run`    | harvest → mine → replay → report, but **stage nothing** (no-staging preview) |
+| `run`        | full cycle: **stage** a validation report and any accepted proposal; only explicit `--auto-adopt` may also update live files |
+| `adopt`      | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) |
+| `harvest`    | debug: print the recurring tasks mined from recent sessions |
+| `schedule`   | install a nightly cron entry for this project (`--hour --minute`, off-:00 by default) |
+| `unschedule` | remove the nightly cron entry (`--all` to remove every managed entry) |
+
+Default backend is `mock` (deterministic, no API spend). To use real budget for
+model-driven optimization, add `--backend claude` or `--backend codex`. An
+accepted gain is evidence on this run's held-out tasks, not a guarantee of
+general improvement; results depend on the tasks, model, and checks. To steer
+what the optimizer writes, add `--preferences "<your house rules>"`.
+
+## Steps to follow
+
+1. **Run the requested action** via the bundled runner above. Capture stdout and
+   stderr.
+2. **For `run`:** if it prints a staging directory, `Read` its `report.md` and
+   show the user:
+   - held-out score: baseline → candidate (evidence on this run's held-out tasks)
+   - the gate decision (accept/reject) and the exact edits it proposes
+   - where the proposal is staged
+3. **For `dry-run`:** no staging directory or `report.md` is created. Summarize
+   the score, gate decision, and edits from stdout (or request `--json` when
+   machine-readable output is useful).
+4. **For `run` that produced an accepted proposal:** inspect whether stdout says
+   it was auto-adopted. If not, tell the user nothing live changed and offer
+   `/skillopt-sleep adopt`; if it was, report the updated paths explicitly.
+5. **For `adopt`:** confirm which live files were updated and that backups were
+   written under the staging dir's `backup/`.
+6. **Never** edit `CLAUDE.md` or `SKILL.md` yourself — let the engine's explicit
+   `adopt` or user-requested `--auto-adopt` path apply its manifest and backup
+   behavior. Respect the review gate.
+
+## Safety reminders
+
+- Harvest is **read-only** over `~/.claude`. Replay in `mock` mode runs no
+  shell side effects.
+- The cycle stages proposals by default; auto-adoption requires explicit opt-in.
+- A real backend sends truncated transcript excerpts and derived tasks to its
+  provider for mining, replay, judging, and reflection. Pattern-based redaction
+  is not a guarantee that outbound prompts are secret-free. For sensitive data,
+  use `mock` or first run `harvest --output <file>`, review/redact the file, set
+  `"reviewed": true`, and then pass it with `--tasks-file`.
+- `schedule` manages a cron entry when `crontab` is available; otherwise it
+  prints a line for manual installation.

plugins/claude-code/hooks/hooks.json

@@ -0,0 +1,16 @@
+{
+  "hooks": {
+    "SessionEnd": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/on-session-end.sh\"",
+            "async": true
+          }
+        ]
+      }
+    ]
+  }
+}

plugins/claude-code/hooks/on-session-end.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep SessionEnd hook (async, best-effort, NON-BLOCKING).
+#
+# This does NOT run the optimizer. It only appends a tiny marker so the next
+# nightly cycle knows there is fresh activity to harvest, and (optionally)
+# nudges the user once that a sleep cycle is available. It must never fail the
+# session or spend API budget.
+set -uo pipefail
+
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+STATE_DIR="${HOME}/.skillopt-sleep"
+mkdir -p "$STATE_DIR" 2>/dev/null || exit 0
+
+# Record that a session just ended (cheap; used for "is there new data?").
+printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${PWD}" \
+  >> "$STATE_DIR/session-end.log" 2>/dev/null || true
+
+exit 0

plugins/claude-code/scripts/install-cron.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Print (does NOT install) a crontab line that runs SkillOpt-Sleep nightly.
+# The user copies the line into `crontab -e` if they want it.
+set -euo pipefail
+
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+RUNNER="$PLUGIN_ROOT/scripts/sleep.sh"
+PROJECT="${1:-$(pwd)}"
+BACKEND="${2:-mock}"
+
+# 3:17am local — deliberately off the :00 mark so many users don't all hit the
+# API at once (and we leave room for jitter).
+MIN=17
+HOUR=3
+
+cat <<EOF
+# ── SkillOpt-Sleep nightly cycle ────────────────────────────────────────────
+# Review past sessions, replay tasks, stage validated memory/skill updates.
+# Runs at ${HOUR}:$(printf '%02d' $MIN) local every day. Output goes to the project's
+# .skillopt-sleep/ dir; nothing live is changed until you run '/skillopt-sleep adopt'
+# (unless you pass --auto-adopt below).
+#
+# Copy the next line into 'crontab -e':
+${MIN} ${HOUR} * * *  "${RUNNER}" run --project "${PROJECT}" --scope invoked --backend ${BACKEND} >> "${PROJECT}/.skillopt-sleep/cron.log" 2>&1
+#
+# For fully-autonomous adoption (power users), append: --auto-adopt
+# To use the authenticated Claude CLI for model-driven optimization, set
+# BACKEND=claude above.
+# ────────────────────────────────────────────────────────────────────────────
+EOF

plugins/claude-code/scripts/run-sleep.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code,
+# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep
+# package), picks a Python >= 3.10, and execs the engine CLI.
+#
+# Usage: run-sleep.sh <run|dry-run|status|adopt|harvest|...> [args...]
+set -euo pipefail
+
+# This script lives at <repo>/plugins/run-sleep.sh, so the repo root (which
+# holds skillopt_sleep/) is one level up. CLAUDE_PLUGIN_ROOT (if set by Claude
+# Code) points at the plugin dir; the engine is then two levels above it.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)"
+elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then
+  REPO_ROOT="$SKILLOPT_SLEEP_REPO"
+else
+  # last resort: search upward from CWD
+  d="$PWD"
+  while [ "$d" != "/" ]; do
+    [ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; }
+    d="$(dirname "$d")"
+  done
+fi
+if [ "$#" -eq 0 ]; then set -- status; fi
+
+if [ -n "${REPO_ROOT:-}" ]; then
+  # Source checkout: run from repo root so skillopt_sleep/ is importable.
+  PY=""
+  # Allow explicit Python override (useful on macOS with old system Python).
+  if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then
+    PY="$SKILLOPT_SLEEP_PYTHON"
+  else
+    for cand in python3.12 python3.11 python3.10 python3; do
+      if command -v "$cand" >/dev/null 2>&1; then
+        ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+        if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi
+      fi
+    done
+  fi
+  if [ -z "$PY" ]; then
+    echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2
+    exit 1
+  fi
+  cd "$REPO_ROOT"
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+# No source checkout found — fall back to an installed engine.
+
+# Fallback 1: skillopt-sleep CLI on PATH (uv tool install / pipx / pip install).
+# Checked before the import fallback because uv tool install / pipx isolate the
+# package from the system Python's import path, so `python -c "import
+# skillopt_sleep"` would fail even though the CLI is available.
+if command -v skillopt-sleep >/dev/null 2>&1; then
+  exec skillopt-sleep "$@"
+fi
+
+# Fallback 2: importable as a module (pip install into the active Python).
+# Pick a Python >= 3.10 and check importability.
+PY=""
+for cand in python3.12 python3.11 python3.10 python3; do
+  if command -v "$cand" >/dev/null 2>&1; then
+    ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+    if [ "${ver:-0}" -ge 310 ] && "$cand" -c "import skillopt_sleep" >/dev/null 2>&1; then
+      PY="$cand"; break
+    fi
+  fi
+done
+if [ -n "$PY" ]; then
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+echo "[sleep] ERROR: could not locate the skillopt_sleep package." >&2
+echo "[sleep] Install it with 'uv tool install skillopt' or 'pip install skillopt'," >&2
+echo "[sleep] or set SKILLOPT_SLEEP_REPO to a clone of the SkillOpt repo." >&2
+exit 1

plugins/claude-code/scripts/sleep.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Claude Code plugin runner — thin wrapper over the shared runner so all
+# platform plugins share one engine launcher.
+#
+# After marketplace install the plugin is isolated in a cache directory and
+# the repo-relative path no longer works.  We try four locations:
+#   1. Co-located run-sleep.sh (bundled copy — works in marketplace cache)
+#   2. Repo-relative ../../run-sleep.sh (dev checkout)
+#   3. CLAUDE_PLUGIN_ROOT/../run-sleep.sh (plugin env variable)
+#   4. SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh (explicit env)
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+SHARED=""
+if [ -f "$HERE/run-sleep.sh" ]; then
+  SHARED="$HERE/run-sleep.sh"
+elif [ -f "$(cd "$HERE/../.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
+  SHARED="$(cd "$HERE/../.." && pwd)/run-sleep.sh"
+elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -f "$(cd "$CLAUDE_PLUGIN_ROOT/.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
+  SHARED="$(cd "$CLAUDE_PLUGIN_ROOT/.." && pwd)/run-sleep.sh"
+elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -f "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" ]; then
+  SHARED="$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh"
+fi
+
+if [ -z "$SHARED" ]; then
+  echo "[sleep] ERROR: cannot locate run-sleep.sh." >&2
+  echo "[sleep] Set SKILLOPT_SLEEP_REPO to the SkillOpt repo root, or pip install skillopt." >&2
+  exit 1
+fi
+exec bash "$SHARED" "$@"

plugins/claude-code/skills/skillopt-sleep/SKILL.md

@@ -0,0 +1,155 @@
+---
+name: skillopt-sleep
+description: "Use when the user wants their Claude agent to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, memory/skill consolidation, or says things like 'make my agent better the more I use it', 'review my past sessions', 'learn my preferences', 'consolidate what you learned', 'run the sleep cycle', or wants to schedule background self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay through a selected backend -> consolidate validated CLAUDE.md/SKILL.md behind a held-out gate."
+---
+
+# SkillOpt-Sleep: usage-driven self-evolution for a local Claude agent
+
+SkillOpt-Sleep gives the user's agent a **sleep cycle**. On demand or on a
+nightly schedule, it reviews real past Claude Code sessions, re-runs recurring
+tasks through the selected backend, and consolidates what it
+learns into **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`). With the
+default validation gate enabled, it keeps only changes that improve a held-out
+score. Live files change only through explicit adoption or a user-requested
+`--auto-adopt`. It aims to improve this user's recurring work, while making
+each accepted proposal measurable on the run's held-out tasks,
+with no model-weight training. It is the deployment-time analogue of training:
+short-term experience → long-term competence.
+
+It synthesizes three ideas:
+- **SkillOpt** — the skill/memory doc is trainable text; bounded add/delete/replace
+  edits; accepted only through a held-out gate; rejected edits are recorded in
+  the run report for review.
+- **Claude Dreams** — consolidation that reads past sessions and proposes changes
+  inside protected learned blocks; the input is never mutated, and output is
+  reviewed before adoption.
+- **Agent sleep** — periodic background replay turns episodes into durable skill.
+
+## When to use this skill
+
+Trigger when the user wants any of:
+- "make my agent learn from how I use it" / "get better the more I use it" / "remember my preferences across sessions"
+- a nightly/scheduled or on-demand **offline self-improvement / dream / sleep** run
+- to **review past sessions/trajectories** and distill recurring tasks
+- to **consolidate** feedback into `CLAUDE.md` or a managed skill
+- to **schedule** the cycle (cron) or **adopt** a staged proposal
+
+## The cycle (six stages)
+
+1. **Harvest** — read `~/.claude/projects/*/<session>.jsonl` + `~/.claude/history.jsonl` (READ-ONLY) → session digests.
+2. **Mine** — digests → `TaskRecord`s (recurring intents + outcome labels + checkable refs where possible).
+3. **Replay** — re-run tasks through the selected backend under the *current*
+   skill+memory → (hard, soft) scores.
+4. **Consolidate** — reflect on failures → propose bounded edits → **gate** on a held-out slice; with the default gate enabled, accept only if it strictly improves.
+5. **Stage** — write the accepted `proposed_CLAUDE.md` and/or
+   `proposed_SKILL.md`, plus `report.md`, `report.json`, `manifest.json`, and
+   `diagnostics.json` into `<project>/.skillopt-sleep/staging/<timestamp>/`.
+   **Nothing live changes.** A rejected run still has a report but no proposed
+   live-file replacement.
+6. **Adopt** — explicit (or opt-in auto): copy staged files over live ones, backing up first.
+
+## How to drive it
+
+Prefer the `/skillopt-sleep` command. Under the hood it calls the bundled runner:
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" status                       # what's happened
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" dry-run --project "$(pwd)"    # no-staging preview
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" run --project "$(pwd)"        # full cycle, stages a proposal
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" adopt --project "$(pwd)"      # apply staged proposal (with backup)
+```
+
+- Default backend is `mock` (deterministic, **no API spend**) — good for trying the plumbing.
+- Add `--backend claude` or `--backend codex` to spend the user's real budget
+  for model-driven optimization. A held-out gain is run-specific evidence, not
+  a guarantee of broader improvement; results depend on the tasks, model, and
+  checks.
+- Scope defaults to the invoked project; `--scope all` harvests every Claude
+  project into the current run's configured targets.
+- A real backend sends truncated transcript/task content to its provider. See
+  the data-boundary rules below before using one with sensitive sessions.
+
+### Scheduling
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" schedule --project "$(pwd)" --hour 3 --minute 17
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" unschedule --project "$(pwd)"
+```
+
+Installs a nightly cron entry. `unschedule --all` removes every managed entry.
+
+## Common CLI flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--project PATH` | cwd | Project directory to evolve |
+| `--scope all\|invoked` | invoked | Harvest scope |
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | mock | Backend (mock = no provider calls) |
+| `--model NAME` | backend default | Override the model used for replay |
+| `--source claude\|codex\|auto` | claude | Transcript source |
+| `--lookback-hours N` | 72 | Harvest window |
+| `--max-sessions N` | derived | Cap harvested sessions; defaults to 3 × max tasks (120 with current defaults) |
+| `--max-tasks N` | 40 | Cap mined tasks |
+| `--target-skill-path PATH` | `~/.claude/skills/skillopt-sleep-learned/SKILL.md` | Explicit SKILL.md to evolve |
+| `--tasks-file PATH` | — | Reviewed TaskRecord JSON (skip harvest) |
+| `--progress` | off | Print phase progress to stderr |
+| `--auto-adopt` | off | Auto-adopt if gate passes |
+| `--edit-budget N` | 4 | Max bounded edits per night |
+| `--preferences TEXT` | empty | Add house rules to the optimizer's reflection prior |
+| `--json` | off | Machine-readable JSON output |
+
+The CLI also has source/runtime path overrides (`--claude-home`, `--codex-home`,
+and `--codex-path`) and action-specific flags. Use
+`python -m skillopt_sleep <action> --help` as the authoritative surface.
+
+## Config keys (`~/.skillopt-sleep/config.json`)
+
+Beyond the CLI flags, advanced behavior is controlled via config:
+
+- **`preferences`** — free-text house rules injected into the optimizer's reflect step (e.g. "Always use async/await", "Answers in `\boxed{}`").
+- **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits).
+- **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores.
+- **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task.
+- **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory).
+- **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation.
+
+## Memory consolidation
+
+The sleep cycle can consolidate both:
+- **SKILL.md** — the managed skill file (bounded edits: add/delete/replace)
+- **CLAUDE.md** — the project memory (same bounded edits)
+
+With the default gate enabled, both are evaluated by the same held-out score.
+Set `evolve_memory: false` to consolidate only skills, or `evolve_skill: false`
+for only memory.
+
+## Hard rules
+
+- **Never** hand-edit the user's `CLAUDE.md` / `SKILL.md` as part of this skill.
+  Let the engine's explicit `adopt` or user-requested `--auto-adopt` path apply
+  the staging manifest and back up existing live files first.
+- Harvest is read-only. `mock` replay has no side effects.
+- Real backends send truncated transcript excerpts and derived tasks to the
+  selected provider for mining, replay, judging, and reflection. The Claude
+  transcript path is not guaranteed to remove every secret before those calls.
+  Review provider policy and session contents first. For sensitive data, use
+  `mock` or run `harvest --output <file>`, inspect/redact the JSON, set
+  `"reviewed": true`, and replay it with `--tasks-file`; real backends refuse an
+  unreviewed task file.
+- Always show the user the **held-out baseline → candidate** score and the
+  exact proposed edits before suggesting adoption. Evidence before adoption.
+- If asked to demonstrate the mechanism without provider calls, run
+  `python -m skillopt_sleep.experiments.run_experiment --persona researcher --json`
+  — a deterministic synthetic demo of held-out lift and gate rejection. It
+  validates the mechanism, not effectiveness on the user's own tasks.
+
+## Validate / demo
+
+```bash
+# deterministic synthetic demo (no API): score rises and the gate blocks a regression
+python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
+python -m skillopt_sleep.experiments.run_experiment --persona programmer  --assert-improves
+```
+
+See the [SkillOpt-Sleep documentation](https://github.com/microsoft/SkillOpt/tree/main/docs/sleep)
+for recorded results, limitations, and the supported integration surface.

plugins/codex/README.md

@@ -0,0 +1,115 @@
+# SkillOpt-Sleep — Codex integration
+
+Give your **Codex** agent a nightly **sleep cycle**: it reviews past sessions
+offline, replays your recurring tasks on your own Codex budget, and consolidates
+what it learns into validated memory + skills behind a held-out gate. Same engine
+as the Claude Code plugin (`skillopt_sleep`), wrapped for Codex.
+
+> **Verified on Codex:** on the public
+> [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`
+> benchmark, a deliberately deficient skill goes **0.00 → 1.00** on a held-out
+> set with the Codex backend (incl. the tool-use seed via a real tool loop).
+> See the recorded results and limitations in
+> [`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+## What Codex supports (and what we use)
+
+Codex (`@openai/codex`) extends via **`AGENTS.md`** instructions, **skills** at
+`~/.agents/skills/<name>/SKILL.md`, and plugins that can distribute skills.
+Custom prompts are deprecated in Codex, so this integration is skill-first: the
+installed `skillopt-sleep` skill contains the launch commands and operating
+rules. The shared runner remains a plain shell entrypoint that the skill calls.
+
+## Install
+
+On Linux/macOS:
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+bash plugins/codex/install.sh          # installs the skill
+export SKILLOPT_SLEEP_REPO="$(pwd)"    # so the runner is found from anywhere
+```
+
+On Windows (PowerShell):
+```powershell
+git clone <repo-url> SkillOpt-Sleep
+cd SkillOpt-Sleep
+powershell -File plugins/codex/install.ps1
+[System.Environment]::SetEnvironmentVariable("SKILLOPT_SLEEP_REPO", "$(pwd)", "User")
+```
+
+If a previous install created `~/.codex/prompts/sleep.md`, the installer moves
+that deprecated prompt aside with a `.skillopt-legacy*.bak` suffix.
+
+Requires Python ≥ 3.10 and the `codex` CLI on PATH.
+
+## Use
+
+Mention `$skillopt-sleep` where Codex supports explicit skill mentions, or ask
+Codex in natural language:
+
+```text
+Use the skillopt-sleep skill to run status for this project.
+Use the skillopt-sleep skill to run a dry-run for this project.
+Use the skillopt-sleep skill to run the full cycle for this project with the Codex backend.
+Use the skillopt-sleep skill to adopt the latest staged proposal.
+```
+
+Or call the engine directly:
+
+```bash
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \
+  --max-sessions 5 --max-tasks 3 --progress
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \
+  --target-skill-path .agents/skills/example/SKILL.md \
+  --max-sessions 5 --max-tasks 3 --progress
+```
+
+`--source codex` reads Codex Desktop archived sessions from
+`~/.codex/archived_sessions`. Use `--codex-home /path/to/.codex` to point at a
+different Codex home, or `--source auto` to try Codex archives first and fall
+back to Claude Code transcripts. Default backend is `mock` (no API spend).
+`--backend codex` uses your Codex budget for model-driven optimization; an
+accepted gain is task-dependent, not guaranteed. Bound live runs
+with `--max-sessions` and `--max-tasks`; add `--progress` because Codex-backed
+mining, replay, and reflection can be slow and otherwise quiet. Use
+`--target-skill-path` to stage/adopt into a repo-scoped Codex skill such as
+`.agents/skills/<name>/SKILL.md`; target runs over-sample mined tasks and
+prefer tasks that match the target skill's path, headings, and content. The
+implemented main-CLI flags work the same across the shared integrations, and
+`--preferences "..."` is available for house rules. Advanced keys such as
+`gate_mode`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the
+nightly CLI does not expose `--gate`, `--rollouts-k`, token/time-budget, or
+optimizer/target-split flags. See the
+[shared CLI reference](../README.md#supported-cli-surface).
+
+For privacy-sensitive projects, split the run into reviewable steps:
+
+```bash
+python -m skillopt_sleep harvest --project "$(pwd)" --source codex \
+  --target-skill-path .agents/skills/example/SKILL.md \
+  --max-sessions 5 --max-tasks 3 \
+  --output reviewed-tasks.json
+
+python -m skillopt_sleep dry-run --project "$(pwd)" --backend codex \
+  --tasks-file reviewed-tasks.json --progress --json
+```
+
+Inspect/redact the JSON and set `"reviewed": true` before using a real backend.
+`--tasks-file` skips archive harvest/mining and replays only the reviewed JSON
+tasks; real backends refuse task files still marked `"reviewed": false`.
+
+This review step matters even though the Codex transcript converter removes
+known secret-shaped strings: pattern-based redaction is not a guarantee. A real
+backend sends truncated transcript/task content to the selected provider, while
+`--backend mock` makes no provider calls.
+
+## Notes / status
+
+- Codex's `exec` runs shell, so the real-tool-loop replay (e.g. the
+  `tool_called: search` benchmark seed) works natively.
+- This integration no longer installs a `.codex/prompts` slash command. Skills
+  are the reusable Codex workflow surface; mention `skillopt-sleep` explicitly
+  or ask for a sleep/dream/offline self-improvement run and Codex can load the
+  skill.

plugins/codex/install.ps1

@@ -0,0 +1,47 @@
+# Install the SkillOpt-Sleep Codex integration as a user-level Codex skill on Windows.
+# Idempotent; prints what it does.
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
+$CodexHome = if ($env:CODEX_HOME) { $env:CODEX_HOME } else { Join-Path $env:USERPROFILE ".codex" }
+$AgentsSkills = Join-Path $env:USERPROFILE ".agents\skills"
+$LegacyPrompt = Join-Path $CodexHome "prompts\sleep.md"
+
+Write-Output "[install] repo: $RepoRoot"
+
+# 1) user-level skill
+$SkillDir = Join-Path $AgentsSkills "skillopt-sleep"
+if (-not (Test-Path $SkillDir)) {
+    New-Item -ItemType Directory -Path $SkillDir -Force | Out-Null
+}
+Copy-Item (Join-Path $RepoRoot "plugins\codex\skills\skillopt-sleep\SKILL.md") (Join-Path $SkillDir "SKILL.md") -Force
+Write-Output "[install] skill           -> $(Join-Path $SkillDir 'SKILL.md')"
+
+# 2) retire the old custom prompt entrypoint from previous installs
+if (Test-Path $LegacyPrompt) {
+    $Backup = "${LegacyPrompt}.skillopt-legacy.bak"
+    if (Test-Path $Backup) {
+        $DateStr = Get-Date -Format "yyyyMMddHHmmss"
+        $Backup = "${LegacyPrompt}.skillopt-legacy.${DateStr}.bak"
+    }
+    Move-Item $LegacyPrompt $Backup -Force
+    Write-Output "[install] legacy prompt  -> $Backup"
+}
+
+# 3) record the repo location so the runner is found from anywhere
+Write-Output "[install] add to your environment variables:"
+Write-Output "    [System.Environment]::SetEnvironmentVariable('SKILLOPT_SLEEP_REPO', '$RepoRoot', 'User')"
+Write-Output "    Or set it via System Properties."
+
+# 4) optional: append an AGENTS.md hint (only if the user opts in)
+Write-Output ""
+Write-Output "[install] Optional — add this to ~/.codex/AGENTS.md so Codex always knows the tool:"
+Write-Output ""
+Write-Output "  ## SkillOpt-Sleep"
+Write-Output "  Use the skillopt-sleep skill when I ask to run a sleep/dream/offline"
+Write-Output "  self-improvement cycle. The runner is:"
+Write-Output "  \`powershell -File `"$RepoRoot\plugins\run-sleep.ps1`" status --project `"\$(pwd)\`"\`."
+Write-Output ""
+Write-Output "Done. Try asking Codex:"
+Write-Output "  Use the skillopt-sleep skill to run status for this project."

plugins/codex/install.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Install the SkillOpt-Sleep Codex integration as a user-level Codex skill.
+# Idempotent; prints what it does.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CODEX_HOME="${CODEX_HOME:-$HOME/.codex}"
+AGENTS_SKILLS="${HOME}/.agents/skills"
+LEGACY_PROMPT="$CODEX_HOME/prompts/sleep.md"
+
+echo "[install] repo: $REPO_ROOT"
+
+# 1) user-level skill
+mkdir -p "$AGENTS_SKILLS/skillopt-sleep"
+cp "$REPO_ROOT/plugins/codex/skills/skillopt-sleep/SKILL.md" "$AGENTS_SKILLS/skillopt-sleep/SKILL.md"
+echo "[install] skill           -> $AGENTS_SKILLS/skillopt-sleep/SKILL.md"
+
+# 2) retire the old custom prompt entrypoint from previous installs
+if [ -f "$LEGACY_PROMPT" ]; then
+  backup="${LEGACY_PROMPT}.skillopt-legacy.bak"
+  if [ -e "$backup" ]; then
+    backup="${LEGACY_PROMPT}.skillopt-legacy.$(date +%Y%m%d%H%M%S).bak"
+  fi
+  mv "$LEGACY_PROMPT" "$backup"
+  echo "[install] legacy prompt  -> $backup"
+fi
+
+# 3) record the repo location so the runner is found from anywhere
+echo "[install] add to your shell profile:"
+echo "    export SKILLOPT_SLEEP_REPO=\"$REPO_ROOT\""
+
+# 4) optional: append an AGENTS.md hint (only if the user opts in)
+cat <<EOF
+
+[install] Optional — add this to ~/.codex/AGENTS.md so Codex always knows the tool:
+
+  ## SkillOpt-Sleep
+  Use the skillopt-sleep skill when I ask to run a sleep/dream/offline
+  self-improvement cycle. The runner is:
+  \`bash "$REPO_ROOT/plugins/run-sleep.sh" status --project "\$(pwd)"\`.
+
+Done. Try asking Codex:
+  Use the skillopt-sleep skill to run status for this project.
+EOF

plugins/codex/skills/skillopt-sleep/SKILL.md

@@ -0,0 +1,185 @@
+---
+name: skillopt-sleep
+description: "Use when the user wants Codex to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, wants Codex to review past sessions, learn preferences, consolidate memory/skills, run dry-run/run/adopt/status for SkillOpt-Sleep, or schedule background self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay through a selected backend -> consolidate validated memory + skills behind a held-out gate."
+---
+
+# SkillOpt-Sleep: usage-driven self-evolution for a local Codex agent
+
+SkillOpt-Sleep gives the user's Codex agent a sleep cycle. On demand or on a
+nightly schedule, it reviews past local sessions, re-runs recurring tasks
+through the selected backend, and proposes changes to a configured skill and to
+the project's `CLAUDE.md`. With the default validation gate enabled, it keeps
+only changes that improve a held-out score. Live files change only through
+explicit adoption or a user-requested `--auto-adopt`. There is no model-weight
+training.
+
+The current shared engine does **not** write `AGENTS.md`. For a Codex-visible
+result, always select a Codex skill explicitly with `--target-skill-path` (for
+example `.agents/skills/<name>/SKILL.md`). If project `CLAUDE.md` is not a
+desired secondary target, set `"evolve_memory": false` in
+`~/.skillopt-sleep/config.json` before running.
+
+## When to use
+
+Trigger when the user wants any of:
+
+- Codex to learn from past sessions or get better the more they use it;
+- a nightly/scheduled or on-demand sleep/dream/offline self-improvement run;
+- to review past sessions and distill recurring tasks;
+- to consolidate feedback into memory or managed skills;
+- to run `status`, `harvest`, `dry-run`, `run`, or `adopt` for SkillOpt-Sleep.
+
+## The cycle
+
+1. **Harvest** - read local session transcripts according to the engine
+   configuration and normalize them into session digests.
+2. **Mine** - turn digests into recurring `TaskRecord`s with outcomes and
+   checkable references where possible.
+3. **Replay** - re-run mined tasks through the selected backend under the
+   current skill and memory.
+4. **Consolidate** - reflect on failures and propose bounded edits.
+5. **Gate** - with the default gate enabled, accept edits only when the held-out
+   validation score improves.
+6. **Stage** - write the proposal under
+   `<project>/.skillopt-sleep/staging/<date>/`; nothing live changes.
+7. **Adopt** - explicitly, or through user-requested auto-adopt, copy staged
+   files over live files with backups for existing targets.
+
+## How to drive it
+
+Invoke the bundled runner via shell (Codex `exec` has shell access). The runner
+finds the engine and a Python >= 3.10 automatically.
+
+```bash
+# point at the repo if it isn't auto-detected from CWD:
+export SKILLOPT_SLEEP_REPO=/path/to/SkillOpt
+TARGET_SKILL=.agents/skills/example/SKILL.md
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)"
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" harvest --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL"
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL" --backend mock
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL" --backend codex \
+  --max-sessions 5 --max-tasks 3 --progress
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" adopt --project "$(pwd)"
+```
+
+On Windows (CMD / PowerShell):
+```cmd
+:: CMD
+set SKILLOPT_SLEEP_REPO=C:\path\to\SkillOpt-Sleep
+"%SKILLOPT_SLEEP_REPO%\plugins\run-sleep.cmd" status --project "%CD%"
+```
+```powershell
+# PowerShell
+$env:SKILLOPT_SLEEP_REPO = "C:\path\to\SkillOpt-Sleep"
+powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" status --project "$(pwd)"
+```
+
+Actions are `status`, `harvest`, `dry-run`, `run`, `adopt`, `schedule`, and `unschedule`.
+
+- Default backend is `mock`, which is deterministic and spends no API budget.
+- `--backend codex` uses the user's Codex budget for model-driven optimization.
+  An accepted held-out gain is run-specific evidence, not a guarantee of
+  broader improvement; results depend on the tasks, model, and checks.
+- `--source codex` reads Codex Desktop archived sessions from `~/.codex/archived_sessions`;
+  use `--codex-home /path/to/.codex` if the archive lives elsewhere.
+- `--target-skill-path` is required for a Codex skill target. Without it, the
+  shared default is a Claude-managed skill under `~/.claude/skills/`, not an
+  `.agents` skill.
+- Keep `dry-run --backend mock` as the first smoke check unless the user
+  explicitly asked for a real optimization run.
+
+### Scheduling
+
+```bash
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" schedule --project "$(pwd)" \
+  --backend codex --hour 3 --minute 17
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" unschedule --project "$(pwd)"
+```
+
+The scheduler persists the project, backend, time, and optional auto-adopt flag;
+it does not persist `--source` or `--target-skill-path` from this command. Before
+scheduling a Codex-targeted run, set `"transcript_source": "codex"` and an
+absolute `"target_skill_path"` in `~/.skillopt-sleep/config.json`. On systems
+without `crontab`, `schedule` prints a line for manual installation.
+`unschedule --all` removes every managed entry.
+
+### All backends
+
+- `--backend mock` — deterministic, no API spend (default)
+- `--backend claude` — uses the Claude CLI
+- `--backend codex` — uses the Codex CLI
+- `--backend copilot` — uses the GitHub Copilot CLI
+- `--backend handoff` — emits prompt/answer files for an interactive session
+- `--backend azure_openai` — uses the configured Azure OpenAI endpoint
+
+### Additional flags
+
+| Flag | Description |
+|------|-------------|
+| `--auto-adopt` | Auto-adopt if the gate passes (default: stage only) |
+| `--edit-budget N` | Max bounded edits per night (default: 4) |
+| `--lookback-hours N` | Harvest window in hours (default: 72) |
+| `--json` | Machine-readable JSON output |
+
+### Config keys (`~/.skillopt-sleep/config.json`)
+
+- **`preferences`** — free-text house rules for the optimizer
+- **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy)
+- **`gate_metric`** — `hard` | `soft` | `mixed` (default)
+- **`dream_rollouts`** — >1 for multi-rollout contrastive reflection
+- **`recall_k`** — >0 recalls similar past tasks from the archive
+
+### Memory consolidation
+
+The shared sleep cycle consolidates project **memory** (`CLAUDE.md`) and the
+selected **skill** (`SKILL.md`) by default. It does not update `AGENTS.md`.
+Each target is independently toggleable through `evolve_memory` /
+`evolve_skill`, and both are gated by the same held-out validation score.
+
+## Steps
+
+1. Run the requested action; capture stdout.
+2. For `dry-run` and `run`, report the held-out baseline -> candidate score,
+   gate action, task count, session count, and exact proposed edits.
+3. If a staging directory is printed, read `report.md` before summarizing.
+4. `run` stages by default; if `--auto-adopt` was explicitly supplied, report
+   the paths it updated instead of claiming nothing changed.
+5. Offer adoption only after the user has reviewed a still-staged proposal.
+6. Never hand-edit the configured `CLAUDE.md` or target skill as a substitute
+   for the engine's adopt path; adoption is the safety boundary and backs up
+   existing targets first.
+
+## Hard rules
+
+- Harvest is read-only. Do not edit archived sessions or raw transcripts.
+- Codex transcript harvesting removes known secret-shaped strings, developer
+  instructions, and raw tool payloads, but pattern-based redaction is not a
+  guarantee. A real backend still sends truncated transcript/task content to
+  its provider. Review sensitive sessions and provider policy first; prefer a
+  reviewed `--tasks-file` workflow when the data boundary matters.
+- Keep raw secrets, credentials, private user data, and transcript contents out
+  of messages, logs, generated artifacts, and commits.
+- Show validation evidence before recommending adoption.
+- Treat generated edits as proposals, not as source of truth.
+- Do not rely on deprecated custom prompts or `/sleep` slash commands for this
+  Codex integration. This skill is the entrypoint.
+
+## Validate
+
+```bash
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex \
+  --target-skill-path .agents/skills/example/SKILL.md --backend mock --json
+python -m skillopt_sleep.experiments.run_gbrain --backend codex \
+  --seeds brief-writer --data-root /path/to/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 2 --limit-replay 3 --limit-holdout 3
+```
+
+In the recorded `brief-writer` gbrain run, the deliberately deficient fixture
+went 0.00 -> 1.00 on that run's held-out set. Treat this as reproducible
+benchmark evidence for that configuration, not a guarantee for other skills,
+tasks, or models; see the
+[recorded results](https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/RESULTS.md)
+for context and limitations.

plugins/copilot/README.md

@@ -0,0 +1,84 @@
+# SkillOpt-Sleep — GitHub Copilot integration
+
+Give **Copilot** (CLI or VS Code) a nightly **sleep cycle** via a tiny **MCP
+server** that exposes the `skillopt_sleep` engine as tools. MCP is GitHub's
+supported way to extend Copilot, so this works across Copilot CLI, VS Code, and
+other MCP clients with the same server.
+
+## What's here
+
+| File | Purpose |
+|---|---|
+| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `sleep_*` tools |
+| `mcp-config.example.json` | drop-in MCP server config |
+| `copilot-instructions.snippet.md` | paste into `.github/copilot-instructions.md` |
+
+## Install
+
+Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib.
+
+1. **Register the MCP server.** Add the server to your Copilot MCP config
+   (Copilot CLI: `~/.copilot/mcp-config.json`; VS Code: your MCP settings).
+   Use `mcp-config.example.json` as a template — set `SKILLOPT_SLEEP_REPO` to
+   this repo's path:
+
+   ```json
+   {
+     "mcpServers": {
+       "skillopt-sleep": {
+         "command": "python3",
+        "args": ["/abs/path/SkillOpt/plugins/copilot/mcp_server.py"],
+        "env": { "SKILLOPT_SLEEP_REPO": "/abs/path/SkillOpt" }
+       }
+     }
+   }
+   ```
+
+2. **(Optional) Tell Copilot about it.** Append
+   `copilot-instructions.snippet.md` to your repo's
+   `.github/copilot-instructions.md` so Copilot reaches for the tools when the
+   user asks to "run the sleep cycle".
+
+## Use
+
+Ask Copilot things like *"run the sleep cycle"*, *"what did the last sleep
+propose?"*, *"adopt the staged sleep proposal"*. The server exposes seven MCP
+tools: `sleep_status`, `sleep_dry_run`, `sleep_run`, `sleep_adopt`,
+`sleep_harvest`, `sleep_schedule`, and `sleep_unschedule`.
+
+Each tool takes optional `project`, `backend` (`mock`/`claude`/`codex`/`copilot`), and
+`scope` arguments. Default backend is `mock` (no API spend). The `copilot`
+backend drives the GitHub Copilot CLI (`copilot -p ... --output-format json`)
+and requires the `copilot` CLI to be installed and authenticated.
+
+Harvesting is local and read-only, and the default `mock` backend makes no
+provider calls. A real backend sends truncated transcript excerpts and derived
+tasks to the selected provider. Outbound prompts are not currently guaranteed
+to be secret-free; review sensitive data and provider policy first. See the
+[shared data-boundary guidance](../README.md#data-boundary).
+
+For speed, the `copilot` backend runs each call against an isolated
+`COPILOT_HOME` with built-in MCP servers and custom instructions disabled, so
+your user MCP servers (including this project's own) are not spawned per call
+(~5x faster). Override with `SKILLOPT_SLEEP_COPILOT_HOME=<dir>`, pick a model
+with `SKILLOPT_SLEEP_COPILOT_MODEL`, or set `SKILLOPT_SLEEP_COPILOT_FULL_ENV=1`
+to use your real Copilot environment instead.
+
+## Verify the server directly (no Copilot needed)
+
+```bash
+printf '%s\n' \
+  '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
+  '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
+  | SKILLOPT_SLEEP_REPO="$(pwd)" python3 plugins/copilot/mcp_server.py
+```
+You should see the server info and all seven `sleep_*` tools.
+
+## Notes / status
+
+- MCP is the stable, official Copilot extension surface, so this is the most
+  portable shared-engine integration (one server → CLI + IDE).
+- The MCP schema exposes the main CLI's implemented controls, including task and
+  session caps, target-skill selection, scheduling, and staged adoption. It does
+  not add experiment-only gate, rollout, token/time-budget, or optimizer/target
+  split flags. See the [shared CLI reference](../README.md#supported-cli-surface).

plugins/copilot/copilot-instructions.snippet.md

@@ -0,0 +1,56 @@
+<!--
+Copy this block into your repo's .github/copilot-instructions.md so Copilot
+knows the SkillOpt-Sleep tools exist. (Copilot reads copilot-instructions.md
+automatically as ambient guidance.)
+-->
+
+## SkillOpt-Sleep (offline self-evolution)
+
+This project has SkillOpt-Sleep available via an MCP server (`skillopt-sleep`).
+It gives the agent a nightly "sleep cycle": it reviews past sessions, replays
+recurring tasks through a selected backend, and stages validation-gated changes
+to project `CLAUDE.md` and a configured `SKILL.md`.
+
+When the user asks to "run the sleep cycle", "review my past sessions", "learn
+my preferences", or "make the agent improve from past usage", use the MCP tools:
+
+- `sleep_status` — what's happened + the latest staged proposal
+- `sleep_dry_run` — no-staging preview; a real backend still makes provider calls
+- `sleep_run` — full cycle, stages a validation-gated proposal by default;
+  explicit `auto_adopt` may update live files
+- `sleep_adopt` — apply the staged proposal (backs up an existing live file first)
+- `sleep_harvest` — list mined recurring tasks
+- `sleep_schedule` — install a nightly cron entry (set `hour`/`minute`)
+- `sleep_unschedule` — remove the nightly cron entry
+
+### Key parameters (pass as MCP tool arguments)
+
+- `backend` — `mock` (default, no provider calls), `claude`, `codex`, or `copilot`
+- `source` — `claude`, `codex`, or `auto` (where to read transcripts)
+- `target_skill_path` — explicit SKILL.md to evolve; use this for a skill that
+  the current agent actually loads
+- `tasks_file` — reviewed TaskRecord JSON (skip harvest); real backends require
+  its metadata to contain `"reviewed": true`
+- `max_tasks` / `max_sessions` — cap workload
+- `auto_adopt` — auto-adopt if the gate passes
+- `json` — machine-readable output for programmatic use
+
+### Advanced config (`~/.skillopt-sleep/config.json`)
+
+- `preferences` — free-text house rules for the optimizer
+- `gate_mode` — `on` (default) or `off`; `dream_rollouts` — >1 for more signal
+- `evolve_memory` / `evolve_skill` — toggle which docs consolidate
+
+Always show the user the held-out baseline → candidate score and the proposed
+edits before suggesting `sleep_adopt`. Never hand-edit the user's memory/skill
+files; use `sleep_adopt` (or an explicitly requested `auto_adopt`) so the engine
+applies its staging manifest and backup behavior.
+
+Harvesting is local and read-only, and `backend: "mock"` makes no provider
+calls. A real backend sends truncated transcript excerpts and derived tasks to
+the selected provider; outbound prompts are not guaranteed to be secret-free.
+Review sensitive data and provider policy before selecting a real backend.
+
+`sleep_schedule` persists only the project, backend, time, and optional
+auto-adopt setting. Put a non-default transcript source or target skill in
+`~/.skillopt-sleep/config.json` before scheduling it.

plugins/copilot/mcp-config.example.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "skillopt-sleep": {
+      "command": "python3",
+      "args": ["plugins/copilot/mcp_server.py"],
+      "env": {
+        "SKILLOPT_SLEEP_REPO": "${workspaceFolder}"
+      }
+    }
+  }
+}