test: add template contract checks and refine benchmark docs

docs/contributing.md

@@ -25,10 +25,10 @@ Open an issue with:
 See [Add a New Benchmark](guide/new-benchmark.md) for the implementation guide.
 
 **Checklist:**
-- [ ] Data loader in `skillopt/envs/<benchmark>/loader.py`
+- [ ] Data loader in `skillopt/envs/<benchmark>/dataloader.py`
-- [ ] Environment adapter in `skillopt/envs/<benchmark>/env.py`
+- [ ] Environment adapter in `skillopt/envs/<benchmark>/adapter.py`
 - [ ] Config file in `configs/<benchmark>/default.yaml`
-- [ ] Registration in `skillopt/envs/__init__.py`
+- [ ] Registration in `scripts/train.py` (`_ENV_REGISTRY`)
 - [ ] Documentation page in `docs/`
 
 ### 🤖 New Model Backend

docs/guide/new-benchmark.md

@@ -1,213 +1,55 @@
 # Add a New Benchmark
 
-Extend SkillOpt with your own benchmark in ~200 lines of code. We will use
+Extend SkillOpt with your own benchmark in ~100 lines of code.
-a tiny worked example, `docfaithful`, that scores a target model on
-how faithfully it answers questions grounded in a small reference doc.
 
-> **Working reference.** The easiest way to copy-cargo-cult a new env is
+## Overview
-> to read [`skillopt/envs/officeqa/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs/officeqa).
-> Everything below is the same shape, simplified.
 
-## What you need to build
+To add a benchmark, you need:
 
-To add a benchmark you implement four things:
+1. **Data Loader** — Subclass `SplitDataLoader` to load your split data
+2. **Environment Adapter** — Subclass `EnvAdapter` and implement rollout/reflect hooks
+3. **Config** — YAML configuration file
+4. **Registration** — Add your adapter to the train script registry
 
-1. **A `SplitDataLoader` subclass** — knows how to load train / val / test
+## Step 1: Create the Benchmark Package
-   item dicts from disk.
-2. **A rollout helper** — runs the target model on a batch of items
-   under the current skill and scores each prediction.
-3. **An `EnvAdapter` subclass** — wires the loader + rollout helper into
-   SkillOpt's lifecycle (`build_*_env`, `rollout`, `reflect`,
-   `get_task_types`).
-4. **A YAML config** — references your env name plus the standard
-   train / optimizer / gradient knobs.
-
-Then one line in `scripts/train.py`'s `_register_builtins()` makes it
-discoverable.
-
----
-
-## Step 1 — Create the package
 
 ```bash
-mkdir -p skillopt/envs/docfaithful
+mkdir -p skillopt/envs/my_benchmark
-touch skillopt/envs/docfaithful/__init__.py
+touch skillopt/envs/my_benchmark/__init__.py
 ```
 
-## Step 2 — Implement the data loader
+## Step 2: Implement the Data Loader
 
-`skillopt/envs/docfaithful/loader.py`:
+Create `skillopt/envs/my_benchmark/dataloader.py`:
 
 ```python
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
 from skillopt.datasets.base import SplitDataLoader
 
 
-def _normalize(raw: dict) -> dict:
+class MyBenchmarkDataLoader(SplitDataLoader):
-    """Make sure every item has an ``id``. Other keys are env-specific."""
+    """Load benchmark items from raw data and/or split directories."""
-    return {
-        "id": str(raw["uid"]),
-        "question": raw["question"],
-        "ground_truth": raw["answer"],
-        "reference_text": raw.get("reference", ""),
-        "task_type": raw.get("category", "docfaithful"),
-    }
 
-
+    def load_raw_items(self, data_path: str) -> list[dict]:
-class DocFaithfulDataLoader(SplitDataLoader):
+        # For ratio mode, parse your source dataset from data_path.
-    """Load DocFaithful items from JSON files inside each split dir."""
+        # Return list[dict] where each item has at least a unique, deterministic "id".
+        return super().load_raw_items(data_path)
 
     def load_split_items(self, split_path: str) -> list[dict]:
-        # split_path is e.g. data/docfaithful_split/train/
+        # For split_dir mode, parse one split directory.
-        json_files = sorted(Path(split_path).glob("*.json"))
+        return super().load_split_items(split_path)
-        if not json_files:
-            raise FileNotFoundError(f"No .json file found in {split_path}")
-        with json_files[0].open(encoding="utf-8") as f:
-            raw = json.load(f)
-        return [_normalize(item) for item in raw]
 ```
 
-Only `load_split_items()` is mandatory. If you also want to support
+## Step 3: Implement the Environment Adapter
-`split_mode="ratio"` (auto-split a single raw file into train/val/test),
-override `load_raw_items(data_path)` as well — see
-`skillopt/datasets/base.py` docstrings.
 
-## Step 3 — Write the rollout helper
+Create `skillopt/envs/my_benchmark/adapter.py`:
-
-`skillopt/envs/docfaithful/rollout.py`:
 
 ```python
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-
-from skillopt.model import chat_target
-
-
-def _score(prediction: str, ground_truth: str) -> tuple[int, float]:
-    """Trivial exact-match scorer. Replace with F1 / ROUGE / LLM-judge."""
-    p = (prediction or "").strip().lower()
-    g = (ground_truth or "").strip().lower()
-    hard = int(p == g and bool(g))
-    soft = 1.0 if hard else 0.0
-    return hard, soft
-
-
-def _rollout_one(item: dict, skill_content: str,
-                 *, max_completion_tokens: int) -> dict:
-    system = skill_content
-    user = (
-        f"Question: {item['question']}\n\n"
-        f"Reference:\n{item.get('reference_text', '')}\n\n"
-        "Answer:"
-    )
-    prediction, _usage = chat_target(
-        system=system,
-        user=user,
-        max_completion_tokens=max_completion_tokens,
-    )
-    hard, soft = _score(prediction, item.get("ground_truth", ""))
-    return {
-        "id": str(item["id"]),
-        "hard": hard,
-        "soft": soft,
-        "predicted_answer": prediction,
-        "question": item.get("question", ""),
-        "reference_text": item.get("reference_text", ""),
-        "task_type": item.get("task_type", "docfaithful"),
-    }
-
-
-def run_batch(*, items: list[dict], skill_content: str, out_root: str,
-              workers: int = 4, max_completion_tokens: int = 4096) -> list[dict]:
-    """Run a batch of episodes sequentially or with a thread pool."""
-    os.makedirs(out_root, exist_ok=True)
-    # For brevity we go sequentially — swap in concurrent.futures.ThreadPoolExecutor
-    # when network / model latency dominates.
-    results = [
-        _rollout_one(item, skill_content,
-                     max_completion_tokens=max_completion_tokens)
-        for item in items
-    ]
-    Path(out_root, "rollouts.json").write_text(
-        json.dumps(results, ensure_ascii=False, indent=2)
-    )
-    return results
-```
-
-Two design points worth flagging:
-
-- **Scoring lives here, not in `EnvAdapter`.** There is no `evaluate()`
-  method on the ABC. Whatever signal you put in `hard` (0/1, or a float
-  in [0, 1] for smoothed reward) and `soft` (float in [0, 1]) is what
-  the optimizer reads.
-- **Use `skillopt.model.chat_target`**, not raw OpenAI/Claude calls.
-  That routes through whichever **chat** target backend the user
-  configured (`openai_chat` / `claude_chat` / `qwen_chat` /
-  `minimax_chat`) without your adapter caring. Exec-style backends
-  (`codex_exec`, `claude_code_exec`) need env-specific rollout code —
-  see `skillopt/envs/swebench/` for an example.
-
-## Step 4 — Implement the environment adapter
-
-`skillopt/envs/docfaithful/adapter.py`:
-
-```python
-from __future__ import annotations
-
-import os
-
-from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
-from skillopt.envs.docfaithful.loader import DocFaithfulDataLoader
+from skillopt.envs.my_benchmark.dataloader import MyBenchmarkDataLoader
-from skillopt.envs.docfaithful.rollout import run_batch
-from skillopt.gradient.reflect import run_minibatch_reflect
 
-
+class MyBenchmarkAdapter(EnvAdapter):
-class DocFaithfulAdapter(EnvAdapter):
+    def __init__(self, split_dir: str = "", data_path: str = "", **kwargs):
-    """SkillOpt adapter for the DocFaithful benchmark."""
+        self.dataloader = MyBenchmarkDataLoader(split_dir=split_dir, data_path=data_path, **kwargs)
-
-    def __init__(
-        self,
-        split_dir: str = "",
-        data_path: str = "",
-        split_mode: str = "split_dir",
-        split_ratio: str = "2:1:7",
-        split_seed: int = 42,
-        split_output_dir: str = "",
-        workers: int = 4,
-        analyst_workers: int = 4,
-        failure_only: bool = False,
-        minibatch_size: int = 8,
-        edit_budget: int = 4,
-        seed: int = 42,
-        limit: int = 0,
-        max_completion_tokens: int = 4096,
-    ) -> None:
-        self.workers = workers
-        self.analyst_workers = analyst_workers
-        self.failure_only = failure_only
-        self.minibatch_size = minibatch_size
-        self.edit_budget = edit_budget
-        self.max_completion_tokens = int(max_completion_tokens)
-        self.dataloader = DocFaithfulDataLoader(
-            split_dir=split_dir,
-            data_path=data_path,
-            split_mode=split_mode,
-            split_ratio=split_ratio,
-            split_seed=split_seed,
-            split_output_dir=split_output_dir,
-            seed=seed,
-            limit=limit,
-        )
-
-    # ── Lifecycle ───────────────────────────────────────────────────────
 
     def setup(self, cfg: dict) -> None:
         super().setup(cfg)
@@ -216,178 +58,73 @@ class DocFaithfulAdapter(EnvAdapter):
     def get_dataloader(self):
         return self.dataloader
 
-    # ── Env construction ────────────────────────────────────────────────
-
-    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
-        # For dataset-backed envs the "manager" is just the items list.
-        return list(batch.payload or [])
-
     def build_train_env(self, batch_size: int, seed: int, **kwargs):
-        batch = self.dataloader.build_train_batch(
+        return self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs).payload
-            batch_size=batch_size, seed=seed, **kwargs
-        )
-        return self.build_env_from_batch(batch, **kwargs)
 
     def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
-        batch = self.dataloader.build_eval_batch(
+        return self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs).payload
-            env_num=env_num, split=split, seed=seed, **kwargs
-        )
-        return self.build_env_from_batch(batch, **kwargs)
 
-    # ── The two real action methods ─────────────────────────────────────
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
+        # env_manager is the payload returned by build_train_env/build_eval_env
+        # (commonly list[dict] task items).
+        # Run target model on each item and return list[dict].
+        # Required keys per row: "id", "hard" (0/1), "soft" (0.0-1.0)
+        raise NotImplementedError
 
-    def rollout(self, env_manager, skill_content: str,
+    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
-                out_dir: str, **kwargs) -> list[dict]:
+        # Convert failure/success analysis into RawPatch-like dicts.
-        items: list[dict] = env_manager
+        raise NotImplementedError
-        return run_batch(
-            items=items,
-            skill_content=skill_content,
-            out_root=out_dir,
-            workers=self.workers,
-            max_completion_tokens=self.max_completion_tokens,
-        )
-
-    def reflect(self, results: list[dict], skill_content: str,
-                out_dir: str, **kwargs) -> list[dict | None]:
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=kwargs.get(
-                "prediction_dir", os.path.join(out_dir, "predictions")
-            ),
-            patches_dir=kwargs.get(
-                "patches_dir", os.path.join(out_dir, "patches")
-            ),
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=kwargs.get("random_seed"),
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=kwargs.get("step_buffer_context", ""),
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
 
     def get_task_types(self) -> list[str]:
-        seen: list[str] = []
+        return ["my_benchmark"]
-        for item in (
-            self.dataloader.train_items
-            + self.dataloader.val_items
-            + self.dataloader.test_items
-        ):
-            tt = str(item.get("task_type") or "docfaithful")
-            if tt not in seen:
-                seen.append(tt)
-        return seen or ["docfaithful"]
 ```
 
-### What the rollout actually does
+## Step 4: Register the Benchmark
 
-Look back at `run_batch` from Step 3 — it sends each `item["question"]`
+Add your adapter to `_register_builtins()` in `scripts/train.py`:
-to the target model with `skill_content` as the system prompt, scores
-the answer against `item["ground_truth"]`, and returns a list of dicts:
 
 ```python
-[
+from skillopt.envs.my_benchmark.adapter import MyBenchmarkAdapter
-    {"id": "ex_001", "hard": 1, "soft": 0.92,
+
-     "predicted_answer": "...", "question": "...",
+_ENV_REGISTRY["my_benchmark"] = MyBenchmarkAdapter
-     "reference_text": item["reference_text"]},
-    {"id": "ex_002", "hard": 0, "soft": 0.13, "fail_reason": "...", ...},
-    ...
-]
 ```
 
-The trainer only requires `id`, `hard`, `soft`. The rest is preserved on
+## Step 5: Create Config
-`RolloutResult.extras` (see `skillopt/types.py`) and is what your
-`reflect()` consumes via `run_minibatch_reflect`.
 
-## Step 5 — Register the adapter
+Create `configs/my_benchmark/default.yaml`:
-
-Edit [`scripts/train.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/train.py)
-and add to `_register_builtins()`:
-
-```python
-    try:
-        from skillopt.envs.docfaithful.adapter import DocFaithfulAdapter
-        _ENV_REGISTRY["docfaithful"] = DocFaithfulAdapter
-    except ImportError:
-        pass  # docfaithful deps not installed — skip
-```
-
-There is **no `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`** —
-the registry lives in `scripts/train.py` and is populated lazily so that
-optional deps don't break `--help`.
-
-## Step 6 — Create the YAML config
-
-`configs/docfaithful/default.yaml`:
 
 ```yaml
-_base_: ../_base_/default.yaml      # NOTE: string, not list
+_base_: ../_base_/default.yaml
 
-model:
+env:
-  reasoning_effort: medium
+  name: my_benchmark
+  data_path: data/my_benchmark
+  split_mode: ratio
+  split_ratio: "2:1:7"
 
 train:
-  batch_size: 16
-  accumulation: 1
   num_epochs: 4
-
+  batch_size: 40
-gradient:
-  minibatch_size: 8
-  merge_batch_size: 8
 
 optimizer:
   learning_rate: 4
+  lr_scheduler: cosine
+  use_slow_update: true
+  use_meta_skill: true
 
-env:
+gradient:
-  name: docfaithful
+  analyst_workers: 16
-  # Optional: a seed skill document. Create this file (or any markdown
-  # file) yourself before the first run, or omit the key to let SkillOpt
-  # start from an empty skill.
-  skill_init: skillopt/envs/docfaithful/skills/initial.md
-  split_mode: split_dir
-  split_dir: data/docfaithful_split
-  workers: 4
-  max_completion_tokens: 4096
-  limit: 0
 ```
 
-> ⚠️ `_base_` is currently parsed as a **string path**, not a list. Write
+## Step 6: Run
-> `_base_: ../_base_/default.yaml`, not `_base_: ['../_base_/default.yaml']`.
-> See [`skillopt/config.py`](https://github.com/microsoft/SkillOpt/blob/main/skillopt/config.py)
-> if you want to add list-form inheritance.
-
-## Step 7 — Run
 
 ```bash
-# If you set skill_init above, create the seed skill first:
+python scripts/train.py --config configs/my_benchmark/default.yaml
-#   mkdir -p skillopt/envs/docfaithful/skills
-#   echo "# DocFaithful initial skill" > skillopt/envs/docfaithful/skills/initial.md
-
-python scripts/train.py --config configs/docfaithful/default.yaml
 ```
 
-If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`,
-you forgot Step 5.
-
-If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`,
-you forgot to implement one of the five abstract methods on `EnvAdapter`:
-`build_train_env`, `build_eval_env`, `rollout`, `reflect`,
-`get_task_types`.
-
 ## Tips
 
-- Start with `train.batch_size: 4` and `limit: 10` while debugging.
+!!! tip
-- The `evaluate` half lives **inside your `rollout`**, not as a separate
+    - Use a small `batch_size` (10-20) for initial testing
-  method — there is no `evaluate()` in the `EnvAdapter` ABC. Score the
+    - Start from `skillopt/envs/_template/` and adapt from there
-  prediction in `run_batch` and put the score on each result dict's
+    - Use an existing adapter (for example `skillopt/envs/officeqa/adapter.py`) as a concrete reference
-  `hard` / `soft`.
-- Noisy scoring kills the optimizer. Spend time on `run_batch`'s scoring
-  before you spend time on prompts.
-- If your benchmark needs heavy optional deps (selenium, vllm, ...),
-  wrap the registration block with `try / except ImportError` (Step 5)
-  so people without those deps can still `--help`.
-- Copy `skillopt/envs/_template/` as a starting skeleton — it now
-  implements the real abstract methods.

docs/reference/api.md

@@ -1,195 +1,75 @@
 # API Reference
 
-This page documents the public Python API SkillOpt exposes for **extending the
-framework** with new environments / benchmarks. For ready-made adapters,
-browse [`skillopt/envs/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs).
-
-> **Source of truth.** The classes below are real Python ABCs defined in
-> `skillopt/envs/base.py`, `skillopt/datasets/base.py`, `skillopt/types.py`,
-> and `skillopt/evaluation/gate.py`. If this page ever drifts, the code
-> wins — please open an issue.
-
----
-
 ## Core Classes
 
 ### `EnvAdapter`
 
-`skillopt/envs/base.py` — abstract adapter that connects the SkillOpt
+Abstract base class for benchmark environments (`skillopt/envs/base.py`).
-trainer to an environment (benchmark, simulator, REST API, ...).
-Subclasses **must** implement the five abstract methods below.
 
 ```python
-from abc import ABC, abstractmethod
-from skillopt.datasets.base import BaseDataLoader, BatchSpec
-
 class EnvAdapter(ABC):
-
+    def setup(self, cfg: dict) -> None
-    # ── Lifecycle hooks (have defaults; override only if needed) ────────
+    def get_dataloader(self) -> BaseDataLoader | None
-
+    def build_train_env(self, batch_size: int, seed: int, **kwargs)
-    def setup(self, cfg: dict) -> None: ...
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs)
-    def get_dataloader(self) -> BaseDataLoader | None: ...
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]
-    def requires_ray(self) -> bool: ...                 # default False
+    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]
-
+    def get_task_types(self) -> list[str]
-    # ── Abstract methods (subclasses MUST implement) ────────────────────
-
-    @abstractmethod
-    def build_train_env(self, batch_size: int, seed: int, **kwargs):
-        """Return an environment-manager object to be passed to rollout()."""
-
-    @abstractmethod
-    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
-        """Like build_train_env() but for a fixed eval split."""
-
-    @abstractmethod
-    def rollout(self, env_manager, skill_content: str,
-                out_dir: str, **kwargs) -> list[dict]:
-        """Run a batch of episodes with the current skill.
-
-        Each returned dict MUST contain:
-          - "id":   str        episode/task identifier
-          - "hard": int (0|1)  pass/fail (may be float 0.0-1.0 if smoothed)
-          - "soft": float      partial-credit score in [0.0, 1.0]
-        It MAY contain env-specific extra keys (parsed into RolloutResult.extras).
-        """
-
-    @abstractmethod
-    def reflect(self, results: list[dict], skill_content: str,
-                out_dir: str, **kwargs) -> list[dict | None]:
-        """Turn rollout results into a list of raw patch dicts.
-
-        Each dict (or None to drop the slot) MUST contain:
-          - "patch":       {"edits": [...]}     a Patch.to_dict() payload
-          - "source_type": "failure" | "success"
-        """
-
-    @abstractmethod
-    def get_task_types(self) -> list[str]:
-        """Distinct task-type strings used for stratified sampling."""
 ```
 
-The trainer also calls a few default-implemented helpers on every adapter:
+The rollout contract expects result rows with at least:
-`build_reference_text`, `get_reference_metadata`, `attach_reference_context`,
+
-`select_representative_items`, and `build_env_from_batch`. Read the docstrings
+```python
-in `skillopt/envs/base.py` if you need to override any of these — most
+{"id": str, "hard": int, "soft": float}
-benchmarks don't.
+```
 
 ### `BaseDataLoader` / `SplitDataLoader`
 
-`skillopt/datasets/base.py` — episode-planning loaders.
+Data loader abstractions (`skillopt/datasets/base.py`).
 
 ```python
 class BaseDataLoader(ABC):
-    def setup(self, cfg: dict) -> None: ...
+    def setup(self, cfg: dict) -> None
-    @abstractmethod
+    def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec
-    def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: ...
+    def build_eval_batch(self, env_num: int, split: str, seed: int, **kwargs) -> BatchSpec
-    @abstractmethod
-    def build_eval_batch(self, env_num: int, split: str, seed: int, **kwargs) -> BatchSpec: ...
 
 class SplitDataLoader(BaseDataLoader):
-    """Concrete base for dataset-backed envs with on-disk train/val/test splits.
+    def load_raw_items(self, data_path: str) -> list[dict]
-
+    def load_split_items(self, split_path: str) -> list[dict]
-    Subclasses only need to implement load_split_items() (and optionally
+    def get_split_items(self, split: str) -> list[dict]
-    load_raw_items() if you also want ``split_mode='ratio'``).
-    """
-    def load_split_items(self, split_path: str) -> list[dict]: ...
-    def load_raw_items(self, data_path: str) -> list[dict]: ...   # optional
 ```
 
-`SplitDataLoader` handles two layout modes:
-
-| `split_mode` | What it expects |
-|---|---|
-| `"split_dir"` | A directory with `train/`, `val/`, `test/` subdirs already split. |
-| `"ratio"` | A raw dataset path + `split_ratio: "2:1:7"` style string. |
-
-In either case the items returned by `load_split_items()` are plain
-`dict` objects with at minimum an `"id"` key.
-
 ### `BatchSpec`
 
-`skillopt/datasets/base.py` — a slotted dataclass describing one batch
+Represents one concrete batch request.
-request the trainer hands to the adapter.
 
 ```python
 @dataclass(slots=True)
 class BatchSpec:
-    phase: str                 # "train" | "eval"
+    phase: str
-    split: str                 # "train" | "val" | "test" | "valid_seen" | ...
+    split: str
     seed: int
     batch_size: int
-    payload: object | None = None     # what the loader produced (e.g. list[dict])
+    payload: object | None = None
-    metadata: dict = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
 ```
 
-### `Edit` / `Patch`
+### `RolloutResult` / `RawPatch`
 
-`skillopt/types.py` — the I/O types Reflect / Aggregate / Update produce
+Typed helpers for stage I/O in `skillopt/types.py`.
-and consume.
 
 ```python
-EditOp = Literal["append", "insert_after", "replace", "delete"]
+@dataclass
+class RolloutResult:
+    id: str
+    hard: int
+    soft: float
+    # optional benchmark-specific fields
 
 @dataclass
-class Edit:
+class RawPatch:
-    op: EditOp
+    patch: Patch
-    content: str = ""
+    source_type: Literal["failure", "success"] = "failure"
-    target: str = ""
-    support_count: int | None = None
-    source_type: Literal["failure", "success"] | None = None
-    merge_level: int | None = None
-    update_origin: str = ""
-    update_target: str = ""
-
-@dataclass
-class Patch:
-    edits: list[Edit] = field(default_factory=list)
-    reasoning: str = ""
-    ranking_details: dict[str, Any] | None = None
 ```
 
-Both types support `to_dict()` / `from_dict()` for serialization.
+For detailed source code, see the [`skillopt/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt) directory.
-
-### `RolloutResult`
-
-`skillopt/types.py` — the normalised rollout return type. The trainer
-calls `RolloutResult.from_dict(...)` on each dict returned from
-`EnvAdapter.rollout()`, so the only **hard** requirement on those dicts is
-the three keys above (`id`, `hard`, `soft`). Extra fields are preserved
-into `RolloutResult.extras`.
-
-### `GateResult` / `GateAction`
-
-`skillopt/evaluation/gate.py` — the validation-gate decision types
-returned each epoch.
-
----
-
-## Registering an environment
-
-Environments are not registered via decorators or a `BENCHMARK_REGISTRY`
-dict. The trainer keeps a lazy registry inside `scripts/train.py` —
-`_ENV_REGISTRY` — populated by `_register_builtins()`. To add a new env
-you append a `try / except ImportError` block there. See
-[Add a New Benchmark](../guide/new-benchmark.md) for the full step-by-step.
-
----
-
-## Backends (model layer)
-
-The model layer lives under `skillopt.model.*`. Backends are selected
-via `model.optimizer_backend` and `model.target_backend` in the config —
-not via a base class subclass. Supported values (as of this writing):
-
-| Backend | Optimizer? | Target? |
-|---|---|---|
-| `openai_chat` | ✓ | ✓ |
-| `claude_chat` | ✓ | ✓ |
-| `qwen_chat` | ✓ | ✓ |
-| `minimax_chat` | ✓ | ✓ |
-| `codex_exec` | — | ✓ |
-| `claude_code_exec` | — | ✓ |
-
-See `skillopt/model/backend_config.py` for the live whitelist and
-[`docs/reference/config.md`](./config.md) for the per-backend
-configuration keys.

skillopt/envs/_template/README.md

@@ -4,40 +4,16 @@ This directory provides scaffold files for adding a new benchmark to SkillOpt.
 
 ## Files
 
-- `env_template.py` — Environment adapter template (subclasses
+- `env_template.py` — Environment adapter template
-  `EnvAdapter`; implements the 5 abstract methods so the file is
+- `loader_template.py` — Data loader template
-  instantiable out of the box).
+- `config_template.yaml` — Config file template
-- `loader_template.py` — Data loader template (subclasses
-  `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`).
-- `config_template.yaml` — Config file template.
 
 ## Usage
 
-1. **Copy the directory:**
+1. Copy this directory: `cp -r skillopt/envs/_template skillopt/envs/your_benchmark`
-   ```bash
+2. Rename files: remove `_template` suffix
-   cp -r skillopt/envs/_template skillopt/envs/your_benchmark
+3. Implement the `TODO` sections
-   ```
+4. Register your adapter in `_ENV_REGISTRY` inside `scripts/train.py`
-2. **Rename the files** (drop the `_template` suffix):
+5. Create config at `configs/your_benchmark/default.yaml`
-   ```bash
-   cd skillopt/envs/your_benchmark
-   mv env_template.py    adapter.py
-   mv loader_template.py loader.py
-   ```
-   …and inside each file rename the classes
-   (`TemplateBenchmarkEnv → YourBenchmarkAdapter`,
-   `TemplateBenchmarkLoader → YourBenchmarkLoader`)
-   and fix the cross-import in `adapter.py`.
-3. **Implement the TODO blocks** inside `adapter.py:rollout` and the
-   `_normalize_item` helper in `loader.py`. If you want real reflection,
-   uncomment the `run_minibatch_reflect` block in `adapter.py:reflect`.
-4. **Register** the adapter — add a `try / except ImportError` block in
-   `scripts/train.py`'s `_register_builtins()` mapping the registry key
-   to your `YourBenchmarkAdapter` class. There is no
-   `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live
-   registry is `_ENV_REGISTRY` in `scripts/train.py`.
-5. **Create the config** at `configs/your_benchmark/default.yaml`
-   (start from `config_template.yaml`). `_base_` is a **string path**,
-   not a list.
 
-See the [Add a New Benchmark guide](../../../docs/guide/new-benchmark.md)
+See the [documentation](../../docs/guide/new-benchmark.md) for the full guide.
-for the full step-by-step with a worked `docfaithful` example.

skillopt/envs/_template/config_template.yaml

@@ -4,36 +4,27 @@
 # Copy this file to configs/<your_benchmark>/default.yaml
 # and customize the values below.
 
-# Inherit global defaults.
+# Inherit global defaults
-# NOTE: `_base_` is a string path, not a list.
 _base_: ../_base_/default.yaml
 
 # ── Environment ──────────────────────────────────
 env:
-  name: your_benchmark            # Must match the key registered in scripts/train.py
+  name: your_benchmark            # Must match _ENV_REGISTRY key in scripts/train.py
-  # Optional: a seed skill document. Create this file yourself before the
+  data_path: data/your_benchmark  # Path to your data
-  # first run, or omit the key to start from an empty skill.
-  # skill_init: skillopt/envs/your_benchmark/skills/initial.md
-  data_path: data/your_benchmark  # Path to your data (for split_mode: ratio)
-  split_dir: ""                   # Set this and use split_mode: split_dir for pre-split data
   split_mode: ratio               # "ratio" or "split_dir"
-  split_ratio: "2:1:7"            # train:val:test (used when split_mode: ratio)
+  split_ratio: "2:1:7"            # train:val:test
-  workers: 4                      # Parallel rollout workers
+  exec_timeout: 120               # Per-task timeout (seconds)
-  max_completion_tokens: 4096     # Cap per target-model call
-  limit: 0                        # 0 = no limit; small int = debug sample
 
 # ── Training ─────────────────────────────────────
 train:
-  num_epochs: 4
+  num_epochs: 4                   # Number of epochs
-  batch_size: 40
+  batch_size: 40                  # Tasks per step (batch size)
-  accumulation: 1
   seed: 42
 
 # ── Gradient (Reflection) ───────────────────────
 gradient:
   analyst_workers: 16             # Parallel reflection workers
   minibatch_size: 8
-  merge_batch_size: 8
 
 # ── Optimizer ────────────────────────────────────
 optimizer:
@@ -48,8 +39,7 @@ evaluation:
   eval_test: true                 # Run test eval after training
 
 # ── Model ────────────────────────────────────────
-# Override only what differs from the inherited defaults.
 model:
-  optimizer_backend: openai_chat  # openai_chat | claude_chat | qwen_chat | minimax_chat
+  backend: azure_openai           # azure_openai | openai_chat | claude_code_exec | qwen
-  target_backend: openai_chat     # … plus codex_exec / claude_code_exec for target only
+  optimizer: gpt-4o
-  reasoning_effort: medium
+  target: gpt-4o

skillopt/envs/_template/env_template.py

@@ -4,59 +4,37 @@ Benchmark Environment Template
 Copy this file and implement the TODO sections to add a new benchmark.
 
 The EnvAdapter is responsible for:
-  1. Building per-batch environment managers (train and eval splits).
+1. Building train/eval environment payloads
-  2. Running rollouts under the current skill document.
+2. Running rollout and returning scored result rows
-  3. Reflecting on those rollouts into raw patch dicts.
+3. Reflecting on results and returning patch candidates
-  4. Reporting the distinct task types in your data (for stratified
-     sampling).
-
-For a fully worked example see ``skillopt/envs/officeqa/``.
 """
 from __future__ import annotations
 
-import os
-
 from skillopt.datasets.base import BatchSpec
+from skillopt.envs._template.loader_template import TemplateBenchmarkDataLoader
 from skillopt.envs.base import EnvAdapter
-from skillopt.envs._template.loader_template import TemplateBenchmarkLoader
-# When you wire in real reflection, also import:
-# from skillopt.gradient.reflect import run_minibatch_reflect
 
 
-class TemplateBenchmarkEnv(EnvAdapter):
+class TemplateBenchmarkAdapter(EnvAdapter):
     """
     Environment adapter for <Your Benchmark Name>.
 
-    Rename this class. Each abstract method below is required by
+    Rename this class and implement the abstract methods below.
-    :class:`skillopt.envs.base.EnvAdapter`. The template implementations
-    are minimal so this file is importable and instantiable; replace the
-    TODOs with real logic.
     """
 
     def __init__(
         self,
         split_dir: str = "",
         data_path: str = "",
-        split_mode: str = "split_dir",
+        split_mode: str = "ratio",
         split_ratio: str = "2:1:7",
         split_seed: int = 42,
         split_output_dir: str = "",
-        workers: int = 4,
-        analyst_workers: int = 4,
-        failure_only: bool = False,
-        minibatch_size: int = 8,
-        edit_budget: int = 4,
         seed: int = 42,
         limit: int = 0,
-        max_completion_tokens: int = 4096,
+        **kwargs,
     ) -> None:
-        self.workers = workers
+        self.dataloader = TemplateBenchmarkDataLoader(
-        self.analyst_workers = analyst_workers
-        self.failure_only = failure_only
-        self.minibatch_size = minibatch_size
-        self.edit_budget = edit_budget
-        self.max_completion_tokens = int(max_completion_tokens)
-        self.dataloader = TemplateBenchmarkLoader(
             split_dir=split_dir,
             data_path=data_path,
             split_mode=split_mode,
@@ -66,8 +44,9 @@ class TemplateBenchmarkEnv(EnvAdapter):
             seed=seed,
             limit=limit,
         )
-
+        # TODO: initialize runtime options, e.g.
-    # ── Lifecycle hooks ────────────────────────────────────────────────
+        # self.max_retries = int(kwargs.get("max_retries", 3))
+        # self.timeout_s = int(kwargs.get("timeout_s", 120))
 
     def setup(self, cfg: dict) -> None:
         super().setup(cfg)
@@ -76,121 +55,29 @@ class TemplateBenchmarkEnv(EnvAdapter):
     def get_dataloader(self):
         return self.dataloader
 
-    # ── Batch → env manager ────────────────────────────────────────────
-
     def build_env_from_batch(self, batch: BatchSpec, **kwargs):
-        # Dataset-backed envs typically just pass items straight through.
         return list(batch.payload or [])
 
     def build_train_env(self, batch_size: int, seed: int, **kwargs):
-        batch = self.dataloader.build_train_batch(
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
-            batch_size=batch_size, seed=seed, **kwargs
-        )
         return self.build_env_from_batch(batch, **kwargs)
 
     def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
-        batch = self.dataloader.build_eval_batch(
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
-            env_num=env_num, split=split, seed=seed, **kwargs
-        )
         return self.build_env_from_batch(batch, **kwargs)
 
-    # ── Rollout: run episodes under current skill ──────────────────────
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
-
-    def rollout(
-        self,
-        env_manager,
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict]:
         """
-        Run a batch of episodes under the current skill.
+        Run one batch and return list[dict] with at least:
-
+        {"id": str, "hard": int, "soft": float}
-        TODO: replace this loop with your real rollout. For each item:
-          1. Build the prompt using `skill_content` as the system message.
-          2. Call your target model.
-          3. Score the prediction.
-          4. Return a dict with at minimum: ``id`` (str), ``hard`` (0|1),
-             ``soft`` (float in [0, 1]). Add any env-specific extras you
-             need for reflect() — they will be preserved on
-             ``RolloutResult.extras``.
         """
-        items: list[dict] = env_manager
+        raise NotImplementedError("Implement rollout() for your benchmark")
-        results: list[dict] = []
-        for item in items:
-            # ── REPLACE THIS BLOCK WITH YOUR REAL ROLLOUT ──
-            results.append(
-                {
-                    "id": str(item.get("id", "")),
-                    "hard": 0,
-                    "soft": 0.0,
-                    "predicted_answer": "",
-                    "question": item.get("question", ""),
-                    "fail_reason": "template rollout — not implemented",
-                }
-            )
-        return results
 
-    # ── Reflect: turn rollout results into patch dicts ─────────────────
+    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
-
-    def reflect(
-        self,
-        results: list[dict],
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
         """
-        Turn rollouts into a list of raw patch dicts (or None to drop).
+        Reflect on rollout results and return patch dicts (or None entries).
-
-        Each non-None dict MUST have:
-          - "patch":       {"edits": [...]}     a Patch.to_dict() payload
-          - "source_type": "failure" | "success"
-
-        Most benchmarks delegate to
-        :func:`skillopt.gradient.reflect.run_minibatch_reflect` which
-        will call the optimizer model with the
-        ``analyst_error_*`` / ``analyst_success_*`` prompts. To enable it,
-        uncomment the import above and call:
-
-            from skillopt.gradient.reflect import run_minibatch_reflect
-            return run_minibatch_reflect(
-                results=results,
-                skill_content=skill_content,
-                prediction_dir=kwargs.get(
-                    "prediction_dir", os.path.join(out_dir, "predictions")
-                ),
-                patches_dir=kwargs.get(
-                    "patches_dir", os.path.join(out_dir, "patches")
-                ),
-                workers=self.analyst_workers,
-                failure_only=self.failure_only,
-                minibatch_size=self.minibatch_size,
-                edit_budget=self.edit_budget,
-                random_seed=kwargs.get("random_seed"),
-                error_system=self.get_error_minibatch_prompt(),
-                success_system=self.get_success_minibatch_prompt(),
-                step_buffer_context=kwargs.get("step_buffer_context", ""),
-                update_mode=getattr(self, "_cfg", {}).get(
-                    "skill_update_mode", "patch"
-                ),
-            )
         """
-        # Template default: produce no patches (no-op trainer step).
+        raise NotImplementedError("Implement reflect() for your benchmark")
-        return [None for _ in results]
-
-    # ── Stratification hint ────────────────────────────────────────────
 
     def get_task_types(self) -> list[str]:
-        """Distinct task-type strings used for stratified sampling."""
+        return ["your_benchmark"]
-        seen: list[str] = []
-        all_items = (
-            self.dataloader.train_items
-            + self.dataloader.val_items
-            + self.dataloader.test_items
-        )
-        for item in all_items:
-            tt = str(item.get("task_type") or "template")
-            if tt not in seen:
-                seen.append(tt)
-        return seen or ["template"]

skillopt/envs/_template/loader_template.py

@@ -1,87 +1,40 @@
 """
 Benchmark Data Loader Template
 ================================
-Copy this file and implement ``load_split_items`` to load your benchmark
+Copy this file and implement the TODO sections to load your benchmark data.
-data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader`
-subclass — the base class handles both ``split_mode="split_dir"`` (read
-an existing train/val/test layout) and ``split_mode="ratio"`` (build the
-splits from a single raw file deterministically).
 
-For a fully worked example see
+The SplitDataLoader is responsible for:
-``skillopt/envs/officeqa/dataloader.py``.
+1. Loading raw data from disk for ratio split mode
+2. Loading items from train/val/test directories for split_dir mode
+3. Returning list[dict] items used by the training loop
 """
 from __future__ import annotations
 
-import json
-from pathlib import Path
-
 from skillopt.datasets.base import SplitDataLoader
 
 
-def _normalize_item(raw: dict) -> dict:
+class TemplateBenchmarkDataLoader(SplitDataLoader):
-    """
-    Normalise one raw entry into the dict shape SkillOpt expects.
-
-    The only **hard** requirement is ``"id"`` (str). Add whatever extra
-    fields your :class:`TemplateBenchmarkEnv.rollout` needs.
-    """
-    return {
-        "id": str(raw.get("uid") or raw.get("id") or ""),
-        "question": str(raw.get("question") or raw.get("prompt") or ""),
-        "ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""),
-        "task_type": str(raw.get("category") or raw.get("task_type") or "template"),
-        # ── add benchmark-specific keys here ──
-    }
-
-
-class TemplateBenchmarkLoader(SplitDataLoader):
     """
     Data loader for <Your Benchmark Name>.
 
-    Subclass note: you usually only need to implement
+    Rename this class and implement the methods below.
-    :meth:`load_split_items`. The base class drives ``setup(cfg)``,
-    materialises ratio-mode splits, exposes ``train_items``,
-    ``val_items``, ``test_items``, and builds ``BatchSpec`` objects on
-    demand.
-
-    If you want to support ``split_mode="ratio"`` (auto-split a single
-    file into train/val/test), also implement
-    :meth:`load_raw_items(data_path)` returning the full list of items.
     """
 
-    def load_split_items(self, split_path: str) -> list[dict]:
+    def load_raw_items(self, data_path: str) -> list[dict]:
-        """Load all items for one split directory.
-
-        ``split_path`` is e.g. ``data/your_benchmark/train/``. Return a
-        list of dicts, each shaped like :func:`_normalize_item`'s output.
         """
-        path = Path(split_path)
+        Parse raw benchmark data for split_mode="ratio".
 
-        json_files = sorted(path.glob("*.json"))
+        Return a list of normalized item dicts.
-        if json_files:
+        """
-            with json_files[0].open(encoding="utf-8") as f:
+        # TODO: parse your raw JSON/JSONL/CSV format and return list[dict]
-                payload = json.load(f)
+        # with deterministic "id" values.
-            if not isinstance(payload, list):
+        return super().load_raw_items(data_path)
-                raise ValueError(
-                    f"Expected JSON array at top level of {json_files[0]}"
-                )
-            return [_normalize_item(row) for row in payload]
 
-        jsonl_files = sorted(path.glob("*.jsonl"))
+    def load_split_items(self, split_path: str) -> list[dict]:
-        if jsonl_files:
+        """
-            items: list[dict] = []
+        Parse one split directory for split_mode="split_dir".
-            with jsonl_files[0].open(encoding="utf-8") as f:
-                for line in f:
-                    line = line.strip()
-                    if not line:
-                        continue
-                    items.append(_normalize_item(json.loads(line)))
-            return items
 
-        raise FileNotFoundError(
+        split_path points to train/, val/, or test/.
-            f"No .json or .jsonl file found in {split_path}"
+        """
-        )
+        # TODO: customize when split directories contain non-standard files.
-
+        return super().load_split_items(split_path)
-    # Optional — only needed if you intend to use ``split_mode='ratio'``.
-    # def load_raw_items(self, data_path: str) -> list[dict]:
-    #     ...

tests/test_env_template.py

@@ -0,0 +1,13 @@
+from skillopt.datasets.base import SplitDataLoader
+from skillopt.envs._template.env_template import TemplateBenchmarkAdapter
+from skillopt.envs._template.loader_template import TemplateBenchmarkDataLoader
+
+
+def test_template_adapter_is_concrete():
+    adapter = TemplateBenchmarkAdapter()
+    assert adapter.get_task_types() == ["your_benchmark"]
+
+
+def test_template_loader_uses_split_dataloader():
+    loader = TemplateBenchmarkDataLoader()
+    assert isinstance(loader, SplitDataLoader)