docs: align benchmark guide and templates with real adapter API

skillopt/envs/_template/env_template.py

@@ -4,89 +4,78 @@ Benchmark Environment Template
 Copy this file and implement the TODO sections to add a new benchmark.
 
 The EnvAdapter is responsible for:
-1. Executing tasks using the target model + current skill document
-2. Evaluating predictions against ground truth
-3. Returning structured results for the training loop
+1. Building train/eval environment payloads
+2. Running rollout and returning scored result rows
+3. Reflecting on results and returning patch candidates
 """
+from __future__ import annotations
+
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs._template.loader_template import TemplateBenchmarkDataLoader
 from skillopt.envs.base import EnvAdapter
 
 
-class TemplateBenchmarkEnv(EnvAdapter):
+class TemplateBenchmarkAdapter(EnvAdapter):
     """
     Environment adapter for <Your Benchmark Name>.
-    
+
     Rename this class and implement the abstract methods below.
     """
 
-    def __init__(self, cfg: dict):
-        super().__init__(cfg)
-        # TODO: Initialize benchmark-specific state
-        # Example: self.tools = load_tools(cfg)
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        seed: int = 42,
+        limit: int = 0,
+        **kwargs,
+    ) -> None:
+        self.dataloader = TemplateBenchmarkDataLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+        # TODO: initialize benchmark-specific runtime options from kwargs
 
-    async def execute(self, item, skill: str, model):
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
         """
-        Execute a single task with the target model.
-
-        Args:
-            item: DataItem with .id, .input, .ground_truth, .metadata
-            skill: Current skill document content (Markdown string)
-            model: Target model backend instance
-
-        Returns:
-            TaskResult with prediction, score, and trajectory
+        Run one batch and return list[dict] with at least:
+        {"id": str, "hard": int, "soft": float}
         """
-        # Step 1: Build the prompt combining skill + task input
-        prompt = self.build_prompt(item, skill)
+        raise NotImplementedError("Implement rollout() for your benchmark")
 
-        # Step 2: Call the target model
-        # TODO: Customize the message format for your benchmark
-        messages = [
-            {"role": "system", "content": skill},
-            {"role": "user", "content": item.input},
-        ]
-        response = await model.generate(messages)
-
-        # Step 3: Parse the model response into a prediction
-        prediction = self.parse_response(response.content)
-
-        # Step 4: Score the prediction
-        score = self.evaluate(prediction, item.ground_truth)
-
-        # Step 5: Return structured result
-        return {
-            "item_id": item.id,
-            "prediction": prediction,
-            "score": score,
-            "trajectory": messages + [{"role": "assistant", "content": response.content}],
-        }
-
-    def evaluate(self, prediction: str, ground_truth: str) -> float:
+    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
         """
-        Score a prediction against the ground truth.
-
-        Returns:
-            Float between 0.0 (wrong) and 1.0 (correct)
-        
-        TODO: Implement your scoring metric. Common options:
-        - Exact match: float(pred.strip().lower() == gt.strip().lower())
-        - F1 score: compute token overlap
-        - ANLS: for document QA tasks
-        - Custom: any float in [0, 1]
+        Reflect on rollout results and return patch dicts (or None entries).
         """
-        # Placeholder — exact match
-        return float(prediction.strip().lower() == ground_truth.strip().lower())
+        raise NotImplementedError("Implement reflect() for your benchmark")
 
-    def build_prompt(self, item, skill: str) -> str:
-        """Combine skill document with task input."""
-        return f"{skill}\n\n---\n\nQuestion: {item.input}"
-
-    def parse_response(self, response: str) -> str:
-        """
-        Extract the answer from the model's raw response.
-        
-        TODO: Implement extraction logic. For example:
-        - Extract text after "Answer:" 
-        - Parse JSON output
-        - Extract from code blocks
-        """
-        return response.strip()
+    def get_task_types(self) -> list[str]:
+        return ["your_benchmark"]