test: add template contract checks and refine benchmark docs

docs/guide/new-benchmark.md

@@ -31,7 +31,7 @@ class MyBenchmarkDataLoader(SplitDataLoader):
 
     def load_raw_items(self, data_path: str) -> list[dict]:
         # For ratio mode, parse your source dataset from data_path.
-        # Return list[dict] where each item has at least a stable "id".
+        # Return list[dict] where each item has at least a unique, deterministic "id".
         return super().load_raw_items(data_path)
 
     def load_split_items(self, split_path: str) -> list[dict]:
@@ -65,7 +65,9 @@ class MyBenchmarkAdapter(EnvAdapter):
         return self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs).payload
 
     def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
-        # Run target model on each item in env_manager and return list[dict].
+        # env_manager is the payload returned by build_train_env/build_eval_env
+        # (commonly list[dict] task items).
+        # Run target model on each item and return list[dict].
         # Required keys per row: "id", "hard" (0/1), "soft" (0.0-1.0)
         raise NotImplementedError
 

skillopt/envs/_template/env_template.py

@@ -44,7 +44,9 @@ class TemplateBenchmarkAdapter(EnvAdapter):
             seed=seed,
             limit=limit,
         )
-        # TODO: initialize benchmark-specific runtime options from kwargs
+        # TODO: initialize runtime options, e.g.
+        # self.max_retries = int(kwargs.get("max_retries", 3))
+        # self.timeout_s = int(kwargs.get("timeout_s", 120))
 
     def setup(self, cfg: dict) -> None:
         super().setup(cfg)

skillopt/envs/_template/loader_template.py

@@ -26,7 +26,8 @@ class TemplateBenchmarkDataLoader(SplitDataLoader):
 
         Return a list of normalized item dicts.
         """
-        # TODO: customize when your raw source format differs.
+        # TODO: parse your raw JSON/JSONL/CSV format and return list[dict]
+        # with deterministic "id" values.
         return super().load_raw_items(data_path)
 
     def load_split_items(self, split_path: str) -> list[dict]:
@@ -35,5 +36,5 @@ class TemplateBenchmarkDataLoader(SplitDataLoader):
 
         split_path points to train/, val/, or test/.
         """
-        # TODO: customize when each split directory has a custom layout.
+        # TODO: customize when split directories contain non-standard files.
         return super().load_split_items(split_path)