diff --git a/docs/guide/new-benchmark.md b/docs/guide/new-benchmark.md index b5d32b5..01b34c4 100644 --- a/docs/guide/new-benchmark.md +++ b/docs/guide/new-benchmark.md @@ -31,7 +31,7 @@ class MyBenchmarkDataLoader(SplitDataLoader): def load_raw_items(self, data_path: str) -> list[dict]: # For ratio mode, parse your source dataset from data_path. - # Return list[dict] where each item has at least a stable "id". + # Return list[dict] where each item has at least a unique, deterministic "id". return super().load_raw_items(data_path) def load_split_items(self, split_path: str) -> list[dict]: @@ -65,7 +65,9 @@ class MyBenchmarkAdapter(EnvAdapter): return self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs).payload def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]: - # Run target model on each item in env_manager and return list[dict]. + # env_manager is the payload returned by build_train_env/build_eval_env + # (commonly list[dict] task items). + # Run target model on each item and return list[dict]. # Required keys per row: "id", "hard" (0/1), "soft" (0.0-1.0) raise NotImplementedError diff --git a/skillopt/envs/_template/env_template.py b/skillopt/envs/_template/env_template.py index 40d39ee..83ca344 100644 --- a/skillopt/envs/_template/env_template.py +++ b/skillopt/envs/_template/env_template.py @@ -44,7 +44,9 @@ class TemplateBenchmarkAdapter(EnvAdapter): seed=seed, limit=limit, ) - # TODO: initialize benchmark-specific runtime options from kwargs + # TODO: initialize runtime options, e.g. + # self.max_retries = int(kwargs.get("max_retries", 3)) + # self.timeout_s = int(kwargs.get("timeout_s", 120)) def setup(self, cfg: dict) -> None: super().setup(cfg) diff --git a/skillopt/envs/_template/loader_template.py b/skillopt/envs/_template/loader_template.py index d82611b..7e0896f 100644 --- a/skillopt/envs/_template/loader_template.py +++ b/skillopt/envs/_template/loader_template.py @@ -26,7 +26,8 @@ class TemplateBenchmarkDataLoader(SplitDataLoader): Return a list of normalized item dicts. """ - # TODO: customize when your raw source format differs. + # TODO: parse your raw JSON/JSONL/CSV format and return list[dict] + # with deterministic "id" values. return super().load_raw_items(data_path) def load_split_items(self, split_path: str) -> list[dict]: @@ -35,5 +36,5 @@ class TemplateBenchmarkDataLoader(SplitDataLoader): split_path points to train/, val/, or test/. """ - # TODO: customize when each split directory has a custom layout. + # TODO: customize when split directories contain non-standard files. return super().load_split_items(split_path)