SkillOpt v0.1.0: initial release
- Skill optimization framework with training loop analogy - 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen) - WebUI for browser-based training control - Pluggable architecture for extending benchmarks and backends
This commit is contained in:
CharlesYang030
@@ -0,0 +1 @@
|
||||
"""LiveMathematicianBench environment package for ReflACT."""
|
||||
@@ -0,0 +1,282 @@
|
||||
"""LiveMathematicianBench environment adapter for ReflACT."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from skillopt.gradient.deep_probe import generate_deep_probe_instruction
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.livemathematicianbench.dataloader import LiveMathematicianBenchDataLoader
|
||||
from skillopt.envs.livemathematicianbench.rollout import run_batch
|
||||
from skillopt.model import get_student_backend
|
||||
|
||||
|
||||
class LiveMathematicianBenchAdapter(EnvAdapter):
|
||||
"""LiveMathematicianBench adapter."""
|
||||
|
||||
def build_reference_text(self, item: dict) -> str:
|
||||
parts: list[str] = []
|
||||
theorem = str(item.get("theorem") or "").strip()
|
||||
sketch = str(item.get("sketch") or "").strip()
|
||||
if theorem:
|
||||
parts.append(f"## Reference Theorem\n{theorem}")
|
||||
if sketch:
|
||||
parts.append(f"## Reference Sketch\n{sketch}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def get_reference_metadata(self, item: dict) -> dict:
|
||||
fields: list[str] = []
|
||||
previews: list[str] = []
|
||||
theorem = str(item.get("theorem") or "").strip()
|
||||
sketch = str(item.get("sketch") or "").strip()
|
||||
if theorem:
|
||||
fields.append("theorem")
|
||||
previews.append(f"[theorem]\n{theorem[:220]}")
|
||||
if sketch:
|
||||
fields.append("sketch")
|
||||
previews.append(f"[sketch]\n{sketch[:220]}")
|
||||
return {
|
||||
"fields": fields,
|
||||
"preview": "\n\n".join(previews)[:500],
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "ratio",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
max_turns: int = 1,
|
||||
exec_timeout: int = 600,
|
||||
workers: int = 64,
|
||||
analyst_workers: int = 16,
|
||||
failure_only: bool = False,
|
||||
minibatch_size: int = 8,
|
||||
edit_budget: int = 4,
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
shuffle_choices: bool = True,
|
||||
use_theorem: bool = False,
|
||||
use_sketch: bool = False,
|
||||
use_deep_reflect: bool = False,
|
||||
deep_reflect_failures: int = 4,
|
||||
deep_reflect_successes: int = 2,
|
||||
) -> None:
|
||||
self.max_turns = max_turns
|
||||
self.exec_timeout = exec_timeout
|
||||
self.workers = workers
|
||||
self.analyst_workers = analyst_workers
|
||||
self.failure_only = failure_only
|
||||
self.minibatch_size = minibatch_size
|
||||
self.edit_budget = edit_budget
|
||||
self.use_theorem = use_theorem
|
||||
self.use_sketch = use_sketch
|
||||
self.use_deep_reflect = use_deep_reflect
|
||||
self.deep_reflect_failures = deep_reflect_failures
|
||||
self.deep_reflect_successes = deep_reflect_successes
|
||||
self.dataloader = LiveMathematicianBenchDataLoader(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
shuffle_choices=shuffle_choices,
|
||||
)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
self.dataloader.setup(cfg)
|
||||
|
||||
def get_dataloader(self):
|
||||
return self.dataloader
|
||||
|
||||
def build_env_from_batch(self, batch: BatchSpec, **kwargs):
|
||||
return list(batch.payload or [])
|
||||
|
||||
def build_train_env(self, batch_size: int, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def rollout(
|
||||
self,
|
||||
env_manager,
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict]:
|
||||
items: list[dict] = env_manager
|
||||
return run_batch(
|
||||
items=items,
|
||||
out_root=out_dir,
|
||||
skill_content=skill_content,
|
||||
max_turns=self.max_turns,
|
||||
exec_timeout=self.exec_timeout,
|
||||
workers=self.workers,
|
||||
use_theorem=self.use_theorem,
|
||||
use_sketch=self.use_sketch,
|
||||
diagnostic_mode=kwargs.get("diagnostic_mode", False),
|
||||
diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
|
||||
diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"),
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def deep_reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
if not self.use_deep_reflect:
|
||||
return []
|
||||
|
||||
env_manager = kwargs.get("env_manager")
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
codex_backend = get_student_backend() == "codex_exec"
|
||||
selected_items = self.select_representative_items(
|
||||
results,
|
||||
env_manager if isinstance(env_manager, list) else None,
|
||||
n_failures=self.deep_reflect_failures,
|
||||
n_successes=self.deep_reflect_successes,
|
||||
seed=random_seed,
|
||||
)
|
||||
if not selected_items:
|
||||
return []
|
||||
selected_ids = {str(item["id"]) for item in selected_items}
|
||||
selected_results = [row for row in results if str(row.get("id")) in selected_ids]
|
||||
selected_examples = self.attach_reference_context(selected_results, selected_items)
|
||||
if codex_backend:
|
||||
selected_examples = self.attach_codex_probe_context(selected_examples, prediction_dir)
|
||||
selected_metadata = []
|
||||
theorem_count = 0
|
||||
sketch_count = 0
|
||||
for item in selected_items:
|
||||
meta = self.get_reference_metadata(item)
|
||||
if "theorem" in meta["fields"]:
|
||||
theorem_count += 1
|
||||
if "sketch" in meta["fields"]:
|
||||
sketch_count += 1
|
||||
selected_metadata.append({
|
||||
"id": str(item["id"]),
|
||||
"task_type": str(item.get("theorem_type", ["math_mcq"])[0] if item.get("theorem_type") else "math_mcq"),
|
||||
"reference_fields": meta["fields"],
|
||||
"reference_preview": meta["preview"],
|
||||
})
|
||||
|
||||
deep_dir = os.path.join(out_dir, "deep_reflect")
|
||||
rollout_dir = os.path.join(deep_dir, "rollout")
|
||||
patches_dir = os.path.join(deep_dir, "patches")
|
||||
os.makedirs(deep_dir, exist_ok=True)
|
||||
print(
|
||||
f" [2b/6 DEEP REFLECT setup] selected={len(selected_items)} "
|
||||
f"reference_fields=theorem({theorem_count}/{len(selected_items)}),"
|
||||
f"sketch({sketch_count}/{len(selected_items)})"
|
||||
)
|
||||
probe = generate_deep_probe_instruction(
|
||||
skill_content=skill_content,
|
||||
items=selected_examples,
|
||||
prediction_dir=prediction_dir,
|
||||
system_prompt=self.get_codex_deep_probe_prompt() if codex_backend else self.get_deep_probe_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
)
|
||||
if not probe:
|
||||
return []
|
||||
diagnostic_trace_context_by_id = None
|
||||
if codex_backend:
|
||||
selected_items, diagnostic_trace_context_by_id, probe = self.resolve_codex_probe_target(
|
||||
selected_items=selected_items,
|
||||
selected_examples=selected_examples,
|
||||
prediction_dir=prediction_dir,
|
||||
probe=probe,
|
||||
)
|
||||
probe_record = {
|
||||
**probe,
|
||||
"reference_summary": {
|
||||
"selected_count": len(selected_items),
|
||||
"field_counts": {
|
||||
"theorem": theorem_count,
|
||||
"sketch": sketch_count,
|
||||
},
|
||||
},
|
||||
"selected_examples": selected_metadata,
|
||||
}
|
||||
with open(os.path.join(deep_dir, "probe.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(probe_record, f, ensure_ascii=False, indent=2)
|
||||
deep_results = run_batch(
|
||||
items=selected_items,
|
||||
out_root=rollout_dir,
|
||||
skill_content=skill_content,
|
||||
max_turns=self.max_turns,
|
||||
workers=min(self.workers, max(len(selected_items), 1)),
|
||||
use_theorem=self.use_theorem,
|
||||
use_sketch=self.use_sketch,
|
||||
diagnostic_mode=True,
|
||||
diagnostic_instruction=probe["probe_instruction"],
|
||||
diagnostic_trace_context_by_id=diagnostic_trace_context_by_id,
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
deep_results = self.attach_reference_context(deep_results, selected_items)
|
||||
return run_minibatch_reflect(
|
||||
results=deep_results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=os.path.join(rollout_dir, "predictions"),
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return self.dataloader.get_task_types()
|
||||
@@ -0,0 +1,308 @@
|
||||
"""LiveMathematicianBench task dataloader."""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
from skillopt.datasets.base import BatchSpec, SplitDataLoader
|
||||
|
||||
|
||||
# ── Raw data loading utilities (for preprocessing / standalone eval) ─────
|
||||
|
||||
_CHOICE_LABELS = ["A", "B", "C", "D", "E", "F", "G"]
|
||||
|
||||
|
||||
def _load_json(path: str) -> Any:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _iter_monthly_files(data_path: str) -> list[str]:
|
||||
if not data_path:
|
||||
return []
|
||||
if os.path.isfile(data_path):
|
||||
return [data_path]
|
||||
if os.path.isdir(data_path):
|
||||
nested = glob.glob(
|
||||
os.path.join(data_path, "**", "qa_*_final.json"),
|
||||
recursive=True,
|
||||
)
|
||||
flat = glob.glob(os.path.join(data_path, "qa_*_final.json"))
|
||||
return sorted(set(nested + flat))
|
||||
return []
|
||||
|
||||
|
||||
def _coerce_choices(raw_choices: Any) -> list[dict]:
|
||||
if isinstance(raw_choices, list):
|
||||
choices: list[dict] = []
|
||||
for idx, item in enumerate(raw_choices):
|
||||
if isinstance(item, dict):
|
||||
label = str(item.get("label") or _CHOICE_LABELS[idx]).strip()
|
||||
text = str(item.get("text") or item.get("content") or "").strip()
|
||||
else:
|
||||
label = _CHOICE_LABELS[idx]
|
||||
text = str(item).strip()
|
||||
if text:
|
||||
choices.append({"label": label, "text": text})
|
||||
return choices
|
||||
|
||||
if isinstance(raw_choices, dict):
|
||||
labels = sorted(raw_choices.keys())
|
||||
return [
|
||||
{"label": str(label).strip(), "text": str(raw_choices[label]).strip()}
|
||||
for label in labels
|
||||
if str(raw_choices[label]).strip()
|
||||
]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _coerce_theorem_types(raw: Any) -> list[str]:
|
||||
if isinstance(raw, list):
|
||||
return [str(x).strip() for x in raw if str(x).strip()]
|
||||
if raw is None:
|
||||
return []
|
||||
text = str(raw).strip()
|
||||
return [text] if text else []
|
||||
|
||||
|
||||
def _normalize_label(text: str) -> str:
|
||||
return str(text).strip().upper().rstrip(".):")
|
||||
|
||||
|
||||
def _normalize_item(item: dict, row_idx: int, source_path: str) -> dict:
|
||||
mcq = item.get("mcq", {}) if isinstance(item.get("mcq"), dict) else {}
|
||||
question = str(mcq.get("question") or item.get("question") or "").strip()
|
||||
choices = _coerce_choices(mcq.get("choices") or item.get("choices") or [])
|
||||
correct = mcq.get("correct_choice") or item.get("correct_choice") or {}
|
||||
|
||||
if isinstance(correct, dict):
|
||||
correct_label = _normalize_label(correct.get("label", ""))
|
||||
correct_text = str(correct.get("text") or "").strip()
|
||||
else:
|
||||
correct_label = _normalize_label(correct)
|
||||
correct_text = ""
|
||||
|
||||
choice_by_label = {
|
||||
_normalize_label(choice["label"]): choice["text"]
|
||||
for choice in choices
|
||||
}
|
||||
if correct_label and not correct_text:
|
||||
correct_text = choice_by_label.get(correct_label, "")
|
||||
if correct_label and correct_text and correct_label not in choice_by_label:
|
||||
choices.append({"label": correct_label, "text": correct_text})
|
||||
choices.sort(key=lambda choice: _CHOICE_LABELS.index(choice["label"]) if choice["label"] in _CHOICE_LABELS else len(_CHOICE_LABELS))
|
||||
choice_by_label[correct_label] = correct_text
|
||||
|
||||
month = str(item.get("month") or "").strip()
|
||||
item_no = item.get("no", row_idx + 1)
|
||||
item_id = f"{month}:{item_no}" if month else str(item_no)
|
||||
|
||||
return {
|
||||
"id": item_id,
|
||||
"month": month,
|
||||
"no": item_no,
|
||||
"paper_link": str(item.get("paper_link") or "").strip(),
|
||||
"theorem": str(item.get("theorem") or "").strip(),
|
||||
"sketch": str(item.get("sketch") or "").strip(),
|
||||
"theorem_type": _coerce_theorem_types(item.get("theorem_type")),
|
||||
"question": question,
|
||||
"choices": choices,
|
||||
"correct_choice": {
|
||||
"label": correct_label,
|
||||
"text": correct_text,
|
||||
},
|
||||
"source_path": source_path,
|
||||
}
|
||||
|
||||
|
||||
def load_items(data_path: str) -> list[dict]:
|
||||
"""Load and normalise LiveMathematicianBench items from JSON files."""
|
||||
files = _iter_monthly_files(data_path)
|
||||
if not files:
|
||||
raise ValueError(
|
||||
"LiveMathematicianBench requires data_path to be a qa_*_final.json file "
|
||||
"or a directory containing monthly qa_*_final.json files."
|
||||
)
|
||||
|
||||
items: list[dict] = []
|
||||
for path in files:
|
||||
raw = _load_json(path)
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError(f"Expected JSON array in {path}, got {type(raw).__name__}")
|
||||
for row_idx, item in enumerate(raw):
|
||||
norm = _normalize_item(item, row_idx=row_idx, source_path=path)
|
||||
if norm["question"] and norm["choices"] and norm["correct_choice"]["label"]:
|
||||
items.append(norm)
|
||||
if not items:
|
||||
raise ValueError(f"No valid LiveMathematicianBench items loaded from {data_path}")
|
||||
return items
|
||||
|
||||
|
||||
# ── Dataloader ───────────────────────────────────────────────────────────
|
||||
|
||||
class LiveMathematicianBenchDataLoader(SplitDataLoader):
|
||||
"""LiveMathematicianBench dataloader with per-seed choice shuffling."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "ratio",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
shuffle_choices: bool = True,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
)
|
||||
self.shuffle_choices = shuffle_choices
|
||||
self._task_types: list[str] = []
|
||||
|
||||
def load_raw_items(self, data_path: str) -> list[dict]:
|
||||
return load_items(data_path)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
all_items = self.train_items + self.val_items + self.test_items
|
||||
task_types: set[str] = set()
|
||||
for item in all_items:
|
||||
for name in item.get("theorem_type", []):
|
||||
if name:
|
||||
task_types.add(name)
|
||||
self._task_types = sorted(task_types)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return list(self._task_types)
|
||||
|
||||
# ── Choice shuffling ─────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _item_shuffle_seed(item_id: str, seed: int) -> int:
|
||||
digest = hashlib.sha256(f"{seed}:{item_id}".encode("utf-8")).hexdigest()
|
||||
return int(digest[:16], 16)
|
||||
|
||||
def _shuffle_item_choices(self, item: dict, seed: int) -> dict:
|
||||
if not self.shuffle_choices:
|
||||
return {
|
||||
**item,
|
||||
"choices": [dict(c) for c in item["choices"]],
|
||||
"correct_choice": dict(item["correct_choice"]),
|
||||
}
|
||||
|
||||
shuffled_choices = [dict(c) for c in item["choices"]]
|
||||
rng = random.Random(self._item_shuffle_seed(str(item["id"]), seed))
|
||||
rng.shuffle(shuffled_choices)
|
||||
|
||||
original_correct = _normalize_label(item["correct_choice"]["label"])
|
||||
remapped_choices: list[dict] = []
|
||||
new_correct_choice = dict(item["correct_choice"])
|
||||
|
||||
for idx, choice in enumerate(shuffled_choices):
|
||||
new_label = _CHOICE_LABELS[idx]
|
||||
old_label = _normalize_label(choice["label"])
|
||||
remapped_choices.append({"label": new_label, "text": choice["text"]})
|
||||
if old_label == original_correct:
|
||||
new_correct_choice = {"label": new_label, "text": choice["text"]}
|
||||
|
||||
transformed = dict(item)
|
||||
transformed["choices"] = remapped_choices
|
||||
transformed["correct_choice"] = new_correct_choice
|
||||
return transformed
|
||||
|
||||
def _materialize_batch(self, items: list[dict], seed: int) -> list[dict]:
|
||||
return [self._shuffle_item_choices(item, seed) for item in items]
|
||||
|
||||
# ── Batch construction (override for choice shuffling) ───────────────
|
||||
|
||||
def plan_train_epoch(
|
||||
self,
|
||||
*,
|
||||
epoch: int,
|
||||
steps_per_epoch: int,
|
||||
accumulation: int,
|
||||
batch_size: int,
|
||||
seed: int,
|
||||
**kwargs,
|
||||
) -> list[BatchSpec]:
|
||||
"""Build a shuffled epoch while preserving per-batch choice shuffling."""
|
||||
epoch_rng = random.Random(seed + epoch * 1000)
|
||||
items = list(self.train_items)
|
||||
epoch_rng.shuffle(items)
|
||||
|
||||
total_batches = steps_per_epoch * accumulation
|
||||
if total_batches <= 0:
|
||||
return []
|
||||
|
||||
batches: list[BatchSpec] = []
|
||||
cursor = 0
|
||||
for batch_idx in range(total_batches):
|
||||
batch_seed = seed + epoch * 1000 + batch_idx + 1
|
||||
batch_items = items[cursor: cursor + batch_size]
|
||||
cursor += len(batch_items)
|
||||
|
||||
if not batch_items and items:
|
||||
refill_rng = random.Random(batch_seed)
|
||||
batch_items = list(items)
|
||||
refill_rng.shuffle(batch_items)
|
||||
batch_items = batch_items[:batch_size]
|
||||
|
||||
batch_items = self._materialize_batch(batch_items, batch_seed)
|
||||
batches.append(
|
||||
BatchSpec(
|
||||
phase="train",
|
||||
split="train",
|
||||
seed=batch_seed,
|
||||
batch_size=len(batch_items),
|
||||
payload=batch_items,
|
||||
)
|
||||
)
|
||||
|
||||
return batches
|
||||
|
||||
def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec:
|
||||
rng = random.Random(seed)
|
||||
items = list(self.train_items)
|
||||
rng.shuffle(items)
|
||||
items = self._materialize_batch(items[:batch_size], seed)
|
||||
return BatchSpec(
|
||||
phase="train",
|
||||
split="train",
|
||||
seed=seed,
|
||||
batch_size=len(items),
|
||||
payload=items,
|
||||
)
|
||||
|
||||
def build_eval_batch(
|
||||
self,
|
||||
env_num: int,
|
||||
split: str,
|
||||
seed: int,
|
||||
**kwargs,
|
||||
) -> BatchSpec:
|
||||
items = self.get_split_items(split)
|
||||
if env_num and env_num < len(items):
|
||||
items = items[:env_num]
|
||||
items = self._materialize_batch(items, seed)
|
||||
return BatchSpec(
|
||||
phase="eval",
|
||||
split=split,
|
||||
seed=seed,
|
||||
batch_size=len(items),
|
||||
payload=items,
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""LiveMathematicianBench evaluation helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def extract_answer(text: str) -> str:
|
||||
matches = re.findall(r"<answer>(.*?)</answer>", text, re.DOTALL | re.IGNORECASE)
|
||||
if matches:
|
||||
return matches[-1].strip()
|
||||
lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()]
|
||||
if lines:
|
||||
return lines[-1]
|
||||
return text.strip()
|
||||
|
||||
|
||||
def normalize_label(text: str) -> str:
|
||||
return str(text).strip().upper().rstrip(".):")
|
||||
|
||||
|
||||
def parse_choice_label(prediction_text: str, choices: list[dict]) -> str:
|
||||
answer = extract_answer(prediction_text)
|
||||
label = normalize_label(answer)
|
||||
valid_labels = {normalize_label(choice.get("label", "")) for choice in choices}
|
||||
if label in valid_labels:
|
||||
return label
|
||||
|
||||
answer_lower = answer.lower()
|
||||
for choice in choices:
|
||||
choice_label = normalize_label(choice.get("label", ""))
|
||||
choice_text = str(choice.get("text", "")).strip()
|
||||
if choice_text and choice_text.lower() == answer_lower:
|
||||
return choice_label
|
||||
|
||||
first_token = normalize_label(answer.split()[0]) if answer.split() else ""
|
||||
if first_token in valid_labels:
|
||||
return first_token
|
||||
return label
|
||||
|
||||
|
||||
def evaluate(prediction_text: str, correct_choice: dict, choices: list[dict]) -> dict:
|
||||
predicted_label = parse_choice_label(prediction_text, choices)
|
||||
correct_label = normalize_label(correct_choice.get("label", ""))
|
||||
predicted_text = ""
|
||||
correct_text = str(correct_choice.get("text", "")).strip()
|
||||
|
||||
for choice in choices:
|
||||
if normalize_label(choice.get("label", "")) == predicted_label:
|
||||
predicted_text = str(choice.get("text", "")).strip()
|
||||
break
|
||||
|
||||
is_correct = float(predicted_label == correct_label)
|
||||
return {
|
||||
"em": is_correct,
|
||||
"f1": is_correct,
|
||||
"sub_em": is_correct,
|
||||
"predicted_answer": predicted_label or extract_answer(prediction_text),
|
||||
"predicted_label": predicted_label,
|
||||
"predicted_text": predicted_text,
|
||||
"correct_label": correct_label,
|
||||
"correct_text": correct_text,
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
You are an expert failure-analysis agent for theorem-grounded mathematical multiple-choice questions.
|
||||
|
||||
You will be given MULTIPLE failed trajectories from a single minibatch and the current skill document.
|
||||
Each trajectory includes the student's response and an evaluation result showing the predicted option
|
||||
versus the correct option.
|
||||
|
||||
Your job is to identify COMMON reasoning failures across the batch and propose concise skill edits.
|
||||
|
||||
## Failure Type Categories
|
||||
- **quantifier_miss**: the agent missed exact quantifiers, scope, or existence/uniqueness conditions
|
||||
- **strength_mismatch**: the agent preferred a weaker or stronger statement than what was proved
|
||||
- **condition_miss**: the agent ignored hypotheses, equality cases, or domain restrictions
|
||||
- **option_confusion**: the agent confused similar answer choices or failed to compare them exactly
|
||||
- **other**: none of the above
|
||||
|
||||
## Rules
|
||||
1. Focus on patterns that recur across the minibatch.
|
||||
2. Prefer edits that improve exact choice discrimination, not theorem-specific memorization.
|
||||
3. Do not hardcode paper-specific content.
|
||||
4. Only patch gaps not already covered by the skill.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number>,
|
||||
"failure_summary": [
|
||||
{"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
|
||||
],
|
||||
"patch": {
|
||||
"reasoning": "<why these edits address the common failures>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
You are an expert success-pattern analyst for theorem-grounded mathematical multiple-choice questions.
|
||||
|
||||
You will be given MULTIPLE successful trajectories from a minibatch and the current skill document.
|
||||
Identify generalizable behavior patterns that are genuinely helping the agent choose the exact correct option.
|
||||
|
||||
## Rules
|
||||
- Focus on broadly useful reasoning behaviors.
|
||||
- Prefer patterns about exact comparison of options, quantifiers, and equality conditions.
|
||||
- Do not add theorem-specific facts.
|
||||
- "edits" may be empty if the skill already captures the useful patterns.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number>,
|
||||
"success_patterns": ["<pattern 1>", "<pattern 2>"],
|
||||
"patch": {
|
||||
"reasoning": "<why these patterns matter>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
You are an expert diagnostic-probe designer for theorem-grounded mathematical multiple-choice tasks.
|
||||
|
||||
You will be shown representative trajectories, the current student skill, and the student's original prompt context.
|
||||
Design one SMALL diagnostic instruction that exposes the student's intermediate judgment without materially changing the original scaffold.
|
||||
|
||||
## Hard Constraints
|
||||
1. Do NOT substantially change the original scaffold.
|
||||
2. Do NOT prescribe a new multi-step theorem-solving procedure.
|
||||
3. Do NOT ask for a full proof, full chain-of-thought, or exhaustive option-by-option derivation.
|
||||
4. Ask only for a short readout of the signals already behind the student's current answer.
|
||||
5. Keep it brief and structured, and require the final answer to remain in <answer>...</answer>.
|
||||
|
||||
## Good Probe Targets
|
||||
- top choice and runner-up
|
||||
- decisive constraint
|
||||
- why the runner-up was rejected
|
||||
- strongest-vs-weaker discrimination signal
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"reasoning": "<why this probe is informative>",
|
||||
"probe_instruction": "<the exact instruction text to append to the student prompt>"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
You are an expert diagnostic-probe designer for theorem-grounded mathematical multiple-choice tasks executed through a Codex trace.
|
||||
|
||||
You will be shown representative trajectories, the current student skill, the student's original prompt context, hidden reference fields, and numbered Codex trace steps.
|
||||
Choose exactly one trajectory and one probe point. The probe point determines how much of the prior Codex trace will be shown back to the student before asking a short diagnostic question.
|
||||
|
||||
## Hard Constraints
|
||||
1. Do NOT reveal or paraphrase the hidden reference directly to the student.
|
||||
2. Do NOT prescribe a new full solving procedure.
|
||||
3. Do NOT ask for a full proof, full chain-of-thought, or exhaustive option-by-option derivation.
|
||||
4. Ask only for a short readout of the signal that should already exist at that point in the student's process.
|
||||
5. The probe instruction must explicitly request a short <analysis>...</analysis> block before the final <answer>...</answer>.
|
||||
6. Select a probe point that is informative about theorem choice, decisive constraint, option elimination, or why a stronger/weaker option should be rejected.
|
||||
|
||||
## Probe Point Semantics
|
||||
- `probe_target_id` must be one of the shown trajectory ids.
|
||||
- `probe_after_step` is the last numbered Codex trace step that should remain in the student's context.
|
||||
- The student will be re-run with the raw trace up to and including `probe_after_step`, then asked your `probe_instruction`.
|
||||
- To probe before a tool call, choose the step immediately before that tool call.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"reasoning": "<why this trajectory and probe point expose the student's intermediate state>",
|
||||
"probe_target_id": "<trajectory id>",
|
||||
"probe_after_step": <integer step number>,
|
||||
"probe_instruction": "<the exact instruction text to append to the student's prompt>"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
You are an expert mathematical reasoning agent solving multiple-choice questions.
|
||||
|
||||
{skill_section}## Task Format
|
||||
You will receive one mathematics multiple-choice question and its answer choices.
|
||||
Reason carefully about quantifiers, hypotheses, extremal wording, and exact equality conditions.
|
||||
|
||||
## Answer Format
|
||||
Think step by step, then provide your final answer inside <answer>...</answer> tags.
|
||||
Inside the tags, output only the single choice label, such as A or C.
|
||||
|
||||
Example:
|
||||
<answer>B</answer>
|
||||
@@ -0,0 +1,4 @@
|
||||
"""LiveMathematicianBench Reflect stage.
|
||||
|
||||
Prompts are now loaded from .md files by the base adapter.
|
||||
"""
|
||||
@@ -0,0 +1,424 @@
|
||||
"""LiveMathematicianBench rollout — theorem-grounded math MCQ agent."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
|
||||
from skillopt.envs.livemathematicianbench.evaluator import evaluate
|
||||
from skillopt.model import chat_student, get_student_backend, is_student_exec_backend
|
||||
from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
|
||||
from skillopt.prompts import load_prompt
|
||||
|
||||
def _build_system(skill_content: str) -> str:
|
||||
if skill_content.strip():
|
||||
skill_section = f"## Skill\n{skill_content.strip()}\n\n"
|
||||
else:
|
||||
skill_section = ""
|
||||
return load_prompt("rollout_system", env="livemathematicianbench").format(skill_section=skill_section)
|
||||
|
||||
|
||||
def _format_choices(choices: list[dict]) -> str:
|
||||
return "\n".join(
|
||||
f"{choice['label']}. {choice['text']}"
|
||||
for choice in choices
|
||||
)
|
||||
|
||||
|
||||
def _build_user(
|
||||
item: dict,
|
||||
*,
|
||||
use_theorem: bool = False,
|
||||
use_sketch: bool = False,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
) -> str:
|
||||
parts = [f"## Question\n{item['question']}", f"## Choices\n{_format_choices(item['choices'])}"]
|
||||
if use_theorem and item.get("theorem"):
|
||||
parts.append(f"## Theorem\n{item['theorem']}")
|
||||
if use_sketch and item.get("sketch"):
|
||||
parts.append(f"## Proof Sketch\n{item['sketch']}")
|
||||
if diagnostic_trace_context.strip():
|
||||
parts.append(
|
||||
"## Previous Codex Trace Snapshot\n"
|
||||
"This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
|
||||
f"{diagnostic_trace_context.strip()}"
|
||||
)
|
||||
if diagnostic_mode and diagnostic_instruction.strip():
|
||||
parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _build_codex_skill(skill_content: str) -> str:
|
||||
return render_skill_md(
|
||||
skill_content,
|
||||
description="Dynamic ReflACT skill for solving the current LiveMathematicianBench multiple-choice question.",
|
||||
preamble=(
|
||||
"Use this skill when solving the current math multiple-choice question.\n"
|
||||
"Inspect the option wording carefully and output only the final choice label inside <answer>...</answer>."
|
||||
),
|
||||
)
|
||||
|
||||
def _run_codex_once(
|
||||
*,
|
||||
pred_dir: str,
|
||||
skill_content: str,
|
||||
item: dict,
|
||||
model: str,
|
||||
timeout: int,
|
||||
use_theorem: bool = False,
|
||||
use_sketch: bool = False,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
previous_response: str = "",
|
||||
) -> tuple[str, str, str, str]:
|
||||
user = _build_user(
|
||||
item,
|
||||
use_theorem=use_theorem,
|
||||
use_sketch=use_sketch,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=diagnostic_trace_context,
|
||||
)
|
||||
task_parts = [user]
|
||||
if previous_response:
|
||||
task_parts.append(
|
||||
"## Previous Attempt\n"
|
||||
f"{previous_response}\n\n"
|
||||
"Re-evaluate the exact option wording. If needed, correct it."
|
||||
)
|
||||
task_text = "\n\n".join(task_parts)
|
||||
skill_md = _build_codex_skill(skill_content)
|
||||
work_dir = os.path.join(pred_dir, "codex_exec")
|
||||
prepare_workspace(work_dir=work_dir, skill_md=skill_md, task_text=task_text)
|
||||
prompt = (
|
||||
"Use the `skillopt-student` skill available in this workspace.\n"
|
||||
"Read `task.md` and solve the multiple-choice problem.\n"
|
||||
"Output only the final choice label inside <answer>...</answer>."
|
||||
)
|
||||
final_message, raw = run_student_exec(
|
||||
work_dir=work_dir,
|
||||
prompt=prompt,
|
||||
model=model,
|
||||
timeout=timeout,
|
||||
)
|
||||
return final_message or raw, raw, skill_md, task_text
|
||||
|
||||
|
||||
def process_one(
|
||||
item: dict,
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
use_theorem: bool = False,
|
||||
use_sketch: bool = False,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
exec_timeout: int = 300,
|
||||
) -> dict:
|
||||
item_id = str(item["id"])
|
||||
result = {
|
||||
"id": item_id,
|
||||
"question": item["question"],
|
||||
"task_type": item.get("theorem_type", ["math_mcq"])[0] if item.get("theorem_type") else "math_mcq",
|
||||
"hard": 0,
|
||||
"soft": 0.0,
|
||||
"predicted_answer": "",
|
||||
"predicted_label": "",
|
||||
"predicted_text": "",
|
||||
"correct_label": item["correct_choice"]["label"],
|
||||
"correct_text": item["correct_choice"]["text"],
|
||||
"response": "",
|
||||
"fail_reason": "",
|
||||
"agent_ok": False,
|
||||
"n_turns": 0,
|
||||
}
|
||||
|
||||
try:
|
||||
pred_dir = os.path.join(out_root, "predictions", item_id)
|
||||
os.makedirs(pred_dir, exist_ok=True)
|
||||
|
||||
if is_student_exec_backend():
|
||||
from skillopt.model import azure_openai as _llm
|
||||
|
||||
conversation: list[dict] = []
|
||||
response = ""
|
||||
system = ""
|
||||
user = ""
|
||||
for turn in range(max_turns):
|
||||
response, raw, system, user = _run_codex_once(
|
||||
pred_dir=pred_dir,
|
||||
skill_content=skill_content,
|
||||
item=item,
|
||||
model=_llm.STUDENT_DEPLOYMENT,
|
||||
timeout=exec_timeout,
|
||||
use_theorem=use_theorem,
|
||||
use_sketch=use_sketch,
|
||||
diagnostic_mode=diagnostic_mode if turn == 0 else False,
|
||||
diagnostic_instruction=diagnostic_instruction if turn == 0 else "",
|
||||
diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "",
|
||||
previous_response=response if turn > 0 else "",
|
||||
)
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": response})
|
||||
if "<answer>" in response.lower():
|
||||
break
|
||||
|
||||
result["response"] = response
|
||||
result["agent_ok"] = True
|
||||
result["n_turns"] = len(conversation)
|
||||
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user)
|
||||
|
||||
eval_result = evaluate(response, item["correct_choice"], item["choices"])
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["predicted_label"] = eval_result["predicted_label"]
|
||||
result["predicted_text"] = eval_result["predicted_text"]
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
|
||||
f"but expected '{eval_result['correct_label']}'"
|
||||
)
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted label: {eval_result['predicted_label']!r}\n"
|
||||
f"Predicted text: {eval_result['predicted_text']!r}\n"
|
||||
f"Correct label: {eval_result['correct_label']!r}\n"
|
||||
f"Correct text: {eval_result['correct_text']!r}\n"
|
||||
f"Exact Match: {eval_result['em']}"
|
||||
)
|
||||
conversation.append({"role": "system", "content": eval_detail})
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
return result
|
||||
|
||||
system = _build_system(skill_content)
|
||||
user = _build_user(
|
||||
item,
|
||||
use_theorem=use_theorem,
|
||||
use_sketch=use_sketch,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=diagnostic_trace_context,
|
||||
)
|
||||
conversation: list[dict] = []
|
||||
response = ""
|
||||
|
||||
for turn in range(max_turns):
|
||||
if turn == 0:
|
||||
resp_text, _ = chat_student(
|
||||
system=system,
|
||||
user=user,
|
||||
max_completion_tokens=16384,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
timeout=exec_timeout,
|
||||
)
|
||||
else:
|
||||
refinement = (
|
||||
f"Your previous answer was:\n{response}\n\n"
|
||||
"Re-evaluate the exact option wording. If needed, correct it. "
|
||||
"Output only the final choice label inside <answer>...</answer>."
|
||||
)
|
||||
resp_text, _ = chat_student(
|
||||
system=system,
|
||||
user=refinement,
|
||||
max_completion_tokens=16384,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
timeout=exec_timeout,
|
||||
)
|
||||
response = resp_text
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": resp_text})
|
||||
if "<answer>" in resp_text.lower():
|
||||
break
|
||||
|
||||
result["response"] = response
|
||||
result["agent_ok"] = True
|
||||
result["n_turns"] = len(conversation)
|
||||
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user)
|
||||
|
||||
eval_result = evaluate(response, item["correct_choice"], item["choices"])
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["predicted_label"] = eval_result["predicted_label"]
|
||||
result["predicted_text"] = eval_result["predicted_text"]
|
||||
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
|
||||
f"but expected '{eval_result['correct_label']}'"
|
||||
)
|
||||
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted label: {eval_result['predicted_label']!r}\n"
|
||||
f"Predicted text: {eval_result['predicted_text']!r}\n"
|
||||
f"Correct label: {eval_result['correct_label']!r}\n"
|
||||
f"Correct text: {eval_result['correct_text']!r}\n"
|
||||
f"Exact Match: {eval_result['em']}"
|
||||
)
|
||||
conversation.append({"role": "system", "content": eval_detail})
|
||||
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["fail_reason"] = f"error: {e}"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def run_batch(
|
||||
items: list[dict],
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
exec_timeout: int = 300,
|
||||
workers: int = 64,
|
||||
use_theorem: bool = False,
|
||||
use_sketch: bool = False,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context_by_id: dict[str, str] | None = None,
|
||||
task_timeout: int = 600,
|
||||
) -> list[dict]:
|
||||
task_timeout = max(int(task_timeout), int(exec_timeout) + 60)
|
||||
results_path = os.path.join(out_root, "results.jsonl")
|
||||
os.makedirs(out_root, exist_ok=True)
|
||||
|
||||
done_ids: set[str] = set()
|
||||
existing: list[dict] = []
|
||||
if os.path.exists(results_path):
|
||||
with open(results_path) as f:
|
||||
for line in f:
|
||||
try:
|
||||
r = json.loads(line)
|
||||
done_ids.add(str(r["id"]))
|
||||
existing.append(r)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pending = [it for it in items if str(it["id"]) not in done_ids]
|
||||
if not pending:
|
||||
return existing
|
||||
|
||||
total = len(existing) + len(pending)
|
||||
completed = len(existing)
|
||||
correct_count = sum(1 for r in existing if r.get("hard", 0))
|
||||
if existing:
|
||||
print(f" [rollout] resuming: {completed}/{total} already done", flush=True)
|
||||
|
||||
results = list(existing)
|
||||
|
||||
started_at: dict[str, float] = {}
|
||||
|
||||
def _run_one(it: dict) -> dict:
|
||||
started_at[str(it["id"])] = time.time()
|
||||
return process_one(
|
||||
it,
|
||||
out_root,
|
||||
skill_content,
|
||||
max_turns=max_turns,
|
||||
exec_timeout=exec_timeout,
|
||||
use_theorem=use_theorem,
|
||||
use_sketch=use_sketch,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=(diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""),
|
||||
)
|
||||
|
||||
def _timeout_result(it: dict) -> dict:
|
||||
correct = it.get("correct_choice") or {}
|
||||
return {
|
||||
"id": str(it["id"]),
|
||||
"question": it.get("question", ""),
|
||||
"task_type": it.get("theorem_type", ["math_mcq"])[0] if it.get("theorem_type") else "math_mcq",
|
||||
"hard": 0,
|
||||
"soft": 0.0,
|
||||
"predicted_answer": "",
|
||||
"predicted_label": "",
|
||||
"predicted_text": "",
|
||||
"correct_label": correct.get("label", ""),
|
||||
"correct_text": correct.get("text", ""),
|
||||
"response": "",
|
||||
"fail_reason": f"task-timeout-{task_timeout}s",
|
||||
"agent_ok": False,
|
||||
"n_turns": 0,
|
||||
}
|
||||
|
||||
def _error_result(it: dict, exc: Exception) -> dict:
|
||||
res = _timeout_result(it)
|
||||
res["fail_reason"] = f"error: {type(exc).__name__}: {exc}"
|
||||
return res
|
||||
|
||||
with open(results_path, "a") as outf:
|
||||
ex = ThreadPoolExecutor(max_workers=workers)
|
||||
try:
|
||||
futs = {
|
||||
ex.submit(_run_one, it): it
|
||||
for it in pending
|
||||
}
|
||||
pending_futs = set(futs)
|
||||
while pending_futs:
|
||||
done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
|
||||
now = time.time()
|
||||
timed_out = [
|
||||
fut for fut in pending_futs - done
|
||||
if str(futs[fut]["id"]) in started_at
|
||||
and now - started_at[str(futs[fut]["id"])] >= task_timeout
|
||||
]
|
||||
for fut in done:
|
||||
pending_futs.remove(fut)
|
||||
item = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception as e: # noqa: BLE001
|
||||
res = _error_result(item, e)
|
||||
results.append(res)
|
||||
completed += 1
|
||||
if res.get("hard", 0):
|
||||
correct_count += 1
|
||||
acc = correct_count / completed if completed else 0
|
||||
print(
|
||||
f" [rollout] {completed}/{total} "
|
||||
f"(acc={acc:.3f}) id={res['id']} "
|
||||
f"hard={res.get('hard', '?')}",
|
||||
flush=True,
|
||||
)
|
||||
outf.write(json.dumps(res, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
for fut in timed_out:
|
||||
pending_futs.remove(fut)
|
||||
res = _timeout_result(futs[fut])
|
||||
results.append(res)
|
||||
completed += 1
|
||||
acc = correct_count / completed if completed else 0
|
||||
print(
|
||||
f" [rollout] {completed}/{total} "
|
||||
f"(acc={acc:.3f}) id={res['id']} TIMEOUT",
|
||||
flush=True,
|
||||
)
|
||||
outf.write(json.dumps(res, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
finally:
|
||||
ex.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,16 @@
|
||||
# Live Mathematical MCQ Heuristics
|
||||
|
||||
## Option Comparison
|
||||
- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case.
|
||||
- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when".
|
||||
|
||||
## Theorem-Level Precision
|
||||
- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence.
|
||||
- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one.
|
||||
|
||||
## Hypotheses
|
||||
- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions.
|
||||
- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily.
|
||||
|
||||
## Final Answer
|
||||
- Output the final answer as the single option label only.
|
||||
Reference in New Issue
Block a user