Initial commit
This commit is contained in:
carpedkm
@@ -0,0 +1 @@
|
||||
"""BabyVision environment package for ReflACT."""
|
||||
@@ -0,0 +1,267 @@
|
||||
"""BabyVision environment adapter for ReflACT."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from reflact.gradient.deep_probe import generate_deep_probe_instruction
|
||||
from reflact.datasets.base import BatchSpec
|
||||
from reflact.gradient.reflect import run_minibatch_reflect
|
||||
from reflact.envs.base import EnvAdapter
|
||||
from reflact.envs.babyvision.dataloader import BabyVisionDataLoader
|
||||
from reflact.envs.babyvision.rollout import run_batch
|
||||
from reflact.model import get_student_backend
|
||||
|
||||
|
||||
class BabyVisionAdapter(EnvAdapter):
|
||||
"""BabyVision adapter."""
|
||||
|
||||
def build_reference_text(self, item: dict) -> str:
|
||||
cot = str(item.get("cot") or "").strip()
|
||||
if not cot:
|
||||
return ""
|
||||
return f"## Reference CoT\n{cot}"
|
||||
|
||||
def get_reference_metadata(self, item: dict) -> dict:
|
||||
cot = str(item.get("cot") or "").strip()
|
||||
if not cot:
|
||||
return {"fields": [], "preview": ""}
|
||||
return {
|
||||
"fields": ["cot"],
|
||||
"preview": cot[:400],
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "ratio",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
max_turns: int = 1,
|
||||
workers: int = 32,
|
||||
analyst_workers: int = 16,
|
||||
failure_only: bool = False,
|
||||
minibatch_size: int = 8,
|
||||
edit_budget: int = 4,
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
image_detail: str = "auto",
|
||||
judge_model: str = "gpt-5.4",
|
||||
judge_max_completion_tokens: int = 256,
|
||||
judge_retries: int = 5,
|
||||
use_deep_reflect: bool = False,
|
||||
deep_reflect_failures: int = 4,
|
||||
deep_reflect_successes: int = 2,
|
||||
) -> None:
|
||||
self.max_turns = max_turns
|
||||
self.workers = workers
|
||||
self.analyst_workers = analyst_workers
|
||||
self.failure_only = failure_only
|
||||
self.minibatch_size = minibatch_size
|
||||
self.edit_budget = edit_budget
|
||||
self.image_detail = image_detail
|
||||
self.judge_model = judge_model
|
||||
self.judge_max_completion_tokens = judge_max_completion_tokens
|
||||
self.judge_retries = judge_retries
|
||||
self.use_deep_reflect = use_deep_reflect
|
||||
self.deep_reflect_failures = deep_reflect_failures
|
||||
self.deep_reflect_successes = deep_reflect_successes
|
||||
self.dataloader = BabyVisionDataLoader(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
self.dataloader.setup(cfg)
|
||||
|
||||
def get_dataloader(self):
|
||||
return self.dataloader
|
||||
|
||||
def build_env_from_batch(self, batch: BatchSpec, **kwargs):
|
||||
return list(batch.payload or [])
|
||||
|
||||
def build_train_env(self, batch_size: int, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def rollout(
|
||||
self,
|
||||
env_manager,
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict]:
|
||||
items: list[dict] = env_manager
|
||||
return run_batch(
|
||||
items=items,
|
||||
out_root=out_dir,
|
||||
skill_content=skill_content,
|
||||
max_turns=self.max_turns,
|
||||
workers=self.workers,
|
||||
image_detail=self.image_detail,
|
||||
judge_model=self.judge_model,
|
||||
judge_max_completion_tokens=self.judge_max_completion_tokens,
|
||||
judge_retries=self.judge_retries,
|
||||
diagnostic_mode=kwargs.get("diagnostic_mode", False),
|
||||
diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
|
||||
diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"),
|
||||
)
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def deep_reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
if not self.use_deep_reflect:
|
||||
return []
|
||||
|
||||
env_manager = kwargs.get("env_manager")
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
codex_backend = get_student_backend() == "codex_exec"
|
||||
selected_items = self.select_representative_items(
|
||||
results,
|
||||
env_manager if isinstance(env_manager, list) else None,
|
||||
n_failures=self.deep_reflect_failures,
|
||||
n_successes=self.deep_reflect_successes,
|
||||
seed=random_seed,
|
||||
)
|
||||
if not selected_items:
|
||||
return []
|
||||
selected_ids = {str(item["id"]) for item in selected_items}
|
||||
selected_results = [row for row in results if str(row.get("id")) in selected_ids]
|
||||
selected_examples = self.attach_reference_context(selected_results, selected_items)
|
||||
if codex_backend:
|
||||
selected_examples = self.attach_codex_probe_context(selected_examples, prediction_dir)
|
||||
selected_metadata = []
|
||||
cot_count = 0
|
||||
for item in selected_items:
|
||||
meta = self.get_reference_metadata(item)
|
||||
if meta["fields"]:
|
||||
cot_count += 1
|
||||
selected_metadata.append({
|
||||
"id": str(item["id"]),
|
||||
"task_type": str(item.get("subtype") or item.get("task_type") or "babyvision"),
|
||||
"reference_fields": meta["fields"],
|
||||
"reference_preview": meta["preview"],
|
||||
})
|
||||
|
||||
deep_dir = os.path.join(out_dir, "deep_reflect")
|
||||
rollout_dir = os.path.join(deep_dir, "rollout")
|
||||
patches_dir = os.path.join(deep_dir, "patches")
|
||||
os.makedirs(deep_dir, exist_ok=True)
|
||||
print(
|
||||
f" [2b/6 DEEP REFLECT setup] selected={len(selected_items)} "
|
||||
f"reference_fields=cot({cot_count}/{len(selected_items)})"
|
||||
)
|
||||
probe = generate_deep_probe_instruction(
|
||||
skill_content=skill_content,
|
||||
items=selected_examples,
|
||||
prediction_dir=prediction_dir,
|
||||
system_prompt=self.get_codex_deep_probe_prompt() if codex_backend else self.get_deep_probe_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
)
|
||||
if not probe:
|
||||
return []
|
||||
diagnostic_trace_context_by_id = None
|
||||
if codex_backend:
|
||||
selected_items, diagnostic_trace_context_by_id, probe = self.resolve_codex_probe_target(
|
||||
selected_items=selected_items,
|
||||
selected_examples=selected_examples,
|
||||
prediction_dir=prediction_dir,
|
||||
probe=probe,
|
||||
)
|
||||
probe_record = {
|
||||
**probe,
|
||||
"reference_summary": {
|
||||
"selected_count": len(selected_items),
|
||||
"field_counts": {
|
||||
"cot": cot_count,
|
||||
},
|
||||
},
|
||||
"selected_examples": selected_metadata,
|
||||
}
|
||||
with open(os.path.join(deep_dir, "probe.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(probe_record, f, ensure_ascii=False, indent=2)
|
||||
deep_results = run_batch(
|
||||
items=selected_items,
|
||||
out_root=rollout_dir,
|
||||
skill_content=skill_content,
|
||||
max_turns=self.max_turns,
|
||||
workers=min(self.workers, max(len(selected_items), 1)),
|
||||
image_detail=self.image_detail,
|
||||
judge_model=self.judge_model,
|
||||
judge_max_completion_tokens=self.judge_max_completion_tokens,
|
||||
judge_retries=self.judge_retries,
|
||||
diagnostic_mode=True,
|
||||
diagnostic_instruction=probe["probe_instruction"],
|
||||
diagnostic_trace_context_by_id=diagnostic_trace_context_by_id,
|
||||
)
|
||||
deep_results = self.attach_reference_context(deep_results, selected_items)
|
||||
return run_minibatch_reflect(
|
||||
results=deep_results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=os.path.join(rollout_dir, "predictions"),
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return self.dataloader.get_task_types()
|
||||
@@ -0,0 +1,214 @@
|
||||
"""BabyVision task dataloader."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from reflact.datasets.base import SplitDataLoader
|
||||
|
||||
|
||||
# ── Raw data loading utilities (for preprocessing / standalone eval) ─────
|
||||
|
||||
_CHOICE_LABELS = ["A", "B", "C", "D", "E", "F", "G"]
|
||||
|
||||
|
||||
def _iter_jsonl(path: str) -> list[dict]:
|
||||
items: list[dict] = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
items.append(json.loads(line))
|
||||
return items
|
||||
|
||||
|
||||
def _normalize_ans_type(raw: Any, options: list[dict], choice_answer: Any) -> str:
|
||||
text = str(raw or "").strip().lower()
|
||||
if text in {"choice", "multiple_choice", "mcq", "option"}:
|
||||
return "choice"
|
||||
if text in {"blank", "open", "open_ended", "fill_blank", "short_answer"}:
|
||||
return "blank"
|
||||
if options or choice_answer not in (None, "", []):
|
||||
return "choice"
|
||||
return "blank"
|
||||
|
||||
|
||||
def _coerce_options(raw: Any) -> list[dict]:
|
||||
options: list[dict] = []
|
||||
if isinstance(raw, list):
|
||||
for idx, item in enumerate(raw):
|
||||
if isinstance(item, dict):
|
||||
text = str(item.get("text") or item.get("content") or item.get("option") or "").strip()
|
||||
label = str(item.get("label") or _CHOICE_LABELS[idx]).strip()
|
||||
else:
|
||||
text = str(item).strip()
|
||||
label = _CHOICE_LABELS[idx]
|
||||
if text:
|
||||
options.append({"label": label, "text": text})
|
||||
elif isinstance(raw, dict):
|
||||
for idx, (key, value) in enumerate(raw.items()):
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
options.append({"label": str(key).strip() or _CHOICE_LABELS[idx], "text": text})
|
||||
return options
|
||||
|
||||
|
||||
def _normalize_choice_answer(choice_answer: Any, options: list[dict]) -> dict[str, str]:
|
||||
if not options:
|
||||
return {"label": "", "text": ""}
|
||||
|
||||
if isinstance(choice_answer, dict):
|
||||
label = str(choice_answer.get("label") or "").strip().upper()
|
||||
text = str(choice_answer.get("text") or "").strip()
|
||||
for option in options:
|
||||
if label and option["label"].strip().upper() == label:
|
||||
return {"label": option["label"], "text": option["text"]}
|
||||
if text and option["text"] == text:
|
||||
return {"label": option["label"], "text": option["text"]}
|
||||
|
||||
if isinstance(choice_answer, int):
|
||||
idx = choice_answer
|
||||
if 0 <= idx < len(options):
|
||||
return dict(options[idx])
|
||||
if 1 <= idx <= len(options):
|
||||
return dict(options[idx - 1])
|
||||
|
||||
text = str(choice_answer or "").strip()
|
||||
label = text.upper().rstrip(".):")
|
||||
for option in options:
|
||||
if option["label"].strip().upper() == label:
|
||||
return dict(option)
|
||||
if option["text"] == text:
|
||||
return dict(option)
|
||||
|
||||
return {"label": "", "text": ""}
|
||||
|
||||
|
||||
def _coerce_blank_answers(raw: Any) -> list[str]:
|
||||
if isinstance(raw, list):
|
||||
return [str(item).strip() for item in raw if str(item).strip()]
|
||||
if raw is None:
|
||||
return []
|
||||
text = str(raw).strip()
|
||||
return [text] if text else []
|
||||
|
||||
|
||||
def load_items(data_path: str) -> list[dict]:
|
||||
"""Load and normalise BabyVision items from a directory or JSONL file."""
|
||||
if not data_path:
|
||||
raise ValueError("BabyVision requires data_path pointing to a local dataset directory or meta_data.jsonl.")
|
||||
|
||||
if os.path.isdir(data_path):
|
||||
meta_path = os.path.join(data_path, "meta_data.jsonl")
|
||||
image_root = os.path.join(data_path, "images")
|
||||
else:
|
||||
meta_path = data_path
|
||||
image_root = os.path.join(os.path.dirname(data_path), "images")
|
||||
|
||||
if not os.path.exists(meta_path):
|
||||
raise ValueError(
|
||||
"BabyVision expected a meta_data.jsonl file. "
|
||||
f"Could not find: {meta_path}"
|
||||
)
|
||||
|
||||
raw_items = _iter_jsonl(meta_path)
|
||||
items: list[dict] = []
|
||||
for idx, raw in enumerate(raw_items):
|
||||
options = _coerce_options(raw.get("options") or raw.get("choices") or raw.get("choiceOptions"))
|
||||
ans_type = _normalize_ans_type(raw.get("ansType"), options, raw.get("choiceAns"))
|
||||
correct_choice = _normalize_choice_answer(raw.get("choiceAns"), options)
|
||||
blank_answers = _coerce_blank_answers(raw.get("blankAns"))
|
||||
|
||||
image_name = str(
|
||||
raw.get("image")
|
||||
or raw.get("image_path")
|
||||
or raw.get("image_file")
|
||||
or raw.get("img")
|
||||
or ""
|
||||
).strip()
|
||||
if not image_name:
|
||||
continue
|
||||
image_path = image_name if os.path.isabs(image_name) else os.path.join(image_root, image_name)
|
||||
if not os.path.exists(image_path):
|
||||
alt = os.path.join(os.path.dirname(meta_path), image_name)
|
||||
if os.path.exists(alt):
|
||||
image_path = alt
|
||||
else:
|
||||
continue
|
||||
|
||||
task_id = str(raw.get("taskId") or raw.get("id") or idx + 1)
|
||||
task_type = str(raw.get("type") or raw.get("taskType") or "unknown").strip() or "unknown"
|
||||
subtype = str(raw.get("subtype") or raw.get("subType") or task_type).strip() or task_type
|
||||
question = str(raw.get("question") or raw.get("query") or "").strip()
|
||||
if not question:
|
||||
continue
|
||||
|
||||
if ans_type == "choice" and not correct_choice["label"]:
|
||||
continue
|
||||
if ans_type != "choice" and not blank_answers:
|
||||
continue
|
||||
|
||||
items.append({
|
||||
"id": task_id,
|
||||
"task_type": task_type,
|
||||
"subtype": subtype,
|
||||
"question": question,
|
||||
"image_path": os.path.abspath(image_path),
|
||||
"ans_type": ans_type,
|
||||
"choices": options,
|
||||
"correct_choice": correct_choice,
|
||||
"blank_answers": blank_answers,
|
||||
"cot": str(raw.get("coT") or raw.get("cot") or "").strip(),
|
||||
"source_path": os.path.abspath(meta_path),
|
||||
})
|
||||
|
||||
if not items:
|
||||
raise ValueError(f"No valid BabyVision items loaded from {data_path}")
|
||||
return items
|
||||
|
||||
|
||||
# ── Dataloader ───────────────────────────────────────────────────────────
|
||||
|
||||
class BabyVisionDataLoader(SplitDataLoader):
|
||||
"""BabyVision dataloader."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "ratio",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
)
|
||||
self._task_types: list[str] = []
|
||||
|
||||
def load_raw_items(self, data_path: str) -> list[dict]:
|
||||
return load_items(data_path)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
all_items = self.train_items + self.val_items + self.test_items
|
||||
task_types = {
|
||||
item.get("subtype") or item.get("task_type") or "unknown"
|
||||
for item in all_items
|
||||
}
|
||||
self._task_types = sorted(task_types)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return list(self._task_types)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""BabyVision evaluation helpers using the official-style LLM judge."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import string
|
||||
|
||||
import regex
|
||||
|
||||
from reflact.model import chat_with_deployment
|
||||
from reflact.prompts import load_prompt
|
||||
|
||||
_EVAL_MODE = "babyvision_judge_v2_official_style"
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
text = str(text).strip().lower()
|
||||
text = "".join(ch for ch in text if ch not in string.punctuation)
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def extract_boxed_answer(text: str | None) -> str | None:
|
||||
"""Extract the final answer using the official BabyVision rule."""
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
pattern = r'\\boxed\{((?:[^{}]|{(?:[^{}]|{.*})*})*)\}'
|
||||
matches = regex.findall(pattern, text)
|
||||
if matches:
|
||||
return matches[-1]
|
||||
|
||||
pattern_alt = r'<\|begin_of_box\|>(.*?)<\|end_of_box\|>'
|
||||
matches_alt = regex.findall(pattern_alt, text)
|
||||
if matches_alt:
|
||||
return matches_alt[-1].strip()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _token_f1(prediction: str, gold: str) -> float:
|
||||
pred_tokens = normalize_text(prediction).split()
|
||||
gold_tokens = normalize_text(gold).split()
|
||||
if not pred_tokens and not gold_tokens:
|
||||
return 1.0
|
||||
if not pred_tokens or not gold_tokens:
|
||||
return 0.0
|
||||
pred_set = {}
|
||||
gold_set = {}
|
||||
for tok in pred_tokens:
|
||||
pred_set[tok] = pred_set.get(tok, 0) + 1
|
||||
for tok in gold_tokens:
|
||||
gold_set[tok] = gold_set.get(tok, 0) + 1
|
||||
common = 0
|
||||
for tok, count in pred_set.items():
|
||||
common += min(count, gold_set.get(tok, 0))
|
||||
if common == 0:
|
||||
return 0.0
|
||||
precision = common / len(pred_tokens)
|
||||
recall = common / len(gold_tokens)
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
|
||||
def _format_choices(choices: list[dict]) -> str:
|
||||
return "\n".join(f"{choice['label']}. {choice['text']}" for choice in choices)
|
||||
|
||||
|
||||
def _judge_answer(
|
||||
*,
|
||||
item: dict,
|
||||
prediction_text: str,
|
||||
extracted_answer: str,
|
||||
judge_model: str,
|
||||
max_completion_tokens: int,
|
||||
retries: int,
|
||||
) -> dict:
|
||||
if item["ans_type"] == "choice":
|
||||
ground_truth = str(item["correct_choice"]["label"])
|
||||
else:
|
||||
if len(item["blank_answers"]) == 1:
|
||||
ground_truth = item["blank_answers"][0]
|
||||
else:
|
||||
ground_truth = " | ".join(item["blank_answers"])
|
||||
|
||||
question = str(item["question"])
|
||||
if item["ans_type"] == "choice" and item.get("choices"):
|
||||
question = f"{question}\nChoices:\n{_format_choices(item['choices'])}"
|
||||
|
||||
raw, _ = chat_with_deployment(
|
||||
deployment=judge_model,
|
||||
system="You are a careful and strict evaluator.",
|
||||
user=load_prompt("judge", env="babyvision").format(
|
||||
question=question,
|
||||
groundtruth=ground_truth,
|
||||
modeloutput=extracted_answer,
|
||||
),
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=retries,
|
||||
stage="babyvision_judge",
|
||||
)
|
||||
judge_response_clean = str(raw).strip().lower()
|
||||
if "true" in judge_response_clean:
|
||||
correct = True
|
||||
elif "false" in judge_response_clean:
|
||||
correct = False
|
||||
else:
|
||||
correct = False
|
||||
return {
|
||||
"raw": raw,
|
||||
"correct": correct,
|
||||
"reason": judge_response_clean,
|
||||
"matched_gold": ground_truth if correct else "",
|
||||
}
|
||||
|
||||
|
||||
def evaluate_item(
|
||||
*,
|
||||
item: dict,
|
||||
prediction_text: str,
|
||||
judge_model: str,
|
||||
max_completion_tokens: int = 256,
|
||||
retries: int = 5,
|
||||
) -> dict:
|
||||
answer = extract_boxed_answer(prediction_text)
|
||||
judge = _judge_answer(
|
||||
item=item,
|
||||
prediction_text=prediction_text,
|
||||
extracted_answer=answer,
|
||||
judge_model=judge_model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=retries,
|
||||
)
|
||||
hard = 1.0 if judge["correct"] else 0.0
|
||||
|
||||
result = {
|
||||
"evaluation_mode": _EVAL_MODE,
|
||||
"predicted_answer": answer,
|
||||
"em": hard,
|
||||
"f1": hard,
|
||||
"sub_em": hard,
|
||||
"judge_model": judge_model,
|
||||
"judge_raw": judge["raw"],
|
||||
"judge_reason": judge["reason"],
|
||||
"matched_gold": judge["matched_gold"],
|
||||
}
|
||||
|
||||
if item["ans_type"] == "choice":
|
||||
result["predicted_label"] = str(answer or "").strip().upper().rstrip(".):")
|
||||
result["predicted_text"] = ""
|
||||
result["correct_label"] = str(item["correct_choice"].get("label") or "")
|
||||
result["correct_text"] = str(item["correct_choice"].get("text") or "")
|
||||
else:
|
||||
result["gold_answers"] = list(item["blank_answers"])
|
||||
best_f1 = 0.0
|
||||
for gold in item["blank_answers"]:
|
||||
best_f1 = max(best_f1, _token_f1(str(answer or ""), gold))
|
||||
result["string_f1"] = best_f1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def evaluation_mode() -> str:
|
||||
return _EVAL_MODE
|
||||
@@ -0,0 +1,36 @@
|
||||
You are an expert failure-analysis agent for child-level visual reasoning tasks.
|
||||
|
||||
You will be given MULTIPLE failed BabyVision trajectories from a minibatch and the current skill document.
|
||||
Each trajectory includes the text prompt, the model answer, and the evaluation result.
|
||||
You do not have direct access to raw pixel content during reflection, so focus on general reasoning,
|
||||
option-selection, and visual-question-answering behaviors that can be improved through prompting.
|
||||
|
||||
## Failure Type Categories
|
||||
- **visual_detail_miss**: the agent likely overlooked a salient visual attribute, relation, count, or object state
|
||||
- **option_mismatch**: the agent selected the wrong option despite relevant evidence likely being present
|
||||
- **instruction_slip**: the agent ignored output format or answered too vaguely
|
||||
- **answer_granularity**: the agent gave an answer that was too broad, too narrow, or mismatched the expected specificity
|
||||
- **other**: none of the above
|
||||
|
||||
## Rules
|
||||
1. Focus on patterns recurring across the minibatch.
|
||||
2. Prefer reusable behaviors for inspecting images and grounding answers in visible evidence.
|
||||
3. Do not memorize dataset-specific answers.
|
||||
4. Only patch gaps not already covered by the current skill.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number>,
|
||||
"failure_summary": [
|
||||
{"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
|
||||
],
|
||||
"patch": {
|
||||
"reasoning": "<why these edits address the common failures>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
You are an expert success-pattern analyst for child-level visual reasoning tasks.
|
||||
|
||||
You will be given MULTIPLE successful BabyVision trajectories from a minibatch and the current skill document.
|
||||
Identify generalizable behavior patterns that help the agent inspect the image carefully and answer at the right level of specificity.
|
||||
|
||||
## Rules
|
||||
- Focus on broadly useful visual QA behaviors.
|
||||
- Prefer patterns about systematic image inspection, comparing options, and concise grounded answers.
|
||||
- Do not add dataset-specific facts.
|
||||
- "edits" may be empty if the skill already captures the useful patterns.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number>,
|
||||
"success_patterns": ["<pattern 1>", "<pattern 2>"],
|
||||
"patch": {
|
||||
"reasoning": "<why these patterns matter>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
You are an expert diagnostic-probe designer for BabyVision-style visual reasoning tasks.
|
||||
|
||||
You will be shown representative trajectories, the current student skill, and the student's original prompt context.
|
||||
Design one SMALL diagnostic instruction that exposes the student's intermediate visual judgment without materially changing the original scaffold.
|
||||
|
||||
## Hard Constraints
|
||||
1. Do NOT substantially change the original scaffold.
|
||||
2. Do NOT prescribe a new step-by-step solving method.
|
||||
3. You MAY ask for a short structured list of a few intermediate conclusions, candidate cues, or counted units, as long as it stays close to the original scaffold.
|
||||
4. Do NOT ask for exhaustive listing of all cells, all objects, or a full chain-of-thought.
|
||||
5. Ask only for a short readout that reveals the student's current latent state.
|
||||
6. Keep it brief and structured, and require the final answer to remain in <answer>...</answer>.
|
||||
|
||||
## Good Probe Targets
|
||||
- top answer and runner-up
|
||||
- decisive visual cue
|
||||
- suspicious region or compared objects
|
||||
- counting unit or formatting interpretation
|
||||
- 2-4 short intermediate conclusions that directly support the final answer
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"reasoning": "<why this probe is informative>",
|
||||
"probe_instruction": "<the exact instruction text to append to the student prompt>"
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
You are a careful and strict evaluator. You will be given:
|
||||
|
||||
1. **Question**
|
||||
2. **Ground Truth Answer** (correct answer)
|
||||
3. **Model Output** (answer from another model)
|
||||
|
||||
**Your goal:** Determine if the Model Output **accurately matches** the Ground Truth Answer in meaning.
|
||||
|
||||
* Matching means: the facts, entities, and key details are equivalent, even if phrasing differs.
|
||||
* Not matching means: the Model Output is wrong, incomplete, contains extra incorrect facts, or changes the meaning.
|
||||
|
||||
**Process (internal reasoning):**
|
||||
|
||||
1. Read and understand the Question, Ground Truth Answer, and Model Output.
|
||||
2. Ignore small wording differences, formatting, or synonyms.
|
||||
3. If all factual content matches, conclude `1`. Otherwise, conclude `0`.
|
||||
|
||||
**Important:**
|
||||
|
||||
* Think through your decision step-by-step **internally** before responding.
|
||||
* In your final output, return **only** True or False, with no extra text or explanation.
|
||||
|
||||
**Output format:**
|
||||
|
||||
True
|
||||
|
||||
or
|
||||
|
||||
False
|
||||
|
||||
**Input:**
|
||||
|
||||
Question: {question},
|
||||
Ground Truth Answer: {groundtruth},
|
||||
Model Output: {modeloutput}
|
||||
@@ -0,0 +1,13 @@
|
||||
You are an expert visual reasoning agent solving child-level image understanding tasks.
|
||||
|
||||
{skill_section}## Task Format
|
||||
You will receive one image and one question about it.
|
||||
Inspect the image carefully before answering. Ground the answer in visible evidence.
|
||||
|
||||
## Answer Format
|
||||
Think step by step, then provide your final answer in \boxed{{Answer}} format.
|
||||
- For multiple-choice questions, output only the single choice label, such as \boxed{{A}}.
|
||||
- For open questions, output only a short final answer inside \boxed{{...}}.
|
||||
|
||||
Example:
|
||||
\boxed{{B}}
|
||||
@@ -0,0 +1,4 @@
|
||||
"""BabyVision Reflect stage.
|
||||
|
||||
Prompts are now loaded from .md files by the base adapter.
|
||||
"""
|
||||
@@ -0,0 +1,467 @@
|
||||
"""BabyVision rollout — multimodal visual QA with image input."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from reflact.envs.babyvision.evaluator import evaluate_item, evaluation_mode, extract_boxed_answer
|
||||
from reflact.model import chat_student_messages, get_student_backend, is_student_exec_backend
|
||||
from reflact.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
|
||||
from reflact.prompts import load_prompt
|
||||
|
||||
def _build_system(skill_content: str) -> str:
|
||||
if skill_content.strip():
|
||||
skill_section = f"## Skill\n{skill_content.strip()}\n\n"
|
||||
else:
|
||||
skill_section = ""
|
||||
return load_prompt("rollout_system", env="babyvision").format(skill_section=skill_section)
|
||||
|
||||
|
||||
def _format_choices(choices: list[dict]) -> str:
|
||||
return "\n".join(f"{choice['label']}. {choice['text']}" for choice in choices)
|
||||
|
||||
|
||||
def _build_user_text(
|
||||
item: dict,
|
||||
*,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
) -> str:
|
||||
parts = []
|
||||
if diagnostic_trace_context.strip():
|
||||
parts.append(
|
||||
"## Previous Codex Trace Snapshot\n"
|
||||
"This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n"
|
||||
f"{diagnostic_trace_context.strip()}"
|
||||
)
|
||||
parts.append(f"## Question\n{item['question']}")
|
||||
if item["ans_type"] == "choice":
|
||||
parts.append(f"## Choices\n{_format_choices(item['choices'])}")
|
||||
parts.append("Answer using the single correct option label in \\boxed{...}.")
|
||||
else:
|
||||
parts.append("Answer with a short phrase in \\boxed{...}.")
|
||||
if diagnostic_mode and diagnostic_instruction.strip():
|
||||
parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _image_to_data_uri(path: str) -> str:
|
||||
mime = mimetypes.guess_type(path)[0] or "image/png"
|
||||
with open(path, "rb") as f:
|
||||
encoded = base64.b64encode(f.read()).decode("ascii")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
def _build_messages(
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
image_detail: str,
|
||||
*,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
) -> tuple[list[dict], str, str]:
|
||||
system = _build_system(skill_content)
|
||||
user_text = _build_user_text(
|
||||
item,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=diagnostic_trace_context,
|
||||
)
|
||||
image_url = {
|
||||
"url": _image_to_data_uri(item["image_path"]),
|
||||
}
|
||||
if image_detail and image_detail != "auto":
|
||||
image_url["detail"] = image_detail
|
||||
messages = [
|
||||
{"role": "system", "content": system},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": user_text},
|
||||
{"type": "image_url", "image_url": image_url},
|
||||
],
|
||||
},
|
||||
]
|
||||
return messages, system, user_text
|
||||
|
||||
|
||||
def _build_codex_skill(skill_content: str) -> str:
|
||||
return render_skill_md(
|
||||
skill_content,
|
||||
description="Dynamic ReflACT skill for solving the current BabyVision visual reasoning question.",
|
||||
preamble=(
|
||||
"Use this skill when answering the current visual reasoning question.\n"
|
||||
"Inspect the attached image carefully and return the final answer in \\boxed{...}."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_codex_once(
|
||||
*,
|
||||
pred_dir: str,
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
model: str,
|
||||
timeout: int,
|
||||
image_detail: str,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
previous_response: str = "",
|
||||
) -> tuple[str, str, str, str]:
|
||||
user_text = _build_user_text(
|
||||
item,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=diagnostic_trace_context,
|
||||
)
|
||||
task_parts = [user_text]
|
||||
if previous_response:
|
||||
task_parts.append(
|
||||
"## Previous Attempt\n"
|
||||
f"{previous_response}\n\n"
|
||||
"Review the same image and question carefully. If needed, correct the answer."
|
||||
)
|
||||
task_text = "\n\n".join(task_parts)
|
||||
skill_md = _build_codex_skill(skill_content)
|
||||
work_dir = os.path.join(pred_dir, "codex_exec")
|
||||
prepare_workspace(
|
||||
work_dir=work_dir,
|
||||
skill_md=skill_md,
|
||||
task_text=task_text,
|
||||
images=[item["image_path"]],
|
||||
)
|
||||
prompt = (
|
||||
"Use the `reflact-student` skill available in this workspace.\n"
|
||||
"Read `task.md`, inspect the attached image, and answer the question.\n"
|
||||
"Return the final answer in \\boxed{...}."
|
||||
)
|
||||
final_message, raw = run_student_exec(
|
||||
work_dir=work_dir,
|
||||
prompt=prompt,
|
||||
model=model,
|
||||
timeout=timeout,
|
||||
images=[item["image_path"]],
|
||||
)
|
||||
return final_message or raw, raw, skill_md, task_text
|
||||
|
||||
|
||||
def process_one(
|
||||
item: dict,
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
image_detail: str = "auto",
|
||||
judge_model: str = "gpt-5.4",
|
||||
judge_max_completion_tokens: int = 256,
|
||||
judge_retries: int = 5,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context: str = "",
|
||||
) -> dict:
|
||||
item_id = str(item["id"])
|
||||
result = {
|
||||
"id": item_id,
|
||||
"question": item["question"],
|
||||
"task_type": item.get("subtype") or item.get("task_type") or "babyvision",
|
||||
"task_description": item["question"],
|
||||
"hard": 0,
|
||||
"soft": 0.0,
|
||||
"predicted_answer": "",
|
||||
"predicted_label": "",
|
||||
"predicted_text": "",
|
||||
"response": "",
|
||||
"fail_reason": "",
|
||||
"agent_ok": False,
|
||||
"n_turns": 0,
|
||||
"image_path": item["image_path"],
|
||||
"ans_type": item["ans_type"],
|
||||
"evaluation_mode": evaluation_mode(),
|
||||
"judge_model": judge_model,
|
||||
}
|
||||
if item["ans_type"] == "choice":
|
||||
result["correct_label"] = item["correct_choice"]["label"]
|
||||
result["correct_text"] = item["correct_choice"]["text"]
|
||||
else:
|
||||
result["gold_answers"] = item["blank_answers"]
|
||||
|
||||
try:
|
||||
pred_dir = os.path.join(out_root, "predictions", item_id)
|
||||
os.makedirs(pred_dir, exist_ok=True)
|
||||
|
||||
if is_student_exec_backend():
|
||||
from reflact.model import azure_openai as _llm
|
||||
|
||||
response = ""
|
||||
conversation: list[dict] = [
|
||||
{"role": "user", "content": f"{item['question']}\n\n[image] {os.path.basename(item['image_path'])}"}
|
||||
]
|
||||
system_prompt = ""
|
||||
user_text = ""
|
||||
for turn in range(max_turns):
|
||||
response, raw, system_prompt, user_text = _run_codex_once(
|
||||
pred_dir=pred_dir,
|
||||
item=item,
|
||||
skill_content=skill_content,
|
||||
model=_llm.STUDENT_DEPLOYMENT,
|
||||
timeout=120,
|
||||
image_detail=image_detail,
|
||||
diagnostic_mode=diagnostic_mode if turn == 0 else False,
|
||||
diagnostic_instruction=diagnostic_instruction if turn == 0 else "",
|
||||
diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "",
|
||||
previous_response=response if turn > 0 else "",
|
||||
)
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": response})
|
||||
if extract_boxed_answer(response) is not None:
|
||||
break
|
||||
|
||||
result["response"] = response
|
||||
result["agent_ok"] = True
|
||||
result["n_turns"] = len(conversation) - 1
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system_prompt)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user_text)
|
||||
|
||||
eval_result = evaluate_item(
|
||||
item=item,
|
||||
prediction_text=response,
|
||||
judge_model=judge_model,
|
||||
max_completion_tokens=judge_max_completion_tokens,
|
||||
retries=judge_retries,
|
||||
)
|
||||
result["evaluation_mode"] = eval_result["evaluation_mode"]
|
||||
result["judge_raw"] = eval_result["judge_raw"]
|
||||
result["judge_reason"] = eval_result["judge_reason"]
|
||||
result["matched_gold"] = eval_result["matched_gold"]
|
||||
if item["ans_type"] == "choice":
|
||||
result["predicted_label"] = eval_result["predicted_label"]
|
||||
result["predicted_text"] = eval_result["predicted_text"]
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"judge=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
|
||||
f"but expected '{eval_result['correct_label']}' ({eval_result['judge_reason']})"
|
||||
)
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted label: {eval_result['predicted_label']!r}\n"
|
||||
f"Predicted text: {eval_result['predicted_text']!r}\n"
|
||||
f"Correct label: {eval_result['correct_label']!r}\n"
|
||||
f"Correct text: {eval_result['correct_text']!r}\n"
|
||||
f"Judge correct: {eval_result['em']}\n"
|
||||
f"Judge reason: {eval_result['judge_reason']}"
|
||||
)
|
||||
else:
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"judge=0: predicted '{eval_result['predicted_answer']}' "
|
||||
f"but expected {item['blank_answers']} ({eval_result['judge_reason']})"
|
||||
)
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted answer: {eval_result['predicted_answer']!r}\n"
|
||||
f"Gold answers: {item['blank_answers']!r}\n"
|
||||
f"Judge correct: {eval_result['em']}\n"
|
||||
f"Judge reason: {eval_result['judge_reason']}\n"
|
||||
f"String F1: {eval_result.get('string_f1', 0.0):.4f}"
|
||||
)
|
||||
conversation.append({"role": "system", "content": eval_detail})
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
return result
|
||||
|
||||
messages, system_prompt, user_text = _build_messages(
|
||||
item,
|
||||
skill_content,
|
||||
image_detail,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=diagnostic_trace_context,
|
||||
)
|
||||
response = ""
|
||||
conversation: list[dict] = [
|
||||
{"role": "user", "content": f"{user_text}\n\n[image] {os.path.basename(item['image_path'])}"}
|
||||
]
|
||||
|
||||
for turn in range(max_turns):
|
||||
if turn == 0:
|
||||
resp_text, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=768,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
)
|
||||
else:
|
||||
refinement_text = (
|
||||
f"Your previous answer was:\n{response}\n\n"
|
||||
"Review the same image and question carefully. "
|
||||
"If needed, correct your answer. Output the final answer in \\boxed{...}."
|
||||
)
|
||||
refinement_messages = [
|
||||
messages[0],
|
||||
messages[1],
|
||||
{"role": "assistant", "content": response},
|
||||
{"role": "user", "content": refinement_text},
|
||||
]
|
||||
resp_text, _ = chat_student_messages(
|
||||
messages=refinement_messages,
|
||||
max_completion_tokens=512,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
)
|
||||
response = resp_text
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": resp_text})
|
||||
if extract_boxed_answer(resp_text) is not None:
|
||||
break
|
||||
|
||||
result["response"] = response
|
||||
result["agent_ok"] = True
|
||||
result["n_turns"] = len(conversation) - 1
|
||||
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system_prompt)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user_text)
|
||||
|
||||
eval_result = evaluate_item(
|
||||
item=item,
|
||||
prediction_text=response,
|
||||
judge_model=judge_model,
|
||||
max_completion_tokens=judge_max_completion_tokens,
|
||||
retries=judge_retries,
|
||||
)
|
||||
result["evaluation_mode"] = eval_result["evaluation_mode"]
|
||||
result["judge_raw"] = eval_result["judge_raw"]
|
||||
result["judge_reason"] = eval_result["judge_reason"]
|
||||
result["matched_gold"] = eval_result["matched_gold"]
|
||||
|
||||
if item["ans_type"] == "choice":
|
||||
result["predicted_label"] = eval_result["predicted_label"]
|
||||
result["predicted_text"] = eval_result["predicted_text"]
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"judge=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' "
|
||||
f"but expected '{eval_result['correct_label']}' ({eval_result['judge_reason']})"
|
||||
)
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted label: {eval_result['predicted_label']!r}\n"
|
||||
f"Predicted text: {eval_result['predicted_text']!r}\n"
|
||||
f"Correct label: {eval_result['correct_label']!r}\n"
|
||||
f"Correct text: {eval_result['correct_text']!r}\n"
|
||||
f"Judge correct: {eval_result['em']}\n"
|
||||
f"Judge reason: {eval_result['judge_reason']}"
|
||||
)
|
||||
else:
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["hard"] = int(eval_result["em"])
|
||||
result["soft"] = eval_result["f1"]
|
||||
if not result["hard"]:
|
||||
result["fail_reason"] = (
|
||||
f"judge=0: predicted '{eval_result['predicted_answer']}' "
|
||||
f"but expected {item['blank_answers']} ({eval_result['judge_reason']})"
|
||||
)
|
||||
eval_detail = (
|
||||
f"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted answer: {eval_result['predicted_answer']!r}\n"
|
||||
f"Gold answers: {item['blank_answers']!r}\n"
|
||||
f"Judge correct: {eval_result['em']}\n"
|
||||
f"Judge reason: {eval_result['judge_reason']}\n"
|
||||
f"String F1: {eval_result.get('string_f1', 0.0):.4f}"
|
||||
)
|
||||
|
||||
conversation.append({"role": "system", "content": eval_detail})
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["fail_reason"] = f"error: {e}"
|
||||
return result
|
||||
|
||||
|
||||
def run_batch(
|
||||
items: list[dict],
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
workers: int = 32,
|
||||
image_detail: str = "auto",
|
||||
judge_model: str = "gpt-5.4",
|
||||
judge_max_completion_tokens: int = 256,
|
||||
judge_retries: int = 5,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
diagnostic_trace_context_by_id: dict[str, str] | None = None,
|
||||
) -> list[dict]:
|
||||
results_path = os.path.join(out_root, "results.jsonl")
|
||||
os.makedirs(out_root, exist_ok=True)
|
||||
|
||||
expected_eval_mode = evaluation_mode()
|
||||
done_ids: set[str] = set()
|
||||
existing: list[dict] = []
|
||||
rewrite_results = False
|
||||
if os.path.exists(results_path):
|
||||
with open(results_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
if row.get("evaluation_mode") != expected_eval_mode:
|
||||
rewrite_results = True
|
||||
continue
|
||||
done_ids.add(str(row["id"]))
|
||||
existing.append(row)
|
||||
except Exception:
|
||||
rewrite_results = True
|
||||
|
||||
pending = [item for item in items if str(item["id"]) not in done_ids]
|
||||
if not pending and not rewrite_results:
|
||||
return existing
|
||||
|
||||
results = list(existing)
|
||||
file_mode = "w" if rewrite_results else "a"
|
||||
with open(results_path, file_mode, encoding="utf-8") as outf, ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
if rewrite_results:
|
||||
for row in existing:
|
||||
outf.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
futs = {
|
||||
ex.submit(
|
||||
process_one,
|
||||
item,
|
||||
out_root,
|
||||
skill_content,
|
||||
max_turns=max_turns,
|
||||
image_detail=image_detail,
|
||||
judge_model=judge_model,
|
||||
judge_max_completion_tokens=judge_max_completion_tokens,
|
||||
judge_retries=judge_retries,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
diagnostic_trace_context=(diagnostic_trace_context_by_id or {}).get(str(item["id"]), ""),
|
||||
): item
|
||||
for item in pending
|
||||
}
|
||||
for fut in as_completed(futs):
|
||||
row = fut.result()
|
||||
results.append(row)
|
||||
outf.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
return results
|
||||
@@ -0,0 +1,18 @@
|
||||
# BabyVision Visual QA Heuristics
|
||||
|
||||
## Image Inspection
|
||||
- First identify the main objects, their attributes, and their spatial relations before answering.
|
||||
- If the question involves counting, compare all relevant instances carefully instead of stopping after the first match.
|
||||
- If the question asks about color, size, position, or action, verify the specific visible evidence for that attribute.
|
||||
|
||||
## Multiple Choice
|
||||
- Compare every option against the visible image evidence before deciding.
|
||||
- Prefer the option that matches the image exactly; reject options that are only partially true or too vague.
|
||||
- When two options are close, check the smallest discriminating visual detail.
|
||||
|
||||
## Open Answers
|
||||
- Answer with the shortest phrase that is fully supported by the image.
|
||||
- Match the expected level of specificity: not broader than the image evidence, not narrower than the question asks.
|
||||
|
||||
## Final Answer
|
||||
- Output only the final answer inside <answer>...</answer>.
|
||||
Reference in New Issue
Block a user