Ziiii
08916747ef
Runnable SearchQA splits should remain disjoint. A duplicate manifest id across train, val, or test previously collapsed into the wanted-id set and reused the same row in multiple output splits without warning. Constraint: Preserve manifest order and output schema for valid manifests. Rejected: Deduplicate automatically | hiding split overlap would make evaluation contamination harder to notice. Confidence: high Scope-risk: narrow Reversibility: clean Directive: Fail fast on split-manifest integrity problems instead of repairing them silently. Tested: uv run --with pytest pytest tests/test_materialize_searchqa.py -q Tested: uv run --with ruff ruff check scripts/materialize_searchqa.py tests/test_materialize_searchqa.py Not-tested: Loading the live Hugging Face dataset.
80 lines
2.5 KiB
Python
80 lines
2.5 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts.materialize_searchqa import materialize_searchqa_splits
|
|
|
|
|
|
def _write_manifest(root: Path, split_ids: dict[str, list[str]]) -> None:
|
|
for split, ids in split_ids.items():
|
|
split_dir = root / split
|
|
split_dir.mkdir(parents=True)
|
|
(split_dir / "items.json").write_text(
|
|
json.dumps([{"id": item_id} for item_id in ids]),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _row(key: str) -> dict:
|
|
return {
|
|
"key": key,
|
|
"question": f"question {key}",
|
|
"context": f"context {key}",
|
|
"answers": [f"answer {key}"],
|
|
"ignored": "not written",
|
|
}
|
|
|
|
|
|
def test_materialize_searchqa_splits_preserves_manifest_order(tmp_path):
|
|
manifest_dir = tmp_path / "manifest"
|
|
output_dir = tmp_path / "out"
|
|
_write_manifest(manifest_dir, {"train": ["b", "a"], "val": ["c"], "test": ["d"]})
|
|
|
|
counts = materialize_searchqa_splits(
|
|
manifest_dir,
|
|
output_dir,
|
|
{"train": [_row("a"), _row("b")], "validation": [_row("c"), _row("d")]},
|
|
dataset_name="example/searchqa",
|
|
)
|
|
|
|
assert counts == {"train": 2, "val": 1, "test": 1}
|
|
train_items = json.loads((output_dir / "train" / "items.json").read_text(encoding="utf-8"))
|
|
assert [item["id"] for item in train_items] == ["b", "a"]
|
|
assert train_items[0] == {
|
|
"id": "b",
|
|
"question": "question b",
|
|
"context": "context b",
|
|
"answers": ["answer b"],
|
|
}
|
|
|
|
split_manifest = json.loads((output_dir / "split_manifest.json").read_text(encoding="utf-8"))
|
|
assert split_manifest["source_dataset"] == "example/searchqa"
|
|
assert split_manifest["counts"] == counts
|
|
|
|
|
|
def test_materialize_searchqa_splits_fails_on_missing_manifest_id(tmp_path):
|
|
manifest_dir = tmp_path / "manifest"
|
|
_write_manifest(manifest_dir, {"train": ["a"], "val": ["missing"], "test": []})
|
|
|
|
with pytest.raises(RuntimeError, match="missing"):
|
|
materialize_searchqa_splits(
|
|
manifest_dir,
|
|
tmp_path / "out",
|
|
{"train": [_row("a")]},
|
|
dataset_name="example/searchqa",
|
|
)
|
|
|
|
|
|
def test_materialize_searchqa_splits_rejects_duplicate_manifest_ids(tmp_path):
|
|
manifest_dir = tmp_path / "manifest"
|
|
_write_manifest(manifest_dir, {"train": ["a"], "val": ["a"], "test": []})
|
|
|
|
with pytest.raises(ValueError, match="duplicate IDs"):
|
|
materialize_searchqa_splits(
|
|
manifest_dir,
|
|
tmp_path / "out",
|
|
{"train": [_row("a")]},
|
|
dataset_name="example/searchqa",
|
|
)
|