Initial commit

reflact/envs/swebench/__init__.py

@@ -0,0 +1 @@
+"""SWEBench environment for ReflACT."""

reflact/envs/swebench/adapter.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+
+from reflact.datasets.base import BatchSpec
+from reflact.envs.base import EnvAdapter
+from reflact.envs.swebench.dataloader import SWEBenchDataLoader
+from reflact.envs.swebench.rollout import run_batch
+from reflact.gradient.reflect import run_minibatch_reflect
+
+
+class SWEBenchAdapter(EnvAdapter):
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        dataset_name: str = "lite",
+        hf_split: str = "test",
+        workers: int = 8,
+        eval_workers: int = 8,
+        analyst_workers: int = 16,
+        failure_only: bool = False,
+        minibatch_size: int = 4,
+        edit_budget: int = 4,
+        seed: int = 42,
+        limit: int = 0,
+        step_limit: int = 50,
+        cost_limit: float = 3.0,
+        timeout_per_instance: int = 600,
+        student_model: str = "",
+    ) -> None:
+        self.dataset_name = dataset_name
+        self.hf_split = hf_split
+        self.workers = workers
+        self.eval_workers = eval_workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.step_limit = step_limit
+        self.cost_limit = cost_limit
+        self.timeout_per_instance = timeout_per_instance
+        self.student_model = student_model
+        self.dataloader = SWEBenchDataLoader(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+            dataset_name=dataset_name,
+            hf_split=hf_split,
+        )
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.student_model = str(self.student_model or cfg.get("student_model") or "gpt-5.4").strip()
+        self.dataset_name = str(self.dataset_name or cfg.get("dataset_name") or "lite").strip()
+        self.hf_split = str(self.hf_split or cfg.get("hf_split") or "test").strip()
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
+        items: list[dict] = env_manager
+        return run_batch(
+            items=items,
+            out_root=out_dir,
+            skill_content=skill_content,
+            student_model=self.student_model,
+            dataset_name=self.dataset_name,
+            hf_split=self.hf_split,
+            workers=self.workers,
+            eval_workers=self.eval_workers,
+            step_limit=self.step_limit,
+            cost_limit=self.cost_limit,
+            timeout_per_instance=self.timeout_per_instance,
+        )
+
+    def reflect(
+        self,
+        results: list[dict],
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict | None]:
+        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
+        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
+        random_seed = kwargs.get("random_seed")
+        step_buffer_context = kwargs.get("step_buffer_context", "")
+        meta_skill_context = kwargs.get("meta_skill_context", "")
+        return run_minibatch_reflect(
+            results=results,
+            skill_content=skill_content,
+            prediction_dir=prediction_dir,
+            patches_dir=patches_dir,
+            workers=self.analyst_workers,
+            failure_only=self.failure_only,
+            minibatch_size=self.minibatch_size,
+            edit_budget=self.edit_budget,
+            random_seed=random_seed,
+            error_system=self.get_error_minibatch_prompt(),
+            success_system=self.get_success_minibatch_prompt(),
+            step_buffer_context=step_buffer_context,
+            meta_skill_context=meta_skill_context,
+            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
+        )
+
+    def get_task_types(self) -> list[str]:
+        repos = {
+            str(item.get("repo") or "").strip()
+            for item in (
+                self.dataloader.train_items
+                + self.dataloader.val_items
+                + self.dataloader.test_items
+            )
+            if str(item.get("repo") or "").strip()
+        }
+        return sorted(repos) or ["swebench"]

reflact/envs/swebench/dataloader.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import os
+import random
+from collections import defaultdict
+
+from reflact.datasets.base import SplitDataLoader, _parse_split_ratio
+
+
+_DATASET_ALIASES = {
+    "lite": "princeton-nlp/SWE-Bench_Lite",
+    "verified": "princeton-nlp/SWE-Bench_Verified",
+    "full": "princeton-nlp/SWE-Bench",
+}
+
+
+def _normalize_dataset_name(name: str) -> str:
+    key = str(name or "").strip()
+    return _DATASET_ALIASES.get(key.lower(), key or _DATASET_ALIASES["lite"])
+
+
+class SWEBenchDataLoader(SplitDataLoader):
+    def __init__(
+        self,
+        split_dir: str = "",
+        data_path: str = "",
+        split_mode: str = "ratio",
+        split_ratio: str = "2:1:7",
+        split_seed: int = 42,
+        split_output_dir: str = "",
+        seed: int = 42,
+        limit: int = 0,
+        dataset_name: str = "lite",
+        hf_split: str = "test",
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            split_dir=split_dir,
+            data_path=data_path,
+            split_mode=split_mode,
+            split_ratio=split_ratio,
+            split_seed=split_seed,
+            split_output_dir=split_output_dir,
+            seed=seed,
+            limit=limit,
+        )
+        self.dataset_name = dataset_name
+        self.hf_split = hf_split
+
+    def setup(self, cfg: dict) -> None:
+        self.dataset_name = str(
+            self.dataset_name or cfg.get("dataset_name") or "lite"
+        ).strip()
+        self.hf_split = str(self.hf_split or cfg.get("hf_split") or "test").strip()
+        super().setup(cfg)
+
+    def load_raw_items(self, data_path: str) -> list[dict]:
+        dataset_ref = str(data_path or "").strip()
+        if dataset_ref and (os.path.exists(dataset_ref) or dataset_ref.endswith(".json") or dataset_ref.endswith(".jsonl")):
+            return super().load_raw_items(dataset_ref)
+
+        dataset_name = _normalize_dataset_name(dataset_ref or self.dataset_name)
+        from datasets import load_dataset
+
+        ds = load_dataset(dataset_name, split=self.hf_split)
+        return [dict(item) for item in ds]
+
+    def _materialize_ratio_split(self, cfg: dict) -> str:
+        dataset_ref = os.path.abspath(str(self.data_path or "").strip()) if str(self.data_path or "").strip() and os.path.exists(str(self.data_path or "").strip()) else str(self.data_path or "").strip()
+        if not dataset_ref:
+            dataset_ref = _normalize_dataset_name(self.dataset_name)
+
+        items = self.load_raw_items(dataset_ref)
+        if not isinstance(items, list) or not items:
+            raise ValueError(f"No SWE-bench items available from {dataset_ref!r}")
+
+        ratio = _parse_split_ratio(self.split_ratio)
+        parts = list(ratio)
+        total_parts = sum(parts)
+        rng = random.Random(self.split_seed)
+
+        by_repo: dict[str, list[dict]] = defaultdict(list)
+        for item in items:
+            repo = str(item.get("repo") or "unknown").strip() or "unknown"
+            by_repo[repo].append(dict(item))
+
+        train_items: list[dict] = []
+        val_items: list[dict] = []
+        test_items: list[dict] = []
+
+        for repo in sorted(by_repo):
+            group = list(by_repo[repo])
+            rng.shuffle(group)
+            n = len(group)
+            n_train = round(n * parts[0] / total_parts)
+            n_val = round(n * parts[1] / total_parts)
+
+            if n >= 3:
+                n_train = max(1, n_train)
+                n_val = max(1, n_val)
+            elif n == 2:
+                n_train, n_val = 1, 0
+            else:
+                n_train, n_val = 0, 0
+
+            while n_train + n_val >= n and n >= 2:
+                if n_val > 1:
+                    n_val -= 1
+                elif n_train > 1:
+                    n_train -= 1
+                else:
+                    break
+
+            train_items.extend(group[:n_train])
+            val_items.extend(group[n_train:n_train + n_val])
+            test_items.extend(group[n_train + n_val:])
+
+        rng2 = random.Random(self.split_seed + 1)
+        rng2.shuffle(train_items)
+        rng2.shuffle(val_items)
+        rng2.shuffle(test_items)
+
+        split_dir = self._resolve_split_output_dir(cfg)
+        os.makedirs(split_dir, exist_ok=True)
+        self.write_split_items(os.path.join(split_dir, "train"), train_items)
+        self.write_split_items(os.path.join(split_dir, "val"), val_items)
+        self.write_split_items(os.path.join(split_dir, "test"), test_items)
+
+        manifest = {
+            "source_data_path": dataset_ref,
+            "dataset_name": _normalize_dataset_name(self.dataset_name),
+            "hf_split": self.hf_split,
+            "split_mode": "ratio",
+            "split_ratio": self.split_ratio,
+            "split_seed": self.split_seed,
+            "strategy": "stratified_by_repo",
+            "counts": {
+                "train": len(train_items),
+                "val": len(val_items),
+                "test": len(test_items),
+            },
+        }
+        with open(os.path.join(split_dir, "split_manifest.json"), "w", encoding="utf-8") as f:
+            json.dump(manifest, f, ensure_ascii=False, indent=2)
+        print(
+            f"  [SWEBenchDataLoader] generated repo-stratified split {self.split_ratio} "
+            f"at {split_dir} from {dataset_ref}"
+        )
+        return split_dir
+

reflact/envs/swebench/rollout.py

@@ -0,0 +1,346 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+
+_DATASET_ALIASES = {
+    "lite": ("princeton-nlp/SWE-Bench_Lite", "SWE-bench/SWE-bench_Lite"),
+    "verified": ("princeton-nlp/SWE-Bench_Verified", "SWE-bench/SWE-bench_Verified"),
+    "full": ("princeton-nlp/SWE-Bench", "SWE-bench/SWE-bench"),
+}
+
+
+def _normalize_dataset_names(dataset_name: str) -> tuple[str, str]:
+    key = str(dataset_name or "lite").strip()
+    pair = _DATASET_ALIASES.get(key.lower())
+    if pair:
+        return pair
+    return key, key
+
+
+def _setup_litellm_env() -> None:
+    mapping = {
+        "AZURE_API_KEY": os.environ.get("AZURE_API_KEY") or os.environ.get("AZURE_OPENAI_API_KEY", ""),
+        "AZURE_API_BASE": os.environ.get("AZURE_API_BASE") or os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+        "AZURE_API_VERSION": os.environ.get("AZURE_API_VERSION") or os.environ.get("AZURE_OPENAI_API_VERSION", ""),
+    }
+    for key, value in mapping.items():
+        if value and not os.environ.get(key):
+            os.environ[key] = value
+
+
+def _normalize_student_model(student_model: str) -> str:
+    model = str(student_model or "").strip()
+    if not model:
+        return "azure/gpt-5.4"
+    if "/" in model:
+        return model
+    if os.environ.get("AZURE_OPENAI_ENDPOINT"):
+        return f"azure/{model}"
+    return model
+
+
+def _load_json(path: str) -> dict | list | None:
+    if not os.path.exists(path):
+        return None
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _build_agent_config(
+    *,
+    skill_content: str,
+    student_model: str,
+    step_limit: int,
+    cost_limit: float,
+) -> tuple[dict, str]:
+    try:
+        from minisweagent.config import get_config_from_spec
+        from minisweagent.utils.serialize import recursive_merge
+    except ImportError as exc:
+        raise ImportError(
+            "SWEBench rollout requires minisweagent. Install the mini-swe-agent environment first."
+        ) from exc
+
+    base_config = get_config_from_spec("swebench.yaml")
+    system_template = base_config.get("agent", {}).get("system_template", "")
+    rendered_system = system_template
+    if skill_content.strip():
+        rendered_system = (
+            system_template.rstrip()
+            + "\n\n## Skill Document\n"
+            + "The following skill contains learned guidance for SWE-bench style bug-fixing tasks.\n\n"
+            + skill_content.strip()
+            + "\n"
+        )
+
+    agent_override = {
+        "agent": {
+            "system_template": rendered_system,
+            "step_limit": int(step_limit),
+            "cost_limit": float(cost_limit),
+        },
+        "model": {
+            "model_name": _normalize_student_model(student_model),
+            "cost_tracking": "ignore_errors",
+        },
+    }
+    return recursive_merge(base_config, agent_override), rendered_system
+
+
+def _load_messages_from_traj(traj_path: Path) -> list[dict]:
+    traj_data = _load_json(str(traj_path))
+    if not isinstance(traj_data, dict):
+        return []
+    messages = traj_data.get("messages")
+    if not isinstance(messages, list):
+        return []
+    return [msg for msg in messages if isinstance(msg, dict) and msg.get("role") != "system"]
+
+
+def _load_exit_status(traj_path: Path) -> str:
+    traj_data = _load_json(str(traj_path))
+    if not isinstance(traj_data, dict):
+        return "missing_traj"
+    info = traj_data.get("info")
+    if isinstance(info, dict):
+        return str(info.get("exit_status") or "unknown")
+    return "unknown"
+
+
+def _run_rollout(
+    *,
+    items: list[dict],
+    predictions_dir: str,
+    skill_content: str,
+    student_model: str,
+    workers: int,
+    step_limit: int,
+    cost_limit: float,
+) -> tuple[list[dict], str]:
+    try:
+        from minisweagent.run.benchmarks.swebench import process_instance
+        from minisweagent.run.benchmarks.utils.batch_progress import RunBatchProgressManager
+    except ImportError as exc:
+        raise ImportError(
+            "SWEBench rollout requires minisweagent with swebench benchmark support."
+        ) from exc
+
+    _setup_litellm_env()
+    config, system_prompt = _build_agent_config(
+        skill_content=skill_content,
+        student_model=student_model,
+        step_limit=step_limit,
+        cost_limit=cost_limit,
+    )
+
+    out_path = Path(predictions_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    preds_path = out_path / "preds.json"
+    done_ids: set[str] = set()
+    if preds_path.exists():
+        data = _load_json(str(preds_path))
+        if isinstance(data, dict):
+            done_ids = set(data.keys())
+
+    pending = [item for item in items if str(item.get("instance_id")) not in done_ids]
+    progress_manager = RunBatchProgressManager(
+        len(pending),
+        out_path / f"exit_statuses_{int(time.time())}.yaml",
+    )
+
+    task_errors: dict[str, str] = {}
+
+    def _process(instance: dict) -> None:
+        process_instance(instance, out_path, config, progress_manager)
+
+    with ThreadPoolExecutor(max_workers=max(int(workers), 1)) as executor:
+        futures = {
+            executor.submit(_process, item): str(item.get("instance_id"))
+            for item in pending
+        }
+        for fut in as_completed(futures):
+            iid = futures[fut]
+            try:
+                fut.result()
+            except Exception as exc:  # noqa: BLE001
+                task_errors[iid] = str(exc)
+
+    preds_data = _load_json(str(preds_path))
+    preds_dict = preds_data if isinstance(preds_data, dict) else {}
+    results: list[dict] = []
+
+    for item in items:
+        iid = str(item.get("instance_id"))
+        pred = preds_dict.get(iid, {}) if isinstance(preds_dict, dict) else {}
+        traj_path = out_path / iid / f"{iid}.traj.json"
+        messages = _load_messages_from_traj(traj_path)
+        task_dir = out_path / iid
+        task_dir.mkdir(parents=True, exist_ok=True)
+        user_prompt = (
+            f"Repository: {item.get('repo', '')}\n\n"
+            f"Issue:\n{item.get('problem_statement', '').strip()}"
+        ).strip()
+        with open(task_dir / "conversation.json", "w", encoding="utf-8") as f:
+            json.dump(messages, f, ensure_ascii=False, indent=2)
+        with open(task_dir / "student_system_prompt.txt", "w", encoding="utf-8") as f:
+            f.write(system_prompt)
+        with open(task_dir / "student_user_prompt.txt", "w", encoding="utf-8") as f:
+            f.write(user_prompt)
+
+        results.append(
+            {
+                "id": iid,
+                "instance_id": iid,
+                "repo": str(item.get("repo") or "").strip(),
+                "task_type": str(item.get("repo") or "swebench").strip() or "swebench",
+                "task_description": str(item.get("problem_statement") or "").strip(),
+                "instruction": str(item.get("problem_statement") or "").strip(),
+                "hard": 0,
+                "soft": 0.0,
+                "response": str(pred.get("model_patch") or ""),
+                "submission": str(pred.get("model_patch") or ""),
+                "predicted_patch": str(pred.get("model_patch") or ""),
+                "agent_ok": bool(messages),
+                "n_turns": sum(1 for msg in messages if msg.get("role") == "assistant"),
+                "fail_reason": task_errors.get(iid, ""),
+                "exit_status": _load_exit_status(traj_path),
+            }
+        )
+
+    return results, str(preds_path)
+
+
+def _run_evaluation(
+    *,
+    preds_path: str,
+    dataset_name: str,
+    split: str,
+    run_id: str,
+    eval_workers: int,
+    report_dir: str,
+    instance_ids: list[str],
+) -> dict:
+    _, eval_dataset = _normalize_dataset_names(dataset_name)
+    os.makedirs(report_dir, exist_ok=True)
+
+    preds_data = _load_json(preds_path)
+    model_name = "unknown"
+    if isinstance(preds_data, dict) and preds_data:
+        first_pred = next(iter(preds_data.values()))
+        if isinstance(first_pred, dict):
+            model_name = str(first_pred.get("model_name_or_path") or "unknown")
+    expected_report = os.path.join(report_dir, f"{model_name.replace('/', '__')}.{run_id}.json")
+    if os.path.exists(expected_report):
+        cached = _load_json(expected_report)
+        return cached if isinstance(cached, dict) else {}
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "swebench.harness.run_evaluation",
+        "--dataset_name",
+        eval_dataset,
+        "--split",
+        split,
+        "--predictions_path",
+        preds_path,
+        "--max_workers",
+        str(max(int(eval_workers), 1)),
+        "--run_id",
+        run_id,
+    ]
+    if instance_ids:
+        cmd.extend(["--instance_ids"] + instance_ids)
+
+    subprocess.run(
+        cmd,
+        cwd=report_dir,
+        capture_output=True,
+        text=True,
+        timeout=7200,
+        check=False,
+    )
+
+    if os.path.exists(expected_report):
+        report = _load_json(expected_report)
+        return report if isinstance(report, dict) else {}
+
+    for name in sorted(os.listdir(report_dir)):
+        if name.endswith(".json") and run_id in name:
+            report = _load_json(os.path.join(report_dir, name))
+            if isinstance(report, dict):
+                if os.path.join(report_dir, name) != expected_report:
+                    shutil.move(os.path.join(report_dir, name), expected_report)
+                return report
+    return {"resolved_ids": [], "total_instances": len(instance_ids), "resolved_instances": 0}
+
+
+def run_batch(
+    *,
+    items: list[dict],
+    out_root: str,
+    skill_content: str,
+    student_model: str,
+    dataset_name: str,
+    hf_split: str,
+    workers: int,
+    eval_workers: int,
+    step_limit: int,
+    cost_limit: float,
+    timeout_per_instance: int,
+) -> list[dict]:
+    os.makedirs(out_root, exist_ok=True)
+    results_path = os.path.join(out_root, "results.jsonl")
+    if os.path.exists(results_path):
+        cached: list[dict] = []
+        with open(results_path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    cached.append(json.loads(line))
+        if cached:
+            return cached
+
+    predictions_dir = os.path.join(out_root, "predictions")
+    results, preds_path = _run_rollout(
+        items=items,
+        predictions_dir=predictions_dir,
+        skill_content=skill_content,
+        student_model=student_model,
+        workers=workers,
+        step_limit=step_limit,
+        cost_limit=cost_limit,
+    )
+    eval_report = _run_evaluation(
+        preds_path=preds_path,
+        dataset_name=dataset_name,
+        split=hf_split,
+        run_id=f"reflact_{int(time.time())}",
+        eval_workers=eval_workers,
+        report_dir=os.path.join(out_root, "evaluation"),
+        instance_ids=[str(item.get("instance_id")) for item in items],
+    )
+    resolved_ids = set(str(i) for i in eval_report.get("resolved_ids", []))
+    for row in results:
+        resolved = str(row["instance_id"]) in resolved_ids
+        row["hard"] = int(resolved)
+        row["soft"] = float(int(resolved))
+        if not resolved:
+            status = row.get("exit_status") or "not_resolved"
+            base_reason = str(row.get("fail_reason") or "").strip()
+            unresolved = f"swebench unresolved ({status})"
+            row["fail_reason"] = f"{base_reason}; {unresolved}" if base_reason else unresolved
+        row["timeout_per_instance"] = int(timeout_per_instance)
+
+    with open(results_path, "w", encoding="utf-8") as f:
+        for row in results:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+    return results

reflact/envs/swebench/skills/initial.md

@@ -0,0 +1,23 @@
+# SWE-bench Bug Fixing Skill
+
+## Overview
+This skill guides agents in resolving real-world GitHub issues by producing correct patches.
+
+**Goal**: Given a repository and an issue description, produce a minimal, correct `git diff` patch that resolves the issue without modifying test files.
+
+## Workflow
+
+1. Understand the issue. Read the problem statement carefully and restate the expected behavior before editing code.
+2. Locate relevant code. Use targeted search to identify the files, functions, and tests that encode the buggy behavior.
+3. Reproduce the issue. Build a small, local reproduction before changing source files when feasible.
+4. Implement the fix. Make the smallest source change that addresses the root cause.
+5. Verify the fix. Re-run the reproduction and any focused checks needed to confirm the change.
+6. Submit the patch. Generate a clean unified diff of only the source files you modified.
+
+## Key Rules
+
+- Keep changes minimal and directly tied to the bug.
+- Do not modify tests, fixtures, or unrelated configuration unless the issue explicitly requires it.
+- Prefer understanding the code path before patching.
+- Verify behavior after editing instead of relying on intuition.
+- The final submission must be a valid unified diff.