SkillOpt v0.1.0: initial release

- Skill optimization framework with training loop analogy
- 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen)
- WebUI for browser-based training control
- Pluggable architecture for extending benchmarks and backends

skillopt/envs/_template/README.md

@@ -0,0 +1,19 @@
+# Benchmark Template
+
+This directory provides scaffold files for adding a new benchmark to SkillOpt.
+
+## Files
+
+- `env_template.py` — Environment adapter template
+- `loader_template.py` — Data loader template
+- `config_template.yaml` — Config file template
+
+## Usage
+
+1. Copy this directory: `cp -r skillopt/envs/_template skillopt/envs/your_benchmark`
+2. Rename files: remove `_template` suffix
+3. Implement the `TODO` sections
+4. Register in `skillopt/envs/__init__.py`
+5. Create config at `configs/your_benchmark/default.yaml`
+
+See the [documentation](../../docs/guide/new-benchmark.md) for the full guide.

skillopt/envs/_template/config_template.yaml

@@ -0,0 +1,45 @@
+# ──────────────────────────────────────────────────
+# SkillOpt Config Template — <Your Benchmark Name>
+# ──────────────────────────────────────────────────
+# Copy this file to configs/<your_benchmark>/default.yaml
+# and customize the values below.
+
+# Inherit global defaults
+_base_: ['../_base_/default.yaml']
+
+# ── Environment ──────────────────────────────────
+env:
+  name: your_benchmark            # Must match registry key
+  data_path: data/your_benchmark  # Path to your data
+  split_mode: ratio               # "ratio" or "split_dir"
+  split_ratio: "2:1:7"            # train:val:test
+  exec_timeout: 120               # Per-task timeout (seconds)
+
+# ── Training ─────────────────────────────────────
+train:
+  num_epochs: 4                   # Number of epochs
+  batch_size: 40                  # Tasks per step (batch size)
+  seed: 42
+
+# ── Gradient (Reflection) ───────────────────────
+gradient:
+  analyst_workers: 16             # Parallel reflection workers
+  minibatch_size: 8
+
+# ── Optimizer ────────────────────────────────────
+optimizer:
+  learning_rate: 4                # Max edits per step (edit budget)
+  lr_scheduler: cosine            # cosine | linear | constant | autonomous
+  use_slow_update: true           # Epoch-boundary momentum
+  use_meta_skill: true            # Cross-epoch teacher memory
+
+# ── Evaluation ───────────────────────────────────
+evaluation:
+  use_gate: true                  # Validation gating
+  eval_test: true                 # Run test eval after training
+
+# ── Model ────────────────────────────────────────
+model:
+  backend: azure_openai           # azure_openai | openai_chat | claude_code_exec | qwen
+  teacher: gpt-5.5
+  student: gpt-5.5

skillopt/envs/_template/env_template.py

@@ -0,0 +1,92 @@
+"""
+Benchmark Environment Template
+===============================
+Copy this file and implement the TODO sections to add a new benchmark.
+
+The EnvAdapter is responsible for:
+1. Executing tasks using the student model + current skill document
+2. Evaluating predictions against ground truth
+3. Returning structured results for the training loop
+"""
+from skillopt.envs.base import EnvAdapter
+
+
+class TemplateBenchmarkEnv(EnvAdapter):
+    """
+    Environment adapter for <Your Benchmark Name>.
+    
+    Rename this class and implement the abstract methods below.
+    """
+
+    def __init__(self, cfg: dict):
+        super().__init__(cfg)
+        # TODO: Initialize benchmark-specific state
+        # Example: self.tools = load_tools(cfg)
+
+    async def execute(self, item, skill: str, model):
+        """
+        Execute a single task with the student model.
+
+        Args:
+            item: DataItem with .id, .input, .ground_truth, .metadata
+            skill: Current skill document content (Markdown string)
+            model: Student model backend instance
+
+        Returns:
+            TaskResult with prediction, score, and trajectory
+        """
+        # Step 1: Build the prompt combining skill + task input
+        prompt = self.build_prompt(item, skill)
+
+        # Step 2: Call the student model
+        # TODO: Customize the message format for your benchmark
+        messages = [
+            {"role": "system", "content": skill},
+            {"role": "user", "content": item.input},
+        ]
+        response = await model.generate(messages)
+
+        # Step 3: Parse the model response into a prediction
+        prediction = self.parse_response(response.content)
+
+        # Step 4: Score the prediction
+        score = self.evaluate(prediction, item.ground_truth)
+
+        # Step 5: Return structured result
+        return {
+            "item_id": item.id,
+            "prediction": prediction,
+            "score": score,
+            "trajectory": messages + [{"role": "assistant", "content": response.content}],
+        }
+
+    def evaluate(self, prediction: str, ground_truth: str) -> float:
+        """
+        Score a prediction against the ground truth.
+
+        Returns:
+            Float between 0.0 (wrong) and 1.0 (correct)
+        
+        TODO: Implement your scoring metric. Common options:
+        - Exact match: float(pred.strip().lower() == gt.strip().lower())
+        - F1 score: compute token overlap
+        - ANLS: for document QA tasks
+        - Custom: any float in [0, 1]
+        """
+        # Placeholder — exact match
+        return float(prediction.strip().lower() == ground_truth.strip().lower())
+
+    def build_prompt(self, item, skill: str) -> str:
+        """Combine skill document with task input."""
+        return f"{skill}\n\n---\n\nQuestion: {item.input}"
+
+    def parse_response(self, response: str) -> str:
+        """
+        Extract the answer from the model's raw response.
+        
+        TODO: Implement extraction logic. For example:
+        - Extract text after "Answer:" 
+        - Parse JSON output
+        - Extract from code blocks
+        """
+        return response.strip()

skillopt/envs/_template/loader_template.py

@@ -0,0 +1,103 @@
+"""
+Benchmark Data Loader Template
+================================
+Copy this file and implement the TODO sections to load your benchmark data.
+
+The DataLoader is responsible for:
+1. Loading raw data from disk
+2. Splitting into train / validation / test sets
+3. Providing DataItem objects to the training loop
+"""
+from pathlib import Path
+
+
+class TemplateBenchmarkLoader:
+    """
+    Data loader for <Your Benchmark Name>.
+    
+    Rename this class and implement the methods below.
+    """
+
+    def __init__(self, data_dir: str = "data/your_benchmark", **kwargs):
+        self.data_dir = Path(data_dir)
+        self.items = []
+        self.splits = {}
+
+    def setup(self, cfg: dict):
+        """
+        Initialize the loader with config.
+        
+        Called once before training starts.
+        
+        Args:
+            cfg: Dict with keys like 'split_mode', 'train_ratio', 'val_ratio', etc.
+        """
+        # Step 1: Load raw data
+        self.items = self._load_items()
+
+        # Step 2: Create splits
+        split_mode = cfg.get("split_mode", "ratio")
+        if split_mode == "ratio":
+            self._split_by_ratio(
+                train_ratio=cfg.get("train_ratio", 0.7),
+                val_ratio=cfg.get("val_ratio", 0.15),
+            )
+        elif split_mode == "split_dir":
+            self._load_predefined_splits(cfg.get("split_dir", self.data_dir))
+
+    def _load_items(self) -> list:
+        """
+        Load raw data into structured items.
+        
+        TODO: Implement data loading. Each item should have at minimum:
+        - id: unique identifier
+        - input: the task input (question, instruction, etc.)
+        - ground_truth: the expected answer
+        - metadata: optional dict with extra info
+        
+        Example:
+            items = []
+            for path in self.data_dir.glob("*.json"):
+                data = json.loads(path.read_text())
+                for entry in data:
+                    items.append({
+                        "id": entry["id"],
+                        "input": entry["question"],
+                        "ground_truth": entry["answer"],
+                        "metadata": {"source": path.name},
+                    })
+            return items
+        """
+        raise NotImplementedError("Implement _load_items() for your benchmark")
+
+    def _split_by_ratio(self, train_ratio: float, val_ratio: float):
+        """Split items by ratio."""
+        import random
+        random.shuffle(self.items)
+        n = len(self.items)
+        n_train = int(n * train_ratio)
+        n_val = int(n * val_ratio)
+        self.splits = {
+            "train": self.items[:n_train],
+            "valid": self.items[n_train:n_train + n_val],
+            "test": self.items[n_train + n_val:],
+        }
+
+    def _load_predefined_splits(self, split_dir):
+        """Load from pre-split directories."""
+        # TODO: Implement if your benchmark has pre-defined splits
+        raise NotImplementedError
+
+    def get_split_items(self, split: str) -> list:
+        """
+        Return items for a given split.
+        
+        Args:
+            split: One of "train", "valid", "test"
+            
+        Returns:
+            List of data items for the requested split
+        """
+        if split not in self.splits:
+            raise ValueError(f"Unknown split '{split}'. Available: {list(self.splits.keys())}")
+        return self.splits[split]