url/SkillOpt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gate): resolve issue #100 (#102)

Browse Source
This commit is contained in:
Sparsh :)
2026-07-12 21:57:13 +05:30
committed by GitHub
parent da99301210
commit 49a5b617c0
5 changed files with 263 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skillopt/config.py
+3
View File
@@ -125,6 +125,9 @@ _FLATTEN_MAP: dict[str, str] = {
"evaluation.use_gate": "use_gate",
"evaluation.gate_metric": "gate_metric",
"evaluation.gate_mixed_weight": "gate_mixed_weight",
"evaluation.use_semantic_density": "use_semantic_density",
"evaluation.semantic_density_weight": "semantic_density_weight",
"evaluation.leading_words": "leading_words",
"evaluation.sel_env_num": "sel_env_num",
"evaluation.test_env_num": "test_env_num",
"evaluation.eval_test": "eval_test",
skillopt/engine/trainer.py
+27
View File
@@ -964,6 +964,15 @@ class ReflACTTrainer:
f"got {gate_metric!r}"
)
gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5))
use_semantic_density = bool(cfg.get("use_semantic_density", False))
semantic_density_weight = float(cfg.get("semantic_density_weight", 0.05))
leading_words_raw = cfg.get("leading_words", None)
leading_words = None
if leading_words_raw is not None:
if isinstance(leading_words_raw, str):
leading_words = [w.strip() for w in leading_words_raw.split(",") if w.strip()]
else:
leading_words = list(leading_words_raw)
if not 0.0 <= gate_mixed_weight <= 1.0:
raise ValueError(
f"evaluation.gate_mixed_weight must be in [0, 1], "
@@ -1003,6 +1012,10 @@ class ReflACTTrainer:
baseline_hard, baseline_soft = compute_score(baseline_results)
current_score = select_gate_score(
baseline_hard, baseline_soft, gate_metric, gate_mixed_weight,
skill_content=skill_init,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
)
best_score = current_score
sh = skill_hash(skill_init)
@@ -1450,9 +1463,16 @@ class ReflACTTrainer:
cand_soft=cand_soft,
metric=gate_metric,
mixed_weight=gate_mixed_weight,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
) if use_gate else None
cand_gate_score = select_gate_score(
cand_hard, cand_soft, gate_metric, gate_mixed_weight,
skill_content=candidate_skill,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
)
if not use_gate:
# Validation ran (scores recorded above) but the gate is
@@ -1844,6 +1864,9 @@ class ReflACTTrainer:
cand_soft=slow_sel_soft,
metric=gate_metric,
mixed_weight=gate_mixed_weight,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
)
slow_result["selection_hard"] = slow_sel_hard
slow_result["selection_soft"] = slow_sel_soft
@@ -2093,6 +2116,10 @@ class ReflACTTrainer:
final_gate_score = select_gate_score(
final_selection_hard, final_selection_soft,
gate_metric, gate_mixed_weight,
skill_content=current_skill,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
)
print(
f"\n [final skill val] items={fval_n} "
skillopt/evaluation/gate.py
+83 -6
View File
@@ -43,11 +43,55 @@ class GateResult:
best_step: int
def compute_semantic_density(
skill_content: str,
leading_words: list[str] | None = None,
) -> float:
"""Compute the semantic density of leading words in a skill document."""
if not skill_content or not skill_content.strip():
return 0.0
if leading_words is None:
leading_words = [
"MUST", "ALWAYS", "NEVER", "ONLY", "CRITICAL", "IMPORTANT",
"RESOLVE", "PREFER", "ENSURE", "STRICT", "VERIFY"
]
# Strip metadata comments to focus purely on instruction text
skill = skill_content
for start, end in [
("<!-- SLOW_UPDATE_START -->", "<!-- SLOW_UPDATE_END -->"),
("<!-- APPENDIX_START -->", "<!-- APPENDIX_END -->")
]:
while True:
s_idx = skill.find(start)
if s_idx == -1:
break
e_idx = skill.find(end, s_idx)
if e_idx == -1:
skill = skill[:s_idx] + skill[s_idx + len(start):]
break
skill = skill[:s_idx] + skill[e_idx + len(end):]
import re
words = re.findall(r'[a-zA-Z0-9]+', skill.lower())
if not words:
return 0.0
leading_set = {w.lower() for w in leading_words}
leading_count = sum(1 for w in words if w in leading_set)
return leading_count / len(words)
def select_gate_score(
hard: float,
soft: float,
metric: GateMetric = "hard",
mixed_weight: float = 0.5,
*,
skill_content: str = "",
use_semantic_density: bool = False,
semantic_density_weight: float = 0.05,
leading_words: list[str] | None = None,
) -> float:
"""Project (hard, soft) onto a single comparison metric.
@@ -60,18 +104,33 @@ def select_gate_score(
mixed_weight
For ``"mixed"``: weight given to ``soft``. Must be in ``[0, 1]``.
Ignored for ``"hard"`` / ``"soft"``.
skill_content
The raw skill document content.
use_semantic_density
Whether to adjust the score based on semantic density of leading words.
semantic_density_weight
Scaling weight for the semantic density bonus.
leading_words
Optional custom list of high-influence words to prioritize.
"""
if metric == "hard":
return float(hard)
if metric == "soft":
return float(soft)
if metric == "mixed":
score = float(hard)
elif metric == "soft":
score = float(soft)
elif metric == "mixed":
w = max(0.0, min(1.0, float(mixed_weight)))
return (1.0 - w) * float(hard) + w * float(soft)
score = (1.0 - w) * float(hard) + w * float(soft)
else:
raise ValueError(
f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'"
)
if use_semantic_density:
density = compute_semantic_density(skill_content, leading_words)
score += float(semantic_density_weight) * density
return score
def evaluate_gate(
candidate_skill: str,
@@ -86,6 +145,9 @@ def evaluate_gate(
cand_soft: float = 0.0,
metric: GateMetric = "hard",
mixed_weight: float = 0.5,
use_semantic_density: bool = False,
semantic_density_weight: float = 0.05,
leading_words: list[str] | None = None,
) -> GateResult:
"""Pure gate decision: compare candidate score to current/best.
@@ -111,6 +173,12 @@ def evaluate_gate(
the original gate behavior.
mixed_weight
Weight on ``soft`` when ``metric == "mixed"``.
use_semantic_density
Whether to adjust the score based on semantic density of leading words.
semantic_density_weight
Scaling weight for the semantic density bonus.
leading_words
Optional custom list of high-influence words to prioritize.
Returns
-------
@@ -118,7 +186,16 @@ def evaluate_gate(
Updated state; the caller decides what to do with it (print,
mutate trainer state, log, etc.).
"""
cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight)
cand_score = select_gate_score(
cand_hard,
cand_soft,
metric,
mixed_weight,
skill_content=candidate_skill,
use_semantic_density=use_semantic_density,
semantic_density_weight=semantic_density_weight,
leading_words=leading_words,
)
if cand_score > current_score:
if cand_score > best_score:
tests/test_semantic_density.py
+124
View File
@@ -0,0 +1,124 @@
"""Tests for semantic density heuristic in the validation gate."""
from __future__ import annotations
import unittest
from skillopt.evaluation.gate import (
compute_semantic_density,
select_gate_score,
evaluate_gate,
)
class TestSemanticDensity(unittest.TestCase):
"""Test suite for semantic density scoring and gating decisions."""
def test_compute_semantic_density_basic(self) -> None:
"""Verify basic compute_semantic_density behaviour with default words."""
# 10 words, 2 leading words ("always", "never") -> 0.2 density
skill = "Always check the inputs and never mix up proxy values."
density = compute_semantic_density(skill)
self.assertAlmostEqual(density, 0.2)
# Empty skill should have 0 density
self.assertEqual(compute_semantic_density(""), 0.0)
self.assertEqual(compute_semantic_density(" \n "), 0.0)
def test_compute_semantic_density_custom_leading_words(self) -> None:
"""Verify compute_semantic_density with custom leading words."""
skill = "Check the inputs carefully and resolve the equation."
leading = ["check", "resolve"]
# 8 words, 2 custom leading words -> 0.25 density
density = compute_semantic_density(skill, leading_words=leading)
self.assertAlmostEqual(density, 0.25)
def test_compute_semantic_density_with_protected_regions(self) -> None:
"""Verify protected comments are excluded from density calculation."""
skill = (
"Always check inputs.\n"
"<!-- SLOW_UPDATE_START -->\n"
"This contains many words that should not count towards density "
"always and never and only.\n"
"<!-- SLOW_UPDATE_END -->\n"
"<!-- APPENDIX_START -->\n"
"More excluded words.\n"
"<!-- APPENDIX_END -->\n"
)
# Without stripping, there would be many more words and a different density.
# Stripped text: "Always check inputs." -> 3 words, 1 leading word ("always") -> 1/3 density
density = compute_semantic_density(skill)
self.assertAlmostEqual(density, 1.0 / 3.0)
def test_select_gate_score_no_density(self) -> None:
"""Verify select_gate_score without semantic density adjustment."""
# Default behavior: no semantic density adjustment
score_hard = select_gate_score(0.8, 0.6, metric="hard")
self.assertEqual(score_hard, 0.8)
score_soft = select_gate_score(0.8, 0.6, metric="soft")
self.assertEqual(score_soft, 0.6)
score_mixed = select_gate_score(0.8, 0.6, metric="mixed", mixed_weight=0.5)
self.assertAlmostEqual(score_mixed, 0.7)
def test_select_gate_score_with_density(self) -> None:
"""Verify select_gate_score with semantic density adjustment."""
# 10 words, 2 leading words ("always", "never") -> 0.2 density
skill = "Always check the inputs and never mix up proxy values."
# bonus: 0.1 (weight) * 0.2 (density) = 0.02
score = select_gate_score(
hard=0.8,
soft=0.6,
metric="hard",
skill_content=skill,
use_semantic_density=True,
semantic_density_weight=0.1,
)
self.assertAlmostEqual(score, 0.82)
def test_evaluate_gate_with_density_preference(self) -> None:
"""Verify evaluate_gate prefers candidates with higher semantic density."""
# Baseline/current skill:
# "Always do this task step by step and be very careful because errors are bad."
# 15 words, 1 leading ("always") -> 1/15 density = ~0.0667
current_skill = "Always do this task step by step and be very careful because errors are bad."
# Candidate skill (shorter/more steerable):
# "Always verify outputs. Never mix proxy values."
# 7 words, 3 leading ("always", "verify", "never") -> 3/7 density = ~0.4286
candidate_skill = "Always verify outputs. Never mix proxy values."
# Both have same rollout accuracy (hard=0.8, soft=0.8)
# Baseline/current score: 0.8 + 0.1 * (1/15) = ~0.8067
current_score = select_gate_score(
hard=0.8,
soft=0.8,
metric="hard",
skill_content=current_skill,
use_semantic_density=True,
semantic_density_weight=0.1,
)
# Candidate score: 0.8 + 0.1 * (3/7) = ~0.8429
# Even though accuracy is equal, the candidate should be accepted due to higher semantic density
res = evaluate_gate(
candidate_skill=candidate_skill,
cand_hard=0.8,
current_skill=current_skill,
current_score=current_score,
best_skill=current_skill,
best_score=current_score,
best_step=1,
global_step=2,
cand_soft=0.8,
metric="hard",
use_semantic_density=True,
semantic_density_weight=0.1,
)
self.assertEqual(res.action, "accept_new_best")
self.assertEqual(res.current_skill, candidate_skill)
self.assertAlmostEqual(res.current_score, 0.8 + 0.1 * (3.0 / 7.0))
if __name__ == "__main__":
unittest.main()
tests/test_sleep_engine.py
+3 -3
View File
@@ -211,7 +211,7 @@ class TestHarvest(unittest.TestCase):
self.assertTrue(cfg.get("progress"))
self.assertEqual(
cfg.managed_skill_path(),
os.path.join(project, ".agents/skills/taste-skill/SKILL.md"),
os.path.abspath(os.path.join(project, ".agents/skills/taste-skill/SKILL.md")),
)
def test_cli_report_payload_includes_rejected_edits(self):
@@ -281,7 +281,7 @@ class TestHarvest(unittest.TestCase):
self.assertEqual(
cfg.managed_skill_path(),
"/repo/Yoshi/.agents/skills/yoshi-monorepo/SKILL.md",
os.path.abspath("/repo/Yoshi/.agents/skills/yoshi-monorepo/SKILL.md"),
)
def test_cmd_run_uses_tasks_file_without_harvest(self):
@@ -912,7 +912,7 @@ class TestFullCycleAndAdopt(unittest.TestCase):
def test_cycle_can_target_repo_scoped_skill_path(self):
with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home:
target = os.path.join(proj, ".agents/skills/taste-skill/SKILL.md")
target = os.path.abspath(os.path.join(proj, ".agents/skills/taste-skill/SKILL.md"))
cfg = load_config(
invoked_project=proj,
projects="invoked",