woolcoxm
cf126cbcf9
CI runs 'make check' = dependency-free tests, no model downloads (by design, #140). glm_tiny/ is a gitignored generated fixture, so test_inefficiency.py hard-failed on the Windows/macOS/Linux runners with 'config.json: No such file or directory' instead of skipping. _engine_present() now requires BOTH glm.exe AND glm_tiny/config.json, and _skip_reason() names exactly which prerequisite is missing so the skip is actionable. Verified: 8 skipped (0 failed) with the fixture absent; 5 pass + 3 CUDA-skip with it present.
258 lines
12 KiB
Python
258 lines
12 KiB
Python
"""Inefficiency / regression tests for the colibri engine (tiny model, asserted).
|
|
|
|
These run against the bundled glm_tiny model (~0.6 MB resident, ~0.1s/run) and
|
|
gate CI: a regression here means something broke. They run on a plain CPU-only
|
|
`glm.exe` build; the CUDA_* tests auto-skip if the engine wasn't built with
|
|
CUDA_DLL=1 (see tests/README_efficiency.md for the build command).
|
|
|
|
The signals under test, and the inefficiency each catches:
|
|
- tok/s floor : a throughput regression (broken build / bad config)
|
|
- profile phases sum : telemetry accounting bug (other balloons)
|
|
- disk-wait not dominant: a tiny resident model should never be I/O-bound
|
|
- CPU determinism : greedy decode is reproducible (no stray RNG/threading)
|
|
- CUDA init path : COLI_CUDA=1 initializes and does not silently exit 2
|
|
- CUDA dense uses VRAM : CUDA_DENSE=1 actually uploads tensors (no silent CPU fallback)
|
|
- CPU vs CUDA TF-match : identical weights+inputs → identical argmax (kernel bug guard)
|
|
"""
|
|
import os
|
|
import shutil
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from tools.efficiency import (
|
|
parse_run, run_engine, disk_wait_share, tf_agreement,
|
|
TINY_TOK_S_FLOOR, MAX_DISK_WAIT_SHARE, MIN_CPU_CUDA_AGREEMENT,
|
|
)
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
C_DIR = HERE.parent
|
|
ENGINE = C_DIR / "glm.exe"
|
|
TINY = C_DIR / "glm_tiny"
|
|
|
|
|
|
def _engine_present() -> bool:
|
|
"""True iff BOTH the built engine AND the tiny fixture are available.
|
|
|
|
These tests need glm.exe (a build artifact) AND glm_tiny/ (a generated
|
|
fixture, gitignored). CI runs `make check` = "dependency-free tests, no
|
|
model downloads" (workflow .github/workflows/check.yml, by design #140), so
|
|
neither is present there and these tests must SKIP rather than fail. They
|
|
run locally after `make glm.exe` (glm_tiny ships alongside the source, or
|
|
is regenerated by tools/make_glm_oracle.py).
|
|
"""
|
|
return ENGINE.exists() and (TINY / "config.json").exists()
|
|
|
|
|
|
def _skip_reason() -> str:
|
|
"""Name exactly which prerequisite is missing, so the skip is actionable."""
|
|
if not ENGINE.exists():
|
|
return "glm.exe not built (run: make glm.exe)"
|
|
if not (TINY / "config.json").exists():
|
|
return "glm_tiny fixture absent (gitignored; ship it locally or run tools/make_glm_oracle.py)"
|
|
return ""
|
|
|
|
|
|
def _cuda_available() -> bool:
|
|
"""True iff the engine binary has the CUDA loader compiled in AND the DLL is present.
|
|
|
|
The host is built with -DCOLI_CUDA only when CUDA_DLL=1 (Makefile). A binary
|
|
built without it embeds the string "this binary is CPU-only; rebuild" and
|
|
exits 2 on any CUDA env var — so we detect the CPU-only build by scanning
|
|
the binary for that marker (avoids a slow ldd/strings on every import; we
|
|
only read enough to find it). On Windows the DLL is also required
|
|
(backend_loader.c loads it at runtime); on Linux it's direct-linked.
|
|
"""
|
|
if not _engine_present():
|
|
return False
|
|
try:
|
|
# Read once; the marker is near the read-only string table. 256 KB is
|
|
# plenty for this string and avoids loading a 1 MB binary into memory.
|
|
with open(ENGINE, "rb") as f:
|
|
blob = f.read(2 * 1024 * 1024)
|
|
if b"this binary is CPU-only" in blob:
|
|
return False # CPU-only build: COLI_CUDA=1 would exit 2.
|
|
except OSError:
|
|
pass
|
|
# Windows: DLL required at runtime. Linux: direct-linked (no DLL).
|
|
if os.name == "nt":
|
|
return (C_DIR / "coli_cuda.dll").exists()
|
|
import subprocess
|
|
try:
|
|
out = subprocess.run(["ldd", str(ENGINE)], capture_output=True, text=True)
|
|
return "libcudart" in out.stdout
|
|
except (FileNotFoundError, OSError):
|
|
return False
|
|
|
|
|
|
@unittest.skipUnless(_engine_present(), _skip_reason() or "glm.exe + glm_tiny required")
|
|
class TinyEfficiencyTest(unittest.TestCase):
|
|
"""Asserted regression tests on the resident tiny model. Gates CI."""
|
|
|
|
def _run(self, **overlay):
|
|
return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0]
|
|
|
|
# -- telemetry contract ---------------------------------------------------
|
|
|
|
def test_telemetry_parses(self):
|
|
"""A REPLAY run must emit the throughput + PROFILE lines the suite keys on.
|
|
|
|
If this fails, either the engine changed its output format (update the
|
|
parsers in tools/efficiency.py) or the run crashed early."""
|
|
t = self._run(REPLAY="1", TEMP="0", NGEN="4")
|
|
self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}")
|
|
self.assertIn("tok_s", t["parsed"], f"missing tok/s line:\n{t['stderr']}")
|
|
self.assertIn("profile", t["parsed"], f"missing PROFILE line:\n{t['stderr']}")
|
|
self.assertIn("hit_pct", t["parsed"], f"missing expert hit line:\n{t['stderr']}")
|
|
self.assertIsNotNone(t["tok_s"])
|
|
self.assertIsNotNone(t["profile"])
|
|
|
|
# -- throughput floor -----------------------------------------------------
|
|
|
|
def test_tiny_tok_s_floor(self):
|
|
"""Tiny decode must beat TINY_TOK_S_FLOOR.
|
|
|
|
The tiny model is fully resident and runs ~200 tok/s; the 20 tok/s
|
|
default floor is a 10x margin that catches broken builds or a pathological
|
|
config cascade (the cap=1 trap from ISSUE_new_model_resource_regression.md)
|
|
without flapping on machine noise."""
|
|
t = self._run(REPLAY="1", TEMP="0", NGEN="8")
|
|
self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}")
|
|
self.assertGreaterEqual(
|
|
t["tok_s"], TINY_TOK_S_FLOOR,
|
|
f"tok/s {t['tok_s']:.1f} below floor {TINY_TOK_S_FLOOR} "
|
|
f"(regression, or a config cascade starving the cache)",
|
|
)
|
|
|
|
# -- accounting sanity ----------------------------------------------------
|
|
|
|
def test_profile_phases_present_and_nonneg(self):
|
|
"""Every PROFILE phase must be present and non-negative.
|
|
|
|
`other` can go slightly negative from timer overhead (the engine allows
|
|
it), but a large negative means the timers are double-counting."""
|
|
t = self._run(REPLAY="1", TEMP="0", NGEN="4")
|
|
p = t["profile"]
|
|
for phase in ("disk", "expert_matmul", "attention", "lm_head"):
|
|
self.assertGreaterEqual(p[phase], -0.01, f"{phase} went negative: {p}")
|
|
# 'other' is a residual; allow a small negative from timer overlap.
|
|
self.assertGreaterEqual(p["other"], -0.05, f"other too negative (double-count): {p}")
|
|
|
|
# -- disk-wait not dominant on a resident model ---------------------------
|
|
|
|
def test_disk_wait_not_dominant(self):
|
|
"""A fully-resident tiny model must NOT be I/O-bound.
|
|
|
|
Everything fits in RAM; the expert-disk wait share should be ~0. If it
|
|
exceeds MAX_DISK_WAIT_SHARE, the cache/I/O path regressed — on a real
|
|
model this same regression would make decode I/O-bound (the exact
|
|
failure mode the [PROF] verdict flags)."""
|
|
t = self._run(REPLAY="1", TEMP="0", NGEN="8", PROF="1")
|
|
self.assertIn("time_shares", t["parsed"], f"missing [PROF] time shares:\n{t['stderr']}")
|
|
share = disk_wait_share(t)
|
|
self.assertIsNotNone(share)
|
|
self.assertLess(
|
|
share, MAX_DISK_WAIT_SHARE,
|
|
f"expert-I/O share {share:.0%} on a resident model — I/O path regressed",
|
|
)
|
|
|
|
# -- determinism ----------------------------------------------------------
|
|
|
|
def test_cpu_vs_cpu_determinism(self):
|
|
"""Two greedy REPLAY runs with the same seed produce identical telemetry.
|
|
|
|
TEMP=0 = greedy (no sampling), so tok/s and hit-rate must be reproducible.
|
|
A drift here means non-determinism crept into the decode path (stray
|
|
threading, uninitialized state) — which on a real model would make A/B
|
|
comparisons meaningless."""
|
|
a = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1")
|
|
b = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1")
|
|
self.assertEqual(a["returncode"], 0)
|
|
self.assertEqual(b["returncode"], 0)
|
|
self.assertEqual(a["hit_pct"], b["hit_pct"], "greedy hit-rate drifted between runs")
|
|
# tok/s within 25% — exact equality is too strict across scheduler noise.
|
|
self.assertLess(abs(a["tok_s"] - b["tok_s"]) / max(a["tok_s"], b["tok_s"]), 0.25)
|
|
|
|
|
|
@unittest.skipUnless(_cuda_available(),
|
|
_skip_reason() or "CUDA build not present (run: make clean && make glm.exe CUDA_DLL=1 && make cuda-dll)")
|
|
class TinyCudaEfficiencyTest(unittest.TestCase):
|
|
"""CUDA-path regression tests on the tiny model. Skip unless CUDA built.
|
|
|
|
glm_tiny is small and fully resident, so CUDA here is fast and exercises the
|
|
real GPU code path (init, dense upload, kernel correctness) without the
|
|
long load time or memory pressure of the full model. These guard the
|
|
silent-failure modes that are otherwise invisible:
|
|
- COLI_CUDA=1 silently falling back to CPU (loader/DLL missing)
|
|
- CUDA_DENSE=1 uploading nothing
|
|
- a CUDA kernel producing different argmax than CPU on identical inputs
|
|
"""
|
|
|
|
def _run(self, **overlay):
|
|
return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0]
|
|
|
|
def test_cuda_init_path(self):
|
|
"""COLI_CUDA=1 must initialize the device and NOT exit 2.
|
|
|
|
Exit 2 is the engine's "requested backend is unavailable" path
|
|
(glm.c: g_cuda_enabled check). A clean init prints the [CUDA] device
|
|
banner to stderr. If this fails, the DLL is broken or the loader can't
|
|
resolve symbols (ABI drift between backend_cuda.h and the dll)."""
|
|
t = self._run(COLI_CUDA="1", COLI_GPU="0", REPLAY="1", TEMP="0", NGEN="4")
|
|
self.assertNotEqual(t["returncode"], 2,
|
|
f"engine refused CUDA backend:\n{t['stderr']}")
|
|
self.assertTrue(t["cuda"]["enabled"],
|
|
f"no [CUDA] device banner on stderr:\n{t['stderr']}")
|
|
|
|
def test_cuda_dense_uses_vram(self):
|
|
"""CUDA_DENSE=1 must actually upload dense tensors to VRAM.
|
|
|
|
The minimal GPU-exercising config (per backend_loader.c analysis):
|
|
COLI_CUDA=1 + CUDA_DENSE=1. Without CUDA_DENSE the dense path stays on
|
|
CPU and [CUDA] resident set reports 0 tensors — a silent no-op. This
|
|
catches that regression: after the run, resident_tensors > 0."""
|
|
t = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1",
|
|
REPLAY="1", TEMP="0", NGEN="4")
|
|
self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}")
|
|
rt = t["cuda"]["resident_tensors"]
|
|
self.assertIsNotNone(rt, f"no [CUDA] resident set line:\n{t['stderr']}")
|
|
self.assertGreater(rt, 0,
|
|
f"CUDA_DENSE=1 but {rt} tensors resident — silent CPU fallback")
|
|
|
|
def test_cpu_vs_cuda_tf_match(self):
|
|
"""CPU and CUDA teacher-forcing must agree on most positions (DIRECTLY).
|
|
|
|
Both paths prefill the SAME oracle sequence on the SAME weights, so their
|
|
argmaxes should match position-for-position — but not exactly: the two
|
|
backends accumulate dot-products in different orders (x86 SIMD vs CUDA
|
|
kernel), so a few near-tied logits flip. That divergence is expected
|
|
numeric behavior, not a kernel bug. A *catastrophic* kernel regression
|
|
(wrong GEMM, wrong scale, wrong fmt) would collapse agreement toward
|
|
random (~1/vocab = ~4%); the MIN_CPU_CUDA_AGREEMENT floor (default 70%)
|
|
catches that while tolerating harmless drift.
|
|
|
|
We compare CPU-vs-CUDA *directly* (not via the oracle match-counts),
|
|
because both backends differ from the oracle at different positions and
|
|
the summary line can't tell "CPU≠CUDA" from "CPU≠oracle"."""
|
|
import json
|
|
ref = json.loads((C_DIR / "ref_glm.json").read_text())
|
|
oracle = ref["tf_pred"]
|
|
|
|
cpu = self._run(TF="1", TEMP="0")
|
|
cuda = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", TF="1", TEMP="0")
|
|
self.assertEqual(cpu["returncode"], 0, f"CPU TF run failed:\n{cpu['stderr']}")
|
|
self.assertEqual(cuda["returncode"], 0, f"CUDA TF run failed:\n{cuda['stderr']}")
|
|
self.assertIn("tf_mismatches", cpu["parsed"], "CPU run missing per-position mismatches")
|
|
self.assertIn("tf_mismatches", cuda["parsed"], "CUDA run missing per-position mismatches")
|
|
|
|
agree, diff = tf_agreement(cpu, cuda, oracle)
|
|
self.assertGreaterEqual(
|
|
agree, MIN_CPU_CUDA_AGREEMENT,
|
|
f"CPU-vs-CUDA argmax agreement {agree:.0%} below floor "
|
|
f"{MIN_CPU_CUDA_AGREEMENT:.0%} — CUDA kernel likely regressed. "
|
|
f"Differing positions: {diff[:10]}{'...' if len(diff)>10 else ''}",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|