"""Inefficiency / regression tests for the colibri engine (tiny model, asserted). These run against the bundled glm_tiny model (~0.6 MB resident, ~0.1s/run) and gate CI: a regression here means something broke. They run on a plain CPU-only `glm.exe` build; the CUDA_* tests auto-skip if the engine wasn't built with CUDA_DLL=1 (see tests/README_efficiency.md for the build command). The signals under test, and the inefficiency each catches: - tok/s floor : a throughput regression (broken build / bad config) - profile phases sum : telemetry accounting bug (other balloons) - disk-wait not dominant: a tiny resident model should never be I/O-bound - CPU determinism : greedy decode is reproducible (no stray RNG/threading) - CUDA init path : COLI_CUDA=1 initializes and does not silently exit 2 - CUDA dense uses VRAM : CUDA_DENSE=1 actually uploads tensors (no silent CPU fallback) - CPU vs CUDA TF-match : identical weights+inputs → identical argmax (kernel bug guard) """ import os import shutil import unittest from pathlib import Path from tools.efficiency import ( parse_run, run_engine, disk_wait_share, tf_agreement, TINY_TOK_S_FLOOR, MAX_DISK_WAIT_SHARE, MIN_CPU_CUDA_AGREEMENT, ) HERE = Path(__file__).resolve().parent C_DIR = HERE.parent ENGINE = C_DIR / "glm.exe" TINY = C_DIR / "glm_tiny" def _engine_present() -> bool: return ENGINE.exists() def _cuda_available() -> bool: """True iff the engine binary has the CUDA loader compiled in AND the DLL is present. The host is built with -DCOLI_CUDA only when CUDA_DLL=1 (Makefile). A binary built without it embeds the string "this binary is CPU-only; rebuild" and exits 2 on any CUDA env var — so we detect the CPU-only build by scanning the binary for that marker (avoids a slow ldd/strings on every import; we only read enough to find it). On Windows the DLL is also required (backend_loader.c loads it at runtime); on Linux it's direct-linked. """ if not _engine_present(): return False try: # Read once; the marker is near the read-only string table. 256 KB is # plenty for this string and avoids loading a 1 MB binary into memory. with open(ENGINE, "rb") as f: blob = f.read(2 * 1024 * 1024) if b"this binary is CPU-only" in blob: return False # CPU-only build: COLI_CUDA=1 would exit 2. except OSError: pass # Windows: DLL required at runtime. Linux: direct-linked (no DLL). if os.name == "nt": return (C_DIR / "coli_cuda.dll").exists() import subprocess try: out = subprocess.run(["ldd", str(ENGINE)], capture_output=True, text=True) return "libcudart" in out.stdout except (FileNotFoundError, OSError): return False @unittest.skipUnless(_engine_present(), "glm.exe not built (run: make glm.exe)") class TinyEfficiencyTest(unittest.TestCase): """Asserted regression tests on the resident tiny model. Gates CI.""" def _run(self, **overlay): return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] # -- telemetry contract --------------------------------------------------- def test_telemetry_parses(self): """A REPLAY run must emit the throughput + PROFILE lines the suite keys on. If this fails, either the engine changed its output format (update the parsers in tools/efficiency.py) or the run crashed early.""" t = self._run(REPLAY="1", TEMP="0", NGEN="4") self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") self.assertIn("tok_s", t["parsed"], f"missing tok/s line:\n{t['stderr']}") self.assertIn("profile", t["parsed"], f"missing PROFILE line:\n{t['stderr']}") self.assertIn("hit_pct", t["parsed"], f"missing expert hit line:\n{t['stderr']}") self.assertIsNotNone(t["tok_s"]) self.assertIsNotNone(t["profile"]) # -- throughput floor ----------------------------------------------------- def test_tiny_tok_s_floor(self): """Tiny decode must beat TINY_TOK_S_FLOOR. The tiny model is fully resident and runs ~200 tok/s; the 20 tok/s default floor is a 10x margin that catches broken builds or a pathological config cascade (the cap=1 trap from ISSUE_new_model_resource_regression.md) without flapping on machine noise.""" t = self._run(REPLAY="1", TEMP="0", NGEN="8") self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") self.assertGreaterEqual( t["tok_s"], TINY_TOK_S_FLOOR, f"tok/s {t['tok_s']:.1f} below floor {TINY_TOK_S_FLOOR} " f"(regression, or a config cascade starving the cache)", ) # -- accounting sanity ---------------------------------------------------- def test_profile_phases_present_and_nonneg(self): """Every PROFILE phase must be present and non-negative. `other` can go slightly negative from timer overhead (the engine allows it), but a large negative means the timers are double-counting.""" t = self._run(REPLAY="1", TEMP="0", NGEN="4") p = t["profile"] for phase in ("disk", "expert_matmul", "attention", "lm_head"): self.assertGreaterEqual(p[phase], -0.01, f"{phase} went negative: {p}") # 'other' is a residual; allow a small negative from timer overlap. self.assertGreaterEqual(p["other"], -0.05, f"other too negative (double-count): {p}") # -- disk-wait not dominant on a resident model --------------------------- def test_disk_wait_not_dominant(self): """A fully-resident tiny model must NOT be I/O-bound. Everything fits in RAM; the expert-disk wait share should be ~0. If it exceeds MAX_DISK_WAIT_SHARE, the cache/I/O path regressed — on a real model this same regression would make decode I/O-bound (the exact failure mode the [PROF] verdict flags).""" t = self._run(REPLAY="1", TEMP="0", NGEN="8", PROF="1") self.assertIn("time_shares", t["parsed"], f"missing [PROF] time shares:\n{t['stderr']}") share = disk_wait_share(t) self.assertIsNotNone(share) self.assertLess( share, MAX_DISK_WAIT_SHARE, f"expert-I/O share {share:.0%} on a resident model — I/O path regressed", ) # -- determinism ---------------------------------------------------------- def test_cpu_vs_cpu_determinism(self): """Two greedy REPLAY runs with the same seed produce identical telemetry. TEMP=0 = greedy (no sampling), so tok/s and hit-rate must be reproducible. A drift here means non-determinism crept into the decode path (stray threading, uninitialized state) — which on a real model would make A/B comparisons meaningless.""" a = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") b = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") self.assertEqual(a["returncode"], 0) self.assertEqual(b["returncode"], 0) self.assertEqual(a["hit_pct"], b["hit_pct"], "greedy hit-rate drifted between runs") # tok/s within 25% — exact equality is too strict across scheduler noise. self.assertLess(abs(a["tok_s"] - b["tok_s"]) / max(a["tok_s"], b["tok_s"]), 0.25) @unittest.skipUnless(_cuda_available(), "CUDA build not present (run: make clean && make glm.exe CUDA_DLL=1 && make cuda-dll)") class TinyCudaEfficiencyTest(unittest.TestCase): """CUDA-path regression tests on the tiny model. Skip unless CUDA built. glm_tiny is small and fully resident, so CUDA here is fast and exercises the real GPU code path (init, dense upload, kernel correctness) without the long load time or memory pressure of the full model. These guard the silent-failure modes that are otherwise invisible: - COLI_CUDA=1 silently falling back to CPU (loader/DLL missing) - CUDA_DENSE=1 uploading nothing - a CUDA kernel producing different argmax than CPU on identical inputs """ def _run(self, **overlay): return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] def test_cuda_init_path(self): """COLI_CUDA=1 must initialize the device and NOT exit 2. Exit 2 is the engine's "requested backend is unavailable" path (glm.c: g_cuda_enabled check). A clean init prints the [CUDA] device banner to stderr. If this fails, the DLL is broken or the loader can't resolve symbols (ABI drift between backend_cuda.h and the dll).""" t = self._run(COLI_CUDA="1", COLI_GPU="0", REPLAY="1", TEMP="0", NGEN="4") self.assertNotEqual(t["returncode"], 2, f"engine refused CUDA backend:\n{t['stderr']}") self.assertTrue(t["cuda"]["enabled"], f"no [CUDA] device banner on stderr:\n{t['stderr']}") def test_cuda_dense_uses_vram(self): """CUDA_DENSE=1 must actually upload dense tensors to VRAM. The minimal GPU-exercising config (per backend_loader.c analysis): COLI_CUDA=1 + CUDA_DENSE=1. Without CUDA_DENSE the dense path stays on CPU and [CUDA] resident set reports 0 tensors — a silent no-op. This catches that regression: after the run, resident_tensors > 0.""" t = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", REPLAY="1", TEMP="0", NGEN="4") self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") rt = t["cuda"]["resident_tensors"] self.assertIsNotNone(rt, f"no [CUDA] resident set line:\n{t['stderr']}") self.assertGreater(rt, 0, f"CUDA_DENSE=1 but {rt} tensors resident — silent CPU fallback") def test_cpu_vs_cuda_tf_match(self): """CPU and CUDA teacher-forcing must agree on most positions (DIRECTLY). Both paths prefill the SAME oracle sequence on the SAME weights, so their argmaxes should match position-for-position — but not exactly: the two backends accumulate dot-products in different orders (x86 SIMD vs CUDA kernel), so a few near-tied logits flip. That divergence is expected numeric behavior, not a kernel bug. A *catastrophic* kernel regression (wrong GEMM, wrong scale, wrong fmt) would collapse agreement toward random (~1/vocab = ~4%); the MIN_CPU_CUDA_AGREEMENT floor (default 70%) catches that while tolerating harmless drift. We compare CPU-vs-CUDA *directly* (not via the oracle match-counts), because both backends differ from the oracle at different positions and the summary line can't tell "CPU≠CUDA" from "CPU≠oracle".""" import json ref = json.loads((C_DIR / "ref_glm.json").read_text()) oracle = ref["tf_pred"] cpu = self._run(TF="1", TEMP="0") cuda = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", TF="1", TEMP="0") self.assertEqual(cpu["returncode"], 0, f"CPU TF run failed:\n{cpu['stderr']}") self.assertEqual(cuda["returncode"], 0, f"CUDA TF run failed:\n{cuda['stderr']}") self.assertIn("tf_mismatches", cpu["parsed"], "CPU run missing per-position mismatches") self.assertIn("tf_mismatches", cuda["parsed"], "CUDA run missing per-position mismatches") agree, diff = tf_agreement(cpu, cuda, oracle) self.assertGreaterEqual( agree, MIN_CPU_CUDA_AGREEMENT, f"CPU-vs-CUDA argmax agreement {agree:.0%} below floor " f"{MIN_CPU_CUDA_AGREEMENT:.0%} — CUDA kernel likely regressed. " f"Differing positions: {diff[:10]}{'...' if len(diff)>10 else ''}", ) if __name__ == "__main__": unittest.main()