From ac04b37af9081c7ed3c087b9c80625e2893b90cf Mon Sep 17 00:00:00 2001 From: Mohamed Mastouri Date: Tue, 21 Jul 2026 02:12:49 +0300 Subject: [PATCH] fix(plan): respect explicit COLI_CUDA_MTP=1 - skip the DRAFT=0 export so the engine's auto path can engage The documented CUDA-MTP opt-in was silently inert through the wrapper: _auto_tune exported DRAFT=0 (compute-bound, or disk-bound with projected hit < 0.90), which preempts the engine's DRAFT=-1 auto path - the only place COLI_CUDA_MTP is consulted (colibri.c). Found while running the #467 A/B: COLI_CUDA_MTP=1 alone measured pure baseline noise until DRAFT=3 was also set. Now an explicit COLI_CUDA_MTP=1 makes the planner leave DRAFT unset (engine resolves draft=3); unset keeps DRAFT=0 -> MTP off under CUDA, the measured-correct default (#389 -42 streaming-bound). ENVIRONMENT.md documents the behaviour and the measured trade-off. --- c/resource_plan.py | 11 ++++++++++- docs/ENVIRONMENT.md | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/c/resource_plan.py b/c/resource_plan.py index 1374a3b..4a1014a 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -296,7 +296,16 @@ def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_meta n_gpu = len(gpus) # MTP: costs more than it saves when compute-bound (#389 measured 42% loss) - if bottleneck_class == "compute": + # or streaming-bound (#467 measured 32% loss under CUDA at 85% hit). + # EXCEPTION: an explicit COLI_CUDA_MTP=1 in the environment is a documented + # opt-in to test speculation under CUDA (glm.c resolves DRAFT=-1 -> 3 only + # when it sees the var). Exporting DRAFT=0 here preempted that auto path, + # so the opt-in was silently inert on the Windows bare-run/auto-tier flows + # (#467): respect it and let the engine's auto path take over. Unset still + # gets DRAFT=0 -> MTP off, which is the measured-correct default. + if os.environ.get("COLI_CUDA_MTP") == "1": + pass # explicit opt-in: leave DRAFT to the engine's auto resolution + elif bottleneck_class == "compute": tune["DRAFT"] = {"value": "0", "reason": "compute-bound: MTP batch overhead exceeds yield"} elif bottleneck_class == "disk" and projected_hit < 0.90: diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 14615a5..2684753 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -103,7 +103,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE | `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | | `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | | `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | -| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292 — though #467 measured acceptance holding at 49% on sm_120). When set explicitly, the resource planner skips its `DRAFT=0` export so the engine's auto path can engage draft=3 — no need to also set `DRAFT`. Note the measured trade-off (#467): at ~85% hit the widened S=4 expert union costs more than speculation saves (−32%); the opt-in pays only near-full residency (~99% hit). | | `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | | `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | | `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. |