Windows: pipe2 resident pipeline at decode + EXPERT_BUDGET ws_b cache fix (#274)

Windows: 1.07 tok/s decode via GPU resident pipeline + disk-I/O tuning (3.2x over stock)

c/glm.c

@@ -3549,17 +3549,30 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in
     if(!coli_cuda_pipe_rmsnorm(dev,nrm_d,x_dev,w_post,S,D,c->eps)) return 0;
     if(!coli_cuda_pipe_download(dev,nrm_d,nrm_host,xb)) return 0;
     m->t_attn+=now_s()-ta;
-    /* expert routed su CPU/gruppi GPU come oggi (shared saltata: la fa il device) */
+    /* OVERLAP: issue the shared expert on the GPU BEFORE moe() runs on the CPU.
-    moe(m,l,li,nrm_host,S,out_host,0);
+     * The shared expert reads nrm_d (valid after the download above) and writes its
+     * residual into x_dev (async). While the GPU computes this, the CPU enters moe()
+     * for routing + expert disk loads + matmul — ~50ms of work that previously left
+     * the GPU idle. The shared expert (~0.5ms) finishes early in that window.
+     *
+     * After moe(), the routed-expert result is uploaded (sync pipe_upload) and added
+     * to x_dev (async). Both residual adds (shared + routed) are ordered on the same
+     * stream — the next layer's pipe_rmsnorm reads x_dev after both complete.
+     *
+     * No pipe_sync at the end: the next layer's pipe_download (sync cudaMemcpy)
+     * provides the implicit sync point. The fallback path (caller downloads x_dev)
+     * also uses pipe_download which syncs. This lets GPU work chain across layers
+     * without a per-layer stall. */
     double te=now_s();
-    if(!coli_cuda_pipe_upload(dev,y_d,out_host,xb)) return 0;
-    if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0;
     if(!coli_cuda_pipe_gemm(l->sh_gate.cuda,sg_d,nrm_d,S)) return 0;
     if(!coli_cuda_pipe_gemm(l->sh_up.cuda,su_d,nrm_d,S)) return 0;
     if(!coli_cuda_pipe_silu_mul(dev,sg_d,su_d,(size_t)S*sI)) return 0;
     if(!coli_cuda_pipe_gemm(l->sh_down.cuda,y_d,sg_d,S)) return 0;
-    if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0;
+    if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0;  /* shared residual (async) */
-    if(!coli_cuda_pipe_sync(dev)) return 0;
+    /* expert routed su CPU/gruppi GPU come oggi (shared saltata: la fa il device) */
+    moe(m,l,li,nrm_host,S,out_host,0);
+    if(!coli_cuda_pipe_upload(dev,y_d,out_host,xb)) return 0;     /* sync: waits for moe */
+    if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0;  /* routed residual (async) */
     m->t_emm+=now_s()-te;
     return 1;
 }
@@ -3656,10 +3669,20 @@ static void layers_forward_rows(Model *m, float *x, int S, int pos_base,
     float *nrm=falloc((int64_t)S*D), *tmp=falloc((int64_t)S*D);
 #ifdef COLI_CUDA
     /* PIPE2 (Inc.2a): il residuo resta sul device del layer, saltando tra le schede
-     * ai confini di layer. x host diventa STALE finche' la residenza e' attiva. */
+     * ai confini di layer. x host diventa STALE finche' la residenza e' attiva.
+     *
+     * S threshold is device-count-dependent (#273): on a single GPU the resident
+     * stream wins at S=1 (evicts the CPU round-trips that dominate small-batch
+     * decode — +49% on a 5070 Ti). With layers sharded across multiple GPUs each
+     * resident forward crosses P2P per layer group, and at one token per forward
+     * those hops don't amortize — A/B on 6x5090 showed S=1 is a wash there. So:
+     * single-GPU engages at S=1, multi-GPU keeps the original S>=8 prefill gate.
+     * COLI_CUDA_PIPE_S_MIN overrides for anyone who wants to measure. */
     float *x_dev=NULL; int x_dev_on=-1;
     size_t xb=(size_t)S*(size_t)D*4;
-    int pipe2 = g_cuda_pipe>=2 && !kvs && S>=8 && g_cuda_enabled && c->kv_lora<=512 &&
+    int pipe_s_min = getenv("COLI_CUDA_PIPE_S_MIN") ? atoi(getenv("COLI_CUDA_PIPE_S_MIN"))
+                                                     : (g_cuda_ndev<=1 ? 1 : 8);
+    int pipe2 = g_cuda_pipe>=2 && !kvs && S>=pipe_s_min && g_cuda_enabled && c->kv_lora<=512 &&
                 !(m->has_dsa && pos_base+S>c->index_topk);
 #endif
     for(int i=0;i<c->n_layers;i++){
@@ -4386,6 +4409,14 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
         for(int i=0;i<4;i++) printf("  %-42s %5.1f%%  (%lld/%lld)\n", nm[i],
             la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);
     }
+    /* TOKENS=1: dump the generated token ids (newline-separated) to stderr,
+     * for exact A/B comparison across decode paths (e.g. resident vs CPU).
+     * The ids are all[np .. np+produced-1]. */
+    if(getenv("TOKENS") && atoi(getenv("TOKENS"))){
+        fprintf(stderr,"[TOKENS] %d generated:",produced);
+        for(int i=np;i<np+produced;i++) fprintf(stderr," %d",all[i]);
+        fprintf(stderr,"\n");
+    }
     free(pids); free(all);
     usage_save(m);
 }
@@ -5421,7 +5452,8 @@ static double kv_pool_bytes(Model *m, int max_ctx){
 static double expert_avail(Model *m, double ram_gb, int ebits, int max_ctx){
     Cfg *c=&m->c; int64_t eb=expert_bytes_probe(m,ebits);
     if(ram_gb<=0){ ram_gb=g_mem_avail_boot*0.88; if(ram_gb<4) ram_gb=8; }
-    double slack = 1.2e9 + 2.5e9 + 64.0*(double)eb
+    double ws_b = (g_expert_budget>0 && g_expert_budget<64) ? (double)(g_expert_budget+4)*(double)eb : 64.0*(double)eb;
+    double slack = 1.2e9 + 2.5e9 + ws_b
         + kv_pool_bytes(m,max_ctx)
         + (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;
     return ram_gb*1e9 - (double)m->resident_bytes - slack;
@@ -5443,11 +5475,22 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
      *  KV cache a max_ctx, kvb_all della ricostruzione k/v in attention,
      *  attivazioni+logits+overhead ~1.2 GB */
     double ws_b  = 64.0*(double)eb;
+    /* Under EXPERT_BUDGET, the block-of-64 working set is capped at budget experts
+     * per layer — only ws[0..budget-1] are populated, not all 64. The 64×eb reserve
+     * overcounts by 16x at budget=4, starving the LRU cache (cap 3 instead of 4).
+     * Cap=4 matches budget=4, eliminating LRU thrashing that causes excessive disk
+     * re-reads. Clamp ws_b to the actual budget (min 8 for non-budgeted / prefill). */
+    if(g_expert_budget>0 && g_expert_budget<64) ws_b = (double)(g_expert_budget+4) * (double)eb;
     double kv_b  = kv_pool_bytes(m,max_ctx);
     double kvb_b = (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;
-    /* RISERVA PAGE-CACHE (misurato 2026-07-06): strangolarla fa crollare le pread
+    /* RISERVA PAGE-CACHE (misurato 2026-07-06 su Linux): strangolarla fa crollare
-     * buffered da ~800 a ~180 MB/s — gli ultimi GB di LRU rendono MENO di quanto
+     * le pread buffered da ~800 a ~180 MB/s — gli ultimi GB di LRU rendono MENO di
-     * costino in banda disco persa. 2.5 GB restano SEMPRE al kernel. */
+     * quanto costino in banda disco persa. 2.5 GB restano SEMPRE al kernel.
+     * NOTE: tested removing this under Windows+DIRECT (it should be dead weight when
+     * O_DIRECT bypasses the buffer cache). Result: cap went 4->5 but RSS hit 24 GB
+     * on a 32 GB machine, causing memory pressure that DROPPED the hit rate (73%->57%)
+     * and slowed decode (1.03->0.83 tok/s). The reserve is a legitimate safety margin
+     * for OS + CUDA + file metadata, not just buffered pread throughput. Keep it. */
     double pc_b  = 2.5e9;
     double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b;
     double avail = ram_gb*1e9 - (double)m->resident_bytes - slack;

issue_diskio.md

@@ -240,3 +240,70 @@ different cache-budget model that excludes mapped-file pages — left as future
 - [microsoft/Windows-Dev-Performance#108 — PrefetchVirtualMemory inconsistency](https://github.com/microsoft/Windows-Dev-Performance/issues/108)
 - [llama.cpp #18758 — mmap faster than O_DIRECT for MoE (Linux)](https://github.com/ggml-org/llama.cpp/discussions/18758)
 - [HN#35426679 — Why MMAP in llama.cpp hides true memory usage](https://news.ycombinator.com/item?id=35426679)
+
+---
+
+# CACHE_ROUTE: miss elimination via cache-aware routing (2026-07-15)
+
+## The breakthrough
+
+Adding `CACHE_ROUTE=1 ROUTE_J=2 ROUTE_M=12` to the optimized stack pushed
+throughput to **1.41 tok/s** (4.3× over stock) by directly reducing the
+miss rate from 27% to 17%.
+
+## How it works
+
+The engine's `CACHE_ROUTE` feature (paper: max-rank routing, arXiv 2412.00099)
+steers the MoE router to prefer experts that are already cache-resident:
+
+- `ROUTE_J=2`: keep the top-2 true router picks (always, even if uncached)
+- `ROUTE_M=12`: fill the remaining 2 of 4 slots with the highest-ranked experts
+  that are ALREADY in the LRU cache (from the top-12 candidates)
+
+This guarantees 2 of 4 expert slots per layer per token are cache hits. The
+`route_agree=94.6%` metric confirms minimal quality cost — 94.6% of cache-steered
+picks match the true top-K the router would have chosen.
+
+## Why this is the right fix for the miss problem
+
+Analysis of route traces showed GLM-5.2's routing is nearly uniform — 75 tokens
+use 237 of 256 experts per layer, with the top-4 capturing only 4.3% of selections.
+This means:
+
+- PIN (hot-expert pre-loading) is ineffective — there are no hot experts
+- PILOT_REAL prefetch can't keep up under the fast pipe2 GPU pipeline
+- A bigger cache helps marginally but can't cover 237 unique experts per layer
+- CACHE_ROUTE is the only lever that reduces misses without more RAM or faster disk
+
+## Results (pipe2 + full stack + ws_b fix, budget=4, RAM_GB=28)
+
+| metric | without CACHE_ROUTE | with CACHE_ROUTE |
+|---|---|---|
+| tok/s | 1.03 | **1.41** (+37%) |
+| hit rate | 73% | **83%** |
+| expert-disk | 12.4s | **8.5s** (−31%) |
+| decode | 31.0s | **22.7s** (−27%) |
+| route_agree | n/a | 94.6% |
+
+## Full optimization journey
+
+| step | tok/s | hit% | disk |
+|---|---|---|---|
+| stock budget=4 | 0.33 | 9% | 65.9s |
+| + disk stack | 0.63 | 72% | 21.5s |
+| + CUDA dense+attn | 0.72 | 75% | 18.4s |
+| + pipe2 GPU pipeline | 0.85 | 57% | 18.2s |
+| + ws_b cache fix | 1.03 | 73% | 12.4s |
+| **+ CACHE_ROUTE** | **1.41** | **83%** | **8.5s** |
+
+**4.3× total speedup.** Disk I/O reduced 7.7× (65.9s → 8.5s).
+
+## Recommended config
+
+```
+EXPERT_BUDGET=4 PIPE=1 RAM_GB=28 PILOT_REAL=1 DIRECT=1
+COLI_CUDA=1 CUDA_DENSE=1 COLI_CUDA_ATTN=1 COLI_CUDA_PIPE=2 CUDA_EXPERT_GB=0
+CACHE_ROUTE=1 ROUTE_J=2 ROUTE_M=12
+```
+
+CACHE_ROUTE is an existing engine feature (no code change) — opt-in via env vars.