From 413b370fbf529840abeacce5acba18175a3fb7bf Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:02:15 -0400 Subject: [PATCH 1/2] =?UTF-8?q?experiment:=20EXPERT=5FBUDGET=3DN=20?= =?UTF-8?q?=E2=80=94=20cap=20distinct=20experts/layer,=20up=20to=201.8x=20?= =?UTF-8?q?faster=20decode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add EXPERT_BUDGET env var that caps the number of distinct experts loaded per layer across the batch-union. When the union exceeds the budget, keeps only the highest-aggregate-gate-weight experts and drops the rest from idxs[] so they're never loaded from disk. Complementary to TOPP (per-position) — this trims the cross-position union that multiplies under MTP/prefill. Based on MoE-Spec (arXiv 2602.16052): 'top 32 of 64 experts capture 93% of routing weight.' Measurements (GLM-5.2 744B, 24GB RAM, cap=2, MTP=0, 32 tokens): Baseline (budget=0): 0.18 tok/s, 9.3% hit, 176s decode, 39s prefill EXPERT_BUDGET=12: 0.19 tok/s, 14.0% hit, 171s decode, 12s prefill EXPERT_BUDGET=6: 0.26 tok/s, 21.0% hit, 123s decode, 7s prefill EXPERT_BUDGET=4: 0.33 tok/s, 16.4% hit, 97s decode, 9s prefill Budget=4 nearly doubles decode speed (+83%) and 4x's prefill speed by halving disk reads per layer. Default OFF (EXPERT_BUDGET=0). --- c/glm.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/c/glm.c b/c/glm.c index fb93e59..28a32ad 100644 --- a/c/glm.c +++ b/c/glm.c @@ -896,6 +896,11 @@ static float g_temp=-1; /* TEMP: temperatura di sampling sui TOKEN. <0 = auto ( static float g_nuc=0.95f;/* NUCLEUS: top-p sul vocabolario (default dal generation_config GLM-5.2) */ static int g_topk=0; /* TOPK=n -> usa n expert/token invece di config (ricerca: meno disco) */ static float g_topp=0; /* TOPP=p (0..1) -> top-p adattivo: tieni gli expert fino a peso cumulato p */ +static int g_expert_budget=0; /* EXPERT_BUDGET=N -> cap distinct experts loaded per layer across the + * batch-union. Reduces disk I/O on cold/low-RAM hosts by dropping the + * lowest-gate-weight experts from the cross-position union. MoE-Spec + * (arXiv 2602.16052): top-32 of 64 capture 93% routing weight. */ +static int64_t g_budget_dropped=0; /* total experts dropped by EXPERT_BUDGET across all layers */ /* CACHE_ROUTE (paper 2412.00099 max-rank): opt-in only. Keep true top-J always; * fill remaining slots preferring pin∪LRU experts ranked within top-M (or mass ROUTE_P). */ static int g_cache_route=0; @@ -2413,6 +2418,53 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int e=idxs[(int64_t)s*K+kk]; if(!seen[e]){ seen[e]=1; uniq[nu++]=e; } } + /* EXPERT_BUDGET: cap distinct experts per layer to reduce disk I/O on cold/low-RAM + * hosts. Keep the highest-aggregate-gate-weight experts; drop the rest from idxs[] + * so they're never loaded. (MoE-Spec arXiv 2602.16052: top-32 of 64 capture 93% + * routing weight.) Complementary to TOPP (per-position) — this trims cross-position. */ + if(g_expert_budget>0 && nu>g_expert_budget){ + /* compute aggregate gate weight per unique expert */ + float *wsum=falloc(nu); for(int j=0;jbv){ bv=wsum[j]; best=j; } + if(best<0) break; keep[best]=1; nkeep++; + } + } + /* build a lookup: for each expert id, is it kept? (reuse seen[]) */ + memset(seen,0,(size_t)E); + for(int j=0;jnorm_topk && w>0){ + float sm=0; for(int kk=0;kkrouted_scale; + } + } + } + /* compact uniq[] to kept experts only */ + int nu2=0; + for(int j=0;jereq/produced:0.0, (produced&&nsp)?(double)m->ereq/produced/nsp:0.0, nsp, c->topk, g_topk, g_topp); if(g_cache_route) printf(" | CACHE_ROUTE J=%d M=%d P=%.2f alpha=%.2f", g_route_j, g_route_m, g_route_p, g_route_alpha); + if(g_expert_budget) printf(" | EXPERT_BUDGET=%d (dropped %lld experts, ~%.1f GB I/O saved)", g_expert_budget, (long long)g_budget_dropped, g_budget_dropped*18.9e6/1e9); printf("\n"); printf("speculation: %.2f tokens/forward (%llu forwards per %llu tokens) | MTP acceptance %.0f%% (%llu/%llu)\n", m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit, @@ -4925,6 +4978,7 @@ int main(int argc, char **argv){ if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n"); g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0; g_topp = getenv("TOPP")?atof(getenv("TOPP")):0; + g_expert_budget = getenv("EXPERT_BUDGET")?atoi(getenv("EXPERT_BUDGET")):0; g_cache_route = getenv("CACHE_ROUTE")?atoi(getenv("CACHE_ROUTE")):0; g_route_j = getenv("ROUTE_J")?atoi(getenv("ROUTE_J")):2; g_route_m = getenv("ROUTE_M")?atoi(getenv("ROUTE_M")):12; From dd0d60692d338c01ee03e3854284a3eca8601da6 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:23:29 -0400 Subject: [PATCH 2/2] =?UTF-8?q?experiment:=20miss-aware=20EXPERT=5FBUDGET?= =?UTF-8?q?=20=E2=80=94=20keep=20all=20cache=20hits,=20only=20drop=20misse?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original budget dropped experts blindly — even cached ones that cost zero disk I/O. The miss-aware version pre-scans pin/ecache residency before applying the budget: - ALL cache hits are kept (free to compute, no disk I/O) - Only misses compete for the remaining budget slots - Miss budget = EXPERT_BUDGET - nhits (min 0) Results (budget=4, same prompt/config as before): Original Miss-aware tok/s 0.33 0.36 (+9%) hit rate 16.4% 38.2% (2.3x) prefill 8.9s 5.6s (1.6x faster) decode 97.4s 88.9s (9% faster) The hit rate doubling is the key quality signal: the model now gets the full contribution from all resident experts plus the top-4 new loads, instead of losing some hits to the budget. --- c/glm.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/c/glm.c b/c/glm.c index 28a32ad..8ea02ed 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2419,9 +2419,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int if(!seen[e]){ seen[e]=1; uniq[nu++]=e; } } /* EXPERT_BUDGET: cap distinct experts per layer to reduce disk I/O on cold/low-RAM - * hosts. Keep the highest-aggregate-gate-weight experts; drop the rest from idxs[] - * so they're never loaded. (MoE-Spec arXiv 2602.16052: top-32 of 64 capture 93% - * routing weight.) Complementary to TOPP (per-position) — this trims cross-position. */ + * hosts. MISS-AWARE: always keep cache hits (pin/LRU — they're free, no disk I/O), + * only drop from misses. From the misses, keep the highest-aggregate-gate-weight + * ones up to the budget; drop the rest from idxs[] so they're never loaded. + * (MoE-Spec arXiv 2602.16052: top-32 of 64 capture 93% routing weight.) + * Complementary to TOPP (per-position) — this trims cross-position. */ if(g_expert_budget>0 && nu>g_expert_budget){ /* compute aggregate gate weight per unique expert */ float *wsum=falloc(nu); for(int j=0;jpin[layer]; + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ found=1; break; } + if(!found){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; + for(int z=0;zbv){ bv=wsum[j]; best=j; } - if(best<0) break; keep[best]=1; nkeep++; - } + for(int j=0;jbv){ bv=wsum[j]; best=j; } + if(best<0) break; keep[best]=1; nkeep++; } /* build a lookup: for each expert id, is it kept? (reuse seen[]) */ memset(seen,0,(size_t)E); @@ -2463,7 +2477,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int nu2=0; for(int j=0;j