From 453d1401bae95c1b522ad255853f2d9fb9eea873 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Mon, 20 Jul 2026 05:15:45 +0800 Subject: [PATCH] pin: with CUDA_RELEASE_HOST the VRAM prefix must not consume the RAM pin budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pin_load computed npin from the RAM budget FIRST, then carved the VRAM-ranked prefix out of those npin slots. With CUDA_RELEASE_HOST the prefix's host slabs are freed right after upload — so on a multi-GPU host the top-ranked experts consumed the RAM budget without occupying RAM, and the CPU tier pinned only the leftovers. Measured on 6x RTX 5090 (251 GB): 9,280 VRAM + only 1,721 RAM pins (32.5 GB warm) on a box whose RAM fits ~10k more — the cold tail then paid disk on every token and the hit rate ceilinged at 99.0% forever. Move the VRAM budget estimate above the npin finalization and make the release-destined prefix ADDITIVE to the RAM-derived count. Same box, same env plus the fix: [PIN] placement: 9,280 VRAM + 10,176 RAM (192.5 GB warm) expert hit rate 100.0% (pin 100.0% + lru 0.0%) — disk 0 1,024-token greedy decode: 3.62 -> 6.21 tok/s (+72%) warm late segment (t=768-1024): 5.11 -> 5.73 tok/s (+12%) prefill: 10.4 -> 8.8 s The whole-run gain is the LRU warmup penalty disappearing (full residency from the first token); the late-segment gain is the steady ~0.9%-miss disk residue recovered. Non-release configs are untouched: prefix_est stays 0 and the arithmetic reduces to today's exactly. --- c/colibri.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/c/colibri.c b/c/colibri.c index 03eb878..f2e24b4 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -4774,6 +4774,29 @@ static void pin_load(Model *m, const char *statspath, double gb){ double avail=expert_avail(m,ram_env,m->ebits,est_ctx); npin=avail>0?(int)(avail/eb):0; } else npin=(int)(gb*1e9/eb); +#ifdef COLI_CUDA + /* The VRAM budget must be known BEFORE npin is finalized: with + * CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right + * after upload, so those slots must NOT consume the RAM pin budget. + * Before this, a 6-GPU host lost its top ~9k ranked experts from the RAM + * count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM + * on a box whose RAM fits ~11k — the cold tail then paid disk forever). */ + double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; + int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0; + double budget=g_cuda_expert_gb*1e9, safe_total=0; + if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; + if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ + prefix_est=(int)(budget/eb)+g_cuda_ndev; + npin+=prefix_est; /* additive: prefix RAM is returned after upload */ + } +#endif if(npin>n) npin=n; if(npin<1){ free(r); return; } int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */ @@ -4784,18 +4807,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i]; double t0=now_s(); #ifdef COLI_CUDA - double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; - int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0; - double budget=g_cuda_expert_gb*1e9, safe_total=0; - if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; - if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; } + if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; } #else int gpu_prefix=0; #endif