url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pin: with CUDA_RELEASE_HOST the VRAM prefix must not consume the RAM pin budget

Browse Source 
pin_load computed npin from the RAM budget FIRST, then carved the
VRAM-ranked prefix out of those npin slots. With CUDA_RELEASE_HOST the
prefix's host slabs are freed right after upload — so on a multi-GPU
host the top-ranked experts consumed the RAM budget without occupying
RAM, and the CPU tier pinned only the leftovers. Measured on 6x RTX 5090
(251 GB): 9,280 VRAM + only 1,721 RAM pins (32.5 GB warm) on a box whose
RAM fits ~10k more — the cold tail then paid disk on every token and the
hit rate ceilinged at 99.0% forever.

Move the VRAM budget estimate above the npin finalization and make the
release-destined prefix ADDITIVE to the RAM-derived count. Same box,
same env plus the fix:

    [PIN] placement: 9,280 VRAM + 10,176 RAM (192.5 GB warm)
    expert hit rate 100.0% (pin 100.0% + lru 0.0%)  — disk 0

    1,024-token greedy decode: 3.62 -> 6.21 tok/s  (+72%)
    warm late segment (t=768-1024): 5.11 -> 5.73 tok/s  (+12%)
    prefill: 10.4 -> 8.8 s

The whole-run gain is the LRU warmup penalty disappearing (full
residency from the first token); the late-segment gain is the steady
~0.9%-miss disk residue recovered. Non-release configs are untouched:
prefix_est stays 0 and the arithmetic reduces to today's exactly.
This commit is contained in:
ZacharyZcR
2026-07-20 05:15:45 +08:00
parent 61004dcb84
commit 453d1401ba
1 changed files with 24 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/colibri.c
+24 -12
View File
@@ -4774,6 +4774,29 @@ static void pin_load(Model *m, const char *statspath, double gb){
double avail=expert_avail(m,ram_env,m->ebits,est_ctx);
npin=avail>0?(int)(avail/eb):0;
} else npin=(int)(gb*1e9/eb);
#ifdef COLI_CUDA
/* The VRAM budget must be known BEFORE npin is finalized: with
* CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right
* after upload, so those slots must NOT consume the RAM pin budget.
* Before this, a 6-GPU host lost its top ~9k ranked experts from the RAM
* count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM
* on a box whose RAM fits ~11k — the cold tail then paid disk forever). */
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0;
double budget=g_cuda_expert_gb*1e9, safe_total=0;
if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;i<g_cuda_ndev;i++){
size_t free_b=0,total_b=0;
if(coli_cuda_mem_info(g_cuda_devices[i],&free_b,&total_b)){
remaining[i]=(double)free_b-(double)g_cuda_dense_projected[i]-2e9;
if(remaining[i]<0) remaining[i]=0; safe_total+=remaining[i];
}
}
if(g_cuda_expert_auto||budget>safe_total) budget=safe_total;
if(g_cuda_enabled&&g_cuda_release_host&&budget>0){
prefix_est=(int)(budget/eb)+g_cuda_ndev;
npin+=prefix_est; /* additive: prefix RAM is returned after upload */
}
#endif
if(npin>n) npin=n;
if(npin<1){ free(r); return; }
int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */
@@ -4784,18 +4807,7 @@ static void pin_load(Model *m, const char *statspath, double gb){
for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i];
double t0=now_s();
#ifdef COLI_CUDA
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0;
double budget=g_cuda_expert_gb*1e9, safe_total=0;
if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;i<g_cuda_ndev;i++){
size_t free_b=0,total_b=0;
if(coli_cuda_mem_info(g_cuda_devices[i],&free_b,&total_b)){
remaining[i]=(double)free_b-(double)g_cuda_dense_projected[i]-2e9;
if(remaining[i]<0) remaining[i]=0; safe_total+=remaining[i];
}
}
if(g_cuda_expert_auto||budget>safe_total) budget=safe_total;
if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; }
if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; }
#else
int gpu_prefix=0;
#endif