url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge #474: LFRU-aware pilot eviction guard — a speculation never evicts a warm demand expert

Browse Source 
Addresses the eviction-thrash half of #441 (cdhdt's controlled A/B: speculative pilot loads evicted just-demand-loaded warm experts → +9-18% bytes for +0.5-0.7pt hit). The pilot now evicts a resident only if the predicted expert is hotter than the LRU victim by tier_pick_lfru's hysteresis (25%+4 freq), else drops the speculation. Applied to both pilot_realload and pilot_uring_batch; demand-promotion and rss_guard untouched. Cache placement only → output byte-identical (author proved sha256-identical on full GLM-5.2 744B cold-streamed: PILOT off / guard ON / guard OFF all 43d9126f…). Default on, PILOT_EVICT_GUARD=0 for the A/B. CI 8/8; verified locally clean build + token-exact. Thanks @cdhdt.

Note: this fixes eviction quality; whether the multi-worker pilot becomes a net win still needs the good-hardware A/B tracked in #441.
This commit is contained in:
Vincenzo Fornaro
2026-07-21 02:00:39 +02:00
committed by GitHub
parent 1b8f3076b6 eaa68f4005
commit 4aca05900f
1 changed files with 24 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/colibri.c
+24 -1
View File
@@ -733,6 +733,11 @@ static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer de
static int g_pilot_two=0; /* PILOT_TWO=1: two-step prefetch — before running L+1's router,
* approximate MoE(L) using only the shared expert (resident, no disk)
* and add it to the state. Trades 3 small matmuls for +2.3% recall. */
static int g_pilot_evict_guard=1;/* PILOT_EVICT_GUARD=0 -> old behavior (a speculation evicts the plain LRU).
* Default ON: a speculative pilot load may evict a RESIDENT expert only if the
* predicted expert is historically HOTTER than the victim (same LFRU hysteresis as
* tier_pick_lfru); otherwise it drops the speculation rather than thrash a warm
* demand-loaded expert. Cache placement only -> output byte-identical. (#441) */
/* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti:
* 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer;
* il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe()
@@ -3427,7 +3432,16 @@ static void pilot_realload(Model *m, int layer, int eid){
for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; }
int slot,isnew; /* cresci se c'e' posto, altrimenti LRU */
if(nn<m->ecap){ slot=nn; isnew=1; }
else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0; }
else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0;
/* LFRU eviction guard (#441): a speculation must not drop a WARM demand-loaded expert.
* Evict only if the predicted expert's history is hotter than the victim by tier_pick_lfru's
* hysteresis (25% + 4 freq); else drop the speculation. Cache placement only -> output unchanged. */
if(g_pilot_evict_guard && m->eheat && m->elast && Sl[lru].eid>=0){
int vid=Sl[lru].eid;
uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);
uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);
if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);
pthread_mutex_unlock(&g_pilot_mx); return; } } }
ESlot *dst=&Sl[slot];
dst->eid=-1; /* nascondi dagli scan-hint mentre carica */
g_pilot_inflight[layer]++;
@@ -3478,6 +3492,14 @@ static void pilot_uring_batch(Model *m){
if(Sl[z].eid< -1) continue; /* URING reservation in flight */
if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;
}
/* LFRU eviction guard (#441): don't drop a warm resident for a speculation (see pilot_realload) */
if(slot>=0 && Sl[slot].eid>=0 && g_pilot_evict_guard && m->eheat && m->elast){
int vid=Sl[slot].eid;
uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);
uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);
if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);
pthread_mutex_unlock(&g_pilot_mx); continue; }
}
}
if(slot<0){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; }
ESlot *dst=&Sl[slot];
@@ -6105,6 +6127,7 @@ int main(int argc, char **argv){
if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */
g_pilot_two = getenv("PILOT_TWO")?atoi(getenv("PILOT_TWO")):0; /* 1 = two-step: shared-expert-corrected router prediction (+2.3% recall, 3 extra matmuls) */
if(g_pilot_two) g_pilot=1; /* PILOT_TWO implies PILOT active */
g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD")?atoi(getenv("PILOT_EVICT_GUARD")):1; /* 0 = old LRU eviction (A/B) */
/* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction).
* Under PILOT_REAL the speculative loads are REAL and create LRU eviction
* pressure, so at ~28% mispredict a large K thrashes the cache — default to 6