diff --git a/c/glm.c b/c/glm.c index 2ee12ed..1446606 100644 --- a/c/glm.c +++ b/c/glm.c @@ -196,6 +196,7 @@ typedef struct { uint64_t mtp_prop, mtp_acc; /* statistica acceptance */ int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */ uint64_t eclock, hits, miss, ereq; + uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */ uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes; uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */ uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ @@ -312,6 +313,7 @@ static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } typedef struct { double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; + uint64_t hit_pin,hit_ecache; } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; @@ -319,6 +321,7 @@ static void prof_base(Model *m, ProfBase *b){ b->attn=m->t_attn; b->head=m->t_head; b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; + b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; } @@ -3132,9 +3135,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int ESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0; for(int j=0;jpin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; } + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } + for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } @@ -4674,11 +4677,12 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; } double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ + uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " - "hit %.1f%% (%llu hit / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", + "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, g_mmap?"; COLI_MMAP=1: page cache may serve part":"", - hitp,(unsigned long long)dh,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, + hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, io_svc,io_w); fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); @@ -4724,7 +4728,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; } kv_alloc(m,nfull+2); float *logit=step(m,full,np-1,0); free(logit); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); ProfBase pb; prof_base(m,&pb); double t0=now_s(); int steps=0; @@ -4774,7 +4778,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ } prefill_t=now_s()-prefill_t; printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; m->n_emit=m->n_fw=0; g_last_repin=0; profile_reset(m); @@ -4787,8 +4791,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% | RSS %.2f GB", - np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb()); + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */ + np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, + tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb()); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -6535,8 +6540,9 @@ int main(int argc, char **argv){ double tot=m.hits+m.miss; printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n", g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit); - printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu) | RSS: %.2f GB | %.1f tok/s\n", - tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss, rss_gb(), n_new/dt); + printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n", + tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache, + (unsigned long long)m.miss, rss_gb(), n_new/dt); profile_print(&m,dt); if(g_prof) prof_report(&m,&pb,dt,n_new,stdout); #ifdef COLI_CUDA